diff options
Diffstat (limited to '')
64 files changed, 20195 insertions, 0 deletions
diff --git a/toolkit/components/translations/README.md b/toolkit/components/translations/README.md new file mode 100644 index 0000000000..0ebabee6cf --- /dev/null +++ b/toolkit/components/translations/README.md @@ -0,0 +1,3 @@ +# Firefox Translations (Work in Progress) + +This folder contains the work in progress integration of Firefox Translations directly into Firefox. See Bug 971044. diff --git a/toolkit/components/translations/TranslationsTelemetry.sys.mjs b/toolkit/components/translations/TranslationsTelemetry.sys.mjs new file mode 100644 index 0000000000..f01ee1d144 --- /dev/null +++ b/toolkit/components/translations/TranslationsTelemetry.sys.mjs @@ -0,0 +1,379 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Switch this to true to help with local debugging. +// Not intended for use in production. +const LOG_TELEMETRY_EVENTS_FOR_DEBUGGING = false; + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "TranslationsTelemetry", + }); +}); + +/** + * Telemetry functions for Translations actors + */ +export class TranslationsTelemetry { + /** + * A cached value to hold the current flowId. + */ + static #flowId = null; + + /** + * Logs the telemetry event to the console if enabled by + * the LOG_TELEMETRY_EVENTS constant. + */ + static logEventToConsole(eventInfo) { + if (!LOG_TELEMETRY_EVENTS_FOR_DEBUGGING) { + return; + } + lazy.console.debug( + `${ + Panel.isFirstUserInteraction() ? "[FIRST OPEN] " : "" + }flowId(${TranslationsTelemetry.getOrCreateFlowId()}): ${eventInfo}` + ); + } + + /** + * Telemetry functions for the Translations panel. + * @returns {Panel} + */ + static panel() { + return Panel; + } + + /** + * Forces the creation of a new Translations telemetry flowId and returns it. + * @returns {string} + */ + static createFlowId() { + const flowId = crypto.randomUUID(); + TranslationsTelemetry.#flowId = flowId; + return flowId; + } + + /** + * Returns a Translations telemetry flowId by retrieving the cached value + * if available, or creating a new one otherwise. + * @returns {string} + */ + static getOrCreateFlowId() { + // If we have the flowId cached, return it. + if (TranslationsTelemetry.#flowId) { + return TranslationsTelemetry.#flowId; + } + + // If no flowId exists, create one. + return TranslationsTelemetry.createFlowId(); + } + + /** + * Records a telemetry event when full page translation fails. + * + * @param {string} errorMessage + */ + static onError(errorMessage) { + Glean.translations.errorRate.addToNumerator(1); + Glean.translations.error.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + reason: errorMessage, + }); + TranslationsTelemetry.logEventToConsole("onError"); + } + + /** + * Records a telemetry event when a translation request is sent. + * + * @param {object} data + * @param {string} data.docLangTag + * @param {string} data.fromLanguage + * @param {string} data.toLanguage + * @param {string} data.topPreferredLanguage + * @param {boolean} data.autoTranslate + */ + static onTranslate(data) { + const { + docLangTag, + fromLanguage, + toLanguage, + autoTranslate, + topPreferredLanguage, + } = data; + Glean.translations.requestsCount.add(1); + Glean.translations.translationRequest.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + from_language: fromLanguage, + to_language: toLanguage, + auto_translate: autoTranslate, + document_language: docLangTag, + top_preferred_language: topPreferredLanguage, + }); + TranslationsTelemetry.logEventToConsole( + `onTranslate[page(${docLangTag}), preferred(${topPreferredLanguage})](${ + autoTranslate ? "auto" : "manual" + }, ${fromLanguage}-${toLanguage})` + ); + } + + static onRestorePage() { + Glean.translations.restorePage.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onRestorePage"); + } +} + +/** + * Telemetry functions for the Translations panel + */ +class Panel { + /** + * A value to retain whether this is the user's first time + * interacting with the translations panel. It is propagated + * to all events. + * + * This value is set only through the onOpen() function. + */ + static #isFirstUserInteraction = false; + + /** + * True if this is the user's first time interacting with the + * Translations panel, otherwise false. + * + * @returns {boolean} + */ + static isFirstUserInteraction() { + return Panel.#isFirstUserInteraction; + } + + /** + * Records a telemetry event when the translations panel is opened. + * + * @param {object} data + * @param {string} data.viewName + * @param {string} data.docLangTag + * @param {boolean} data.autoShow + * @param {boolean} data.maintainFlow + * @param {boolean} data.openedFromAppMenu + * @param {boolean} data.isFirstUserInteraction + */ + static onOpen({ + viewName = null, + autoShow = null, + docLangTag = null, + maintainFlow = false, + openedFromAppMenu = false, + isFirstUserInteraction = null, + }) { + if (isFirstUserInteraction !== null || !maintainFlow) { + Panel.#isFirstUserInteraction = isFirstUserInteraction ?? false; + } + Glean.translationsPanel.open.record({ + flow_id: maintainFlow + ? TranslationsTelemetry.getOrCreateFlowId() + : TranslationsTelemetry.createFlowId(), + auto_show: autoShow, + view_name: viewName, + document_language: docLangTag, + first_interaction: Panel.isFirstUserInteraction(), + opened_from: openedFromAppMenu ? "appMenu" : "translationsButton", + }); + TranslationsTelemetry.logEventToConsole( + `onOpen[${autoShow ? "auto" : "manual"}, ${ + openedFromAppMenu ? "appMenu" : "translationsButton" + }, ${viewName ? viewName : "NULL"}](${docLangTag})` + ); + } + + static onClose() { + Glean.translationsPanel.close.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onClose"); + } + + static onOpenFromLanguageMenu() { + Glean.translationsPanel.openFromLanguageMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onOpenFromLanguageMenu"); + } + + static onChangeFromLanguage(langTag) { + Glean.translationsPanel.changeFromLanguage.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + language: langTag, + }); + TranslationsTelemetry.logEventToConsole(`onChangeFromLanguage(${langTag})`); + } + + static onCloseFromLanguageMenu() { + Glean.translationsPanel.closeFromLanguageMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onCloseFromLanguageMenu"); + } + + static onOpenToLanguageMenu() { + Glean.translationsPanel.openToLanguageMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onOpenToLanguageMenu"); + } + + static onChangeToLanguage(langTag) { + Glean.translationsPanel.changeToLanguage.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + language: langTag, + }); + TranslationsTelemetry.logEventToConsole(`onChangeToLanguage(${langTag})`); + } + + static onCloseToLanguageMenu() { + Glean.translationsPanel.closeToLanguageMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onCloseToLanguageMenu"); + } + + static onOpenSettingsMenu() { + Glean.translationsPanel.openSettingsMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onOpenSettingsMenu"); + } + + static onCloseSettingsMenu() { + Glean.translationsPanel.closeSettingsMenu.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onCloseSettingsMenu"); + } + + static onCancelButton() { + Glean.translationsPanel.cancelButton.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onCancelButton"); + } + + static onChangeSourceLanguageButton() { + Glean.translationsPanel.changeSourceLanguageButton.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onChangeSourceLanguageButton"); + } + + static onDismissErrorButton() { + Glean.translationsPanel.dismissErrorButton.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onDismissErrorButton"); + } + + static onRestorePageButton() { + Glean.translationsPanel.restorePageButton.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onRestorePageButton"); + } + + static onTranslateButton() { + Glean.translationsPanel.translateButton.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onTranslateButton"); + } + + static onAlwaysOfferTranslations(toggledOn) { + Glean.translationsPanel.alwaysOfferTranslations.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + toggled_on: toggledOn, + }); + TranslationsTelemetry.logEventToConsole( + `[${toggledOn ? "✔" : "x"}] onAlwaysOfferTranslations` + ); + } + + static onAlwaysTranslateLanguage(langTag, toggledOn) { + Glean.translationsPanel.alwaysTranslateLanguage.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + language: langTag, + toggled_on: toggledOn, + }); + TranslationsTelemetry.logEventToConsole( + `[${toggledOn ? "✔" : "x"}] onAlwaysTranslateLanguage(${langTag})` + ); + } + + static onNeverTranslateLanguage(langTag, toggledOn) { + Glean.translationsPanel.neverTranslateLanguage.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + language: langTag, + toggled_on: toggledOn, + }); + TranslationsTelemetry.logEventToConsole( + `[${toggledOn ? "✔" : "x"}] onNeverTranslateLanguage(${langTag})` + ); + } + + static onNeverTranslateSite(toggledOn) { + Glean.translationsPanel.neverTranslateSite.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + toggled_on: toggledOn, + }); + TranslationsTelemetry.logEventToConsole( + `[${toggledOn ? "✔" : "x"}] onNeverTranslateSite` + ); + } + + static onManageLanguages() { + Glean.translationsPanel.manageLanguages.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onManageLanguages"); + } + + static onAboutTranslations() { + Glean.translationsPanel.aboutTranslations.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onAboutTranslations"); + } + + static onLearnMoreLink() { + Glean.translationsPanel.learnMore.record({ + flow_id: TranslationsTelemetry.getOrCreateFlowId(), + first_interaction: Panel.isFirstUserInteraction(), + }); + TranslationsTelemetry.logEventToConsole("onLearnMoreLink"); + } +} diff --git a/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs new file mode 100644 index 0000000000..c501c1b0cd --- /dev/null +++ b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs @@ -0,0 +1,264 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +ChromeUtils.defineESModuleGetters(lazy, { + LanguageDetector: + "resource://gre/modules/translation/LanguageDetector.sys.mjs", +}); + +/** + * @typedef {import("./TranslationsChild.sys.mjs").TranslationsEngine} TranslationsEngine + * @typedef {import("./TranslationsChild.sys.mjs").SupportedLanguages} SupportedLanguages + */ + +/** + * The AboutTranslationsChild is responsible for coordinating what privileged APIs + * are exposed to the un-privileged scope of the about:translations page. + */ +export class AboutTranslationsChild extends JSWindowActorChild { + /** + * The translations engine uses text translations by default in about:translations, + * but it can be changed to translate HTML by setting this pref to true. This is + * useful for manually testing HTML translation behavior, but is not useful to surface + * as a user-facing feature. + * + * @type {bool} + */ + #isHtmlTranslation = Services.prefs.getBoolPref( + "browser.translations.useHTML" + ); + + handleEvent(event) { + if (event.type === "DOMDocElementInserted") { + this.#exportFunctions(); + } + + if ( + event.type === "DOMContentLoaded" && + Services.prefs.getBoolPref("browser.translations.enable") + ) { + this.#sendEventToContent({ type: "enable" }); + } + } + + receiveMessage({ name, data }) { + switch (name) { + case "AboutTranslations:SendTranslationsPort": + const { fromLanguage, toLanguage, port } = data; + const transferables = [port]; + this.contentWindow.postMessage( + { + type: "GetTranslationsPort", + fromLanguage, + toLanguage, + port, + }, + "*", + transferables + ); + break; + default: + throw new Error("Unknown AboutTranslations message: " + name); + } + } + + /** + * @param {object} detail + */ + #sendEventToContent(detail) { + this.contentWindow.dispatchEvent( + new this.contentWindow.CustomEvent("AboutTranslationsChromeToContent", { + detail: Cu.cloneInto(detail, this.contentWindow), + }) + ); + } + + /** + * @returns {TranslationsChild} + */ + #getTranslationsChild() { + const child = this.contentWindow.windowGlobalChild.getActor("Translations"); + if (!child) { + throw new Error("Unable to find the TranslationsChild"); + } + return child; + } + + /** + * A privileged promise can't be used in the content page, so convert a privileged + * promise into a content one. + * + * @param {Promise<any>} promise + * @returns {Promise<any>} + */ + #convertToContentPromise(promise) { + return new this.contentWindow.Promise((resolve, reject) => + promise.then(resolve, error => { + let contentWindow; + try { + contentWindow = this.contentWindow; + } catch (error) { + // The content window is no longer available. + reject(); + return; + } + // Create an error in the content window, if the content window is still around. + let message = "An error occured in the AboutTranslations actor."; + if (typeof error === "string") { + message = error; + } + if (typeof error?.message === "string") { + message = error.message; + } + if (typeof error?.stack === "string") { + message += `\n\nOriginal stack:\n\n${error.stack}\n`; + } + + reject(new contentWindow.Error(message)); + }) + ); + } + + /** + * Export any of the child functions that start with "AT_" to the unprivileged content + * page. This restricts the security capabilities of the the content page. + */ + #exportFunctions() { + const window = this.contentWindow; + + const fns = [ + "AT_log", + "AT_logError", + "AT_getAppLocale", + "AT_getSupportedLanguages", + "AT_isTranslationEngineSupported", + "AT_isHtmlTranslation", + "AT_createTranslationsPort", + "AT_identifyLanguage", + "AT_getScriptDirection", + ]; + for (const name of fns) { + Cu.exportFunction(this[name].bind(this), window, { defineAs: name }); + } + } + + /** + * Log messages if "browser.translations.logLevel" is set to "All". + * + * @param {...any} args + */ + AT_log(...args) { + lazy.console.log(...args); + } + + /** + * Report an error to the console. + * + * @param {...any} args + */ + AT_logError(...args) { + lazy.console.error(...args); + } + + /** + * Returns the app's locale. + * + * @returns {Intl.Locale} + */ + AT_getAppLocale() { + return Services.locale.appLocaleAsBCP47; + } + + /** + * Wire this function to the TranslationsChild. + * + * @returns {Promise<SupportedLanguages>} + */ + AT_getSupportedLanguages() { + return this.#convertToContentPromise( + this.sendQuery("AboutTranslations:GetSupportedLanguages").then(data => + Cu.cloneInto(data, this.contentWindow) + ) + ); + } + + /** + * Does this device support the translation engine? + * @returns {Promise<boolean>} + */ + AT_isTranslationEngineSupported() { + return this.#convertToContentPromise( + this.sendQuery("AboutTranslations:IsTranslationsEngineSupported") + ); + } + + /** + * Expose the #isHtmlTranslation property. + * + * @returns {bool} + */ + AT_isHtmlTranslation() { + return this.#isHtmlTranslation; + } + + /** + * Requests a port to the TranslationsEngine process. An engine will be created on + * the fly for translation requests through this port. This port is unique to its + * language pair. In order to translate a different language pair, a new port must be + * created for that pair. The lifecycle of the engine is managed by the + * TranslationsEngine. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {void} + */ + AT_createTranslationsPort(fromLanguage, toLanguage) { + this.sendAsyncMessage("AboutTranslations:GetTranslationsPort", { + fromLanguage, + toLanguage, + }); + } + + /** + * Attempts to identify the human language in which the message is written. + * + * @param {string} message + * @returns {Promise<{ langTag: string, confidence: number }>} + */ + AT_identifyLanguage(message) { + return this.#convertToContentPromise( + lazy.LanguageDetector.detectLanguage(message).then(data => + Cu.cloneInto( + // This language detector reports confidence as a boolean instead of + // a percentage, so we need to map the confidence to 0.0 or 1.0. + { langTag: data.language, confidence: data.confident ? 1.0 : 0.0 }, + this.contentWindow + ) + ) + ); + } + + /** + * TODO - Remove this when Intl.Locale.prototype.textInfo is available to + * content scripts. + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/textInfo + * https://bugzilla.mozilla.org/show_bug.cgi?id=1693576 + * + * @param {string} locale + * @returns {string} + */ + AT_getScriptDirection(locale) { + return Services.intl.getScriptDirection(locale); + } +} diff --git a/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs b/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs new file mode 100644 index 0000000000..4680dbeef5 --- /dev/null +++ b/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + TranslationsParent: "resource://gre/actors/TranslationsParent.sys.mjs", +}); + +/** + * This parent is blank because the Translations actor handles most of the features + * needed in AboutTranslations. + */ +export class AboutTranslationsParent extends JSWindowActorParent { + #isDestroyed = false; + + didDestroy() { + this.#isDestroyed = true; + } + + async receiveMessage({ name, data }) { + switch (name) { + case "AboutTranslations:GetTranslationsPort": { + const { fromLanguage, toLanguage } = data; + const engineProcess = await lazy.TranslationsParent.getEngineProcess(); + if (this.#isDestroyed) { + return undefined; + } + const { port1, port2 } = new MessageChannel(); + engineProcess.actor.startTranslation( + fromLanguage, + toLanguage, + port1, + this.browsingContext.top.embedderElement.innerWindowID + ); + + // At the time of writing, you can't return a port via the `sendQuery` API, + // so results can't just be returned. The `sendAsyncMessage` method must be + // invoked. Additionally, in the AboutTranslationsChild, the port must + // be transfered to the content page with `postMessage`. + this.sendAsyncMessage( + "AboutTranslations:SendTranslationsPort", + { + fromLanguage, + toLanguage, + port: port2, + }, + [port2] // Mark the port as transerable. + ); + return undefined; + } + case "AboutTranslations:GetSupportedLanguages": { + return lazy.TranslationsParent.getSupportedLanguages(); + } + case "AboutTranslations:IsTranslationsEngineSupported": { + return lazy.TranslationsParent.getIsTranslationsEngineSupported(); + } + default: + throw new Error("Unknown AboutTranslations message: " + name); + } + } +} diff --git a/toolkit/components/translations/actors/TranslationsChild.sys.mjs b/toolkit/components/translations/actors/TranslationsChild.sys.mjs new file mode 100644 index 0000000000..a3f8d15c85 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsChild.sys.mjs @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + TranslationsDocument: + "chrome://global/content/translations/translations-document.sys.mjs", + LRUCache: + "chrome://global/content/translations/translations-document.sys.mjs", + LanguageDetector: + "resource://gre/modules/translation/LanguageDetector.sys.mjs", +}); + +/** + * This file is extremely sensitive to memory size and performance! + */ +export class TranslationsChild extends JSWindowActorChild { + /** + * @type {TranslationsDocument | null} + */ + #translatedDoc = null; + + /** + * This cache is shared across TranslationsChild instances. This means + * that it will be shared across multiple page loads in the same origin. + * @type {LRUCache | null} + */ + static #translationsCache = null; + + handleEvent(event) { + switch (event.type) { + case "DOMContentLoaded": + this.sendAsyncMessage("Translations:ReportLangTags", { + documentElementLang: this.document.documentElement.lang, + }); + break; + } + } + + addProfilerMarker(message) { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.contentWindow.windowGlobalChild.innerWindowId }, + message + ); + } + + async receiveMessage({ name, data }) { + switch (name) { + case "Translations:TranslatePage": { + if (this.#translatedDoc?.translator.engineStatus === "error") { + this.#translatedDoc.destroy(); + this.#translatedDoc = null; + } + + if (this.#translatedDoc) { + console.error("This page was already translated."); + return undefined; + } + + const { fromLanguage, toLanguage, port, translationsStart } = data; + if ( + !TranslationsChild.#translationsCache || + !TranslationsChild.#translationsCache.matches( + fromLanguage, + toLanguage + ) + ) { + TranslationsChild.#translationsCache = new lazy.LRUCache( + fromLanguage, + toLanguage + ); + } + + this.#translatedDoc = new lazy.TranslationsDocument( + this.document, + fromLanguage, + toLanguage, + this.contentWindow.windowGlobalChild.innerWindowId, + port, + () => this.sendAsyncMessage("Translations:RequestPort"), + translationsStart, + () => this.docShell.now(), + TranslationsChild.#translationsCache + ); + + return undefined; + } + case "Translations:GetDocumentElementLang": + return this.document.documentElement.lang; + case "Translations:IdentifyLanguage": { + // Wait for idle callback as the page will be more settled if it has + // dynamic content, like on a React app. + if (this.contentWindow) { + await new Promise(resolve => { + this.contentWindow.requestIdleCallback(resolve); + }); + } + + try { + return lazy.LanguageDetector.detectLanguageFromDocument( + this.document + ); + } catch (error) { + return null; + } + } + case "Translations:AcquirePort": { + this.addProfilerMarker("Acquired a port, resuming translations"); + this.#translatedDoc.translator.acquirePort(data.port); + return undefined; + } + default: + throw new Error("Unknown message.", name); + } + } +} diff --git a/toolkit/components/translations/actors/TranslationsEngineChild.sys.mjs b/toolkit/components/translations/actors/TranslationsEngineChild.sys.mjs new file mode 100644 index 0000000000..a4ab8e2640 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsEngineChild.sys.mjs @@ -0,0 +1,207 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +/** + * The engine child is responsible for exposing privileged code to the un-privileged + * space the engine runs in. + */ +export class TranslationsEngineChild extends JSWindowActorChild { + /** + * The resolve function for the Promise returned by the + * "TranslationsEngine:ForceShutdown" message. + * @type {null | () => {}} + */ + #resolveForceShutdown = null; + + actorCreated() { + this.#exportFunctions(); + } + + handleEvent(event) { + switch (event.type) { + case "DOMContentLoaded": + this.sendAsyncMessage("TranslationsEngine:Ready"); + break; + } + } + + // eslint-disable-next-line consistent-return + async receiveMessage({ name, data }) { + switch (name) { + case "TranslationsEngine:StartTranslation": { + const { fromLanguage, toLanguage, innerWindowId, port } = data; + const transferables = [port]; + const message = { + type: "StartTranslation", + fromLanguage, + toLanguage, + innerWindowId, + port, + }; + this.contentWindow.postMessage(message, "*", transferables); + break; + } + case "TranslationsEngine:DiscardTranslations": { + const { innerWindowId } = data; + this.contentWindow.postMessage({ + type: "DiscardTranslations", + innerWindowId, + }); + break; + } + case "TranslationsEngine:ForceShutdown": { + this.contentWindow.postMessage({ + type: "ForceShutdown", + }); + return new Promise(resolve => { + this.#resolveForceShutdown = resolve; + }); + } + default: + console.error("Unknown message received", name); + } + } + + /** + * Export any of the child functions that start with "TE_" to the unprivileged content + * page. This restricts the security capabilities of the content page. + */ + #exportFunctions() { + const fns = [ + "TE_addProfilerMarker", + "TE_getLogLevel", + "TE_log", + "TE_logError", + "TE_requestEnginePayload", + "TE_reportEngineStatus", + "TE_resolveForceShutdown", + "TE_destroyEngineProcess", + ]; + for (const defineAs of fns) { + Cu.exportFunction(this[defineAs].bind(this), this.contentWindow, { + defineAs, + }); + } + } + + /** + * A privileged promise can't be used in the content page, so convert a privileged + * promise into a content one. + * + * @param {Promise<any>} promise + * @returns {Promise<any>} + */ + #convertToContentPromise(promise) { + return new this.contentWindow.Promise((resolve, reject) => + promise.then(resolve, error => { + let contentWindow; + try { + contentWindow = this.contentWindow; + } catch (error) { + // The content window is no longer available. + reject(); + return; + } + // Create an error in the content window, if the content window is still around. + let message = "An error occured in the TranslationsEngine actor."; + if (typeof error === "string") { + message = error; + } + if (typeof error?.message === "string") { + message = error.message; + } + if (typeof error?.stack === "string") { + message += `\n\nOriginal stack:\n\n${error.stack}\n`; + } + + reject(new contentWindow.Error(message)); + }) + ); + } + + /** + * @param {Object} options + * @param {number?} options.startTime + * @param {string} options.message + */ + TE_addProfilerMarker({ startTime, message, innerWindowId }) { + ChromeUtils.addProfilerMarker( + "TranslationsEngine", + { startTime, innerWindowId }, + message + ); + } + + /** + * Pass the message from content that the engines were shut down. + */ + TE_resolveForceShutdown() { + this.#resolveForceShutdown(); + } + + /** + * @returns {string} + */ + TE_getLogLevel() { + return Services.prefs.getCharPref("browser.translations.logLevel"); + } + + /** + * Log messages if "browser.translations.logLevel" is set to "All". + * + * @param {...any} args + */ + TE_log(...args) { + lazy.console.log(...args); + } + + /** + * Report an error to the console. + * + * @param {...any} args + */ + TE_logError(...args) { + lazy.console.error(...args); + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + TE_requestEnginePayload(fromLanguage, toLanguage) { + return this.#convertToContentPromise( + this.sendQuery("TranslationsEngine:RequestEnginePayload", { + fromLanguage, + toLanguage, + }) + ); + } + + /** + * @param {number} innerWindowId + * @param {"ready" | "error"} status + */ + TE_reportEngineStatus(innerWindowId, status) { + this.sendAsyncMessage("TranslationsEngine:ReportEngineStatus", { + innerWindowId, + status, + }); + } + + /** + * No engines are still alive, destroy the process. + */ + TE_destroyEngineProcess() { + this.sendAsyncMessage("TranslationsEngine:DestroyEngineProcess"); + } +} diff --git a/toolkit/components/translations/actors/TranslationsEngineParent.sys.mjs b/toolkit/components/translations/actors/TranslationsEngineParent.sys.mjs new file mode 100644 index 0000000000..77b16d7ae9 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsEngineParent.sys.mjs @@ -0,0 +1,141 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + TranslationsParent: "resource://gre/actors/TranslationsParent.sys.mjs", +}); + +/** + * The translations engine is in its own content process. This actor handles the + * marshalling of the data such as the engine payload and port passing. + */ +export class TranslationsEngineParent extends JSWindowActorParent { + /** + * Keep track of the live actors by InnerWindowID. + * + * @type {Map<InnerWindowID, TranslationsParent | AboutTranslationsParent>} + */ + #translationsParents = new Map(); + + async receiveMessage({ name, data }) { + switch (name) { + case "TranslationsEngine:Ready": + if (!lazy.TranslationsParent.resolveEngine) { + throw new Error( + "Unable to find the resolve function for when the translations engine is ready." + ); + } + lazy.TranslationsParent.resolveEngine(this); + return undefined; + case "TranslationsEngine:RequestEnginePayload": { + const { fromLanguage, toLanguage } = data; + const payloadPromise = + lazy.TranslationsParent.getTranslationsEnginePayload( + fromLanguage, + toLanguage + ); + payloadPromise.catch(error => { + lazy.TranslationsParent.telemetry().onError(String(error)); + }); + return payloadPromise; + } + case "TranslationsEngine:ReportEngineStatus": { + const { innerWindowId, status } = data; + const translationsParent = this.#translationsParents.get(innerWindowId); + + // about:translations will not have a TranslationsParent associated with + // this call. + if (translationsParent) { + switch (status) { + case "ready": + translationsParent.languageState.isEngineReady = true; + break; + case "error": + translationsParent.languageState.error = "engine-load-failure"; + break; + default: + throw new Error("Unknown engine status: " + status); + } + } + return undefined; + } + case "TranslationsEngine:DestroyEngineProcess": + ChromeUtils.addProfilerMarker( + "TranslationsEngine", + {}, + "Loading bergamot wasm array buffer" + ); + lazy.TranslationsParent.destroyEngineProcess().catch(error => + console.error(error) + ); + return undefined; + default: + return undefined; + } + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {number} innerWindowId + * @param {MessagePort} port + * @param {number} innerWindowId + * @param {TranslationsParent} [translationsParent] + */ + startTranslation( + fromLanguage, + toLanguage, + port, + innerWindowId, + translationsParent + ) { + if (translationsParent) { + this.#translationsParents.set( + translationsParent.innerWindowId, + translationsParent + ); + } + if (this.#isDestroyed) { + throw new Error("The translation engine process was already destroyed."); + } + const transferables = [port]; + this.sendAsyncMessage( + "TranslationsEngine:StartTranslation", + { + fromLanguage, + toLanguage, + innerWindowId, + port, + }, + transferables + ); + } + + /** + * Remove all the translations that are currently queued, and remove + * the communication port. + * + * @param {number} innerWindowId + */ + discardTranslations(innerWindowId) { + this.#translationsParents.delete(innerWindowId); + this.sendAsyncMessage("TranslationsEngine:DiscardTranslations", { + innerWindowId, + }); + } + + /** + * Manually shut down the engines, typically for testing purposes. + */ + forceShutdown() { + return this.sendQuery("TranslationsEngine:ForceShutdown"); + } + + #isDestroyed = false; + + didDestroy() { + this.#isDestroyed = true; + } +} diff --git a/toolkit/components/translations/actors/TranslationsParent.sys.mjs b/toolkit/components/translations/actors/TranslationsParent.sys.mjs new file mode 100644 index 0000000000..44b761e6b0 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsParent.sys.mjs @@ -0,0 +1,2947 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The pivot language is used to pivot between two different language translations + * when there is not a model available to translate directly between the two. In this + * case "en" is common between the various supported models. + * + * For instance given the following two models: + * "fr" -> "en" + * "en" -> "it" + * + * You can accomplish: + * "fr" -> "it" + * + * By doing: + * "fr" -> "en" -> "it" + */ +const PIVOT_LANGUAGE = "en"; + +const TRANSLATIONS_PERMISSION = "translations"; +const ALWAYS_TRANSLATE_LANGS_PREF = + "browser.translations.alwaysTranslateLanguages"; +const NEVER_TRANSLATE_LANGS_PREF = + "browser.translations.neverTranslateLanguages"; + +const lazy = {}; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +if (AppConstants.ENABLE_WEBDRIVER) { + XPCOMUtils.defineLazyServiceGetter( + lazy, + "Marionette", + "@mozilla.org/remote/marionette;1", + "nsIMarionette" + ); + + XPCOMUtils.defineLazyServiceGetter( + lazy, + "RemoteAgent", + "@mozilla.org/remote/agent;1", + "nsIRemoteAgent" + ); +} else { + lazy.Marionette = { running: false }; + lazy.RemoteAgent = { running: false }; +} + +XPCOMUtils.defineLazyServiceGetters(lazy, { + BrowserHandler: ["@mozilla.org/browser/clh;1", "nsIBrowserHandler"], +}); + +ChromeUtils.defineESModuleGetters(lazy, { + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", + TranslationsTelemetry: + "chrome://global/content/translations/TranslationsTelemetry.sys.mjs", + HiddenFrame: "resource://gre/modules/HiddenFrame.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "translationsEnabledPref", + "browser.translations.enable" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "chaosErrorsPref", + "browser.translations.chaos.errors" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "chaosTimeoutMSPref", + "browser.translations.chaos.timeoutMS" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "automaticallyPopupPref", + "browser.translations.automaticallyPopup" +); + +/** + * Returns the always-translate language tags as an array. + */ +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "alwaysTranslateLangTags", + ALWAYS_TRANSLATE_LANGS_PREF, + /* aDefaultPrefValue */ "", + /* onUpdate */ null, + /* aTransform */ rawLangTags => + rawLangTags ? new Set(rawLangTags.split(",")) : new Set() +); + +/** + * Returns the never-translate language tags as an array. + */ +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "neverTranslateLangTags", + NEVER_TRANSLATE_LANGS_PREF, + /* aDefaultPrefValue */ "", + /* onUpdate */ null, + /* aTransform */ rawLangTags => + rawLangTags ? new Set(rawLangTags.split(",")) : new Set() +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "simulateUnsupportedEnginePref", + "browser.translations.simulateUnsupportedEngine" +); + +// At this time the signatures of the files are not being checked when they are being +// loaded from disk. This signature check involves hitting the network, and translations +// are explicitly an offline-capable feature. See Bug 1827265 for re-enabling this +// check. +const VERIFY_SIGNATURES_FROM_FS = false; + +/** + * @typedef {import("../translations").TranslationModelRecord} TranslationModelRecord + * @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient + * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles + * @typedef {import("../translations").WasmRecord} WasmRecord + * @typedef {import("../translations").LangTags} LangTags + * @typedef {import("../translations").LanguagePair} LanguagePair + * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + * @typedef {import("../translations").TranslationErrors} TranslationErrors + */ + +/** + * @typedef {Object} TranslationPair + * @prop {string} fromLanguage + * @prop {string} toLanguage + * @prop {string} [fromDisplayLanguage] + * @prop {string} [toDisplayLanguage] + */ + +/** + * The translations parent is used to orchestrate translations in Firefox. It can + * download the wasm translation engines, and the machine learning language models. + * + * See Bug 971044 for more details of planned work. + */ +export class TranslationsParent extends JSWindowActorParent { + /** + * The following constants control the major version for assets downloaded from + * Remote Settings. When a breaking change is introduced, Nightly will have these + * numbers incremented by one, but Beta and Release will still be on the previous + * version. Remote Settings will ship both versions of the records, and the latest + * asset released in that version will be used. For instance, with a major version + * of "1", assets can be downloaded for "1.0", "1.2", "1.3beta", but assets marked + * as "2.0", "2.1", etc will not be downloaded. + * + * Release docs: + * https://firefox-source-docs.mozilla.org/toolkit/components/translations/resources/03_bergamot.html + */ + static BERGAMOT_MAJOR_VERSION = 1; + static LANGUAGE_MODEL_MAJOR_VERSION = 1; + + /** + * Contains the state that would affect UI. Anytime this state is changed, a dispatch + * event is sent so that UI can react to it. The actor is inside of /toolkit and + * needs a way of notifying /browser code (or other users) of when the state changes. + * + * @type {TranslationsLanguageState} + */ + languageState; + + /** + * Allows the TranslationsEngineParent to resolve an engine once it is ready. + * + * @type {null | () => TranslationsEngineParent} + */ + resolveEngine = null; + + /** + * The cached URI spec where the panel was first ever shown, as determined by the + * browser.translations.panelShown pref. + * + * Holding on to this URI value allows us to show the introductory message in the panel + * when the panel opens, as long as the active panel is open on that particular URI. + * + * @type {string | null} + */ + firstShowUriSpec = null; + + /** + * Do not send queries or do work when the actor is already destroyed. This flag needs + * to be checked after calls to `await`. + */ + #isDestroyed = false; + + /** + * Remember the detected languages on a page reload. This will keep the translations + * button from disappearing and reappearing, which causes the button to lose focus. + * + * @type {LangTags | null} previousDetectedLanguages + */ + static #previousDetectedLanguages = null; + + actorCreated() { + this.innerWindowId = this.browsingContext.top.embedderElement.innerWindowID; + this.languageState = new TranslationsLanguageState( + this, + TranslationsParent.#previousDetectedLanguages + ); + TranslationsParent.#previousDetectedLanguages = null; + + if (TranslationsParent.#translateOnPageReload) { + // The actor was recreated after a page reload, start the translation. + const { fromLanguage, toLanguage } = + TranslationsParent.#translateOnPageReload; + TranslationsParent.#translateOnPageReload = null; + + lazy.console.log( + `Translating on a page reload from "${fromLanguage}" to "${toLanguage}".` + ); + + this.translate( + fromLanguage, + toLanguage, + false // reportAsAutoTranslate + ); + } + } + + /** + * A map of the TranslationModelRecord["id"] to the record of the model in Remote Settings. + * Used to coordinate the downloads. + * + * @type {null | Promise<Map<string, TranslationModelRecord>>} + */ + static #translationModelRecords = null; + + /** + * The RemoteSettingsClient that downloads the translation models. + * + * @type {RemoteSettingsClient | null} + */ + static #translationModelsRemoteClient = null; + + /** + * The RemoteSettingsClient that downloads the wasm binaries. + * + * @type {RemoteSettingsClient | null} + */ + static #translationsWasmRemoteClient = null; + + /** + * The page may auto-translate due to user settings. On a page restore, always + * skip the page restore logic. + */ + static #isPageRestored = false; + + /** + * Allows the actor's behavior to be changed when the translations engine is mocked via + * a dummy RemoteSettingsClient. + * + * @type {bool} + */ + static #isTranslationsEngineMocked = false; + + /** + * @type {null | Promise<boolean>} + */ + static #isTranslationsEngineSupported = null; + + /** + * When reloading the page, store the translation pair that needs translating. + * + * @type {null | TranslationPair} + */ + static #translateOnPageReload = null; + + /** + * An ordered list of preferred languages based on: + * 1. App languages + * 2. Web requested languages + * 3. OS language + * + * @type {null | string[]} + */ + static #preferredLanguages = null; + + /** + * The value of navigator.languages. + * + * @type {null | Set<string>} + */ + static #webContentLanguages = null; + + static #observingLanguages = false; + + // On a fast connection, 10 concurrent downloads were measured to be the fastest when + // downloading all of the language files. + static MAX_CONCURRENT_DOWNLOADS = 10; + static MAX_DOWNLOAD_RETRIES = 3; + + // The set of hosts that have already been offered for translations. + static #hostsOffered = new Set(); + + // Enable the translations popup offer in tests. + static testAutomaticPopup = false; + + /** + * Gecko preference for always translating a language. + * + * @type {string} + */ + static ALWAYS_TRANSLATE_LANGS_PREF = ALWAYS_TRANSLATE_LANGS_PREF; + + /** + * Gecko preference for never translating a language. + * + * @type {string} + */ + static NEVER_TRANSLATE_LANGS_PREF = NEVER_TRANSLATE_LANGS_PREF; + + /** + * Telemetry functions for Translations + * @returns {TranslationsTelemetry} + */ + static telemetry() { + return lazy.TranslationsTelemetry; + } + + /** + * TODO(Bug 1834306) - Cu.isInAutomation doesn't recognize Marionette and RemoteAgent + * tests. + */ + static isInAutomation() { + return ( + Cu.isInAutomation || lazy.Marionette.running || lazy.RemoteAgent.running + ); + } + + /** + * @type {Promise<{ hiddenFrame: HiddenFrame, actor: TranslationsEngineParent }> | null} + */ + static #engine = null; + + static async getEngineProcess() { + if (!TranslationsParent.#engine) { + TranslationsParent.#engine = TranslationsParent.#getEngineProcessImpl(); + } + const enginePromise = TranslationsParent.#engine; + + // Determine if the actor was destroyed, or if there was an error. In this case + // attempt to rebuild the process. + let needsRebuilding = true; + try { + const { actor } = await enginePromise; + needsRebuilding = actor.isDestroyed; + } catch {} + + if ( + TranslationsParent.#engine && + enginePromise !== TranslationsParent.#engine + ) { + // This call lost the race, something else updated the engine promise, return that. + return TranslationsParent.#engine; + } + + if (needsRebuilding) { + // The engine was destroyed, attempt to re-create the engine process. + const rebuild = TranslationsParent.destroyEngineProcess().then(() => + TranslationsParent.#getEngineProcessImpl() + ); + TranslationsParent.#engine = rebuild; + return rebuild; + } + + return enginePromise; + } + + static destroyEngineProcess() { + const enginePromise = this.#engine; + this.#engine = null; + if (enginePromise) { + ChromeUtils.addProfilerMarker( + "TranslationsParent", + {}, + "Destroying the translations engine process" + ); + return enginePromise.then(({ actor, hiddenFrame }) => + actor + .forceShutdown() + .catch(error => { + lazy.console.error( + "There was an error shutting down the engine.", + error + ); + }) + .then(() => { + hiddenFrame.destroy(); + }) + ); + } + return Promise.resolve(); + } + + /** + * @type {Promise<{ hiddenFrame: HiddenFrame, actor: TranslationsEngineParent }> | null} + */ + static async #getEngineProcessImpl() { + ChromeUtils.addProfilerMarker( + "TranslationsParent", + {}, + "Creating the translations engine process" + ); + + // Manages the hidden ChromeWindow. + const hiddenFrame = new lazy.HiddenFrame(); + const chromeWindow = await hiddenFrame.get(); + const doc = chromeWindow.document; + + const actorPromise = new Promise(resolve => { + this.resolveEngine = resolve; + }); + + const browser = doc.createXULElement("browser"); + browser.setAttribute("remote", "true"); + browser.setAttribute("remoteType", "web"); + browser.setAttribute("disableglobalhistory", "true"); + browser.setAttribute("type", "content"); + browser.setAttribute( + "src", + "chrome://global/content/translations/translations-engine.html" + ); + doc.documentElement.appendChild(browser); + + const actor = await actorPromise; + this.resolveEngine = null; + return { hiddenFrame, browser, actor }; + } + + /** + * Offer translations (for instance by automatically opening the popup panel) whenever + * languages are detected, but only do it once per host per session. + * @param {LangTags} detectedLanguages + */ + maybeOfferTranslations(detectedLanguages) { + if (!this.browsingContext.currentWindowGlobal) { + return; + } + if (!lazy.automaticallyPopupPref) { + return; + } + if (lazy.BrowserHandler?.kiosk) { + // Pop-ups should not be shown in kiosk mode. + return; + } + const { documentURI } = this.browsingContext.currentWindowGlobal; + + if ( + TranslationsParent.isInAutomation() && + !TranslationsParent.testAutomaticPopup + ) { + // Do not offer translations in automation, as many tests do not expect this + // behavior. + lazy.console.log( + "maybeOfferTranslations - Do not offer translations in automation.", + documentURI.spec + ); + return; + } + + if ( + !detectedLanguages.docLangTag || + !detectedLanguages.userLangTag || + !detectedLanguages.isDocLangTagSupported + ) { + lazy.console.log( + "maybeOfferTranslations - The detected languages were not supported.", + detectedLanguages + ); + return; + } + + let host; + try { + host = documentURI.host; + } catch { + // nsIURI.host can throw if the URI scheme doesn't have a host. In this case + // do not offer a translation. + return; + } + if (TranslationsParent.#hostsOffered.has(host)) { + // This host was already offered a translation. + lazy.console.log( + "maybeOfferTranslations - Host already offered a translation, so skip.", + documentURI.spec + ); + return; + } + const browser = this.browsingContext.top.embedderElement; + if (!browser) { + return; + } + TranslationsParent.#hostsOffered.add(host); + const { CustomEvent } = browser.ownerGlobal; + + if ( + TranslationsParent.shouldNeverTranslateLanguage( + detectedLanguages.docLangTag + ) + ) { + lazy.console.log( + `maybeOfferTranslations - Should never translate language. "${detectedLanguages.docLangTag}"`, + documentURI.spec + ); + return; + } + if (this.shouldNeverTranslateSite()) { + lazy.console.log( + "maybeOfferTranslations - Should never translate site.", + documentURI.spec + ); + return; + } + + if (detectedLanguages.docLangTag === detectedLanguages.userLangTag) { + lazy.console.error( + "maybeOfferTranslations - The document and user lang tag are the same, not offering a translation.", + documentURI.spec + ); + return; + } + + // Only offer the translation if it's still the current page. + let isCurrentPage = false; + if (AppConstants.platform !== "android") { + isCurrentPage = + documentURI.spec === + this.browsingContext.topChromeWindow.gBrowser.selectedBrowser + .documentURI.spec; + } else { + // In Android, the active window is the active tab. + isCurrentPage = documentURI.spec === browser.documentURI.spec; + } + if (isCurrentPage) { + lazy.console.log( + "maybeOfferTranslations - Offering a translation", + documentURI.spec, + detectedLanguages + ); + + TranslationsParent.getEngineProcess().catch(error => + console.error(error) + ); + + browser.dispatchEvent( + new CustomEvent("TranslationsParent:OfferTranslation", { + bubbles: true, + }) + ); + } + } + + /** + * This is for testing purposes. + */ + static resetHostsOffered() { + TranslationsParent.#hostsOffered = new Set(); + } + + /** + * Detect if Wasm SIMD is supported, and cache the value. It's better to check + * for support before downloading large binary blobs to a user who can't even + * use the feature. This function also respects mocks and simulating unsupported + * engines. + * + * @type {boolean} + */ + static getIsTranslationsEngineSupported() { + if (lazy.simulateUnsupportedEnginePref) { + // Use the non-lazy console.log so that the user is always informed as to why + // the translations engine is not working. + console.log( + "Translations: The translations engine is disabled through the pref " + + '"browser.translations.simulateUnsupportedEngine".' + ); + + // The user is manually testing unsupported engines. + return false; + } + + if (TranslationsParent.#isTranslationsEngineMocked) { + // A mocked translations engine is always supported. + return true; + } + + if (TranslationsParent.#isTranslationsEngineSupported === null) { + TranslationsParent.#isTranslationsEngineSupported = detectSimdSupport(); + } + + return TranslationsParent.#isTranslationsEngineSupported; + } + + /** + * Only translate pages that match certain protocols, that way internal pages like + * about:* pages will not be translated. Keep this logic up to date with the "matches" + * array in the `toolkit/modules/ActorManagerParent.sys.mjs` definition. + * + * @param {string} scheme - The URI spec + * @returns {boolean} + */ + static isRestrictedPage(gBrowser) { + const contentType = gBrowser.selectedBrowser.documentContentType; + const scheme = gBrowser.currentURI.scheme; + + if (contentType === "application/pdf") { + return true; + } + + // Keep this logic up to date with TranslationsChild.prototype.#isRestrictedPage. + switch (scheme) { + case "https": + case "http": + case "file": + return false; + } + return true; + } + + static #resetPreferredLanguages() { + TranslationsParent.#webContentLanguages = null; + TranslationsParent.#preferredLanguages = null; + TranslationsParent.getPreferredLanguages(); + } + + static async observe(_subject, topic, _data) { + switch (topic) { + case "nsPref:changed": + case "intl:app-locales-changed": { + TranslationsParent.#resetPreferredLanguages(); + break; + } + default: + throw new Error("Unknown observer event", topic); + } + } + + /** + * Provide a way for tests to override the system locales. + * @type {null | string[]} + */ + static mockedSystemLocales = null; + + /** + * The "Accept-Language" values that the localizer or user has indicated for + * the preferences for the web. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language + * + * Note that this preference always has English in the fallback chain, even if the + * user doesn't actually speak English, and to other languages they potentially do + * not speak. However, this preference will be used as an indication that a user may + * prefer this language. + * + * https://transvision.flod.org/string/?entity=toolkit/chrome/global/intl.properties:intl.accept_languages&repo=gecko_strings + */ + static getWebContentLanguages() { + if (!TranslationsParent.#webContentLanguages) { + const values = Services.prefs + .getComplexValue("intl.accept_languages", Ci.nsIPrefLocalizedString) + .data.split(/\s*,\s*/g); + + TranslationsParent.#webContentLanguages = new Set(); + + for (const locale of values) { + try { + // Wrap this in a try statement since users can manually edit this pref. + TranslationsParent.#webContentLanguages.add( + new Intl.Locale(locale).language + ); + } catch { + // The locale was invalid, discard it. + } + } + + if ( + !Services.prefs.prefHasUserValue("intl.accept_languages") && + Services.locale.appLocaleAsBCP47 !== "en" && + !Services.locale.appLocaleAsBCP47.startsWith("en-") + ) { + // The user hasn't customized their accept languages, this means that English + // is always provided as a fallback language, even if it is not available. + TranslationsParent.#webContentLanguages.delete("en"); + } + + if (TranslationsParent.#webContentLanguages.size === 0) { + // The user has removed all of their web content languages, default to the + // app locale. + TranslationsParent.#webContentLanguages.add( + new Intl.Locale(Services.locale.appLocaleAsBCP47).language + ); + } + } + + return TranslationsParent.#webContentLanguages; + } + + /** + * An ordered list of preferred languages based on: + * + * 1. App languages + * 2. Web requested languages + * 3. OS language + * + * @returns {string[]} + */ + static getPreferredLanguages() { + if (TranslationsParent.#preferredLanguages) { + return TranslationsParent.#preferredLanguages; + } + + if (!TranslationsParent.#observingLanguages) { + Services.obs.addObserver( + TranslationsParent.#resetPreferredLanguages, + "intl:app-locales-changed" + ); + Services.prefs.addObserver( + "intl.accept_languages", + TranslationsParent.#resetPreferredLanguages + ); + TranslationsParent.#observingLanguages = true; + } + + // The system language could also be a good option for a language to offer the user. + const osPrefs = Cc["@mozilla.org/intl/ospreferences;1"].getService( + Ci.mozIOSPreferences + ); + const systemLocales = + TranslationsParent.mockedSystemLocales ?? osPrefs.systemLocales; + + // Combine the locales together. + const preferredLocales = new Set([ + ...TranslationsParent.getWebContentLanguages(), + ...Services.locale.appLocalesAsBCP47, + ...systemLocales, + ]); + + // Attempt to convert the locales to lang tags. Do not completely trust the + // values coming from preferences and the OS to have been validated as correct + // BCP 47 locale identifiers. + const langTags = new Set(); + for (const locale of preferredLocales) { + try { + langTags.add(new Intl.Locale(locale).language); + } catch (_) { + // The locale was invalid, discard it. + } + } + + // Convert the Set to an array to indicate that it is an ordered listing of languages. + TranslationsParent.#preferredLanguages = [...langTags]; + + return TranslationsParent.#preferredLanguages; + } + + async receiveMessage({ name, data }) { + switch (name) { + case "Translations:ReportLangTags": { + const { documentElementLang, href } = data; + const detectedLanguages = await this.getDetectedLanguages( + documentElementLang, + href + ).catch(error => { + // Detecting the languages can fail if the page gets destroyed before it + // can be completed. This runs on every page that doesn't have a lang tag, + // so only report the error if you have Translations logging turned on to + // avoid console spam. + lazy.console.log("Failed to get the detected languages.", error); + }); + + if (!detectedLanguages) { + // The actor was already destroyed, and the detectedLanguages weren't reported + // in time. + return undefined; + } + + this.languageState.detectedLanguages = detectedLanguages; + + if (this.shouldAutoTranslate(detectedLanguages)) { + this.translate( + detectedLanguages.docLangTag, + detectedLanguages.userLangTag, + true // reportAsAutoTranslate + ); + } else { + this.maybeOfferTranslations(detectedLanguages); + } + return undefined; + } + case "Translations:RequestPort": { + const { requestedTranslationPair } = this.languageState; + if (!requestedTranslationPair) { + lazy.console.error( + "A port was requested but no translation pair was previously requested" + ); + return undefined; + } + + let engineProcess; + try { + engineProcess = await TranslationsParent.getEngineProcess(); + } catch (error) { + console.error("Failed to get the translation engine process", error); + return undefined; + } + + if (this.#isDestroyed) { + // This actor was already destroyed. + return undefined; + } + + if (!this.innerWindowId) { + throw new Error( + "The innerWindowId for the TranslationsParent was not available." + ); + } + + // The MessageChannel will be used for communicating directly between the content + // process and the engine's process. + const { port1, port2 } = new MessageChannel(); + engineProcess.actor.startTranslation( + requestedTranslationPair.fromLanguage, + requestedTranslationPair.toLanguage, + port1, + this.innerWindowId, + this + ); + + this.sendAsyncMessage( + "Translations:AcquirePort", + { port: port2 }, + [port2] // Mark the port as transferable. + ); + + return undefined; + } + } + return undefined; + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + static async getTranslationsEnginePayload(fromLanguage, toLanguage) { + const wasmStartTime = Cu.now(); + const bergamotWasmArrayBufferPromise = + TranslationsParent.#getBergamotWasmArrayBuffer(); + bergamotWasmArrayBufferPromise + .then(() => { + ChromeUtils.addProfilerMarker( + "TranslationsParent", + { innerWindowId: this.innerWindowId, startTime: wasmStartTime }, + "Loading bergamot wasm array buffer" + ); + }) + .catch(() => { + // Do nothing. + }); + + const modelStartTime = Cu.now(); + let files = await TranslationsParent.getLanguageTranslationModelFiles( + fromLanguage, + toLanguage + ); + + let languageModelFiles; + if (files) { + languageModelFiles = [files]; + } else { + // No matching model was found, try to pivot between English. + const [files1, files2] = await Promise.all([ + TranslationsParent.getLanguageTranslationModelFiles( + fromLanguage, + PIVOT_LANGUAGE + ), + TranslationsParent.getLanguageTranslationModelFiles( + PIVOT_LANGUAGE, + toLanguage + ), + ]); + if (!files1 || !files2) { + throw new Error( + `No language models were found for ${fromLanguage} to ${toLanguage}` + ); + } + languageModelFiles = [files1, files2]; + } + + ChromeUtils.addProfilerMarker( + "TranslationsParent", + { innerWindowId: this.innerWindowId, startTime: modelStartTime }, + "Loading translation model files" + ); + + const bergamotWasmArrayBuffer = await bergamotWasmArrayBufferPromise; + + return { + bergamotWasmArrayBuffer, + languageModelFiles, + isMocked: TranslationsParent.#isTranslationsEngineMocked, + }; + } + + /** + * Returns true if translations should auto-translate from the given + * language, otherwise returns false. + * + * @param {LangTags} langTags + * @returns {boolean} + */ + static #maybeAutoTranslate(langTags) { + if (TranslationsParent.#isPageRestored) { + // The user clicked the restore button. Respect it for one page load. + TranslationsParent.#isPageRestored = false; + + // Skip this auto-translation. + return false; + } + + return TranslationsParent.shouldAlwaysTranslateLanguage(langTags); + } + + /** + * Creates a lookup key that is unique to each fromLanguage-toLanguage pair. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {string} + */ + static languagePairKey(fromLanguage, toLanguage) { + return `${fromLanguage},${toLanguage}`; + } + + /** + * The cached language pairs. + * @type {Promise<Array<LanguagePair>> | null} + */ + static #languagePairs = null; + + /** + * Get the list of translation pairs supported by the translations engine. + * + * @returns {Promise<Array<LanguagePair>>} + */ + static getLanguagePairs() { + if (!TranslationsParent.#languagePairs) { + TranslationsParent.#languagePairs = + TranslationsParent.#getTranslationModelRecords().then(records => { + const languagePairMap = new Map(); + + for (const { fromLang, toLang } of records.values()) { + const key = TranslationsParent.languagePairKey(fromLang, toLang); + if (!languagePairMap.has(key)) { + languagePairMap.set(key, { fromLang, toLang }); + } + } + return Array.from(languagePairMap.values()); + }); + } + return TranslationsParent.#languagePairs; + } + + /** + * Get the list of languages and their display names, sorted by their display names. + * This is more expensive of a call than getLanguagePairs since the display names + * are looked up. + * + * This is all of the information needed to render dropdowns for translation + * language selection. + * + * @returns {Promise<SupportedLanguages>} + */ + static async getSupportedLanguages() { + const languagePairs = await TranslationsParent.getLanguagePairs(); + + /** @type {Set<string>} */ + const fromLanguages = new Set(); + /** @type {Set<string>} */ + const toLanguages = new Set(); + + for (const { fromLang, toLang } of languagePairs) { + fromLanguages.add(fromLang); + toLanguages.add(toLang); + } + + // Build a map of the langTag to the display name. + /** @type {Map<string, string>} */ + const displayNames = new Map(); + { + const dn = new Services.intl.DisplayNames(undefined, { + type: "language", + }); + + for (const langTagSet of [fromLanguages, toLanguages]) { + for (const langTag of langTagSet.keys()) { + if (displayNames.has(langTag)) { + continue; + } + displayNames.set(langTag, dn.of(langTag)); + } + } + } + + const addDisplayName = langTag => ({ + langTag, + displayName: displayNames.get(langTag), + }); + + const sort = (a, b) => a.displayName.localeCompare(b.displayName); + + return { + languagePairs, + fromLanguages: Array.from(fromLanguages.keys()) + .map(addDisplayName) + .sort(sort), + toLanguages: Array.from(toLanguages.keys()) + .map(addDisplayName) + .sort(sort), + }; + } + + /** + * Create a unique list of languages, sorted by the display name. + * + * @param {Object} supportedLanguages + * @returns {Array<{ langTag: string, displayName: string}} + */ + static getLanguageList(supportedLanguages) { + const displayNames = new Map(); + for (const languages of [ + supportedLanguages.fromLanguages, + supportedLanguages.toLanguages, + ]) { + for (const { langTag, displayName } of languages) { + displayNames.set(langTag, displayName); + } + } + + let appLangTag = new Intl.Locale(Services.locale.appLocaleAsBCP47).language; + + // Don't offer to download the app's language. + displayNames.delete(appLangTag); + + // Sort the list of languages by the display names. + return [...displayNames.entries()] + .map(([langTag, displayName]) => ({ + langTag, + displayName, + })) + .sort((a, b) => a.displayName.localeCompare(b.displayName)); + } + + /** + * @param {Object} event + * @param {Object} event.data + * @param {TranslationModelRecord[]} event.data.created + * @param {TranslationModelRecord[]} event.data.updated + * @param {TranslationModelRecord[]} event.data.deleted + */ + static async #handleTranslationsModelsSync({ + data: { created, updated, deleted }, + }) { + const client = TranslationsParent.#translationModelsRemoteClient; + if (!client) { + lazy.console.error( + "Translations client was not present when receiving a sync event." + ); + return; + } + + // Language model attachments will only be downloaded when they are used. + lazy.console.log( + `Remote Settings "sync" event for remote language models `, + { + created, + updated, + deleted, + } + ); + + const records = await TranslationsParent.#getTranslationModelRecords(); + + // Remove all the deleted records. + for (const record of deleted) { + await client.attachments.deleteDownloaded(record); + records.delete(record.id); + } + + // Pre-emptively remove the old downloads, and set the new updated record. + for (const { old: oldRecord, new: newRecord } of updated) { + await client.attachments.deleteDownloaded(oldRecord); + // The language pairs should be the same on the update, but use the old + // record just in case. + records.delete(oldRecord.id); + records.set(newRecord.id, newRecord); + } + + // Add the new records, but don't download any attachments. + for (const record of created) { + records.set(record.id, record); + } + + // Invalidate cached data. + TranslationsParent.#languagePairs = null; + } + + /** + * Lazily initializes the RemoteSettingsClient for the language models. + * + * @returns {RemoteSettingsClient} + */ + static #getTranslationModelsRemoteClient() { + if (TranslationsParent.#translationModelsRemoteClient) { + return TranslationsParent.#translationModelsRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-models"); + TranslationsParent.#translationModelsRemoteClient = client; + client.on("sync", TranslationsParent.#handleTranslationsModelsSync); + return client; + } + + /** + * Retrieves the maximum major version of each record in the RemoteSettingsClient. + * + * If the client contains two different-version copies of the same record (e.g. 1.0 and 1.1) + * then only the 1.1-version record will be returned in the resulting collection. + * + * @param {RemoteSettingsClient} remoteSettingsClient + * @param {Object} [options] + * @param {Object} [options.filters={}] + * The filters to apply when retrieving the records from RemoteSettings. + * Filters should correspond to properties on the RemoteSettings records themselves. + * For example, A filter to retrieve only records with a `fromLang` value of "en" and a `toLang` value of "es": + * { filters: { fromLang: "en", toLang: "es" } } + * @param {Function} [options.lookupKey=(record => record.name)] + * The function to use to extract a lookup key from each record. + * This function should take a record as input and return a string that represents the lookup key for the record. + * For most record types, the name (default) is sufficient, however if a collection contains records with + * non-unique name values, it may be necessary to provide an alternative function here. + * @returns {Array<TranslationModelRecord | WasmRecord>} + */ + static async getMaxVersionRecords( + remoteSettingsClient, + { filters = {}, majorVersion, lookupKey = record => record.name } = {} + ) { + if (!majorVersion) { + throw new Error("Expected the records to have a major version."); + } + try { + await chaosMode(1 / 4); + } catch (_error) { + // Simulate an error by providing empty records. + return []; + } + const retrievedRecords = await remoteSettingsClient.get({ + // Pull the records from the network if empty. + syncIfEmpty: true, + // Do not load the JSON dump if it is newer. + // + // The JSON dump comes from the Prod RemoteSettings channel + // so we shouldn't ever have an issue with the Prod server + // being older than the JSON dump itself (this is good). + // + // However, setting this to true will prevent us from + // testing RemoteSettings on the Dev and Stage + // environments if they happen to be older than the + // most recent JSON dump from Prod. + loadDumpIfNewer: false, + // Don't verify the signature if the client is mocked. + verifySignature: VERIFY_SIGNATURES_FROM_FS, + // Apply any filters for retrieving the records. + filters, + }); + + // Create a mapping to only the max version of each record discriminated by + // the result of the lookupKey() function. + const keyToRecord = new Map(); + + for (const record of retrievedRecords) { + const key = lookupKey(record); + const existing = keyToRecord.get(key); + + if (!record.version) { + lazy.console.error(record); + throw new Error("Expected the record to have a version."); + } + if ( + TranslationsParent.isBetterRecordVersion( + majorVersion, + record.version, + existing?.version + ) + ) { + keyToRecord.set(key, record); + } + } + + return Array.from(keyToRecord.values()); + } + + /** + * Applies the constraint of matching for the best matching major version. + * + * @param {number} majorVersion + * @param {string} nextVersion + * @param {string} [existingVersion] + * + */ + static isBetterRecordVersion(majorVersion, nextVersion, existingVersion) { + return ( + // Check that this is a major version record we can support. + Services.vc.compare(`${majorVersion}.0a`, nextVersion) <= 0 && + Services.vc.compare(`${majorVersion + 1}.0a`, nextVersion) > 0 && + // Check that the new record is bigger version number + (!existingVersion || + Services.vc.compare(existingVersion, nextVersion) < 0) + ); + } + + /** + * Lazily initializes the model records, and returns the cached ones if they + * were already retrieved. The key of the returned `Map` is the record id. + * + * @returns {Promise<Map<string, TranslationModelRecord>>} + */ + static async #getTranslationModelRecords() { + if (!TranslationsParent.#translationModelRecords) { + // Place the records into a promise to prevent any races. + TranslationsParent.#translationModelRecords = (async () => { + const records = new Map(); + const now = Date.now(); + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + + // Load the models. If no data is present, then there will be an initial sync. + // Rely on Remote Settings for the syncing strategy for receiving updates. + lazy.console.log(`Getting remote language models.`); + + /** @type {TranslationModelRecord[]} */ + const translationModelRecords = + await TranslationsParent.getMaxVersionRecords(client, { + majorVersion: TranslationsParent.LANGUAGE_MODEL_MAJOR_VERSION, + // Names in this collection are not unique, so we are appending the languagePairKey + // to guarantee uniqueness. + lookupKey: record => + `${record.name}${TranslationsParent.languagePairKey( + record.fromLang, + record.toLang + )}`, + }); + + if (translationModelRecords.length === 0) { + throw new Error("Unable to retrieve the translation models."); + } + + for (const record of TranslationsParent.ensureLanguagePairsHavePivots( + translationModelRecords + )) { + records.set(record.id, record); + } + + const duration = (Date.now() - now) / 1000; + lazy.console.log( + `Remote language models loaded in ${duration} seconds.`, + records + ); + + return records; + })(); + + TranslationsParent.#translationModelRecords.catch(() => { + TranslationsParent.#translationModelRecords = null; + }); + } + + return TranslationsParent.#translationModelRecords; + } + + /** + * This implementation assumes that every language pair has access to the + * pivot language. If any languages are added without a pivot language, or the + * pivot language is changed, then this implementation will need a more complicated + * language solver. This means that any UI pickers would need to be updated, and + * the pivot language selection would need a solver. + * + * @param {TranslationModelRecord[] | LanguagePair[]} records + */ + static ensureLanguagePairsHavePivots(records) { + if (!AppConstants.DEBUG) { + // Only run this check on debug builds as it's in the performance critical first + // page load path. + return records; + } + // lang -> pivot + const hasToPivot = new Set(); + // pivot -> en + const hasFromPivot = new Set(); + + const fromLangs = new Set(); + const toLangs = new Set(); + + for (const { fromLang, toLang } of records) { + fromLangs.add(fromLang); + toLangs.add(toLang); + + if (toLang === PIVOT_LANGUAGE) { + // lang -> pivot + hasToPivot.add(fromLang); + } + if (fromLang === PIVOT_LANGUAGE) { + // pivot -> en + hasFromPivot.add(toLang); + } + } + + const fromLangsToRemove = new Set(); + const toLangsToRemove = new Set(); + + for (const lang of fromLangs) { + if (lang === PIVOT_LANGUAGE) { + continue; + } + // Check for "lang -> pivot" + if (!hasToPivot.has(lang)) { + TranslationsParent.reportError( + new Error( + `The "from" language model "${lang}" is being discarded as it doesn't have a pivot language.` + ) + ); + fromLangsToRemove.add(lang); + } + } + + for (const lang of toLangs) { + if (lang === PIVOT_LANGUAGE) { + continue; + } + // Check for "pivot -> lang" + if (!hasFromPivot.has(lang)) { + TranslationsParent.reportError( + new Error( + `The "to" language model "${lang}" is being discarded as it doesn't have a pivot language.` + ) + ); + toLangsToRemove.add(lang); + } + } + + const after = records.filter(record => { + if (fromLangsToRemove.has(record.fromLang)) { + return false; + } + if (toLangsToRemove.has(record.toLang)) { + return false; + } + return true; + }); + return after; + } + + /** + * Lazily initializes the RemoteSettingsClient for the downloaded wasm binary data. + * + * @returns {RemoteSettingsClient} + */ + static #getTranslationsWasmRemoteClient() { + if (TranslationsParent.#translationsWasmRemoteClient) { + return TranslationsParent.#translationsWasmRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-wasm"); + + TranslationsParent.#translationsWasmRemoteClient = client; + + client.on("sync", async ({ data: { created, updated, deleted } }) => { + lazy.console.log(`"sync" event for remote bergamot wasm `, { + created, + updated, + deleted, + }); + + // Remove all the deleted records. + for (const record of deleted) { + await client.attachments.deleteDownloaded(record); + } + + // Remove any updated records, and download the new ones. + for (const { old: oldRecord } of updated) { + await client.attachments.deleteDownloaded(oldRecord); + } + + // Do nothing for the created records. + }); + + return client; + } + + /** @type {Promise<WasmRecord> | null} */ + static #bergamotWasmRecord = null; + + /** @type {boolean} */ + static #lookForLocalWasmBuild = true; + + /** + * This is used to load a local copy of the Bergamot translations engine, if it exists. + * From a local build of Firefox: + * + * 1. Run the python script: + * ./toolkit/components/translations/bergamot-translator/build-bergamot.py --debug + * + * 2. Uncomment the .wasm file in: toolkit/components/translations/jar.mn + * 3. Run: ./mach build + * 4. Run: ./mach run + */ + static async #maybeFetchLocalBergamotWasmArrayBuffer() { + if (TranslationsParent.#lookForLocalWasmBuild) { + // Attempt to get a local copy of the translator. Most likely this will be a 404. + try { + const response = await fetch( + "chrome://global/content/translations/bergamot-translator-worker.wasm" + ); + const arrayBuffer = response.arrayBuffer(); + lazy.console.log(`Using a local copy of Bergamot.`); + return arrayBuffer; + } catch { + // Only attempt to fetch once, if it fails don't try again. + TranslationsParent.#lookForLocalWasmBuild = false; + } + } + return null; + } + + /** + * Bergamot is the translation engine that has been compiled to wasm. It is shipped + * to the user via Remote Settings. + * + * https://github.com/mozilla/bergamot-translator/ + */ + /** + * @returns {Promise<ArrayBuffer>} + */ + static async #getBergamotWasmArrayBuffer() { + const start = Date.now(); + const client = TranslationsParent.#getTranslationsWasmRemoteClient(); + + const localCopy = + await TranslationsParent.#maybeFetchLocalBergamotWasmArrayBuffer(); + if (localCopy) { + return localCopy; + } + + if (!TranslationsParent.#bergamotWasmRecord) { + // Place the records into a promise to prevent any races. + TranslationsParent.#bergamotWasmRecord = (async () => { + // Load the wasm binary from remote settings, if it hasn't been already. + lazy.console.log(`Getting remote bergamot-translator wasm records.`); + + /** @type {WasmRecord[]} */ + const wasmRecords = await TranslationsParent.getMaxVersionRecords( + client, + { + filters: { name: "bergamot-translator" }, + majorVersion: TranslationsParent.BERGAMOT_MAJOR_VERSION, + } + ); + + if (wasmRecords.length === 0) { + // The remote settings client provides an empty list of records when there is + // an error. + throw new Error( + "Unable to get the bergamot translator from Remote Settings." + ); + } + + if (wasmRecords.length > 1) { + TranslationsParent.reportError( + new Error( + "Expected the bergamot-translator to only have 1 record." + ), + wasmRecords + ); + } + const [record] = wasmRecords; + lazy.console.log( + `Using ${record.name}@${record.release} release version ${record.version} first released on Fx${record.fx_release}`, + record + ); + return record; + })(); + } + // Unlike the models, greedily download the wasm. It will pull it from a locale + // cache on disk if it's already been downloaded. Do not retain a copy, as + // this will be running in the parent process. It's not worth holding onto + // this much memory, so reload it every time it is needed. + + try { + await chaosModeError(1 / 3); + + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download( + await TranslationsParent.#bergamotWasmRecord + ); + + const duration = Date.now() - start; + lazy.console.log( + `"bergamot-translator" wasm binary loaded in ${duration / 1000} seconds` + ); + + return buffer; + } catch (error) { + TranslationsParent.#bergamotWasmRecord = null; + throw error; + } + } + + /** + * Deletes language files that match a language. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + static async deleteLanguageFiles(language) { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + const isForDeletion = true; + return Promise.all( + Array.from( + await TranslationsParent.getRecordsForTranslatingToAndFromAppLanguage( + language, + isForDeletion + ) + ).map(record => { + lazy.console.log("Deleting record", record); + return client.attachments.deleteDownloaded(record); + }) + ); + } + + /** + * Download language files that match a language. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + static async downloadLanguageFiles(language) { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + + const queue = []; + + for (const record of await TranslationsParent.getRecordsForTranslatingToAndFromAppLanguage( + language + )) { + const download = () => { + lazy.console.log("Downloading record", record.name, record.id); + return client.attachments.download(record); + }; + queue.push({ download }); + } + + return downloadManager(queue); + } + + /** + * Download all files used for translations. + */ + static async downloadAllFiles() { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + + const queue = []; + + for (const record of ( + await TranslationsParent.#getTranslationModelRecords() + ).values()) { + queue.push({ + // The download may be attempted multiple times. + onFailure: () => { + console.error("Failed to download", record.name); + }, + download: () => client.attachments.download(record), + }); + } + + queue.push({ + download: () => TranslationsParent.#getBergamotWasmArrayBuffer(), + }); + + return downloadManager(queue); + } + + /** + * Delete all language model files. + * @returns {Promise<string[]>} A list of record IDs. + */ + static async deleteAllLanguageFiles() { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + await chaosMode(); + await client.attachments.deleteAll(); + return [...(await TranslationsParent.#getTranslationModelRecords()).keys()]; + } + + /** + * Only returns true if all language files are present for a requested language. + * It's possible only half the files exist for a pivot translation into another + * language, or there was a download error, and we're still missing some files. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + static async hasAllFilesForLanguage(requestedLanguage) { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + for (const record of await TranslationsParent.getRecordsForTranslatingToAndFromAppLanguage( + requestedLanguage, + true + )) { + if (!(await client.attachments.isDownloaded(record))) { + return false; + } + } + + return true; + } + + /** + * Get the necessary files for translating to and from the app language and a + * requested language. This may require the files for a pivot language translation + * if there is no language model for a direct translation. + * + * @param {string} requestedLanguage The BCP 47 language tag. + * @param {boolean} isForDeletion - Return a more restrictive set of languages, as + * these files are marked for deletion. We don't want to remove + * files that are needed for some other language's pivot translation. + * @returns {Set<TranslationModelRecord>} + */ + static async getRecordsForTranslatingToAndFromAppLanguage( + requestedLanguage, + isForDeletion = false + ) { + const records = await TranslationsParent.#getTranslationModelRecords(); + const appLanguage = new Intl.Locale(Services.locale.appLocaleAsBCP47) + .language; + + let matchedRecords = new Set(); + + if (requestedLanguage === appLanguage) { + // There are no records if the requested language and app language are the same. + return matchedRecords; + } + + const addLanguagePair = (fromLang, toLang) => { + let matchFound = false; + for (const record of records.values()) { + if (record.fromLang === fromLang && record.toLang === toLang) { + matchedRecords.add(record); + matchFound = true; + } + } + return matchFound; + }; + + if ( + // Is there a direct translation? + !addLanguagePair(requestedLanguage, appLanguage) + ) { + // This is no direct translation, get the pivot files. + addLanguagePair(requestedLanguage, PIVOT_LANGUAGE); + // These files may be required for other pivot translations, so don't remove + // them if we are deleting records. + if (!isForDeletion) { + addLanguagePair(PIVOT_LANGUAGE, appLanguage); + } + } + + if ( + // Is there a direct translation? + !addLanguagePair(appLanguage, requestedLanguage) + ) { + // This is no direct translation, get the pivot files. + addLanguagePair(PIVOT_LANGUAGE, requestedLanguage); + // These files may be required for other pivot translations, so don't remove + // them if we are deleting records. + if (!isForDeletion) { + addLanguagePair(appLanguage, PIVOT_LANGUAGE); + } + } + + return matchedRecords; + } + + /** + * Gets the language model files in an array buffer by downloading attachments from + * Remote Settings, or retrieving them from the local cache. Each translation + * requires multiple files. + * + * Results are only returned if the model is found. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {boolean} withQualityEstimation + * @returns {null | LanguageTranslationModelFiles} + */ + static async getLanguageTranslationModelFiles( + fromLanguage, + toLanguage, + withQualityEstimation = false + ) { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + + lazy.console.log( + `Beginning model downloads: "${fromLanguage}" to "${toLanguage}"` + ); + + const records = [ + ...(await TranslationsParent.#getTranslationModelRecords()).values(), + ]; + + /** @type {LanguageTranslationModelFiles} */ + let results; + + // Use Promise.all to download (or retrieve from cache) the model files in parallel. + await Promise.all( + records.map(async record => { + if (record.fileType === "qualityModel" && !withQualityEstimation) { + // Do not include the quality models if they aren't needed. + return; + } + + if (record.fromLang !== fromLanguage || record.toLang !== toLanguage) { + // Only use models that match. + return; + } + + if (!results) { + results = {}; + } + + const start = Date.now(); + + // Download or retrieve from the local cache: + + await chaosMode(1 / 3); + + /** @type {{buffer: ArrayBuffer }} */ + const { buffer } = await client.attachments.download(record); + + results[record.fileType] = { + buffer, + record, + }; + + const duration = Date.now() - start; + lazy.console.log( + `Translation model fetched in ${duration / 1000} seconds:`, + record.fromLang, + record.toLang, + record.fileType + ); + }) + ); + + if (!results) { + // No model files were found, pivoting will be required. + return null; + } + + // Validate that all of the files we expected were actually available and + // downloaded. + + if (!results.model) { + throw new Error( + `No model file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (!results.lex) { + throw new Error( + `No lex file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (withQualityEstimation && !results.qualityModel) { + throw new Error( + `No quality file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (results.vocab) { + if (results.srcvocab) { + throw new Error( + `A srcvocab and vocab file were both included for "${fromLanguage}" to "${toLanguage}." Only one is needed.` + ); + } + if (results.trgvocab) { + throw new Error( + `A trgvocab and vocab file were both included for "${fromLanguage}" to "${toLanguage}." Only one is needed.` + ); + } + } else if (!results.srcvocab || !results.srcvocab) { + throw new Error( + `No vocab files were provided for "${fromLanguage}" to "${toLanguage}."` + ); + } + + return results; + } + + /** + * Gets the expected download size that will occur (if any) if translate is called on two given languages for display purposes. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {boolean} withQualityEstimation + * @returns {Promise<long>} Size in bytes of the expected download. A result of 0 indicates no download is expected for the request. + */ + static async getExpectedTranslationDownloadSize( + fromLanguage, + toLanguage, + withQualityEstimation = false + ) { + const directSize = await this.#getModelDownloadSize( + fromLanguage, + toLanguage, + withQualityEstimation + ); + + // If a direct model is not found, then check pivots. + if (directSize.downloadSize == 0 && !directSize.modelFound) { + const indirectFrom = await TranslationsParent.#getModelDownloadSize( + fromLanguage, + PIVOT_LANGUAGE, + withQualityEstimation + ); + + const indirectTo = await TranslationsParent.#getModelDownloadSize( + PIVOT_LANGUAGE, + toLanguage, + withQualityEstimation + ); + + // Note, will also return 0 due to the models not being available as well. + return ( + parseInt(indirectFrom.downloadSize) + parseInt(indirectTo.downloadSize) + ); + } + return directSize.downloadSize; + } + + /** + * Determines the language model download size for a specified translation for display purposes. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {boolean} withQualityEstimation + * @returns {Promise<{downloadSize: long, modelFound: boolean}> Download size is the size in bytes of the estimated download for display purposes. Model found indicates a model was found. + * e.g., a result of {size: 0, modelFound: false} indicates no bytes to download, because a model wasn't located. + */ + static async #getModelDownloadSize( + fromLanguage, + toLanguage, + withQualityEstimation = false + ) { + const client = TranslationsParent.#getTranslationModelsRemoteClient(); + const records = [ + ...(await TranslationsParent.#getTranslationModelRecords()).values(), + ]; + + let downloadSize = 0; + let modelFound = false; + + await Promise.all( + records.map(async record => { + if (record.fileType === "qualityModel" && !withQualityEstimation) { + return; + } + + if (record.fromLang !== fromLanguage || record.toLang !== toLanguage) { + return; + } + + modelFound = true; + const isDownloaded = await client.attachments.isDownloaded(record); + if (!isDownloaded) { + downloadSize += parseInt(record.attachment.size); + } + }) + ); + return { downloadSize, modelFound }; + } + + /** + * For testing purposes, allow the Translations Engine to be mocked. If called + * with `null` the mock is removed. + * + * @param {null | RemoteSettingsClient} [translationModelsRemoteClient] + * @param {null | RemoteSettingsClient} [translationsWasmRemoteClient] + */ + static mockTranslationsEngine( + translationModelsRemoteClient, + translationsWasmRemoteClient + ) { + lazy.console.log("Mocking RemoteSettings for the translations engine."); + TranslationsParent.#translationModelsRemoteClient = + translationModelsRemoteClient; + TranslationsParent.#translationsWasmRemoteClient = + translationsWasmRemoteClient; + TranslationsParent.#isTranslationsEngineMocked = true; + + translationModelsRemoteClient.on( + "sync", + TranslationsParent.#handleTranslationsModelsSync + ); + } + + /** + * Most values are cached for performance, in tests we want to be able to clear them. + */ + static clearCache() { + // Records. + TranslationsParent.#bergamotWasmRecord = null; + TranslationsParent.#translationModelRecords = null; + + // Clients. + TranslationsParent.#translationModelsRemoteClient = null; + TranslationsParent.#translationsWasmRemoteClient = null; + + // Derived data. + TranslationsParent.#preferredLanguages = null; + TranslationsParent.#languagePairs = null; + TranslationsParent.#isTranslationsEngineSupported = null; + } + + /** + * Remove the mocks for the translations engine, make sure and call clearCache after + * to remove the cached values. + */ + static unmockTranslationsEngine() { + lazy.console.log( + "Removing RemoteSettings mock for the translations engine." + ); + TranslationsParent.#translationModelsRemoteClient.off( + "sync", + TranslationsParent.#handleTranslationsModelsSync + ); + + TranslationsParent.#isTranslationsEngineMocked = false; + } + + /** + * Report an error. Having this as a method allows tests to check that an error + * was properly reported. + * @param {Error} error - Providing an Error object makes sure the stack is properly + * reported. + * @param {any[]} args - Any args to pass on to console.error. + */ + static reportError(error, ...args) { + lazy.console.log(error, ...args); + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {boolean} reportAsAutoTranslate - In telemetry, report this as + * an auto-translate. + */ + async translate(fromLanguage, toLanguage, reportAsAutoTranslate) { + if (fromLanguage === toLanguage) { + lazy.console.error( + "A translation was requested where the from and to language match.", + { fromLanguage, toLanguage, reportAsAutoTranslate } + ); + return; + } + if (!fromLanguage || !toLanguage) { + lazy.console.error( + "A translation was requested but the fromLanguage or toLanguage was not set.", + { fromLanguage, toLanguage, reportAsAutoTranslate } + ); + return; + } + if (this.languageState.requestedTranslationPair) { + // This page has already been translated, restore it and translate it + // again once the actor has been recreated. + TranslationsParent.#translateOnPageReload = { fromLanguage, toLanguage }; + this.restorePage(fromLanguage); + } else { + const { docLangTag } = this.languageState.detectedLanguages; + + let engineProcess; + try { + engineProcess = await TranslationsParent.getEngineProcess(); + } catch (error) { + console.error("Failed to get the translation engine process", error); + return; + } + + if (!this.innerWindowId) { + throw new Error( + "The innerWindowId for the TranslationsParent was not available." + ); + } + + // The MessageChannel will be used for communicating directly between the content + // process and the engine's process. + const { port1, port2 } = new MessageChannel(); + engineProcess.actor.startTranslation( + fromLanguage, + toLanguage, + port1, + this.innerWindowId, + this + ); + + this.languageState.requestedTranslationPair = { + fromLanguage, + toLanguage, + }; + + const preferredLanguages = TranslationsParent.getPreferredLanguages(); + const topPreferredLanguage = + preferredLanguages && preferredLanguages.length + ? preferredLanguages[0] + : null; + + TranslationsParent.telemetry().onTranslate({ + docLangTag, + fromLanguage, + toLanguage, + topPreferredLanguage, + autoTranslate: reportAsAutoTranslate, + }); + + this.sendAsyncMessage( + "Translations:TranslatePage", + { + fromLanguage, + toLanguage, + port: port2, + }, + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects + // Mark the MessageChannel port as transferable. + [port2] + ); + } + } + + /** + * Restore the page to the original language by doing a hard reload. + */ + restorePage() { + TranslationsParent.telemetry().onRestorePage(); + // Skip auto-translate for one page load. + TranslationsParent.#isPageRestored = true; + this.languageState.requestedTranslationPair = null; + TranslationsParent.#previousDetectedLanguages = + this.languageState.detectedLanguages; + + const browser = this.browsingContext.embedderElement; + browser.reload(); + } + + /** + * Keep track of when the location changes. + */ + static #locationChangeId = 0; + + static onLocationChange(browser) { + if (!lazy.translationsEnabledPref) { + // The pref isn't enabled, so don't attempt to get the actor. + return; + } + let windowGlobal = browser.browsingContext.currentWindowGlobal; + TranslationsParent.#locationChangeId++; + let actor; + try { + actor = windowGlobal.getActor("Translations"); + } catch (_) { + // The actor may not be supported on this page. + } + if (actor) { + actor.languageState.locationChangeId = + TranslationsParent.#locationChangeId; + } + } + + /** + * Is this actor active for the current location change? + * + * @param {number} locationChangeId - The id sent by the "TranslationsParent:LanguageState" event. + * @returns {boolean} + */ + static isActiveLocation(locationChangeId) { + return locationChangeId === TranslationsParent.#locationChangeId; + } + + async queryIdentifyLanguage() { + if ( + TranslationsParent.isInAutomation() && + !TranslationsParent.#isTranslationsEngineMocked + ) { + return null; + } + return this.sendQuery("Translations:IdentifyLanguage").catch(error => { + if (this.#isDestroyed) { + // The actor was destroyed while this message was still being resolved. + return null; + } + return Promise.reject(error); + }); + } + + /** + * Returns the language from the document element. + * + * @returns {Promise<string>} + */ + queryDocumentElementLang() { + return this.sendQuery("Translations:GetDocumentElementLang"); + } + + /** + * @param {LangTags} langTags + */ + shouldAutoTranslate(langTags) { + if ( + langTags.docLangTag && + langTags.userLangTag && + langTags.isDocLangTagSupported && + TranslationsParent.#maybeAutoTranslate(langTags) && + !TranslationsParent.shouldNeverTranslateLanguage(langTags.docLangTag) && + !this.shouldNeverTranslateSite() + ) { + return true; + } + + return false; + } + + /** + * Returns the lang tags that should be offered for translation. This is in the parent + * rather than the child to remove the per-content process memory allocation amount. + * + * @param {string} [documentElementLang] + * @param {string} [href] + * @returns {Promise<LangTags | null>} - Returns null if the actor was destroyed before + * the result could be resolved. + */ + async getDetectedLanguages(documentElementLang, href) { + if (this.languageState.detectedLanguages) { + return this.languageState.detectedLanguages; + } + const langTags = { + docLangTag: null, + userLangTag: null, + isDocLangTagSupported: false, + }; + if (!TranslationsParent.getIsTranslationsEngineSupported()) { + return null; + } + + if (documentElementLang === undefined) { + documentElementLang = await this.queryDocumentElementLang(); + if (this.#isDestroyed) { + return null; + } + } + + let languagePairs = await TranslationsParent.getLanguagePairs(); + if (this.#isDestroyed) { + return null; + } + + const determineIsDocLangTagSupported = () => + Boolean( + languagePairs.find(({ fromLang }) => fromLang === langTags.docLangTag) + ); + + // First try to get the langTag from the document's markup. + try { + const docLocale = new Intl.Locale(documentElementLang); + langTags.docLangTag = docLocale.language; + langTags.isDocLangTagSupported = determineIsDocLangTagSupported(); + } catch (error) {} + + if (langTags.docLangTag) { + // If it's not supported, try it again with a canonicalized version. + if (!langTags.isDocLangTagSupported) { + langTags.docLangTag = Intl.getCanonicalLocales(langTags.docLangTag)[0]; + langTags.isDocLangTagSupported = determineIsDocLangTagSupported(); + } + + // If it's still not supported, map macro language codes to specific ones. + // https://en.wikipedia.org/wiki/ISO_639_macrolanguage + if (!langTags.isDocLangTagSupported) { + // If more macro language codes are needed, this logic can be expanded. + if (langTags.docLangTag === "no") { + // Choose "Norwegian Bokmål" over "Norwegian Nynorsk" as it is more widely used. + // + // https://en.wikipedia.org/wiki/Norwegian_language#Bokm%C3%A5l_and_Nynorsk + // + // > A 2005 poll indicates that 86.3% use primarily Bokmål as their daily + // > written language, 5.5% use both Bokmål and Nynorsk, and 7.5% use + // > primarily Nynorsk. + langTags.docLangTag = "nb"; + langTags.isDocLangTagSupported = determineIsDocLangTagSupported(); + } + } + } else { + // If the document's markup had no specified langTag, attempt to identify the page's language. + langTags.docLangTag = await this.queryIdentifyLanguage(); + if (this.#isDestroyed) { + return null; + } + langTags.isDocLangTagSupported = determineIsDocLangTagSupported(); + } + + const preferredLanguages = TranslationsParent.getPreferredLanguages(); + + if (!langTags.docLangTag) { + const message = "No valid language detected."; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, href); + + const languagePairs = await TranslationsParent.getLanguagePairs(); + if (this.#isDestroyed) { + return null; + } + + // Attempt to find a good language to select for the user. + langTags.userLangTag = + preferredLanguages.find(langTag => langTag === languagePairs.toLang) ?? + null; + + return langTags; + } + + if (TranslationsParent.getWebContentLanguages().has(langTags.docLangTag)) { + // The doc language has been marked as a known language by the user, do not + // offer a translation. + const message = + "The app and document languages match, so not translating."; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, href); + // The docLangTag will be set, while the userLangTag will be null. + return langTags; + } + + // Attempt to find a matching language pair for a preferred language. + for (const preferredLangTag of preferredLanguages) { + if (!langTags.isDocLangTagSupported) { + if (languagePairs.some(({ toLang }) => toLang === preferredLangTag)) { + // Only match the "to" language, since the "from" is not supported. + langTags.userLangTag = preferredLangTag; + } + break; + } + + // Is there a direct language pair match? + if ( + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === langTags.docLangTag && toLang === preferredLangTag + ) + ) { + // A match was found in one of the preferred languages. + langTags.userLangTag = preferredLangTag; + break; + } + + // Is there a pivot language match? + if ( + // Match doc -> pivot + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === langTags.docLangTag && toLang === PIVOT_LANGUAGE + ) && + // Match pivot -> preferred language + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === PIVOT_LANGUAGE && toLang === preferredLangTag + ) + ) { + langTags.userLangTag = preferredLangTag; + break; + } + } + + if (!langTags.userLangTag) { + // No language pairs match. + const message = `No matching translation pairs were found for translating from "${langTags.docLangTag}".`; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, languagePairs); + } + + return langTags; + } + + /** + * The pref for if we can always offer a translation when it's available. + */ + static shouldAlwaysOfferTranslations() { + return lazy.automaticallyPopupPref; + } + + /** + * Returns true if the given language tag is present in the always-translate + * languages preference, otherwise false. + * + * @param {LangTags} langTags + * @returns {boolean} + */ + static shouldAlwaysTranslateLanguage(langTags) { + const { docLangTag, userLangTag } = langTags; + if (docLangTag === userLangTag || !userLangTag) { + // Do not auto-translate when the docLangTag matches the userLangTag, or when + // the userLangTag is not set. The "always translate" is exposed via about:confg. + // In case of users putting in non-sensical things here, we don't want to break + // the experience. This behavior can lead to a "language degradation machine" + // where we go from a source language -> pivot language -> source language. + return false; + } + return lazy.alwaysTranslateLangTags.has(docLangTag); + } + + /** + * Returns true if the given language tag is present in the never-translate + * languages preference, otherwise false. + * + * @param {string} langTag - A BCP-47 language tag + * @returns {boolean} + */ + static shouldNeverTranslateLanguage(langTag) { + return lazy.neverTranslateLangTags.has(langTag); + } + + /** + * Returns true if the current site is denied permissions to translate, + * otherwise returns false. + * + * @returns {Promise<boolean>} + */ + shouldNeverTranslateSite() { + const perms = Services.perms; + const permission = perms.getPermissionObject( + this.browsingContext.currentWindowGlobal.documentPrincipal, + TRANSLATIONS_PERMISSION, + /* exactHost */ false + ); + return permission?.capability === perms.DENY_ACTION; + } + + /** + * Removes the given language tag from the given preference. + * + * @param {string} langTag - A BCP-47 language tag + * @param {string} prefName - The pref name + */ + static removeLangTagFromPref(langTag, prefName) { + const langTags = + prefName === ALWAYS_TRANSLATE_LANGS_PREF + ? lazy.alwaysTranslateLangTags + : lazy.neverTranslateLangTags; + const newLangTags = [...langTags].filter(tag => tag !== langTag); + Services.prefs.setCharPref(prefName, [...newLangTags].join(",")); + } + + /** + * Adds the given language tag to the given preference. + * + * @param {string} langTag - A BCP-47 language tag + * @param {string} prefName - The pref name + */ + static addLangTagToPref(langTag, prefName) { + const langTags = + prefName === ALWAYS_TRANSLATE_LANGS_PREF + ? lazy.alwaysTranslateLangTags + : lazy.neverTranslateLangTags; + if (!langTags.has(langTag)) { + langTags.add(langTag); + } + Services.prefs.setCharPref(prefName, [...langTags].join(",")); + } + + /** + * Toggles the always-translate language preference by adding the language + * to the pref list if it is not present, or removing it if it is present. + * + * @param {LangTags} langTags + * @returns {boolean} + * True if always-translate was enabled for this language. + * False if always-translate was disabled for this language. + */ + static toggleAlwaysTranslateLanguagePref(langTags) { + const { docLangTag, appLangTag } = langTags; + + if (appLangTag === docLangTag) { + // In case somehow the user attempts to toggle this when the app and doc language + // are the same, just remove the lang tag. + this.removeLangTagFromPref(appLangTag, ALWAYS_TRANSLATE_LANGS_PREF); + return false; + } + + if (TranslationsParent.shouldAlwaysTranslateLanguage(langTags)) { + // The pref was toggled off for this langTag + this.removeLangTagFromPref(docLangTag, ALWAYS_TRANSLATE_LANGS_PREF); + return false; + } + + // The pref was toggled on for this langTag + this.addLangTagToPref(docLangTag, ALWAYS_TRANSLATE_LANGS_PREF); + this.removeLangTagFromPref(docLangTag, NEVER_TRANSLATE_LANGS_PREF); + return true; + } + + /** + * Toggle the automatically popup pref, which will either + * enable or disable translations being offered to the user. + * + * @returns {boolean} + * True if offering translations was enabled by this call. + * False if offering translations was disabled by this call. + */ + static toggleAutomaticallyPopupPref() { + const prefValueBeforeToggle = lazy.automaticallyPopupPref; + Services.prefs.setBoolPref( + "browser.translations.automaticallyPopup", + !prefValueBeforeToggle + ); + return !prefValueBeforeToggle; + } + + /** + * Toggles the never-translate language preference by adding the language + * to the pref list if it is not present, or removing it if it is present. + * + * @param {string} langTag - A BCP-47 language tag + * @returns {boolean} Whether the pref was toggled on or off for this langTag. + * True if never-translate was enabled for this language. + * False if never-translate was disabled for this language. + */ + static toggleNeverTranslateLanguagePref(langTag) { + if (TranslationsParent.shouldNeverTranslateLanguage(langTag)) { + // The pref was toggled off for this langTag + this.removeLangTagFromPref(langTag, NEVER_TRANSLATE_LANGS_PREF); + return false; + } + + // The pref was toggled on for this langTag + this.addLangTagToPref(langTag, NEVER_TRANSLATE_LANGS_PREF); + this.removeLangTagFromPref(langTag, ALWAYS_TRANSLATE_LANGS_PREF); + return true; + } + + /** + * Toggles the never-translate site permissions by adding DENY_ACTION to + * the site principal if it is not present, or removing it if it is present. + * + * @returns {boolean} + * True if never-translate was enabled for this site. + * False if never-translate was disabled for this site. + */ + toggleNeverTranslateSitePermissions() { + if (this.shouldNeverTranslateSite()) { + return this.setNeverTranslateSitePermissions(false); + } + + return this.setNeverTranslateSitePermissions(true); + } + + /** + * Sets the never-translate site permissions by adding DENY_ACTION to + * the site principal. + * + * @param {string} neverTranslate - The never translate setting. + * @returns {boolean} + * True if never-translate was enabled for this site. + * False if never-translate was disabled for this site. + */ + setNeverTranslateSitePermissions(neverTranslate) { + const { documentPrincipal } = this.browsingContext.currentWindowGlobal; + return TranslationsParent.#setNeverTranslateSiteByPrincipal( + neverTranslate, + documentPrincipal + ); + } + + /** + * Sets the never-translate site permissions by creating a principal from the URL origin + * and setting or unsetting the DENY_ACTION on the permission. + * + * @param {string} neverTranslate - The never translate setting to use. + * @param {string} urlOrigin - The url origin to set the permission for. + * @returns {boolean} + * True if never-translate was enabled for this origin. + * False if never-translate was disabled for this origin. + */ + static setNeverTranslateSiteByOrigin(neverTranslate, urlOrigin) { + const principal = + Services.scriptSecurityManager.createContentPrincipalFromOrigin( + urlOrigin + ); + return TranslationsParent.#setNeverTranslateSiteByPrincipal( + neverTranslate, + principal + ); + } + + /** + * Sets the never-translate site permissions by adding DENY_ACTION to + * the specified site principal. + * + * @param {string} neverTranslate - The never translate setting. + * @param {string} principal - The principal that should have the permission attached. + * @returns {boolean} + * True if never-translate was enabled for this principal. + * False if never-translate was disabled for this principal. + */ + static #setNeverTranslateSiteByPrincipal(neverTranslate, principal) { + const perms = Services.perms; + + if (!neverTranslate) { + perms.removeFromPrincipal(principal, TRANSLATIONS_PERMISSION); + return false; + } + + perms.addFromPrincipal( + principal, + TRANSLATIONS_PERMISSION, + perms.DENY_ACTION + ); + return true; + } + + /** + * Creates a list of URLs that have a translations permission set on the resource. + * These are the sites to never translate. + * + * @returns {Array<string>} String array with the URL of the sites that have the never translate permission. + */ + static listNeverTranslateSites() { + const neverTranslateSites = []; + for (const perm of Services.perms.getAllByTypes([ + TRANSLATIONS_PERMISSION, + ])) { + if (perm.capability === Services.perms.DENY_ACTION) { + neverTranslateSites.push(perm.principal.origin); + } + } + let stripProtocol = s => s?.replace(/^\w+:/, "") || ""; + return neverTranslateSites.sort((a, b) => { + return stripProtocol(a).localeCompare(stripProtocol(b)); + }); + } + + /** + * Ensure that the translations are always destroyed, even if the content translations + * are misbehaving. + */ + #ensureTranslationsDiscarded() { + if (!TranslationsParent.#engine) { + return; + } + TranslationsParent.#engine + // If the engine fails to load, ignore it since we are ending translations. + .catch(() => null) + .then(engineProcess => { + if (engineProcess && this.languageState.requestedTranslationPair) { + engineProcess.actor.discardTranslations(this.innerWindowId); + } + }) + // This error will be one from the endTranslation code, which we need to + // surface. + .catch(error => lazy.console.error(error)); + } + + didDestroy() { + if (!this.innerWindowId) { + throw new Error( + "The innerWindowId for the TranslationsParent was not available." + ); + } + + this.#ensureTranslationsDiscarded(); + + this.#isDestroyed = true; + } +} + +/** + * Validate some simple Wasm that uses a SIMD operation. + */ +function detectSimdSupport() { + return WebAssembly.validate( + new Uint8Array( + // ``` + // ;; Detect SIMD support. + // ;; Compile by running: wat2wasm --enable-all simd-detect.wat + // + // (module + // (func (result v128) + // i32.const 0 + // i8x16.splat + // i8x16.popcnt + // ) + // ) + // ``` + + // prettier-ignore + [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x05, 0x01, 0x60, 0x00, + 0x01, 0x7b, 0x03, 0x02, 0x01, 0x00, 0x0a, 0x0a, 0x01, 0x08, 0x00, 0x41, 0x00, + 0xfd, 0x0f, 0xfd, 0x62, 0x0b + ] + ) + ); +} + +/** + * State that affects the UI. Any of the state that gets set triggers a dispatch to update + * the UI. + */ +class TranslationsLanguageState { + /** + * @param {TranslationsParent} actor + * @param {LangTags | null} previousDetectedLanguages + */ + constructor(actor, previousDetectedLanguages = null) { + this.#actor = actor; + this.#detectedLanguages = previousDetectedLanguages; + this.dispatch(); + } + + /** + * The data members for TranslationsLanguageState, see the getters for their + * documentation. + */ + + /** @type {TranslationsParent} */ + #actor; + + /** @type {TranslationPair | null} */ + #requestedTranslationPair = null; + + /** @type {LangTags | null} */ + #detectedLanguages = null; + + /** @type {number} */ + #locationChangeId = -1; + + /** @type {null | TranslationErrors} */ + #error = null; + + #isEngineReady = false; + + /** + * Dispatch anytime the language details change, so that any UI can react to it. + */ + dispatch() { + if (!TranslationsParent.isActiveLocation(this.#locationChangeId)) { + // Do not dispatch as this location is not active. + return; + } + + const browser = this.#actor.browsingContext.top.embedderElement; + if (!browser) { + return; + } + const { CustomEvent } = browser.ownerGlobal; + browser.dispatchEvent( + new CustomEvent("TranslationsParent:LanguageState", { + bubbles: true, + detail: { + actor: this.#actor, + }, + }) + ); + } + + /** + * When a translation is requested, this contains the translation pair. This means + * that the TranslationsChild should be creating a TranslationsDocument and keep + * the page updated with the target language. + * + * @returns {TranslationPair | null} + */ + get requestedTranslationPair() { + return this.#requestedTranslationPair; + } + + set requestedTranslationPair(requestedTranslationPair) { + if (this.#requestedTranslationPair === requestedTranslationPair) { + return; + } + + this.#error = null; + this.#isEngineReady = false; + this.#requestedTranslationPair = requestedTranslationPair; + this.dispatch(); + } + + /** + * The TranslationsChild will detect languages and offer them up for translation. + * The results are stored here. + * + * @returns {LangTags | null} + */ + get detectedLanguages() { + return this.#detectedLanguages; + } + + set detectedLanguages(detectedLanguages) { + if (this.#detectedLanguages === detectedLanguages) { + return; + } + + this.#detectedLanguages = detectedLanguages; + this.dispatch(); + } + + /** + * This id represents the last location change that happened for this actor. This + * allows the UI to disambiguate when there are races and out of order events that + * are dispatched. Only the most up to date `locationChangeId` is used. + * + * @returns {number} + */ + get locationChangeId() { + return this.#locationChangeId; + } + + set locationChangeId(locationChangeId) { + if (this.#locationChangeId === locationChangeId) { + return; + } + + this.#locationChangeId = locationChangeId; + + // When the location changes remove the previous error. + this.#error = null; + + this.dispatch(); + } + + /** + * The last error that occured during translation. + */ + get error() { + return this.#error; + } + + set error(error) { + if (this.#error === error) { + return; + } + this.#error = error; + // Setting an error invalidates the requested translation pair. + this.#requestedTranslationPair = null; + this.#isEngineReady = false; + this.dispatch(); + } + + /** + * Stores when the translations engine is ready. The wasm and language files must + * be downloaded, which can take some time. + */ + get isEngineReady() { + return this.#isEngineReady; + } + + set isEngineReady(isEngineReady) { + if (this.#isEngineReady === isEngineReady) { + return; + } + this.#isEngineReady = isEngineReady; + this.dispatch(); + } +} + +/** + * @typedef {Object} QueueItem + * @prop {Function} download + * @prop {Function} [onSuccess] + * @prop {Function} [onFailure] + * @prop {number} [retriesLeft] + */ + +/** + * Manage the download of the files by providing a maximum number of concurrent files + * and the ability to retry a file download in case of an error. + * + * @param {QueueItem[]} queue + */ +async function downloadManager(queue) { + const NOOP = () => {}; + + const pendingDownloadAttempts = new Set(); + let failCount = 0; + let index = 0; + const start = Date.now(); + const originalQueueLength = queue.length; + + while (index < queue.length || pendingDownloadAttempts.size > 0) { + // Start new downloads up to the maximum limit + while ( + index < queue.length && + pendingDownloadAttempts.size < TranslationsParent.MAX_CONCURRENT_DOWNLOADS + ) { + lazy.console.log(`Starting download ${index + 1} of ${queue.length}`); + + const { + download, + onSuccess = NOOP, + onFailure = NOOP, + retriesLeft = TranslationsParent.MAX_DOWNLOAD_RETRIES, + } = queue[index]; + + const handleFailedDownload = error => { + // The download failed. Either retry it, or report the failure. + TranslationsParent.reportError( + new Error("Failed to download file."), + error + ); + + const newRetriesLeft = retriesLeft - 1; + + if (retriesLeft > 0) { + lazy.console.log( + `Queueing another attempt. ${newRetriesLeft} attempts left.` + ); + queue.push({ + download, + retriesLeft: newRetriesLeft, + onSuccess, + onFailure, + }); + } else { + // Give up on this download. + failCount++; + onFailure(); + } + }; + + const afterDownloadAttempt = () => { + pendingDownloadAttempts.delete(downloadAttempt); + }; + + // Kick off the download. If it fails, retry it a certain number of attempts. + // This is done asynchronously from the rest of the for loop. + const downloadAttempt = download() + .then(onSuccess, handleFailedDownload) + .then(afterDownloadAttempt); + + pendingDownloadAttempts.add(downloadAttempt); + index++; + } + + // Wait for any active downloads to complete. + await Promise.race(pendingDownloadAttempts); + } + + const duration = ((Date.now() - start) / 1000).toFixed(3); + + if (failCount > 0) { + const message = `Finished downloads in ${duration} seconds, but ${failCount} download(s) failed.`; + lazy.console.log( + `Finished downloads in ${duration} seconds, but ${failCount} download(s) failed.` + ); + throw new Error(message); + } + + lazy.console.log( + `Finished ${originalQueueLength} downloads in ${duration} seconds.` + ); +} + +/** + * The translations code has lots of async code and fallible network requests. To test + * this manually while using the feature, enable chaos mode by setting "errors" to true + * and "timeoutMS" to a positive number of milliseconds. + * prefs to true: + * + * - browser.translations.chaos.timeoutMS + * - browser.translations.chaos.errors + */ +async function chaosMode(probability = 0.5) { + await chaosModeTimer(); + await chaosModeError(probability); +} + +/** + * The translations code has lots of async code that relies on the network. To test + * this manually while using the feature, enable chaos mode by setting the following pref + * to a positive number of milliseconds. + * + * - browser.translations.chaos.timeoutMS + */ +async function chaosModeTimer() { + if (lazy.chaosTimeoutMSPref) { + const timeout = Math.random() * lazy.chaosTimeoutMSPref; + lazy.console.log( + `Chaos mode timer started for ${(timeout / 1000).toFixed(1)} seconds.` + ); + await new Promise(resolve => lazy.setTimeout(resolve, timeout)); + } +} + +/** + * The translations code has lots of async code that is fallible. To test this manually + * while using the feature, enable chaos mode by setting the following pref to true. + * + * - browser.translations.chaos.errors + */ +async function chaosModeError(probability = 0.5) { + if (lazy.chaosErrorsPref && Math.random() < probability) { + lazy.console.trace(`Chaos mode error generated.`); + throw new Error( + `Chaos Mode error from the pref "browser.translations.chaos.errors".` + ); + } +} diff --git a/toolkit/components/translations/actors/moz.build b/toolkit/components/translations/actors/moz.build new file mode 100644 index 0000000000..70743e82fd --- /dev/null +++ b/toolkit/components/translations/actors/moz.build @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FINAL_TARGET_FILES.actors += [ + "AboutTranslationsChild.sys.mjs", + "AboutTranslationsParent.sys.mjs", + "TranslationsChild.sys.mjs", + "TranslationsEngineChild.sys.mjs", + "TranslationsEngineParent.sys.mjs", + "TranslationsParent.sys.mjs", +] diff --git a/toolkit/components/translations/bergamot-translator/LICENSE b/toolkit/components/translations/bergamot-translator/LICENSE new file mode 100644 index 0000000000..a612ad9813 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/toolkit/components/translations/bergamot-translator/bergamot-translator.js b/toolkit/components/translations/bergamot-translator/bergamot-translator.js new file mode 100644 index 0000000000..c50eb32dff --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/bergamot-translator.js @@ -0,0 +1,3509 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function loadBergamot(Module) { + var BERGAMOT_VERSION_FULL = "v0.4.5+05a8778"; + null; + + var Module = typeof Module != "undefined" ? Module : {}; + + var moduleOverrides = Object.assign({}, Module); + + var arguments_ = []; + + var thisProgram = "./this.program"; + + var quit_ = (status, toThrow) => { + throw toThrow; + }; + + var ENVIRONMENT_IS_WEB = typeof window == "object"; + + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + + var ENVIRONMENT_IS_NODE = + typeof process == "object" && + typeof process.versions == "object" && + typeof process.versions.node == "string"; + + var scriptDirectory = ""; + + function locateFile(path) { + if (Module.locateFile) { + return Module.locateFile(path, scriptDirectory); + } + return scriptDirectory + path; + } + + var read_, readAsync, readBinary, setWindowTitle; + + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (scriptDirectory.indexOf("blob:") !== 0) { + scriptDirectory = scriptDirectory.substr( + 0, + scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1 + ); + } else { + scriptDirectory = ""; + } + { + read_ = url => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = url => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array(xhr.response); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + setWindowTitle = title => (document.title = title); + } else { + } + + var out = Module.print || console.log.bind(console); + + var err = Module.printErr || console.warn.bind(console); + + Object.assign(Module, moduleOverrides); + + moduleOverrides = null; + + if (Module.arguments) { + arguments_ = Module.arguments; + } + + if (Module.thisProgram) { + thisProgram = Module.thisProgram; + } + + if (Module.quit) { + quit_ = Module.quit; + } + + var tempRet0 = 0; + + var setTempRet0 = value => { + tempRet0 = value; + }; + + var wasmBinary; + + if (Module.wasmBinary) { + wasmBinary = Module.wasmBinary; + } + + var noExitRuntime = Module.noExitRuntime || true; + + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + + var wasmMemory; + + var ABORT = false; + + var EXITSTATUS; + + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + + var UTF8Decoder = + typeof TextDecoder != "undefined" ? new TextDecoder("utf8") : undefined; + + function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead) { + var endIdx = idx + maxBytesToRead; + var endPtr = idx; + while (heapOrArray[endPtr] && !(endPtr >= endIdx)) { + ++endPtr; + } + if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) { + return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr)); + } + var str = ""; + while (idx < endPtr) { + var u0 = heapOrArray[idx++]; + if (!(u0 & 128)) { + str += String.fromCharCode(u0); + continue; + } + var u1 = heapOrArray[idx++] & 63; + if ((u0 & 224) == 192) { + str += String.fromCharCode(((u0 & 31) << 6) | u1); + continue; + } + var u2 = heapOrArray[idx++] & 63; + if ((u0 & 240) == 224) { + u0 = ((u0 & 15) << 12) | (u1 << 6) | u2; + } else { + u0 = + ((u0 & 7) << 18) | (u1 << 12) | (u2 << 6) | (heapOrArray[idx++] & 63); + } + if (u0 < 65536) { + str += String.fromCharCode(u0); + } else { + var ch = u0 - 65536; + str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023)); + } + } + + return str; + } + + function UTF8ToString(ptr, maxBytesToRead) { + return ptr ? UTF8ArrayToString(HEAPU8, ptr, maxBytesToRead) : ""; + } + + function stringToUTF8Array(str, heap, outIdx, maxBytesToWrite) { + if (!(maxBytesToWrite > 0)) { + return 0; + } + var startIdx = outIdx; + var endIdx = outIdx + maxBytesToWrite - 1; + for (var i = 0; i < str.length; ++i) { + var u = str.charCodeAt(i); + if (u >= 55296 && u <= 57343) { + var u1 = str.charCodeAt(++i); + u = (65536 + ((u & 1023) << 10)) | (u1 & 1023); + } + if (u <= 127) { + if (outIdx >= endIdx) { + break; + } + heap[outIdx++] = u; + } else if (u <= 2047) { + if (outIdx + 1 >= endIdx) { + break; + } + heap[outIdx++] = 192 | (u >> 6); + heap[outIdx++] = 128 | (u & 63); + } else if (u <= 65535) { + if (outIdx + 2 >= endIdx) { + break; + } + heap[outIdx++] = 224 | (u >> 12); + heap[outIdx++] = 128 | ((u >> 6) & 63); + heap[outIdx++] = 128 | (u & 63); + } else { + if (outIdx + 3 >= endIdx) { + break; + } + heap[outIdx++] = 240 | (u >> 18); + heap[outIdx++] = 128 | ((u >> 12) & 63); + heap[outIdx++] = 128 | ((u >> 6) & 63); + heap[outIdx++] = 128 | (u & 63); + } + } + heap[outIdx] = 0; + return outIdx - startIdx; + } + + function stringToUTF8(str, outPtr, maxBytesToWrite) { + return stringToUTF8Array(str, HEAPU8, outPtr, maxBytesToWrite); + } + + function lengthBytesUTF8(str) { + var len = 0; + for (var i = 0; i < str.length; ++i) { + var u = str.charCodeAt(i); + if (u >= 55296 && u <= 57343) { + u = (65536 + ((u & 1023) << 10)) | (str.charCodeAt(++i) & 1023); + } + if (u <= 127) { + ++len; + } else if (u <= 2047) { + len += 2; + } else if (u <= 65535) { + len += 3; + } else { + len += 4; + } + } + return len; + } + + var UTF16Decoder = + typeof TextDecoder != "undefined" ? new TextDecoder("utf-16le") : undefined; + + function UTF16ToString(ptr, maxBytesToRead) { + var endPtr = ptr; + var idx = endPtr >> 1; + var maxIdx = idx + maxBytesToRead / 2; + while (!(idx >= maxIdx) && HEAPU16[idx]) { + ++idx; + } + endPtr = idx << 1; + if (endPtr - ptr > 32 && UTF16Decoder) { + return UTF16Decoder.decode(HEAPU8.subarray(ptr, endPtr)); + } + var str = ""; + for (var i = 0; !(i >= maxBytesToRead / 2); ++i) { + var codeUnit = HEAP16[(ptr + i * 2) >> 1]; + if (codeUnit == 0) { + break; + } + str += String.fromCharCode(codeUnit); + } + return str; + } + + function stringToUTF16(str, outPtr, maxBytesToWrite) { + if (maxBytesToWrite === undefined) { + maxBytesToWrite = 2147483647; + } + if (maxBytesToWrite < 2) { + return 0; + } + maxBytesToWrite -= 2; + var startPtr = outPtr; + var numCharsToWrite = + maxBytesToWrite < str.length * 2 ? maxBytesToWrite / 2 : str.length; + for (var i = 0; i < numCharsToWrite; ++i) { + var codeUnit = str.charCodeAt(i); + HEAP16[outPtr >> 1] = codeUnit; + outPtr += 2; + } + HEAP16[outPtr >> 1] = 0; + return outPtr - startPtr; + } + + function lengthBytesUTF16(str) { + return str.length * 2; + } + + function UTF32ToString(ptr, maxBytesToRead) { + var i = 0; + var str = ""; + while (!(i >= maxBytesToRead / 4)) { + var utf32 = HEAP32[(ptr + i * 4) >> 2]; + if (utf32 == 0) { + break; + } + ++i; + if (utf32 >= 65536) { + var ch = utf32 - 65536; + str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023)); + } else { + str += String.fromCharCode(utf32); + } + } + return str; + } + + function stringToUTF32(str, outPtr, maxBytesToWrite) { + if (maxBytesToWrite === undefined) { + maxBytesToWrite = 2147483647; + } + if (maxBytesToWrite < 4) { + return 0; + } + var startPtr = outPtr; + var endPtr = startPtr + maxBytesToWrite - 4; + for (var i = 0; i < str.length; ++i) { + var codeUnit = str.charCodeAt(i); + if (codeUnit >= 55296 && codeUnit <= 57343) { + var trailSurrogate = str.charCodeAt(++i); + codeUnit = + (65536 + ((codeUnit & 1023) << 10)) | (trailSurrogate & 1023); + } + HEAP32[outPtr >> 2] = codeUnit; + outPtr += 4; + if (outPtr + 4 > endPtr) { + break; + } + } + HEAP32[outPtr >> 2] = 0; + return outPtr - startPtr; + } + + function lengthBytesUTF32(str) { + var len = 0; + for (var i = 0; i < str.length; ++i) { + var codeUnit = str.charCodeAt(i); + if (codeUnit >= 55296 && codeUnit <= 57343) { + ++i; + } + len += 4; + } + return len; + } + + function allocateUTF8(str) { + var size = lengthBytesUTF8(str) + 1; + var ret = _malloc(size); + if (ret) { + stringToUTF8Array(str, HEAP8, ret, size); + } + return ret; + } + + function writeArrayToMemory(array, buffer) { + HEAP8.set(array, buffer); + } + + function writeAsciiToMemory(str, buffer, dontAddNull) { + for (var i = 0; i < str.length; ++i) { + HEAP8[buffer++ >> 0] = str.charCodeAt(i); + } + if (!dontAddNull) { + HEAP8[buffer >> 0] = 0; + } + } + + var buffer, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + + function updateGlobalBufferAndViews(buf) { + const mb = (buf.byteLength / 1_000_000).toFixed(); + Module.print(`Growing wasm buffer to ${mb}MB (${buf.byteLength} bytes).`); + + buffer = buf; + Module.HEAP8 = HEAP8 = new Int8Array(buf); + Module.HEAP16 = HEAP16 = new Int16Array(buf); + Module.HEAP32 = HEAP32 = new Int32Array(buf); + Module.HEAPU8 = HEAPU8 = new Uint8Array(buf); + Module.HEAPU16 = HEAPU16 = new Uint16Array(buf); + Module.HEAPU32 = HEAPU32 = new Uint32Array(buf); + Module.HEAPF32 = HEAPF32 = new Float32Array(buf); + Module.HEAPF64 = HEAPF64 = new Float64Array(buf); + } + + var INITIAL_MEMORY = Module.INITIAL_MEMORY || 16777216; + + if (Module.wasmMemory) { + wasmMemory = Module.wasmMemory; + } else { + wasmMemory = new WebAssembly.Memory({ + initial: INITIAL_MEMORY / 65536, + maximum: 2147483648 / 65536, + }); + } + + if (wasmMemory) { + buffer = wasmMemory.buffer; + } + + INITIAL_MEMORY = buffer.byteLength; + + updateGlobalBufferAndViews(buffer); + + var wasmTable; + + var __ATPRERUN__ = []; + + var __ATINIT__ = []; + + var __ATPOSTRUN__ = []; + + var runtimeInitialized = false; + + function keepRuntimeAlive() { + return noExitRuntime; + } + + function preRun() { + if (Module.preRun) { + if (typeof Module.preRun == "function") { + Module.preRun = [Module.preRun]; + } + while (Module.preRun.length) { + addOnPreRun(Module.preRun.shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + + function postRun() { + if (Module.postRun) { + if (typeof Module.postRun == "function") { + Module.postRun = [Module.postRun]; + } + while (Module.postRun.length) { + addOnPostRun(Module.postRun.shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + + var runDependencies = 0; + + var runDependencyWatcher = null; + + var dependenciesFulfilled = null; + + function addRunDependency(id) { + runDependencies++; + if (Module.monitorRunDependencies) { + Module.monitorRunDependencies(runDependencies); + } + } + + function removeRunDependency(id) { + runDependencies--; + if (Module.monitorRunDependencies) { + Module.monitorRunDependencies(runDependencies); + } + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + + Module.preloadedImages = {}; + + Module.preloadedAudios = {}; + + function abort(what) { + { + if (Module.onAbort) { + Module.onAbort(what); + } + } + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -s ASSERTIONS=1 for more info."; + var e = new WebAssembly.RuntimeError(what); + throw e; + } + + var dataURIPrefix = "data:application/octet-stream;base64,"; + + function isDataURI(filename) { + return filename.startsWith(dataURIPrefix); + } + + var wasmBinaryFile; + + wasmBinaryFile = "bergamot-translator-worker.wasm"; + + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + + function getBinary(file) { + try { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } catch (err) { + abort(err); + } + } + + function getBinaryPromise() { + if (!wasmBinary && (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER)) { + if (typeof fetch == "function") { + return fetch(wasmBinaryFile, { + credentials: "same-origin", + }) + .then(function (response) { + if (!response.ok) { + throw ( + "failed to load wasm binary file at '" + wasmBinaryFile + "'" + ); + } + return response.arrayBuffer(); + }) + .catch(function () { + return getBinary(wasmBinaryFile); + }); + } + } + return Promise.resolve().then(function () { + return getBinary(wasmBinaryFile); + }); + } + + function createWasm() { + var info = { + env: asmLibraryArg, + wasm_gemm: createWasmGemm(), + wasi_snapshot_preview1: asmLibraryArg, + }; + function receiveInstance(instance, module) { + var exports = instance.exports; + Module.asm = exports; + wasmTable = Module.asm.__indirect_function_table; + addOnInit(Module.asm.__wasm_call_ctors); + exportAsmFunctions(exports); + removeRunDependency("wasm-instantiate"); + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result.instance); + } + function instantiateArrayBuffer(receiver) { + return getBinaryPromise() + .then(function (binary) { + return WebAssembly.instantiate(binary, info); + }) + .then(function (instance) { + return instance; + }) + .then(receiver, function (reason) { + err("failed to asynchronously prepare wasm: " + reason); + abort(reason); + }); + } + function instantiateAsync() { + if ( + !wasmBinary && + typeof WebAssembly.instantiateStreaming == "function" && + !isDataURI(wasmBinaryFile) && + typeof fetch == "function" + ) { + return fetch(wasmBinaryFile, { + credentials: "same-origin", + }).then(function (response) { + var result = WebAssembly.instantiateStreaming(response, info); + return result.then(receiveInstantiationResult, function (reason) { + err("wasm streaming compile failed: " + reason); + err("falling back to ArrayBuffer instantiation"); + return instantiateArrayBuffer(receiveInstantiationResult); + }); + }); + } + return instantiateArrayBuffer(receiveInstantiationResult); + } + if (Module.instantiateWasm) { + try { + var exports = Module.instantiateWasm(info, receiveInstance); + return exports; + } catch (e) { + err("Module.instantiateWasm callback failed with error: " + e); + return false; + } + } + instantiateAsync(); + return {}; + } + + function callRuntimeCallbacks(callbacks) { + while (callbacks.length) { + var callback = callbacks.shift(); + if (typeof callback == "function") { + callback(Module); + continue; + } + var func = callback.func; + if (typeof func == "number") { + if (callback.arg === undefined) { + getWasmTableEntry(func)(); + } else { + getWasmTableEntry(func)(callback.arg); + } + } else { + func(callback.arg === undefined ? null : callback.arg); + } + } + } + + function asmjsMangle(x) { + var unmangledSymbols = ["stackAlloc", "stackSave", "stackRestore"]; + return x.indexOf("dynCall_") == 0 || unmangledSymbols.includes(x) + ? x + : "_" + x; + } + + function exportAsmFunctions(asm) { + var global_object = this; + for (var __exportedFunc in asm) { + var jsname = asmjsMangle(__exportedFunc); + global_object[jsname] = Module[jsname] = asm[__exportedFunc]; + } + } + + var wasmTableMirror = []; + + function getWasmTableEntry(funcPtr) { + var func = wasmTableMirror[funcPtr]; + if (!func) { + if (funcPtr >= wasmTableMirror.length) { + wasmTableMirror.length = funcPtr + 1; + } + wasmTableMirror[funcPtr] = func = wasmTable.get(funcPtr); + } + return func; + } + + function ___assert_fail(condition, filename, line, func) { + abort( + "Assertion failed: " + + UTF8ToString(condition) + + ", at: " + + [ + filename ? UTF8ToString(filename) : "unknown filename", + line, + func ? UTF8ToString(func) : "unknown function", + ] + ); + } + + function ___cxa_allocate_exception(size) { + return _malloc(size + 16) + 16; + } + + var exceptionCaught = []; + + var exceptionLast = 0; + + var uncaughtExceptionCount = 0; + + function ___cxa_rethrow() { + var catchInfo = exceptionCaught.pop(); + if (!catchInfo) { + abort("no exception to throw"); + } + var info = catchInfo.get_exception_info(); + var ptr = catchInfo.get_base_ptr(); + if (!info.get_rethrown()) { + exceptionCaught.push(catchInfo); + info.set_rethrown(true); + info.set_caught(false); + uncaughtExceptionCount++; + } else { + catchInfo.free(); + } + exceptionLast = ptr; + throw ptr; + } + + function ExceptionInfo(excPtr) { + this.excPtr = excPtr; + this.ptr = excPtr - 16; + this.set_type = function (type) { + HEAP32[(this.ptr + 4) >> 2] = type; + }; + this.get_type = function () { + return HEAP32[(this.ptr + 4) >> 2]; + }; + this.set_destructor = function (destructor) { + HEAP32[(this.ptr + 8) >> 2] = destructor; + }; + this.get_destructor = function () { + return HEAP32[(this.ptr + 8) >> 2]; + }; + this.set_refcount = function (refcount) { + HEAP32[this.ptr >> 2] = refcount; + }; + this.set_caught = function (caught) { + caught = caught ? 1 : 0; + HEAP8[(this.ptr + 12) >> 0] = caught; + }; + this.get_caught = function () { + return HEAP8[(this.ptr + 12) >> 0] != 0; + }; + this.set_rethrown = function (rethrown) { + rethrown = rethrown ? 1 : 0; + HEAP8[(this.ptr + 13) >> 0] = rethrown; + }; + this.get_rethrown = function () { + return HEAP8[(this.ptr + 13) >> 0] != 0; + }; + this.init = function (type, destructor) { + this.set_type(type); + this.set_destructor(destructor); + this.set_refcount(0); + this.set_caught(false); + this.set_rethrown(false); + }; + this.add_ref = function () { + var value = HEAP32[this.ptr >> 2]; + HEAP32[this.ptr >> 2] = value + 1; + }; + this.release_ref = function () { + var prev = HEAP32[this.ptr >> 2]; + HEAP32[this.ptr >> 2] = prev - 1; + return prev === 1; + }; + } + + function ___cxa_throw(ptr, type, destructor) { + var info = new ExceptionInfo(ptr); + info.init(type, destructor); + exceptionLast = ptr; + uncaughtExceptionCount++; + throw ptr; + } + + var SYSCALLS = { + buffers: [null, [], []], + printChar(stream, curr) { + var buffer = SYSCALLS.buffers[stream]; + if (curr === 0 || curr === 10) { + (stream === 1 ? out : err)(UTF8ArrayToString(buffer, 0)); + buffer.length = 0; + } else { + buffer.push(curr); + } + }, + varargs: undefined, + get() { + SYSCALLS.varargs += 4; + var ret = HEAP32[(SYSCALLS.varargs - 4) >> 2]; + return ret; + }, + getStr(ptr) { + var ret = UTF8ToString(ptr); + return ret; + }, + get64(low, high) { + return low; + }, + }; + + function ___syscall_faccessat(dirfd, path, amode, flags) { + path = SYSCALLS.getStr(path); + path = SYSCALLS.calculateAt(dirfd, path); + return SYSCALLS.doAccess(path, amode); + } + + function ___syscall_fcntl64(fd, cmd, varargs) { + SYSCALLS.varargs = varargs; + return 0; + } + + function ___syscall_fstat64(fd, buf) {} + + function ___syscall_getcwd(buf, size) {} + + function ___syscall_ioctl(fd, op, varargs) { + SYSCALLS.varargs = varargs; + return 0; + } + + function ___syscall_lstat64(path, buf) {} + + function ___syscall_newfstatat(dirfd, path, buf, flags) {} + + function ___syscall_openat(dirfd, path, flags, varargs) { + SYSCALLS.varargs = varargs; + } + + function ___syscall_renameat(olddirfd, oldpath, newdirfd, newpath) {} + + function ___syscall_rmdir(path) {} + + function ___syscall_stat64(path, buf) {} + + function ___syscall_unlinkat(dirfd, path, flags) {} + + var structRegistrations = {}; + + function runDestructors(destructors) { + while (destructors.length) { + var ptr = destructors.pop(); + var del = destructors.pop(); + del(ptr); + } + } + + function simpleReadValueFromPointer(pointer) { + return this.fromWireType(HEAPU32[pointer >> 2]); + } + + var awaitingDependencies = {}; + + var registeredTypes = {}; + + var typeDependencies = {}; + + var char_0 = 48; + + var char_9 = 57; + + function makeLegalFunctionName(name) { + if (undefined === name) { + return "_unknown"; + } + name = name.replace(/[^a-zA-Z0-9_]/g, "$"); + var f = name.charCodeAt(0); + if (f >= char_0 && f <= char_9) { + return "_" + name; + } + return name; + } + + function createNamedFunction(name, body) { + name = makeLegalFunctionName(name); + return function () { + null; + return body.apply(this, arguments); + }; + } + + function extendError(baseErrorType, errorName) { + var errorClass = createNamedFunction(errorName, function (message) { + this.name = errorName; + this.message = message; + var stack = new Error(message).stack; + if (stack !== undefined) { + this.stack = + this.toString() + "\n" + stack.replace(/^Error(:[^\n]*)?\n/, ""); + } + }); + errorClass.prototype = Object.create(baseErrorType.prototype); + errorClass.prototype.constructor = errorClass; + errorClass.prototype.toString = function () { + if (this.message === undefined) { + return this.name; + } + return this.name + ": " + this.message; + }; + return errorClass; + } + + var InternalError = undefined; + + function throwInternalError(message) { + throw new InternalError(message); + } + + function whenDependentTypesAreResolved( + myTypes, + dependentTypes, + getTypeConverters + ) { + myTypes.forEach(function (type) { + typeDependencies[type] = dependentTypes; + }); + function onComplete(typeConverters) { + var myTypeConverters = getTypeConverters(typeConverters); + if (myTypeConverters.length !== myTypes.length) { + throwInternalError("Mismatched type converter count"); + } + for (var i = 0; i < myTypes.length; ++i) { + registerType(myTypes[i], myTypeConverters[i]); + } + } + var typeConverters = new Array(dependentTypes.length); + var unregisteredTypes = []; + var registered = 0; + dependentTypes.forEach((dt, i) => { + if (registeredTypes.hasOwnProperty(dt)) { + typeConverters[i] = registeredTypes[dt]; + } else { + unregisteredTypes.push(dt); + if (!awaitingDependencies.hasOwnProperty(dt)) { + awaitingDependencies[dt] = []; + } + awaitingDependencies[dt].push(() => { + typeConverters[i] = registeredTypes[dt]; + ++registered; + if (registered === unregisteredTypes.length) { + onComplete(typeConverters); + } + }); + } + }); + if (0 === unregisteredTypes.length) { + onComplete(typeConverters); + } + } + + function __embind_finalize_value_object(structType) { + var reg = structRegistrations[structType]; + delete structRegistrations[structType]; + var rawConstructor = reg.rawConstructor; + var rawDestructor = reg.rawDestructor; + var fieldRecords = reg.fields; + var fieldTypes = fieldRecords + .map(field => field.getterReturnType) + .concat(fieldRecords.map(field => field.setterArgumentType)); + whenDependentTypesAreResolved([structType], fieldTypes, fieldTypes => { + var fields = {}; + fieldRecords.forEach((field, i) => { + var fieldName = field.fieldName; + var getterReturnType = fieldTypes[i]; + var getter = field.getter; + var getterContext = field.getterContext; + var setterArgumentType = fieldTypes[i + fieldRecords.length]; + var setter = field.setter; + var setterContext = field.setterContext; + fields[fieldName] = { + read: ptr => { + return getterReturnType.fromWireType(getter(getterContext, ptr)); + }, + write: (ptr, o) => { + var destructors = []; + setter( + setterContext, + ptr, + setterArgumentType.toWireType(destructors, o) + ); + runDestructors(destructors); + }, + }; + }); + return [ + { + name: reg.name, + fromWireType: function (ptr) { + var rv = {}; + for (var i in fields) { + rv[i] = fields[i].read(ptr); + } + rawDestructor(ptr); + return rv; + }, + toWireType: function (destructors, o) { + for (var fieldName in fields) { + if (!(fieldName in o)) { + throw new TypeError('Missing field: "' + fieldName + '"'); + } + } + var ptr = rawConstructor(); + for (fieldName in fields) { + fields[fieldName].write(ptr, o[fieldName]); + } + if (destructors !== null) { + destructors.push(rawDestructor, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: rawDestructor, + }, + ]; + }); + } + + function __embind_register_bigint( + primitiveType, + name, + size, + minRange, + maxRange + ) {} + + function getShiftFromSize(size) { + switch (size) { + case 1: + return 0; + + case 2: + return 1; + + case 4: + return 2; + + case 8: + return 3; + + default: + throw new TypeError("Unknown type size: " + size); + } + } + + function embind_init_charCodes() { + var codes = new Array(256); + for (var i = 0; i < 256; ++i) { + codes[i] = String.fromCharCode(i); + } + embind_charCodes = codes; + } + + var embind_charCodes = undefined; + + function readLatin1String(ptr) { + var ret = ""; + var c = ptr; + while (HEAPU8[c]) { + ret += embind_charCodes[HEAPU8[c++]]; + } + return ret; + } + + var BindingError = undefined; + + function throwBindingError(message) { + throw new BindingError(message); + } + + function registerType(rawType, registeredInstance, options = {}) { + if (!("argPackAdvance" in registeredInstance)) { + throw new TypeError( + "registerType registeredInstance requires argPackAdvance" + ); + } + var name = registeredInstance.name; + if (!rawType) { + throwBindingError( + 'type "' + name + '" must have a positive integer typeid pointer' + ); + } + if (registeredTypes.hasOwnProperty(rawType)) { + if (options.ignoreDuplicateRegistrations) { + return; + } + throwBindingError("Cannot register type '" + name + "' twice"); + } + registeredTypes[rawType] = registeredInstance; + delete typeDependencies[rawType]; + if (awaitingDependencies.hasOwnProperty(rawType)) { + var callbacks = awaitingDependencies[rawType]; + delete awaitingDependencies[rawType]; + callbacks.forEach(cb => cb()); + } + } + + function __embind_register_bool(rawType, name, size, trueValue, falseValue) { + var shift = getShiftFromSize(size); + name = readLatin1String(name); + registerType(rawType, { + name, + fromWireType: function (wt) { + return !!wt; + }, + toWireType: function (destructors, o) { + return o ? trueValue : falseValue; + }, + argPackAdvance: 8, + readValueFromPointer: function (pointer) { + var heap; + if (size === 1) { + heap = HEAP8; + } else if (size === 2) { + heap = HEAP16; + } else if (size === 4) { + heap = HEAP32; + } else { + throw new TypeError("Unknown boolean type size: " + name); + } + return this.fromWireType(heap[pointer >> shift]); + }, + destructorFunction: null, + }); + } + + function ClassHandle_isAliasOf(other) { + if (!(this instanceof ClassHandle)) { + return false; + } + if (!(other instanceof ClassHandle)) { + return false; + } + var leftClass = this.$$.ptrType.registeredClass; + var left = this.$$.ptr; + var rightClass = other.$$.ptrType.registeredClass; + var right = other.$$.ptr; + while (leftClass.baseClass) { + left = leftClass.upcast(left); + leftClass = leftClass.baseClass; + } + while (rightClass.baseClass) { + right = rightClass.upcast(right); + rightClass = rightClass.baseClass; + } + return leftClass === rightClass && left === right; + } + + function shallowCopyInternalPointer(o) { + return { + count: o.count, + deleteScheduled: o.deleteScheduled, + preservePointerOnDelete: o.preservePointerOnDelete, + ptr: o.ptr, + ptrType: o.ptrType, + smartPtr: o.smartPtr, + smartPtrType: o.smartPtrType, + }; + } + + function throwInstanceAlreadyDeleted(obj) { + function getInstanceTypeName(handle) { + return handle.$$.ptrType.registeredClass.name; + } + throwBindingError(getInstanceTypeName(obj) + " instance already deleted"); + } + + var finalizationRegistry = false; + + function detachFinalizer(handle) {} + + function runDestructor($$) { + if ($$.smartPtr) { + $$.smartPtrType.rawDestructor($$.smartPtr); + } else { + $$.ptrType.registeredClass.rawDestructor($$.ptr); + } + } + + function releaseClassHandle($$) { + $$.count.value -= 1; + var toDelete = 0 === $$.count.value; + if (toDelete) { + runDestructor($$); + } + } + + function downcastPointer(ptr, ptrClass, desiredClass) { + if (ptrClass === desiredClass) { + return ptr; + } + if (undefined === desiredClass.baseClass) { + return null; + } + var rv = downcastPointer(ptr, ptrClass, desiredClass.baseClass); + if (rv === null) { + return null; + } + return desiredClass.downcast(rv); + } + + var registeredPointers = {}; + + function getInheritedInstanceCount() { + return Object.keys(registeredInstances).length; + } + + function getLiveInheritedInstances() { + var rv = []; + for (var k in registeredInstances) { + if (registeredInstances.hasOwnProperty(k)) { + rv.push(registeredInstances[k]); + } + } + return rv; + } + + var deletionQueue = []; + + function flushPendingDeletes() { + while (deletionQueue.length) { + var obj = deletionQueue.pop(); + obj.$$.deleteScheduled = false; + obj.delete(); + } + } + + var delayFunction = undefined; + + function setDelayFunction(fn) { + delayFunction = fn; + if (deletionQueue.length && delayFunction) { + delayFunction(flushPendingDeletes); + } + } + + function init_embind() { + Module.getInheritedInstanceCount = getInheritedInstanceCount; + Module.getLiveInheritedInstances = getLiveInheritedInstances; + Module.flushPendingDeletes = flushPendingDeletes; + Module.setDelayFunction = setDelayFunction; + } + + var registeredInstances = {}; + + function getBasestPointer(class_, ptr) { + if (ptr === undefined) { + throwBindingError("ptr should not be undefined"); + } + while (class_.baseClass) { + ptr = class_.upcast(ptr); + class_ = class_.baseClass; + } + return ptr; + } + + function getInheritedInstance(class_, ptr) { + ptr = getBasestPointer(class_, ptr); + return registeredInstances[ptr]; + } + + function makeClassHandle(prototype, record) { + if (!record.ptrType || !record.ptr) { + throwInternalError("makeClassHandle requires ptr and ptrType"); + } + var hasSmartPtrType = !!record.smartPtrType; + var hasSmartPtr = !!record.smartPtr; + if (hasSmartPtrType !== hasSmartPtr) { + throwInternalError("Both smartPtrType and smartPtr must be specified"); + } + record.count = { + value: 1, + }; + return attachFinalizer( + Object.create(prototype, { + $$: { + value: record, + }, + }) + ); + } + + function RegisteredPointer_fromWireType(ptr) { + var rawPointer = this.getPointee(ptr); + if (!rawPointer) { + this.destructor(ptr); + return null; + } + var registeredInstance = getInheritedInstance( + this.registeredClass, + rawPointer + ); + if (undefined !== registeredInstance) { + if (0 === registeredInstance.$$.count.value) { + registeredInstance.$$.ptr = rawPointer; + registeredInstance.$$.smartPtr = ptr; + return registeredInstance.clone(); + } + var rv = registeredInstance.clone(); + this.destructor(ptr); + return rv; + } + function makeDefaultHandle() { + if (this.isSmartPointer) { + return makeClassHandle(this.registeredClass.instancePrototype, { + ptrType: this.pointeeType, + ptr: rawPointer, + smartPtrType: this, + smartPtr: ptr, + }); + } + return makeClassHandle(this.registeredClass.instancePrototype, { + ptrType: this, + ptr, + }); + } + var actualType = this.registeredClass.getActualType(rawPointer); + var registeredPointerRecord = registeredPointers[actualType]; + if (!registeredPointerRecord) { + return makeDefaultHandle.call(this); + } + var toType; + if (this.isConst) { + toType = registeredPointerRecord.constPointerType; + } else { + toType = registeredPointerRecord.pointerType; + } + var dp = downcastPointer( + rawPointer, + this.registeredClass, + toType.registeredClass + ); + if (dp === null) { + return makeDefaultHandle.call(this); + } + if (this.isSmartPointer) { + return makeClassHandle(toType.registeredClass.instancePrototype, { + ptrType: toType, + ptr: dp, + smartPtrType: this, + smartPtr: ptr, + }); + } + return makeClassHandle(toType.registeredClass.instancePrototype, { + ptrType: toType, + ptr: dp, + }); + } + + function attachFinalizer(handle) { + if ("undefined" === typeof FinalizationRegistry) { + attachFinalizer = handle => handle; + return handle; + } + finalizationRegistry = new FinalizationRegistry(info => { + releaseClassHandle(info.$$); + }); + attachFinalizer = handle => { + var $$ = handle.$$; + var hasSmartPtr = !!$$.smartPtr; + if (hasSmartPtr) { + var info = { + $$, + }; + finalizationRegistry.register(handle, info, handle); + } + return handle; + }; + detachFinalizer = handle => finalizationRegistry.unregister(handle); + return attachFinalizer(handle); + } + + function ClassHandle_clone() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.preservePointerOnDelete) { + this.$$.count.value += 1; + return this; + } + var clone = attachFinalizer( + Object.create(Object.getPrototypeOf(this), { + $$: { + value: shallowCopyInternalPointer(this.$$), + }, + }) + ); + clone.$$.count.value += 1; + clone.$$.deleteScheduled = false; + return clone; + } + + function ClassHandle_delete() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { + throwBindingError("Object already scheduled for deletion"); + } + detachFinalizer(this); + releaseClassHandle(this.$$); + if (!this.$$.preservePointerOnDelete) { + this.$$.smartPtr = undefined; + this.$$.ptr = undefined; + } + } + + function ClassHandle_isDeleted() { + return !this.$$.ptr; + } + + function ClassHandle_deleteLater() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { + throwBindingError("Object already scheduled for deletion"); + } + deletionQueue.push(this); + if (deletionQueue.length === 1 && delayFunction) { + delayFunction(flushPendingDeletes); + } + this.$$.deleteScheduled = true; + return this; + } + + function init_ClassHandle() { + ClassHandle.prototype.isAliasOf = ClassHandle_isAliasOf; + ClassHandle.prototype.clone = ClassHandle_clone; + ClassHandle.prototype.delete = ClassHandle_delete; + ClassHandle.prototype.isDeleted = ClassHandle_isDeleted; + ClassHandle.prototype.deleteLater = ClassHandle_deleteLater; + } + + function ClassHandle() {} + + function ensureOverloadTable(proto, methodName, humanName) { + if (undefined === proto[methodName].overloadTable) { + var prevFunc = proto[methodName]; + proto[methodName] = function () { + if (!proto[methodName].overloadTable.hasOwnProperty(arguments.length)) { + throwBindingError( + "Function '" + + humanName + + "' called with an invalid number of arguments (" + + arguments.length + + ") - expects one of (" + + proto[methodName].overloadTable + + ")!" + ); + } + return proto[methodName].overloadTable[arguments.length].apply( + this, + arguments + ); + }; + proto[methodName].overloadTable = []; + proto[methodName].overloadTable[prevFunc.argCount] = prevFunc; + } + } + + function exposePublicSymbol(name, value, numArguments) { + if (Module.hasOwnProperty(name)) { + if ( + undefined === numArguments || + (undefined !== Module[name].overloadTable && + undefined !== Module[name].overloadTable[numArguments]) + ) { + throwBindingError("Cannot register public name '" + name + "' twice"); + } + ensureOverloadTable(Module, name, name); + if (Module.hasOwnProperty(numArguments)) { + throwBindingError( + "Cannot register multiple overloads of a function with the same number of arguments (" + + numArguments + + ")!" + ); + } + Module[name].overloadTable[numArguments] = value; + } else { + Module[name] = value; + if (undefined !== numArguments) { + Module[name].numArguments = numArguments; + } + } + } + + function RegisteredClass( + name, + constructor, + instancePrototype, + rawDestructor, + baseClass, + getActualType, + upcast, + downcast + ) { + this.name = name; + this.constructor = constructor; + this.instancePrototype = instancePrototype; + this.rawDestructor = rawDestructor; + this.baseClass = baseClass; + this.getActualType = getActualType; + this.upcast = upcast; + this.downcast = downcast; + this.pureVirtualFunctions = []; + } + + function upcastPointer(ptr, ptrClass, desiredClass) { + while (ptrClass !== desiredClass) { + if (!ptrClass.upcast) { + throwBindingError( + "Expected null or instance of " + + desiredClass.name + + ", got an instance of " + + ptrClass.name + ); + } + ptr = ptrClass.upcast(ptr); + ptrClass = ptrClass.baseClass; + } + return ptr; + } + + function constNoSmartPtrRawPointerToWireType(destructors, handle) { + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + return 0; + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + return ptr; + } + + function genericPointerToWireType(destructors, handle) { + var ptr; + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + if (this.isSmartPointer) { + ptr = this.rawConstructor(); + if (destructors !== null) { + destructors.push(this.rawDestructor, ptr); + } + return ptr; + } + return 0; + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + if (!this.isConst && handle.$$.ptrType.isConst) { + throwBindingError( + "Cannot convert argument of type " + + (handle.$$.smartPtrType + ? handle.$$.smartPtrType.name + : handle.$$.ptrType.name) + + " to parameter type " + + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + if (this.isSmartPointer) { + if (undefined === handle.$$.smartPtr) { + throwBindingError("Passing raw pointer to smart pointer is illegal"); + } + switch (this.sharingPolicy) { + case 0: + if (handle.$$.smartPtrType === this) { + ptr = handle.$$.smartPtr; + } else { + throwBindingError( + "Cannot convert argument of type " + + (handle.$$.smartPtrType + ? handle.$$.smartPtrType.name + : handle.$$.ptrType.name) + + " to parameter type " + + this.name + ); + } + break; + + case 1: + ptr = handle.$$.smartPtr; + break; + + case 2: + if (handle.$$.smartPtrType === this) { + ptr = handle.$$.smartPtr; + } else { + var clonedHandle = handle.clone(); + ptr = this.rawShare( + ptr, + Emval.toHandle(function () { + clonedHandle.delete(); + }) + ); + if (destructors !== null) { + destructors.push(this.rawDestructor, ptr); + } + } + break; + + default: + throwBindingError("Unsupporting sharing policy"); + } + } + return ptr; + } + + function nonConstNoSmartPtrRawPointerToWireType(destructors, handle) { + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + return 0; + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + if (handle.$$.ptrType.isConst) { + throwBindingError( + "Cannot convert argument of type " + + handle.$$.ptrType.name + + " to parameter type " + + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + return ptr; + } + + function RegisteredPointer_getPointee(ptr) { + if (this.rawGetPointee) { + ptr = this.rawGetPointee(ptr); + } + return ptr; + } + + function RegisteredPointer_destructor(ptr) { + if (this.rawDestructor) { + this.rawDestructor(ptr); + } + } + + function RegisteredPointer_deleteObject(handle) { + if (handle !== null) { + handle.delete(); + } + } + + function init_RegisteredPointer() { + RegisteredPointer.prototype.getPointee = RegisteredPointer_getPointee; + RegisteredPointer.prototype.destructor = RegisteredPointer_destructor; + RegisteredPointer.prototype.argPackAdvance = 8; + RegisteredPointer.prototype.readValueFromPointer = + simpleReadValueFromPointer; + RegisteredPointer.prototype.deleteObject = RegisteredPointer_deleteObject; + RegisteredPointer.prototype.fromWireType = RegisteredPointer_fromWireType; + } + + function RegisteredPointer( + name, + registeredClass, + isReference, + isConst, + isSmartPointer, + pointeeType, + sharingPolicy, + rawGetPointee, + rawConstructor, + rawShare, + rawDestructor + ) { + this.name = name; + this.registeredClass = registeredClass; + this.isReference = isReference; + this.isConst = isConst; + this.isSmartPointer = isSmartPointer; + this.pointeeType = pointeeType; + this.sharingPolicy = sharingPolicy; + this.rawGetPointee = rawGetPointee; + this.rawConstructor = rawConstructor; + this.rawShare = rawShare; + this.rawDestructor = rawDestructor; + if (!isSmartPointer && registeredClass.baseClass === undefined) { + if (isConst) { + this.toWireType = constNoSmartPtrRawPointerToWireType; + this.destructorFunction = null; + } else { + this.toWireType = nonConstNoSmartPtrRawPointerToWireType; + this.destructorFunction = null; + } + } else { + this.toWireType = genericPointerToWireType; + } + } + + function replacePublicSymbol(name, value, numArguments) { + if (!Module.hasOwnProperty(name)) { + throwInternalError("Replacing nonexistant public symbol"); + } + if ( + undefined !== Module[name].overloadTable && + undefined !== numArguments + ) { + Module[name].overloadTable[numArguments] = value; + } else { + Module[name] = value; + Module[name].argCount = numArguments; + } + } + + function dynCallLegacy(sig, ptr, args) { + var f = Module["dynCall_" + sig]; + return args && args.length + ? f.apply(null, [ptr].concat(args)) + : f.call(null, ptr); + } + + function dynCall(sig, ptr, args) { + if (sig.includes("j")) { + return dynCallLegacy(sig, ptr, args); + } + return getWasmTableEntry(ptr).apply(null, args); + } + + function getDynCaller(sig, ptr) { + var argCache = []; + return function () { + argCache.length = 0; + Object.assign(argCache, arguments); + return dynCall(sig, ptr, argCache); + }; + } + + function embind__requireFunction(signature, rawFunction) { + signature = readLatin1String(signature); + function makeDynCaller() { + if (signature.includes("j")) { + return getDynCaller(signature, rawFunction); + } + return getWasmTableEntry(rawFunction); + } + var fp = makeDynCaller(); + if (typeof fp != "function") { + throwBindingError( + "unknown function pointer with signature " + + signature + + ": " + + rawFunction + ); + } + return fp; + } + + var UnboundTypeError = undefined; + + function getTypeName(type) { + var ptr = ___getTypeName(type); + var rv = readLatin1String(ptr); + _free(ptr); + return rv; + } + + function throwUnboundTypeError(message, types) { + var unboundTypes = []; + var seen = {}; + function visit(type) { + if (seen[type]) { + return; + } + if (registeredTypes[type]) { + return; + } + if (typeDependencies[type]) { + typeDependencies[type].forEach(visit); + return; + } + unboundTypes.push(type); + seen[type] = true; + } + types.forEach(visit); + throw new UnboundTypeError( + message + ": " + unboundTypes.map(getTypeName).join([", "]) + ); + } + + function __embind_register_class( + rawType, + rawPointerType, + rawConstPointerType, + baseClassRawType, + getActualTypeSignature, + getActualType, + upcastSignature, + upcast, + downcastSignature, + downcast, + name, + destructorSignature, + rawDestructor + ) { + name = readLatin1String(name); + getActualType = embind__requireFunction( + getActualTypeSignature, + getActualType + ); + if (upcast) { + upcast = embind__requireFunction(upcastSignature, upcast); + } + if (downcast) { + downcast = embind__requireFunction(downcastSignature, downcast); + } + rawDestructor = embind__requireFunction(destructorSignature, rawDestructor); + var legalFunctionName = makeLegalFunctionName(name); + exposePublicSymbol(legalFunctionName, function () { + throwUnboundTypeError( + "Cannot construct " + name + " due to unbound types", + [baseClassRawType] + ); + }); + whenDependentTypesAreResolved( + [rawType, rawPointerType, rawConstPointerType], + baseClassRawType ? [baseClassRawType] : [], + function (base) { + base = base[0]; + var baseClass; + var basePrototype; + if (baseClassRawType) { + baseClass = base.registeredClass; + basePrototype = baseClass.instancePrototype; + } else { + basePrototype = ClassHandle.prototype; + } + var constructor = createNamedFunction(legalFunctionName, function () { + if (Object.getPrototypeOf(this) !== instancePrototype) { + throw new BindingError("Use 'new' to construct " + name); + } + if (undefined === registeredClass.constructor_body) { + throw new BindingError(name + " has no accessible constructor"); + } + var body = registeredClass.constructor_body[arguments.length]; + if (undefined === body) { + throw new BindingError( + "Tried to invoke ctor of " + + name + + " with invalid number of parameters (" + + arguments.length + + ") - expected (" + + Object.keys(registeredClass.constructor_body).toString() + + ") parameters instead!" + ); + } + return body.apply(this, arguments); + }); + var instancePrototype = Object.create(basePrototype, { + constructor: { + value: constructor, + }, + }); + constructor.prototype = instancePrototype; + var registeredClass = new RegisteredClass( + name, + constructor, + instancePrototype, + rawDestructor, + baseClass, + getActualType, + upcast, + downcast + ); + var referenceConverter = new RegisteredPointer( + name, + registeredClass, + true, + false, + false + ); + var pointerConverter = new RegisteredPointer( + name + "*", + registeredClass, + false, + false, + false + ); + var constPointerConverter = new RegisteredPointer( + name + " const*", + registeredClass, + false, + true, + false + ); + registeredPointers[rawType] = { + pointerType: pointerConverter, + constPointerType: constPointerConverter, + }; + replacePublicSymbol(legalFunctionName, constructor); + return [referenceConverter, pointerConverter, constPointerConverter]; + } + ); + } + + function heap32VectorToArray(count, firstElement) { + var array = []; + for (var i = 0; i < count; i++) { + array.push(HEAP32[(firstElement >> 2) + i]); + } + return array; + } + + function __embind_register_class_constructor( + rawClassType, + argCount, + rawArgTypesAddr, + invokerSignature, + invoker, + rawConstructor + ) { + assert(argCount > 0); + var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); + invoker = embind__requireFunction(invokerSignature, invoker); + whenDependentTypesAreResolved([], [rawClassType], function (classType) { + classType = classType[0]; + var humanName = "constructor " + classType.name; + if (undefined === classType.registeredClass.constructor_body) { + classType.registeredClass.constructor_body = []; + } + if ( + undefined !== classType.registeredClass.constructor_body[argCount - 1] + ) { + throw new BindingError( + "Cannot register multiple constructors with identical number of parameters (" + + (argCount - 1) + + ") for class '" + + classType.name + + "'! Overload resolution is currently only performed using the parameter count, not actual type info!" + ); + } + classType.registeredClass.constructor_body[argCount - 1] = () => { + throwUnboundTypeError( + "Cannot construct " + classType.name + " due to unbound types", + rawArgTypes + ); + }; + whenDependentTypesAreResolved([], rawArgTypes, function (argTypes) { + argTypes.splice(1, 0, null); + classType.registeredClass.constructor_body[argCount - 1] = + craftInvokerFunction( + humanName, + argTypes, + null, + invoker, + rawConstructor + ); + return []; + }); + return []; + }); + } + + function craftInvokerFunction( + humanName, + argTypes, + classType, + cppInvokerFunc, + cppTargetFunc + ) { + var argCount = argTypes.length; + if (argCount < 2) { + throwBindingError( + "argTypes array size mismatch! Must at least get return value and 'this' types!" + ); + } + var isClassMethodFunc = argTypes[1] !== null && classType !== null; + var needsDestructorStack = false; + for (var i = 1; i < argTypes.length; ++i) { + if ( + argTypes[i] !== null && + argTypes[i].destructorFunction === undefined + ) { + needsDestructorStack = true; + break; + } + } + var returns = argTypes[0].name !== "void"; + var expectedArgCount = argCount - 2; + var argsWired = new Array(expectedArgCount); + var invokerFuncArgs = []; + var destructors = []; + return function () { + if (arguments.length !== expectedArgCount) { + throwBindingError( + "function " + + humanName + + " called with " + + arguments.length + + " arguments, expected " + + expectedArgCount + + " args!" + ); + } + destructors.length = 0; + var thisWired; + invokerFuncArgs.length = isClassMethodFunc ? 2 : 1; + invokerFuncArgs[0] = cppTargetFunc; + if (isClassMethodFunc) { + thisWired = argTypes[1].toWireType(destructors, this); + invokerFuncArgs[1] = thisWired; + } + for (var i = 0; i < expectedArgCount; ++i) { + argsWired[i] = argTypes[i + 2].toWireType(destructors, arguments[i]); + invokerFuncArgs.push(argsWired[i]); + } + var rv = cppInvokerFunc.apply(null, invokerFuncArgs); + function onDone(rv) { + if (needsDestructorStack) { + runDestructors(destructors); + } else { + for (var i = isClassMethodFunc ? 1 : 2; i < argTypes.length; i++) { + var param = i === 1 ? thisWired : argsWired[i - 2]; + if (argTypes[i].destructorFunction !== null) { + argTypes[i].destructorFunction(param); + } + } + } + if (returns) { + return argTypes[0].fromWireType(rv); + } + } + return onDone(rv); + }; + } + + function __embind_register_class_function( + rawClassType, + methodName, + argCount, + rawArgTypesAddr, + invokerSignature, + rawInvoker, + context, + isPureVirtual + ) { + var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); + methodName = readLatin1String(methodName); + rawInvoker = embind__requireFunction(invokerSignature, rawInvoker); + whenDependentTypesAreResolved([], [rawClassType], function (classType) { + classType = classType[0]; + var humanName = classType.name + "." + methodName; + if (methodName.startsWith("@@")) { + methodName = Symbol[methodName.substring(2)]; + } + if (isPureVirtual) { + classType.registeredClass.pureVirtualFunctions.push(methodName); + } + function unboundTypesHandler() { + throwUnboundTypeError( + "Cannot call " + humanName + " due to unbound types", + rawArgTypes + ); + } + var proto = classType.registeredClass.instancePrototype; + var method = proto[methodName]; + if ( + undefined === method || + (undefined === method.overloadTable && + method.className !== classType.name && + method.argCount === argCount - 2) + ) { + unboundTypesHandler.argCount = argCount - 2; + unboundTypesHandler.className = classType.name; + proto[methodName] = unboundTypesHandler; + } else { + ensureOverloadTable(proto, methodName, humanName); + proto[methodName].overloadTable[argCount - 2] = unboundTypesHandler; + } + whenDependentTypesAreResolved([], rawArgTypes, function (argTypes) { + var memberFunction = craftInvokerFunction( + humanName, + argTypes, + classType, + rawInvoker, + context + ); + if (undefined === proto[methodName].overloadTable) { + memberFunction.argCount = argCount - 2; + proto[methodName] = memberFunction; + } else { + proto[methodName].overloadTable[argCount - 2] = memberFunction; + } + return []; + }); + return []; + }); + } + + var emval_free_list = []; + + var emval_handle_array = [ + {}, + { + value: undefined, + }, + { + value: null, + }, + { + value: true, + }, + { + value: false, + }, + ]; + + function __emval_decref(handle) { + if (handle > 4 && 0 === --emval_handle_array[handle].refcount) { + emval_handle_array[handle] = undefined; + emval_free_list.push(handle); + } + } + + function count_emval_handles() { + var count = 0; + for (var i = 5; i < emval_handle_array.length; ++i) { + if (emval_handle_array[i] !== undefined) { + ++count; + } + } + return count; + } + + function get_first_emval() { + for (var i = 5; i < emval_handle_array.length; ++i) { + if (emval_handle_array[i] !== undefined) { + return emval_handle_array[i]; + } + } + return null; + } + + function init_emval() { + Module.count_emval_handles = count_emval_handles; + Module.get_first_emval = get_first_emval; + } + + var Emval = { + toValue: handle => { + if (!handle) { + throwBindingError("Cannot use deleted val. handle = " + handle); + } + return emval_handle_array[handle].value; + }, + toHandle: value => { + switch (value) { + case undefined: + return 1; + + case null: + return 2; + + case true: + return 3; + + case false: + return 4; + + default: { + var handle = emval_free_list.length + ? emval_free_list.pop() + : emval_handle_array.length; + emval_handle_array[handle] = { + refcount: 1, + value, + }; + return handle; + } + } + }, + }; + + function __embind_register_emval(rawType, name) { + name = readLatin1String(name); + registerType(rawType, { + name, + fromWireType: function (handle) { + var rv = Emval.toValue(handle); + __emval_decref(handle); + return rv; + }, + toWireType: function (destructors, value) { + return Emval.toHandle(value); + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: null, + }); + } + + function _embind_repr(v) { + if (v === null) { + return "null"; + } + var t = typeof v; + if (t === "object" || t === "array" || t === "function") { + return v.toString(); + } + return "" + v; + } + + function floatReadValueFromPointer(name, shift) { + switch (shift) { + case 2: + return function (pointer) { + return this.fromWireType(HEAPF32[pointer >> 2]); + }; + + case 3: + return function (pointer) { + return this.fromWireType(HEAPF64[pointer >> 3]); + }; + + default: + throw new TypeError("Unknown float type: " + name); + } + } + + function __embind_register_float(rawType, name, size) { + var shift = getShiftFromSize(size); + name = readLatin1String(name); + registerType(rawType, { + name, + fromWireType: function (value) { + return value; + }, + toWireType: function (destructors, value) { + return value; + }, + argPackAdvance: 8, + readValueFromPointer: floatReadValueFromPointer(name, shift), + destructorFunction: null, + }); + } + + function integerReadValueFromPointer(name, shift, signed) { + switch (shift) { + case 0: + return signed + ? function readS8FromPointer(pointer) { + return HEAP8[pointer]; + } + : function readU8FromPointer(pointer) { + return HEAPU8[pointer]; + }; + + case 1: + return signed + ? function readS16FromPointer(pointer) { + return HEAP16[pointer >> 1]; + } + : function readU16FromPointer(pointer) { + return HEAPU16[pointer >> 1]; + }; + + case 2: + return signed + ? function readS32FromPointer(pointer) { + return HEAP32[pointer >> 2]; + } + : function readU32FromPointer(pointer) { + return HEAPU32[pointer >> 2]; + }; + + default: + throw new TypeError("Unknown integer type: " + name); + } + } + + function __embind_register_integer( + primitiveType, + name, + size, + minRange, + maxRange + ) { + name = readLatin1String(name); + if (maxRange === -1) { + maxRange = 4294967295; + } + var shift = getShiftFromSize(size); + var fromWireType = value => value; + if (minRange === 0) { + var bitshift = 32 - 8 * size; + fromWireType = value => (value << bitshift) >>> bitshift; + } + var isUnsignedType = name.includes("unsigned"); + var checkAssertions = (value, toTypeName) => {}; + var toWireType; + if (isUnsignedType) { + toWireType = function (destructors, value) { + checkAssertions(value, this.name); + return value >>> 0; + }; + } else { + toWireType = function (destructors, value) { + checkAssertions(value, this.name); + return value; + }; + } + registerType(primitiveType, { + name, + fromWireType: fromWireType, + toWireType: toWireType, + argPackAdvance: 8, + readValueFromPointer: integerReadValueFromPointer( + name, + shift, + minRange !== 0 + ), + destructorFunction: null, + }); + } + + function __embind_register_memory_view(rawType, dataTypeIndex, name) { + var typeMapping = [ + Int8Array, + Uint8Array, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + Float32Array, + Float64Array, + ]; + var TA = typeMapping[dataTypeIndex]; + function decodeMemoryView(handle) { + handle = handle >> 2; + var heap = HEAPU32; + var size = heap[handle]; + var data = heap[handle + 1]; + return new TA(buffer, data, size); + } + name = readLatin1String(name); + registerType( + rawType, + { + name, + fromWireType: decodeMemoryView, + argPackAdvance: 8, + readValueFromPointer: decodeMemoryView, + }, + { + ignoreDuplicateRegistrations: true, + } + ); + } + + function __embind_register_smart_ptr( + rawType, + rawPointeeType, + name, + sharingPolicy, + getPointeeSignature, + rawGetPointee, + constructorSignature, + rawConstructor, + shareSignature, + rawShare, + destructorSignature, + rawDestructor + ) { + name = readLatin1String(name); + rawGetPointee = embind__requireFunction(getPointeeSignature, rawGetPointee); + rawConstructor = embind__requireFunction( + constructorSignature, + rawConstructor + ); + rawShare = embind__requireFunction(shareSignature, rawShare); + rawDestructor = embind__requireFunction(destructorSignature, rawDestructor); + whenDependentTypesAreResolved( + [rawType], + [rawPointeeType], + function (pointeeType) { + pointeeType = pointeeType[0]; + var registeredPointer = new RegisteredPointer( + name, + pointeeType.registeredClass, + false, + false, + true, + pointeeType, + sharingPolicy, + rawGetPointee, + rawConstructor, + rawShare, + rawDestructor + ); + return [registeredPointer]; + } + ); + } + + function __embind_register_std_string(rawType, name) { + name = readLatin1String(name); + var stdStringIsUTF8 = name === "std::string"; + registerType(rawType, { + name, + fromWireType: function (value) { + var length = HEAPU32[value >> 2]; + var str; + if (stdStringIsUTF8) { + var decodeStartPtr = value + 4; + for (var i = 0; i <= length; ++i) { + var currentBytePtr = value + 4 + i; + if (i == length || HEAPU8[currentBytePtr] == 0) { + var maxRead = currentBytePtr - decodeStartPtr; + var stringSegment = UTF8ToString(decodeStartPtr, maxRead); + if (str === undefined) { + str = stringSegment; + } else { + str += String.fromCharCode(0); + str += stringSegment; + } + decodeStartPtr = currentBytePtr + 1; + } + } + } else { + var a = new Array(length); + for (var i = 0; i < length; ++i) { + a[i] = String.fromCharCode(HEAPU8[value + 4 + i]); + } + str = a.join(""); + } + _free(value); + return str; + }, + toWireType: function (destructors, value) { + if (value instanceof ArrayBuffer) { + value = new Uint8Array(value); + } + var getLength; + var valueIsOfTypeString = typeof value == "string"; + if ( + !( + valueIsOfTypeString || + value instanceof Uint8Array || + value instanceof Uint8ClampedArray || + value instanceof Int8Array + ) + ) { + throwBindingError("Cannot pass non-string to std::string"); + } + if (stdStringIsUTF8 && valueIsOfTypeString) { + getLength = () => lengthBytesUTF8(value); + } else { + getLength = () => value.length; + } + var length = getLength(); + var ptr = _malloc(4 + length + 1); + HEAPU32[ptr >> 2] = length; + if (stdStringIsUTF8 && valueIsOfTypeString) { + stringToUTF8(value, ptr + 4, length + 1); + } else if (valueIsOfTypeString) { + for (var i = 0; i < length; ++i) { + var charCode = value.charCodeAt(i); + if (charCode > 255) { + _free(ptr); + throwBindingError( + "String has UTF-16 code units that do not fit in 8 bits" + ); + } + HEAPU8[ptr + 4 + i] = charCode; + } + } else { + for (var i = 0; i < length; ++i) { + HEAPU8[ptr + 4 + i] = value[i]; + } + } + if (destructors !== null) { + destructors.push(_free, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction(ptr) { + _free(ptr); + }, + }); + } + + function __embind_register_std_wstring(rawType, charSize, name) { + name = readLatin1String(name); + var decodeString, encodeString, getHeap, lengthBytesUTF, shift; + if (charSize === 2) { + decodeString = UTF16ToString; + encodeString = stringToUTF16; + lengthBytesUTF = lengthBytesUTF16; + getHeap = () => HEAPU16; + shift = 1; + } else if (charSize === 4) { + decodeString = UTF32ToString; + encodeString = stringToUTF32; + lengthBytesUTF = lengthBytesUTF32; + getHeap = () => HEAPU32; + shift = 2; + } + registerType(rawType, { + name, + fromWireType: function (value) { + var length = HEAPU32[value >> 2]; + var HEAP = getHeap(); + var str; + var decodeStartPtr = value + 4; + for (var i = 0; i <= length; ++i) { + var currentBytePtr = value + 4 + i * charSize; + if (i == length || HEAP[currentBytePtr >> shift] == 0) { + var maxReadBytes = currentBytePtr - decodeStartPtr; + var stringSegment = decodeString(decodeStartPtr, maxReadBytes); + if (str === undefined) { + str = stringSegment; + } else { + str += String.fromCharCode(0); + str += stringSegment; + } + decodeStartPtr = currentBytePtr + charSize; + } + } + _free(value); + return str; + }, + toWireType: function (destructors, value) { + if (!(typeof value == "string")) { + throwBindingError( + "Cannot pass non-string to C++ string type " + name + ); + } + var length = lengthBytesUTF(value); + var ptr = _malloc(4 + length + charSize); + HEAPU32[ptr >> 2] = length >> shift; + encodeString(value, ptr + 4, length + charSize); + if (destructors !== null) { + destructors.push(_free, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction(ptr) { + _free(ptr); + }, + }); + } + + function __embind_register_value_object( + rawType, + name, + constructorSignature, + rawConstructor, + destructorSignature, + rawDestructor + ) { + structRegistrations[rawType] = { + name: readLatin1String(name), + rawConstructor: embind__requireFunction( + constructorSignature, + rawConstructor + ), + rawDestructor: embind__requireFunction( + destructorSignature, + rawDestructor + ), + fields: [], + }; + } + + function __embind_register_value_object_field( + structType, + fieldName, + getterReturnType, + getterSignature, + getter, + getterContext, + setterArgumentType, + setterSignature, + setter, + setterContext + ) { + structRegistrations[structType].fields.push({ + fieldName: readLatin1String(fieldName), + getterReturnType, + getter: embind__requireFunction(getterSignature, getter), + getterContext, + setterArgumentType, + setter: embind__requireFunction(setterSignature, setter), + setterContext, + }); + } + + function __embind_register_void(rawType, name) { + name = readLatin1String(name); + registerType(rawType, { + isVoid: true, + name, + argPackAdvance: 0, + fromWireType: function () { + return undefined; + }, + toWireType: function (destructors, o) { + return undefined; + }, + }); + } + + function __emscripten_date_now() { + return Date.now(); + } + + var nowIsMonotonic = true; + + function __emscripten_get_now_is_monotonic() { + return nowIsMonotonic; + } + + function requireRegisteredType(rawType, humanName) { + var impl = registeredTypes[rawType]; + if (undefined === impl) { + throwBindingError( + humanName + " has unknown type " + getTypeName(rawType) + ); + } + return impl; + } + + function __emval_lookupTypes(argCount, argTypes) { + var a = new Array(argCount); + for (var i = 0; i < argCount; ++i) { + a[i] = requireRegisteredType( + HEAP32[(argTypes >> 2) + i], + "parameter " + i + ); + } + return a; + } + + function __emval_call(handle, argCount, argTypes, argv) { + handle = Emval.toValue(handle); + var types = __emval_lookupTypes(argCount, argTypes); + var args = new Array(argCount); + for (var i = 0; i < argCount; ++i) { + var type = types[i]; + args[i] = type.readValueFromPointer(argv); + argv += type.argPackAdvance; + } + var rv = handle.apply(undefined, args); + return Emval.toHandle(rv); + } + + function __emval_incref(handle) { + if (handle > 4) { + emval_handle_array[handle].refcount += 1; + } + } + + function __emval_take_value(type, argv) { + type = requireRegisteredType(type, "_emval_take_value"); + var v = type.readValueFromPointer(argv); + return Emval.toHandle(v); + } + + function __localtime_js(time, tmPtr) { + var date = new Date(HEAP32[time >> 2] * 1e3); + HEAP32[tmPtr >> 2] = date.getSeconds(); + HEAP32[(tmPtr + 4) >> 2] = date.getMinutes(); + HEAP32[(tmPtr + 8) >> 2] = date.getHours(); + HEAP32[(tmPtr + 12) >> 2] = date.getDate(); + HEAP32[(tmPtr + 16) >> 2] = date.getMonth(); + HEAP32[(tmPtr + 20) >> 2] = date.getFullYear() - 1900; + HEAP32[(tmPtr + 24) >> 2] = date.getDay(); + var start = new Date(date.getFullYear(), 0, 1); + var yday = ((date.getTime() - start.getTime()) / (1e3 * 60 * 60 * 24)) | 0; + HEAP32[(tmPtr + 28) >> 2] = yday; + HEAP32[(tmPtr + 36) >> 2] = -(date.getTimezoneOffset() * 60); + var summerOffset = new Date(date.getFullYear(), 6, 1).getTimezoneOffset(); + var winterOffset = start.getTimezoneOffset(); + var dst = + (summerOffset != winterOffset && + date.getTimezoneOffset() == Math.min(winterOffset, summerOffset)) | 0; + HEAP32[(tmPtr + 32) >> 2] = dst; + } + + function __mmap_js(addr, len, prot, flags, fd, off, allocated, builtin) { + return -52; + } + + function __munmap_js(addr, len, prot, flags, fd, offset) {} + + function _tzset_impl(timezone, daylight, tzname) { + var currentYear = new Date().getFullYear(); + var winter = new Date(currentYear, 0, 1); + var summer = new Date(currentYear, 6, 1); + var winterOffset = winter.getTimezoneOffset(); + var summerOffset = summer.getTimezoneOffset(); + var stdTimezoneOffset = Math.max(winterOffset, summerOffset); + HEAP32[timezone >> 2] = stdTimezoneOffset * 60; + HEAP32[daylight >> 2] = Number(winterOffset != summerOffset); + function extractZone(date) { + var match = date.toTimeString().match(/\(([A-Za-z ]+)\)$/); + return match ? match[1] : "GMT"; + } + var winterName = extractZone(winter); + var summerName = extractZone(summer); + var winterNamePtr = allocateUTF8(winterName); + var summerNamePtr = allocateUTF8(summerName); + if (summerOffset < winterOffset) { + HEAP32[tzname >> 2] = winterNamePtr; + HEAP32[(tzname + 4) >> 2] = summerNamePtr; + } else { + HEAP32[tzname >> 2] = summerNamePtr; + HEAP32[(tzname + 4) >> 2] = winterNamePtr; + } + } + + function __tzset_js(timezone, daylight, tzname) { + if (__tzset_js.called) { + return; + } + __tzset_js.called = true; + _tzset_impl(timezone, daylight, tzname); + } + + function _abort() { + abort(""); + } + + function _emscripten_get_heap_max() { + return 2147483648; + } + + var _emscripten_get_now; + + _emscripten_get_now = () => performance.now(); + + function _emscripten_memcpy_big(dest, src, num) { + HEAPU8.copyWithin(dest, src, src + num); + } + + function emscripten_realloc_buffer(size) { + try { + wasmMemory.grow((size - buffer.byteLength + 65535) >>> 16); + updateGlobalBufferAndViews(wasmMemory.buffer); + return 1; + } catch (e) {} + } + + function _emscripten_resize_heap(requestedSize) { + var oldSize = HEAPU8.length; + requestedSize = requestedSize >>> 0; + var maxHeapSize = _emscripten_get_heap_max(); + if (requestedSize > maxHeapSize) { + return false; + } + let alignUp = (x, multiple) => x + ((multiple - (x % multiple)) % multiple); + for (var cutDown = 1; cutDown <= 4; cutDown *= 2) { + var overGrownHeapSize = oldSize * (1 + 0.2 / cutDown); + overGrownHeapSize = Math.min( + overGrownHeapSize, + requestedSize + 100663296 + ); + var newSize = Math.min( + maxHeapSize, + alignUp(Math.max(requestedSize, overGrownHeapSize), 65536) + ); + var replacement = emscripten_realloc_buffer(newSize); + if (replacement) { + return true; + } + } + return false; + } + + var ENV = {}; + + function getExecutableName() { + return thisProgram || "./this.program"; + } + + function getEnvStrings() { + if (!getEnvStrings.strings) { + var lang = + ( + (typeof navigator == "object" && + navigator.languages && + navigator.languages[0]) || + "C" + ).replace("-", "_") + ".UTF-8"; + var env = { + USER: "web_user", + LOGNAME: "web_user", + PATH: "/", + PWD: "/", + HOME: "/home/web_user", + LANG: lang, + _: getExecutableName(), + }; + for (var x in ENV) { + if (ENV[x] === undefined) { + delete env[x]; + } else { + env[x] = ENV[x]; + } + } + var strings = []; + for (var x in env) { + strings.push(x + "=" + env[x]); + } + getEnvStrings.strings = strings; + } + return getEnvStrings.strings; + } + + function _environ_get(__environ, environ_buf) { + var bufSize = 0; + getEnvStrings().forEach(function (string, i) { + var ptr = environ_buf + bufSize; + HEAP32[(__environ + i * 4) >> 2] = ptr; + writeAsciiToMemory(string, ptr); + bufSize += string.length + 1; + }); + return 0; + } + + function _environ_sizes_get(penviron_count, penviron_buf_size) { + var strings = getEnvStrings(); + HEAP32[penviron_count >> 2] = strings.length; + var bufSize = 0; + strings.forEach(function (string) { + bufSize += string.length + 1; + }); + HEAP32[penviron_buf_size >> 2] = bufSize; + return 0; + } + + function _exit(status) { + exit(status); + } + + function _fd_close(fd) { + return 0; + } + + function _fd_read(fd, iov, iovcnt, pnum) { + var stream = SYSCALLS.getStreamFromFD(fd); + var num = SYSCALLS.doReadv(stream, iov, iovcnt); + HEAP32[pnum >> 2] = num; + return 0; + } + + function _fd_seek(fd, offset_low, offset_high, whence, newOffset) {} + + function _fd_write(fd, iov, iovcnt, pnum) { + var num = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[iov >> 2]; + var len = HEAP32[(iov + 4) >> 2]; + iov += 8; + for (var j = 0; j < len; j++) { + SYSCALLS.printChar(fd, HEAPU8[ptr + j]); + } + num += len; + } + HEAP32[pnum >> 2] = num; + return 0; + } + + function getRandomDevice() { + if ( + typeof crypto == "object" && + typeof crypto.getRandomValues == "function" + ) { + var randomBuffer = new Uint8Array(1); + return function () { + crypto.getRandomValues(randomBuffer); + return randomBuffer[0]; + }; + } + return function () { + abort("randomDevice"); + }; + } + + function _getentropy(buffer, size) { + if (!_getentropy.randomDevice) { + _getentropy.randomDevice = getRandomDevice(); + } + for (var i = 0; i < size; i++) { + HEAP8[(buffer + i) >> 0] = _getentropy.randomDevice(); + } + return 0; + } + + function _pclose() { + err("missing function: pclose"); + abort(-1); + } + + function _setTempRet0(val) { + setTempRet0(val); + } + + function __isLeapYear(year) { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + } + + function __arraySum(array, index) { + var sum = 0; + for (var i = 0; i <= index; sum += array[i++]) {} + return sum; + } + + var __MONTH_DAYS_LEAP = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + var __MONTH_DAYS_REGULAR = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + function __addDays(date, days) { + var newDate = new Date(date.getTime()); + while (days > 0) { + var leap = __isLeapYear(newDate.getFullYear()); + var currentMonth = newDate.getMonth(); + var daysInCurrentMonth = ( + leap ? __MONTH_DAYS_LEAP : __MONTH_DAYS_REGULAR + )[currentMonth]; + if (days > daysInCurrentMonth - newDate.getDate()) { + days -= daysInCurrentMonth - newDate.getDate() + 1; + newDate.setDate(1); + if (currentMonth < 11) { + newDate.setMonth(currentMonth + 1); + } else { + newDate.setMonth(0); + newDate.setFullYear(newDate.getFullYear() + 1); + } + } else { + newDate.setDate(newDate.getDate() + days); + return newDate; + } + } + return newDate; + } + + function _strftime(s, maxsize, format, tm) { + var tm_zone = HEAP32[(tm + 40) >> 2]; + var date = { + tm_sec: HEAP32[tm >> 2], + tm_min: HEAP32[(tm + 4) >> 2], + tm_hour: HEAP32[(tm + 8) >> 2], + tm_mday: HEAP32[(tm + 12) >> 2], + tm_mon: HEAP32[(tm + 16) >> 2], + tm_year: HEAP32[(tm + 20) >> 2], + tm_wday: HEAP32[(tm + 24) >> 2], + tm_yday: HEAP32[(tm + 28) >> 2], + tm_isdst: HEAP32[(tm + 32) >> 2], + tm_gmtoff: HEAP32[(tm + 36) >> 2], + tm_zone: tm_zone ? UTF8ToString(tm_zone) : "", + }; + var pattern = UTF8ToString(format); + var EXPANSION_RULES_1 = { + "%c": "%a %b %d %H:%M:%S %Y", + "%D": "%m/%d/%y", + "%F": "%Y-%m-%d", + "%h": "%b", + "%r": "%I:%M:%S %p", + "%R": "%H:%M", + "%T": "%H:%M:%S", + "%x": "%m/%d/%y", + "%X": "%H:%M:%S", + "%Ec": "%c", + "%EC": "%C", + "%Ex": "%m/%d/%y", + "%EX": "%H:%M:%S", + "%Ey": "%y", + "%EY": "%Y", + "%Od": "%d", + "%Oe": "%e", + "%OH": "%H", + "%OI": "%I", + "%Om": "%m", + "%OM": "%M", + "%OS": "%S", + "%Ou": "%u", + "%OU": "%U", + "%OV": "%V", + "%Ow": "%w", + "%OW": "%W", + "%Oy": "%y", + }; + for (var rule in EXPANSION_RULES_1) { + pattern = pattern.replace(new RegExp(rule, "g"), EXPANSION_RULES_1[rule]); + } + var WEEKDAYS = [ + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + ]; + var MONTHS = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ]; + function leadingSomething(value, digits, character) { + var str = typeof value == "number" ? value.toString() : value || ""; + while (str.length < digits) { + str = character[0] + str; + } + return str; + } + function leadingNulls(value, digits) { + return leadingSomething(value, digits, "0"); + } + function compareByDay(date1, date2) { + function sgn(value) { + return value < 0 ? -1 : value > 0 ? 1 : 0; + } + var compare; + if ((compare = sgn(date1.getFullYear() - date2.getFullYear())) === 0) { + if ((compare = sgn(date1.getMonth() - date2.getMonth())) === 0) { + compare = sgn(date1.getDate() - date2.getDate()); + } + } + return compare; + } + function getFirstWeekStartDate(janFourth) { + switch (janFourth.getDay()) { + case 0: + return new Date(janFourth.getFullYear() - 1, 11, 29); + + case 1: + return janFourth; + + case 2: + return new Date(janFourth.getFullYear(), 0, 3); + + case 3: + return new Date(janFourth.getFullYear(), 0, 2); + + case 4: + return new Date(janFourth.getFullYear(), 0, 1); + + case 5: + return new Date(janFourth.getFullYear() - 1, 11, 31); + + case 6: + return new Date(janFourth.getFullYear() - 1, 11, 30); + } + } + function getWeekBasedYear(date) { + var thisDate = __addDays( + new Date(date.tm_year + 1900, 0, 1), + date.tm_yday + ); + var janFourthThisYear = new Date(thisDate.getFullYear(), 0, 4); + var janFourthNextYear = new Date(thisDate.getFullYear() + 1, 0, 4); + var firstWeekStartThisYear = getFirstWeekStartDate(janFourthThisYear); + var firstWeekStartNextYear = getFirstWeekStartDate(janFourthNextYear); + if (compareByDay(firstWeekStartThisYear, thisDate) <= 0) { + if (compareByDay(firstWeekStartNextYear, thisDate) <= 0) { + return thisDate.getFullYear() + 1; + } + return thisDate.getFullYear(); + } + return thisDate.getFullYear() - 1; + } + var EXPANSION_RULES_2 = { + "%a": function (date) { + return WEEKDAYS[date.tm_wday].substring(0, 3); + }, + "%A": function (date) { + return WEEKDAYS[date.tm_wday]; + }, + "%b": function (date) { + return MONTHS[date.tm_mon].substring(0, 3); + }, + "%B": function (date) { + return MONTHS[date.tm_mon]; + }, + "%C": function (date) { + var year = date.tm_year + 1900; + return leadingNulls((year / 100) | 0, 2); + }, + "%d": function (date) { + return leadingNulls(date.tm_mday, 2); + }, + "%e": function (date) { + return leadingSomething(date.tm_mday, 2, " "); + }, + "%g": function (date) { + return getWeekBasedYear(date).toString().substring(2); + }, + "%G": function (date) { + return getWeekBasedYear(date); + }, + "%H": function (date) { + return leadingNulls(date.tm_hour, 2); + }, + "%I": function (date) { + var twelveHour = date.tm_hour; + if (twelveHour == 0) { + twelveHour = 12; + } else if (twelveHour > 12) { + twelveHour -= 12; + } + return leadingNulls(twelveHour, 2); + }, + "%j": function (date) { + return leadingNulls( + date.tm_mday + + __arraySum( + __isLeapYear(date.tm_year + 1900) + ? __MONTH_DAYS_LEAP + : __MONTH_DAYS_REGULAR, + date.tm_mon - 1 + ), + 3 + ); + }, + "%m": function (date) { + return leadingNulls(date.tm_mon + 1, 2); + }, + "%M": function (date) { + return leadingNulls(date.tm_min, 2); + }, + "%n": function () { + return "\n"; + }, + "%p": function (date) { + if (date.tm_hour >= 0 && date.tm_hour < 12) { + return "AM"; + } + return "PM"; + }, + "%S": function (date) { + return leadingNulls(date.tm_sec, 2); + }, + "%t": function () { + return "\t"; + }, + "%u": function (date) { + return date.tm_wday || 7; + }, + "%U": function (date) { + var days = date.tm_yday + 7 - date.tm_wday; + return leadingNulls(Math.floor(days / 7), 2); + }, + "%V": function (date) { + var val = Math.floor((date.tm_yday + 7 - ((date.tm_wday + 6) % 7)) / 7); + if ((date.tm_wday + 371 - date.tm_yday - 2) % 7 <= 2) { + val++; + } + if (!val) { + val = 52; + var dec31 = (date.tm_wday + 7 - date.tm_yday - 1) % 7; + if ( + dec31 == 4 || + (dec31 == 5 && __isLeapYear((date.tm_year % 400) - 1)) + ) { + val++; + } + } else if (val == 53) { + var jan1 = (date.tm_wday + 371 - date.tm_yday) % 7; + if (jan1 != 4 && (jan1 != 3 || !__isLeapYear(date.tm_year))) { + val = 1; + } + } + return leadingNulls(val, 2); + }, + "%w": function (date) { + return date.tm_wday; + }, + "%W": function (date) { + var days = date.tm_yday + 7 - ((date.tm_wday + 6) % 7); + return leadingNulls(Math.floor(days / 7), 2); + }, + "%y": function (date) { + return (date.tm_year + 1900).toString().substring(2); + }, + "%Y": function (date) { + return date.tm_year + 1900; + }, + "%z": function (date) { + var off = date.tm_gmtoff; + var ahead = off >= 0; + off = Math.abs(off) / 60; + off = (off / 60) * 100 + (off % 60); + return (ahead ? "+" : "-") + String("0000" + off).slice(-4); + }, + "%Z": function (date) { + return date.tm_zone; + }, + "%%": function () { + return "%"; + }, + }; + pattern = pattern.replace(/%%/g, "\0\0"); + for (var rule in EXPANSION_RULES_2) { + if (pattern.includes(rule)) { + pattern = pattern.replace( + new RegExp(rule, "g"), + EXPANSION_RULES_2[rule](date) + ); + } + } + pattern = pattern.replace(/\0\0/g, "%"); + var bytes = intArrayFromString(pattern, false); + if (bytes.length > maxsize) { + return 0; + } + writeArrayToMemory(bytes, s); + return bytes.length - 1; + } + + function _strftime_l(s, maxsize, format, tm) { + return _strftime(s, maxsize, format, tm); + } + + InternalError = Module.InternalError = extendError(Error, "InternalError"); + + embind_init_charCodes(); + + BindingError = Module.BindingError = extendError(Error, "BindingError"); + + init_ClassHandle(); + + init_embind(); + + init_RegisteredPointer(); + + UnboundTypeError = Module.UnboundTypeError = extendError( + Error, + "UnboundTypeError" + ); + + init_emval(); + + function intArrayFromString(stringy, dontAddNull, length) { + var len = length > 0 ? length : lengthBytesUTF8(stringy) + 1; + var u8array = new Array(len); + var numBytesWritten = stringToUTF8Array( + stringy, + u8array, + 0, + u8array.length + ); + if (dontAddNull) { + u8array.length = numBytesWritten; + } + return u8array; + } + + var asmLibraryArg = { + __assert_fail: ___assert_fail, + __cxa_allocate_exception: ___cxa_allocate_exception, + __cxa_rethrow: ___cxa_rethrow, + __cxa_throw: ___cxa_throw, + __syscall_faccessat: ___syscall_faccessat, + __syscall_fcntl64: ___syscall_fcntl64, + __syscall_fstat64: ___syscall_fstat64, + __syscall_getcwd: ___syscall_getcwd, + __syscall_ioctl: ___syscall_ioctl, + __syscall_lstat64: ___syscall_lstat64, + __syscall_newfstatat: ___syscall_newfstatat, + __syscall_openat: ___syscall_openat, + __syscall_renameat: ___syscall_renameat, + __syscall_rmdir: ___syscall_rmdir, + __syscall_stat64: ___syscall_stat64, + __syscall_unlinkat: ___syscall_unlinkat, + _embind_finalize_value_object: __embind_finalize_value_object, + _embind_register_bigint: __embind_register_bigint, + _embind_register_bool: __embind_register_bool, + _embind_register_class: __embind_register_class, + _embind_register_class_constructor: __embind_register_class_constructor, + _embind_register_class_function: __embind_register_class_function, + _embind_register_emval: __embind_register_emval, + _embind_register_float: __embind_register_float, + _embind_register_integer: __embind_register_integer, + _embind_register_memory_view: __embind_register_memory_view, + _embind_register_smart_ptr: __embind_register_smart_ptr, + _embind_register_std_string: __embind_register_std_string, + _embind_register_std_wstring: __embind_register_std_wstring, + _embind_register_value_object: __embind_register_value_object, + _embind_register_value_object_field: __embind_register_value_object_field, + _embind_register_void: __embind_register_void, + _emscripten_date_now: __emscripten_date_now, + _emscripten_get_now_is_monotonic: __emscripten_get_now_is_monotonic, + _emval_call: __emval_call, + _emval_decref: __emval_decref, + _emval_incref: __emval_incref, + _emval_take_value: __emval_take_value, + _localtime_js: __localtime_js, + _mmap_js: __mmap_js, + _munmap_js: __munmap_js, + _tzset_js: __tzset_js, + abort: _abort, + emscripten_get_heap_max: _emscripten_get_heap_max, + emscripten_get_now: _emscripten_get_now, + emscripten_memcpy_big: _emscripten_memcpy_big, + emscripten_resize_heap: _emscripten_resize_heap, + environ_get: _environ_get, + environ_sizes_get: _environ_sizes_get, + exit: _exit, + fd_close: _fd_close, + fd_read: _fd_read, + fd_seek: _fd_seek, + fd_write: _fd_write, + getentropy: _getentropy, + memory: wasmMemory, + pclose: _pclose, + setTempRet0: _setTempRet0, + strftime_l: _strftime_l, + }; + + var asm = createWasm(); + + var calledRun; + + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = "Program terminated with exit(" + status + ")"; + this.status = status; + } + + dependenciesFulfilled = function runCaller() { + if (!calledRun) { + run(); + } + if (!calledRun) { + dependenciesFulfilled = runCaller; + } + }; + + function run(args) { + args = args || arguments_; + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) { + return; + } + calledRun = true; + Module.calledRun = true; + if (ABORT) { + return; + } + initRuntime(); + if (Module.onRuntimeInitialized) { + Module.onRuntimeInitialized(); + } + postRun(); + } + if (Module.setStatus) { + Module.setStatus("Running..."); + setTimeout(function () { + setTimeout(function () { + Module.setStatus(""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + + Module.run = run; + + function exit(status, implicit) { + EXITSTATUS = status; + procExit(status); + } + + function procExit(code) { + EXITSTATUS = code; + if (!keepRuntimeAlive()) { + if (Module.onExit) { + Module.onExit(code); + } + ABORT = true; + } + quit_(code, new ExitStatus(code)); + } + + if (Module.preInit) { + if (typeof Module.preInit == "function") { + Module.preInit = [Module.preInit]; + } + while (Module.preInit.length) { + Module.preInit.pop()(); + } + } + + run(); + + /* Use an optimized gemm implementation if available, otherwise use the fallback + * implementation. + */ + function createWasmGemm() { + // A map of expected gemm function to the corresponding fallback gemm function names. + const GEMM_TO_FALLBACK_FUNCTIONS_MAP = { + int8_prepare_a: "int8PrepareAFallback", + int8_prepare_b: "int8PrepareBFallback", + int8_prepare_b_from_transposed: "int8PrepareBFromTransposedFallback", + int8_prepare_b_from_quantized_transposed: + "int8PrepareBFromQuantizedTransposedFallback", + int8_prepare_bias: "int8PrepareBiasFallback", + int8_multiply_and_add_bias: "int8MultiplyAndAddBiasFallback", + int8_select_columns_of_b: "int8SelectColumnsOfBFallback", + }; + + // Name of the optimized gemm implementation. + const OPTIMIZED_GEMM = "mozIntGemm"; + + const optimizedGemmModule = WebAssembly[OPTIMIZED_GEMM]; + if (!optimizedGemmModule) { + return fallbackGemm(GEMM_TO_FALLBACK_FUNCTIONS_MAP); + } + + const optimizedGemmModuleExports = new WebAssembly.Instance( + optimizedGemmModule(), + { "": { memory: wasmMemory } } + ).exports; + for (let key in GEMM_TO_FALLBACK_FUNCTIONS_MAP) { + if (!optimizedGemmModuleExports[key]) { + return fallbackGemm(GEMM_TO_FALLBACK_FUNCTIONS_MAP); + } + } + Module.print(`Using optimized gemm (${OPTIMIZED_GEMM}) implementation`); + return optimizedGemmModuleExports; + } + + // Return the fallback gemm implementation. + function fallbackGemm(gemmToFallbackFunctionsMap) { + // The fallback gemm implementation + const FALLBACK_GEMM = "asm"; + + let fallbackGemmModuleExports = {}; + for (let key in gemmToFallbackFunctionsMap) { + fallbackGemmModuleExports[key] = (...a) => + Module[FALLBACK_GEMM][gemmToFallbackFunctionsMap[key]](...a); + } + Module.print(`Using fallback gemm implementation`); + return fallbackGemmModuleExports; + } + return Module; +} diff --git a/toolkit/components/translations/bergamot-translator/build-bergamot.py b/toolkit/components/translations/bergamot-translator/build-bergamot.py new file mode 100755 index 0000000000..6c55237f22 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/build-bergamot.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Builds the Bergamot translations engine for integration with Firefox. + +If you wish to test the Bergamot engine locally, then uncomment the .wasm line in +the toolkit/components/translations/jar.mn after building the file. Just make sure +not to check the code change in. +""" + +import argparse +import multiprocessing +import os +import shutil +import subprocess +from collections import namedtuple + +import yaml + +DIR_PATH = os.path.realpath(os.path.dirname(__file__)) +THIRD_PARTY_PATH = os.path.join(DIR_PATH, "thirdparty") +MOZ_YAML_PATH = os.path.join(DIR_PATH, "moz.yaml") +PATCHES_PATH = os.path.join(DIR_PATH, "patches") +BERGAMOT_PATH = os.path.join(THIRD_PARTY_PATH, "bergamot-translator") +MARIAN_PATH = os.path.join(BERGAMOT_PATH, "3rd_party/marian-dev") +GEMM_SCRIPT = os.path.join(BERGAMOT_PATH, "wasm/patch-artifacts-import-gemm-module.sh") +BUILD_PATH = os.path.join(THIRD_PARTY_PATH, "build-wasm") +EMSDK_PATH = os.path.join(THIRD_PARTY_PATH, "emsdk") +EMSDK_ENV_PATH = os.path.join(EMSDK_PATH, "emsdk_env.sh") +WASM_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.wasm") +JS_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.js") +FINAL_JS_PATH = os.path.join(DIR_PATH, "bergamot-translator.js") +ROOT_PATH = os.path.join(DIR_PATH, "../../../..") + +# 3.1.47 had an error compiling sentencepiece. +EMSDK_VERSION = "3.1.8" +EMSDK_REVISION = "2346baa7bb44a4a0571cc75f1986ab9aaa35aa03" + +patches = [ + (BERGAMOT_PATH, os.path.join(PATCHES_PATH, "allocation-bergamot.patch")), + (MARIAN_PATH, os.path.join(PATCHES_PATH, "allocation-marian.patch")), +] + +parser = argparse.ArgumentParser( + description=__doc__, + # Preserves whitespace in the help text. + formatter_class=argparse.RawTextHelpFormatter, +) +parser.add_argument( + "--clobber", action="store_true", help="Clobber the build artifacts" +) +parser.add_argument( + "--debug", + action="store_true", + help="Build with debug symbols, useful for profiling", +) + +ArgNamespace = namedtuple("ArgNamespace", ["clobber", "debug"]) + + +def git_clone_update(name: str, repo_path: str, repo_url: str, revision: str): + if not os.path.exists(repo_path): + print(f"\n⬇️ Clone the {name} repo into {repo_path}\n") + subprocess.check_call( + ["git", "clone", repo_url], + cwd=THIRD_PARTY_PATH, + ) + + local_head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=repo_path, + text=True, + ).strip() + + def run(command): + return subprocess.check_call(command, cwd=repo_path) + + if local_head != revision: + print(f"The head ({local_head}) and revision ({revision}) don't match.") + print(f"\n🔎 Fetching the latest from {name}.\n") + run(["git", "fetch", "--recurse-submodules"]) + + print(f"🛒 Checking out the revision {revision}") + run(["git", "checkout", revision]) + run(["git", "submodule", "update", "--init", "--recursive"]) + + +def install_and_activate_emscripten(args: ArgNamespace): + git_clone_update( + name="emsdk", + repo_path=EMSDK_PATH, + repo_url="https://github.com/emscripten-core/emsdk.git", + revision=EMSDK_REVISION, + ) + + # Run these commands in the shell so that the configuration is saved. + def run_shell(command): + return subprocess.run(command, cwd=EMSDK_PATH, shell=True, check=True) + + print(f"\n🛠️ Installing EMSDK version {EMSDK_VERSION}\n") + run_shell("./emsdk install " + EMSDK_VERSION) + + print("\n🛠️ Activating emsdk\n") + run_shell("./emsdk activate " + EMSDK_VERSION) + + +def install_bergamot(): + with open(MOZ_YAML_PATH, "r", encoding="utf8") as file: + text = file.read() + + moz_yaml = yaml.safe_load(text) + + git_clone_update( + name="bergamot", + repo_path=BERGAMOT_PATH, + repo_url=moz_yaml["origin"]["url"], + revision=moz_yaml["origin"]["revision"], + ) + + +def to_human_readable(size): + """Convert sizes to human-readable format""" + size_in_mb = size / 1048576 + return f"{size_in_mb:.2f}M ({size} bytes)" + + +def apply_git_patch(repo_path, patch_path): + print(f"Applying patch {patch_path} to {os.path.basename(repo_path)}") + subprocess.check_call(["git", "apply", "--reject", patch_path], cwd=repo_path) + + +def revert_git_patch(repo_path, patch_path): + print(f"Reverting patch {patch_path} from {os.path.basename(repo_path)}") + subprocess.check_call(["git", "apply", "-R", "--reject", patch_path], cwd=repo_path) + + +def build_bergamot(args: ArgNamespace): + if args.clobber and os.path.exists(BUILD_PATH): + shutil.rmtree(BUILD_PATH) + + if not os.path.exists(BUILD_PATH): + os.mkdir(BUILD_PATH) + + print("\n 🖌️ Applying source code patches\n") + for repo_path, patch_path in patches: + apply_git_patch(repo_path, patch_path) + + # These commands require the emsdk environment variables to be set up. + def run_shell(command): + if '"' in command or "'" in command: + raise Exception("This run_shell utility does not support quotes.") + + return subprocess.run( + # "source" is not available in all shells so explicitly + f"bash -c 'source {EMSDK_ENV_PATH} && {command}'", + cwd=BUILD_PATH, + shell=True, + check=True, + ) + + try: + flags = "" + if args.debug: + flags = "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + + print("\n 🏃 Running CMake for Bergamot\n") + run_shell( + f"emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off {flags} {BERGAMOT_PATH}" + ) + + print("\n 🏃 Building Bergamot with emmake\n") + run_shell(f"emmake make -j {multiprocessing.cpu_count()}") + + print("\n 🪚 Patching Bergamot for gemm support\n") + subprocess.check_call(["bash", GEMM_SCRIPT, BUILD_PATH]) + + print("\n✅ Build complete\n") + print(" " + JS_PATH) + print(" " + WASM_PATH) + + # Get the sizes of the build artifacts. + wasm_size = os.path.getsize(WASM_PATH) + gzip_size = int( + subprocess.run( + f"gzip -c {WASM_PATH} | wc -c", + check=True, + shell=True, + capture_output=True, + ).stdout.strip() + ) + print(f" Uncompressed wasm size: {to_human_readable(wasm_size)}") + print(f" Compressed wasm size: {to_human_readable(gzip_size)}") + finally: + print("\n🖌️ Reverting the source code patches\n") + for repo_path, patch_path in patches[::-1]: + revert_git_patch(repo_path, patch_path) + + +def write_final_bergamot_js_file(): + """ + The generated JS file requires some light patching for integration. + """ + + source = "\n".join( + [ + "/* This Source Code Form is subject to the terms of the Mozilla Public", + " * License, v. 2.0. If a copy of the MPL was not distributed with this", + " * file, You can obtain one at http://mozilla.org/MPL/2.0/. */", + "", + "function loadBergamot(Module) {", + "", + ] + ) + + with open(JS_PATH, "r", encoding="utf8") as file: + for line in file.readlines(): + source += " " + line + + source += " return Module;\n}" + + # Use the Module's printing. + source = source.replace("console.log(", "Module.print(") + + # Add some instrumentation to the module's memory size. + source = source.replace( + "function updateGlobalBufferAndViews(buf) {", + """ + function updateGlobalBufferAndViews(buf) { + const mb = (buf.byteLength / 1_000_000).toFixed(); + Module.print( + `Growing wasm buffer to ${mb}MB (${buf.byteLength} bytes).` + ); + """, + ) + + print("\n Formatting the final bergamot file") + # Create the file outside of this directory so it's not ignored by eslint. + temp_path = os.path.join(DIR_PATH, "../temp-bergamot.js") + with open(temp_path, "w", encoding="utf8") as file: + file.write(source) + + subprocess.run( + f"./mach eslint --fix {temp_path}", + cwd=ROOT_PATH, + check=True, + shell=True, + capture_output=True, + ) + + print(f"\n Writing out final bergamot file: {FINAL_JS_PATH}") + shutil.move(temp_path, FINAL_JS_PATH) + + +def main(): + args: ArgNamespace = parser.parse_args() + + if not os.path.exists(THIRD_PARTY_PATH): + os.mkdir(THIRD_PARTY_PATH) + + install_and_activate_emscripten(args) + install_bergamot() + build_bergamot(args) + write_final_bergamot_js_file() + + +if __name__ == "__main__": + main() diff --git a/toolkit/components/translations/bergamot-translator/moz.yaml b/toolkit/components/translations/bergamot-translator/moz.yaml new file mode 100644 index 0000000000..23a4cc8c27 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/moz.yaml @@ -0,0 +1,51 @@ +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Firefox + component: Translation + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: bergamot-translator + + description: The JavaScript emscripten worker to run Project Bergamot. + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://github.com/browsermt/bergamot-translator.git + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: v0.4.5 + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: 05a87784973b6e1cc591f1f1a9a05c5873d9971e + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: MPL-2.0 + + notes: > + The generated emscripten code contains many global variables. When updating + the code, paste it into the following function to protect the global scope. + + ``` + /* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + function loadBergamot(Module) { + {insertCodeHere} + return Module; + } + ``` + + Then replace `console.log()` calls with `log()` calls so that logging preferences + are respected. diff --git a/toolkit/components/translations/bergamot-translator/patches/allocation-bergamot.patch b/toolkit/components/translations/bergamot-translator/patches/allocation-bergamot.patch new file mode 100644 index 0000000000..a8dca5b7e1 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/patches/allocation-bergamot.patch @@ -0,0 +1,26 @@ +commit dfa705777729fd084f0187a90f9712eb76ea9209 +parent 05a87784973b6e1cc591f1f1a9a05c5873d9971e +Author: Greg Tatum <tatum.creative@gmail.com> +Date: Tue Nov 7 10:57:07 2023 -0600 + + Change allocation strategy + + This fixes an issue where the memory would grow to 500mb by pre-allocating large + workspaces. For some reason the "workspace" configuration for the Wasm build wasn't + fixing this, but hard-coding the value does. Perhaps the configuration file in Bergamot + is not working correctly, or it was just a mistake on the author's part. Empirically + this value keeps memory from growing too rapidly, and does not degrade Wasm performance. + +diff --git a/src/translator/translation_model.cpp b/src/translator/translation_model.cpp +index 3f91ebb..61a299f 100644 +--- a/src/translator/translation_model.cpp ++++ b/src/translator/translation_model.cpp +@@ -59,7 +59,7 @@ void TranslationModel::loadBackend(size_t idx) { + graph->setDefaultElementType(typeFromString(prec[0])); + graph->setDevice(device_); + graph->getBackend()->configureDevice(options_); +- graph->reserveWorkspaceMB(options_->get<size_t>("workspace")); ++ graph->reserveWorkspaceMB(5); + + // Marian Model: Load from memoryBundle or shortList + if (memory_.model.size() > 0 && diff --git a/toolkit/components/translations/bergamot-translator/patches/allocation-marian.patch b/toolkit/components/translations/bergamot-translator/patches/allocation-marian.patch new file mode 100644 index 0000000000..4fe2616d07 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/patches/allocation-marian.patch @@ -0,0 +1,25 @@ +commit 31a05b47381a5b22b57fe9af7805fa40a5c5e384 +parent 11c6ae7c46be21ef96ed10c60f28022fa968939f +Author: Greg Tatum <tatum.creative@gmail.com> +Date: Mon Nov 6 14:01:32 2023 -0600 + + Change allocation strategy for tensors + + When tensors grow, they would pre-emptively allocate large amounts of memory, and + would allocate ~500mb of memory for a single translation. Adjusting this value + down appears to fix this issue. Empirically this value keeps memory from growing too + rapidly, and does not degrade Wasm performance. + +diff --git a/src/tensors/tensor_allocator.h b/src/tensors/tensor_allocator.h +index e3bc79f9..66f8e44d 100644 +--- a/src/tensors/tensor_allocator.h ++++ b/src/tensors/tensor_allocator.h +@@ -13,7 +13,7 @@ class TensorAllocator { + private: + const size_t CHUNK = 128; + const size_t MBYTE = 1024 * 1024; +- const size_t GROW = CHUNK * MBYTE; ++ const size_t GROW = MBYTE; + const size_t ALIGN = 256; + + Ptr<Backend> backend_; diff --git a/toolkit/components/translations/bergamot-translator/upload-bergamot.py b/toolkit/components/translations/bergamot-translator/upload-bergamot.py new file mode 100755 index 0000000000..712eae0e2c --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/upload-bergamot.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Uploads the bergamot wasm file to Remote Settings. This just uploads the build artifact, +approval and deployment will still need to be done through Remote Settings. You must +run ./build-bergamot.py first to generate the wasm artifact. + +Log in to Remote Settings via LDAP to either dev or prod: + + Dev: https://remote-settings-dev.allizom.org/v1/admin + Prod: https://remote-settings.mozilla.org/v1/admin + +In the header click the little clipboard icon to get the authentication header. +Set the BEARER_TOKEN environment variable to use the bearer token. In zsh this can +be done privately via the command line with the `setopt HIST_IGNORE_SPACE` and +adding a space in front of the command, e.g. + +` export BEARER_TOKEN="Bearer uLdb-Yafefe....2Hyl5_w"` +""" + +import argparse +import asyncio +import json +import os +import pprint +import sys +import uuid +from collections import namedtuple +from textwrap import dedent + +import requests +import yaml + +# When running upload-bergamot.py, this number should be bumped for new uploads. +# A minor version bump means that there is no breaking change. A major version +# bump means that the upload is a breaking change. Firefox will only download +# records that match the TranslationsParent.BERGAMOT_MAJOR_VERSION. +REMOTE_SETTINGS_VERSION = "1.1" + +COLLECTION_NAME = "translations-wasm" +DEV_SERVER = "https://remote-settings-dev.allizom.org/v1" +PROD_SERVER = "https://remote-settings.mozilla.org/v1" + +DIR_PATH = os.path.realpath(os.path.dirname(__file__)) +MOZ_YAML_PATH = os.path.join(DIR_PATH, "moz.yaml") +THIRD_PARTY_PATH = os.path.join(DIR_PATH, "thirdparty") +BUILD_PATH = os.path.join(THIRD_PARTY_PATH, "build-wasm") +WASM_PATH = os.path.join(BUILD_PATH, "bergamot-translator-worker.wasm") +ROOT_PATH = os.path.join(DIR_PATH, "../../../..") +BROWSER_VERSION_PATH = os.path.join(ROOT_PATH, "browser/config/version.txt") +RECORDS_PATH = "/admin/#/buckets/main-workspace/collections/translations-wasm/records" + +parser = argparse.ArgumentParser( + description=__doc__, + # Preserves whitespace in the help text. + formatter_class=argparse.RawTextHelpFormatter, +) +parser.add_argument("--server", help='Set to either "dev" or "prod"') +parser.add_argument( + "--dry_run", action="store_true", help="Verify the login, but do not upload" +) +ArgNamespace = namedtuple("ArgNamespace", ["server", "dry_run"]) + +pp = pprint.PrettyPrinter(indent=2) + + +def print_error(message): + """This is a simple util function.""" + red = "\033[91m" + reset = "\033[0m" + print(f"{red}Error:{reset} {message}\n", file=sys.stderr) + + +class RemoteSettings: + """ + After validating the arguments, this class controls the REST operations for + communicating with Remote Settings. + """ + + def __init__(self, server: str, bearer_token: str): + self.server: str = server + self.bearer_token: str = bearer_token + + with open(MOZ_YAML_PATH, "r", encoding="utf8") as file: + moz_yaml_text = file.read() + self.moz_yaml = yaml.safe_load(moz_yaml_text) + + self.version: str = REMOTE_SETTINGS_VERSION + if not isinstance(self.version, str): + print_error("The bergamot remote settings version must be a string.") + sys.exit(1) + + async def fetch_json(self, path: str): + """Perfrom a simple GET operation and return the JSON results""" + url = self.server + path + response = requests.get(url, headers=self.get_headers()) + + if not response.ok: + print_error(f"❌ Failed fetching {url}\nStatus: {response.status_code}") + try: + print(response.json()) + except Exception: + print_error("Unable to read response") + raise Exception() + + return response.json() + + async def verify_login(self): + """Before performing any operations, verify the login credentials.""" + try: + json = await self.fetch_json("/") + except Exception: + print_error("Your login information could not be verified") + parser.print_help(sys.stderr) + sys.exit(0) + + if "user" not in json: + print_error("Your bearer token has expired or is not valid.") + parser.print_help(sys.stderr) + sys.exit(1) + + print( + f"✅ Authorized to use {self.server} as user {json['user']['profile']['email']}" + ) + + def get_headers(self): + return {"Authorization": self.bearer_token} + + async def verify_record_version(self): + try: + main_records = await self.fetch_json( + f"/buckets/main/collections/{COLLECTION_NAME}/records", + ) + except Exception: + print_error("Failed to get the main records") + sys.exit(1) + + for record in main_records["data"]: + if ( + record["name"] == "bergamot-translator" + and record["version"] == self.version + ): + print("Conflicting record in Remote Settings:", record) + print_error( + dedent( + f""" + The version {self.version} already existed in the published records. + You need to bump the major or minor version number in the moz.yaml file. + """ + ) + ) + sys.exit(1) + + try: + workspace_records = await self.fetch_json( + f"/buckets/main-workspace/collections/{COLLECTION_NAME}/records", + ) + except Exception: + print_error("Failed to get the workspace records") + sys.exit(1) + + for record in workspace_records["data"]: + if ( + record["name"] == "bergamot-translator" + and record["version"] == self.version + ): + print("Conflicting record in Remote Settings:", record) + print_error( + dedent( + f""" + The version {self.version} already existed in the workspace records. + You need to delete the file in the workspace before uploading again. + + {self.server + RECORDS_PATH} + """ + ) + ) + sys.exit(1) + + print("📦 Packages in the workspace:") + for record in workspace_records["data"]: + if record["name"] == "bergamot-translator": + print(f' - bergamot-translator@{record["version"]}') + + print(f"✅ Version {self.version} does not conflict, ready for uploading.") + + def create_record(self): + name = self.moz_yaml["origin"]["name"] + release = self.moz_yaml["origin"]["release"] + revision = self.moz_yaml["origin"]["revision"] + license = self.moz_yaml["origin"]["license"] + version = REMOTE_SETTINGS_VERSION + + if not name or not release or not revision or not license or not version: + print_error("Some of the required record data is not in the moz.yaml file.") + sys.exit(1) + + with open(BROWSER_VERSION_PATH, "r", encoding="utf8") as file: + fx_release = file.read().strip() + + files = [ + ( + "attachment", + ( + os.path.basename(WASM_PATH), # filename + open(WASM_PATH, "rb"), # file handle + "application/wasm", # mimetype + ), + ) + ] + + data = { + "name": name, + "release": release, + "revision": revision, + "license": license, + "version": version, + "fx_release": fx_release, + # Default to nightly and local builds. + "filter_expression": "env.channel == 'nightly' || env.channel == 'default'", + } + + print("📬 Posting record") + print("✉️ Attachment: ", WASM_PATH) + print("📀 Record: ", end="") + pp.pprint(data) + + return files, data + + def upload_record(self, files, data): + id = str(uuid.uuid4()) + url = f"{self.server}/buckets/main-workspace/collections/{COLLECTION_NAME}/records/{id}/attachment" + + print(f"\n⬆️ POSTing the record to: {url}\n") + response = requests.post( + url, + headers=self.get_headers(), + data={"data": json.dumps(data)}, + files=files, + ) + + if response.status_code >= 200 and response.status_code < 300: + print("✅ Record created:", self.server + RECORDS_PATH) + print("✉️ Attachment details: ", end="") + pp.pprint(json.loads(response.text)) + else: + print_error( + f"Error creating record: (Error code {response.status_code})\n{response.text}" + ) + raise Exception() + + +async def main(): + if len(sys.argv) == 1: + parser.print_help(sys.stderr) + sys.exit(1) + + args: ArgNamespace = parser.parse_args() + + bearer_token = os.environ.get("BEARER_TOKEN") + if not bearer_token: + print_error('A "BEARER_TOKEN" environment variable must be set.') + parser.print_help(sys.stderr) + sys.exit(1) + + if args.server == "prod": + server = PROD_SERVER + elif args.server == "dev": + server = DEV_SERVER + else: + print_error('The server must either be "prod" or "dev"') + parser.print_help(sys.stderr) + sys.exit(1) + + remote_settings = RemoteSettings(server, bearer_token) + + await remote_settings.verify_login() + await remote_settings.verify_record_version() + + files, data = remote_settings.create_record() + if args.dry_run: + return + + remote_settings.upload_record(files, data) + + +if __name__ == "__main__": + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/toolkit/components/translations/content/translations-document.sys.mjs b/toolkit/components/translations/content/translations-document.sys.mjs new file mode 100644 index 0000000000..7f436575d8 --- /dev/null +++ b/toolkit/components/translations/content/translations-document.sys.mjs @@ -0,0 +1,2140 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + setTimeout: "resource://gre/modules/Timer.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +/** + * Map the NodeFilter enums that are used by the TreeWalker into enums that make + * sense for determining the status of the nodes for the TranslationsDocument process. + * This aligns the meanings of the filtering for the translations process. + */ +const NodeStatus = { + // This node is ready to translate as is. + READY_TO_TRANSLATE: NodeFilter.FILTER_ACCEPT, + + // This node is a shadow host and needs to be subdivided further. + SHADOW_HOST: NodeFilter.FILTER_ACCEPT, + + // This node contains too many block elements and needs to be subdivided further. + SUBDIVIDE_FURTHER: NodeFilter.FILTER_SKIP, + + // This node should not be considered for translation. + NOT_TRANSLATABLE: NodeFilter.FILTER_REJECT, +}; + +/** + * @typedef {import("../translations").NodeVisibility} NodeVisibility + * @typedef {(message: string) => Promise<string>} TranslationFunction + */ + +/** + * Create a translation cache with a limit. It implements a "least recently used" strategy + * to remove old translations. After `#cacheExpirationMS` the cache will be emptied. + * This cache is owned statically by the TranslationsChild. This means that it will be + * re-used on page reloads if the origin of the site does not change. + */ +export class LRUCache { + /** @type {Map<string, string>} */ + #htmlCache = new Map(); + /** @type {Map<string, string>} */ + #textCache = new Map(); + /** @type {string} */ + #fromLanguage; + /** @type {string} */ + #toLanguage; + + /** + * This limit is used twice, once for Text translations, and once for HTML translations. + */ + #cacheLimit = 5_000; + + /** + * This cache will self-destruct after 10 minutes. + */ + #cacheExpirationMS = 10 * 60_000; + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + constructor(fromLanguage, toLanguage) { + this.#fromLanguage = fromLanguage; + this.#toLanguage = toLanguage; + } + + /** + * @param {boolean} isHTML + * @returns {boolean} + */ + #getCache(isHTML) { + return isHTML ? this.#htmlCache : this.#textCache; + } + + /** + * Get a translation if it exists from the cache, and move it to the end of the cache + * to keep it alive longer. + * + * @param {string} sourceString + * @param {boolean} isHTML + * @returns {string} + */ + get(sourceString, isHTML) { + const cache = this.#getCache(isHTML); + const targetString = cache.get(sourceString); + + if (targetString === undefined) { + return undefined; + } + + // Maps are ordered, move this item to the end of the list so it will stay + // alive longer. + cache.delete(sourceString); + cache.set(sourceString, targetString); + + this.keepAlive(); + + return targetString; + } + + /** + * @param {string} sourceString + * @param {string} targetString + * @param {boolean} isHTML + */ + set(sourceString, targetString, isHTML) { + const cache = this.#getCache(isHTML); + if (cache.size === this.#cacheLimit) { + // If the cache is at the limit, get the least recently used translation and + // remove it. This works since Maps have keys ordered by insertion order. + const key = cache.keys().next().value; + cache.delete(key); + } + cache.set(sourceString, targetString); + this.keepAlive(); + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + matches(fromLanguage, toLanguage) { + return ( + this.#fromLanguage === fromLanguage && this.#toLanguage === toLanguage + ); + } + + /** + * @type {number} + */ + #timeoutId = 0; + + #pendingKeepAlive = false; + + /** + * Clear out the cache on a timer. + */ + keepAlive() { + if (this.#timeoutId) { + lazy.clearTimeout(this.#timeoutId); + } + if (!this.#pendingKeepAlive) { + // Rather than continuously creating new functions in a tight loop, only schedule + // one keepAlive timeout on the next tick. + this.#pendingKeepAlive = true; + + lazy.setTimeout(() => { + this.#pendingKeepAlive = false; + this.#timeoutId = lazy.setTimeout(() => { + this.#htmlCache = new Map(); + this.#textCache = new Map(); + }, this.#cacheExpirationMS); + }, 0); + } + } +} + +/** + * How often the DOM is updated with translations, in milliseconds. + */ +const DOM_UPDATE_INTERVAL_MS = 50; + +/** + * These tags are excluded from translation. + */ +const EXCLUDED_TAGS = new Set([ + // The following are elements that semantically should not be translated. + "CODE", + "KBD", + "SAMP", + "VAR", + "ACRONYM", + + // The following are deprecated tags. + "DIR", + "APPLET", + + // The following are embedded elements, and are not supported (yet). + "MATH", + "EMBED", + "OBJECT", + "IFRAME", + + // This is an SVG tag that can contain arbitrary XML, ignore it. + "METADATA", + + // These are elements that are treated as opaque by Firefox which causes their + // innerHTML property to be just the raw text node behind it. Any text that is sent as + // HTML must be valid, and there is no guarantee that the innerHTML is valid. + "NOSCRIPT", + "NOEMBED", + "NOFRAMES", + + // The title is handled separately, and a HEAD tag should not be considered. + "HEAD", + + // These are not user-visible tags. + "STYLE", + "SCRIPT", + "TEMPLATE", + + // Textarea elements contain user content, which should not be translated. + "TEXTAREA", +]); + +/** + * Attributes to be translated + */ +const TRANSLATABLE_ATTRIBUTES = ["title", "placeholder"]; + +/** + * Selector to get all the attributes + * ["[attribute1]", "[attribute2]", ...]; + */ +const TRANSLATABLE_ATTRIBUTES_SELECTOR = TRANSLATABLE_ATTRIBUTES.map( + attribute => "[" + attribute + "]" +); + +/** + * Options used by the mutation observer + */ +const MUTATION_OBSERVER_OPTIONS = { + characterData: true, + childList: true, + subtree: true, + attributes: true, + attributeFilter: TRANSLATABLE_ATTRIBUTES, +}; + +/** + * This class manages the process of translating the DOM from one language to another. + * A translateHTML and a translateText function are injected into the constructor. This + * class is responsible for subdividing a Node into small enough pieces to where it + * contains a reasonable amount of text and inline elements for the translations engine + * to translate. Once a node has been identified as a small enough chunk, its innerHTML + * is read, and sent for translation. The async translation result comes back as an HTML + * string. The DOM node is updated with the new text and potentially changed DOM ordering. + * + * This class also handles mutations of the DOM and will translate nodes as they are added + * to the page, or the when the node's text is changed by content scripts. + */ +export class TranslationsDocument { + /** + * The BCP 47 language tag that is used on the page. + * + * @type {string} */ + documentLanguage; + + /** + * The timeout between the first translation received and the call to update the DOM + * with translations. + */ + #updateTimeout = null; + #attributeUpdateTimeout = null; + + /** + * The nodes that need translations. They are queued when the document tree is walked, + * and then they are dispatched for translation based on their visibility. The viewport + * nodes are given the highest priority. + * + * @type {Map<Node, NodeVisibility>} + */ + #queuedNodes = new Map(); + + /** + * The nodes that need Attribute translations. They are queued when the document tree is walked, + * and then they are dispatched for translation based on their visibility. The viewport + * nodes are given the highest priority. + * + * @type {Map<Node, { attributeList: string[], visibility: NodeVisibility }>} + */ + #queuedAttributeNodes = new Map(); + + /** + * The count of how many pending translations have been sent to the translations + * engine. + */ + #pendingTranslationsCount = 0; + + /** + * The list of nodes that need updating with the translated HTML. These are batched + * into an update. + * + * @type {Set<{ node: Node, translatedHTML: string }} + */ + #nodesWithTranslatedHTML = new Set(); + + /** + * The list of nodes that need updating with the translated Attribute HTML. These are batched + * into an update. + * + * @type {Set<{ node: Node, translation: string, attribute: string }} + */ + #nodesWithTranslatedAttributes = new Set(); + + /** + * The set of nodes that have been subdivided and processed for translation. They + * should not be submitted again unless their contents have been changed. + * + * @type {WeakSet<Node>} + */ + #processedNodes = new WeakSet(); + + /** + * All root elements we're trying to translate. This should be the `document.body` + * and the the `title` element. + * + * @type {Set<Node>} + */ + #rootNodes = new Set(); + + /** + * This promise gets resolved when the initial viewport translations are done. + * This is a key user-visible performance metric. It represents what the user + * actually sees. + * + * @type {Promise<void> | null} + */ + viewportTranslated = null; + + isDestroyed = false; + + /** + * Construct a new TranslationsDocument. It is tied to a specific Document and cannot + * be re-used. The translation functions are injected since this class shouldn't + * manage the life cycle of the translations engines. + * + * @param {Document} document + * @param {string} documentLanguage - The BCP 47 tag of the source language. + * @param {string} toLanguage - The BCP 47 tag of the destination language. + * @param {number} innerWindowId - This is used for better profiler marker reporting. + * @param {MessagePort} port - The port to the translations engine. + * @param {() => void} requestNewPort - Used when an engine times out and a new + * translation request comes in. + * @param {number} translationsStart + * @param {() => number} now + * @param {LRUCache} translationsCache + */ + constructor( + document, + documentLanguage, + toLanguage, + innerWindowId, + port, + requestNewPort, + translationsStart, + now, + translationsCache + ) { + /** + * The language of the document. If elements are found that do not match this language, + * then they are skipped. + * + * @type {string} + */ + this.documentLanguage = documentLanguage; + if (documentLanguage.length !== 2) { + throw new Error( + "Expected the document language to be a valid 2 letter BCP 47 language tag: " + + documentLanguage + ); + } + if (toLanguage.length !== 2) { + throw new Error( + "Expected the destination language to be a valid 2 letter BCP 47 language tag: " + + toLanguage + ); + } + + /** @type {QueuedTranslator} */ + this.translator = new QueuedTranslator(port, requestNewPort); + + /** @type {number} */ + this.innerWindowId = innerWindowId; + + /** @type {DOMParser} */ + this.domParser = new document.ownerGlobal.DOMParser(); + + /** @type {Document} */ + this.document = document; + + /** @type {LRUCache} */ + this.translationsCache = translationsCache; + + /** + * This selector runs to find child nodes that should be excluded. It should be + * basically the same implementation of `isExcludedNode`, but as a selector. + * + * @type {string} + */ + this.excludedNodeSelector = [ + // Use: [lang|=value] to match language codes. + // + // Per: https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors + // + // The elements with an attribute name of attr whose value can be exactly + // value or can begin with value immediately followed by a hyphen, - (U+002D). + // It is often used for language subcode matches. + `[lang]:not([lang|="${this.documentLanguage}"])`, + `[translate=no]`, + `.notranslate`, + `[contenteditable="true"]`, + `[contenteditable=""]`, + [...EXCLUDED_TAGS].join(","), + ].join(","); + + this.observer = new document.ownerGlobal.MutationObserver(mutationsList => { + for (const mutation of mutationsList) { + switch (mutation.type) { + case "childList": + for (const node of mutation.addedNodes) { + this.#processedNodes.delete(node); + this.subdivideNodeForTranslations(node); + if (node.nodeType === Node.ELEMENT_NODE) { + this.translateAttributes(node); + } + } + break; + case "characterData": + this.#processedNodes.delete(mutation); + this.subdivideNodeForTranslations(mutation.target); + break; + case "attributes": + this.queueAttributeNodeForTranslation(mutation.target, [ + mutation.attributeName, + ]); + this.dispatchQueuedAttributeTranslations(); + break; + default: + break; + } + } + }); + + this.document.addEventListener( + "visibilitychange", + this.handleVisibilityChange + ); + + this.addRootElement(document.querySelector("title")); + this.addRootElement(document.body, true /* reportWordsInViewport */); + + this.viewportTranslated?.then(() => { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId, startTime: now() }, + "Viewport translations" + ); + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId, startTime: translationsStart }, + "Time to first translation" + ); + }); + + document.documentElement.lang = toLanguage; + + lazy.console.log( + "Beginning to translate.", + // The defaultView may not be there on tests. + document.defaultView?.location.href + ); + } + + /** + * Queue a node for translation of attributes. + * @param {Node} node + * @param {Array<String>} + */ + queueAttributeNodeForTranslation(node, attributeList) { + /** @type {NodeVisibility} */ + let visibility = "out-of-viewport"; + if (isNodeHidden(node)) { + visibility = "hidden"; + } else if (isNodeInViewport(node)) { + visibility = "in-viewport"; + } + this.#queuedAttributeNodes.set(node, { attributeList, visibility }); + } + + /** + * Start and stop the translator as the page is shown. For instance, this will + * transition into "hidden" when the user tabs away from a document. + */ + handleVisibilityChange = () => { + if (this.document.visibilityState === "visible") { + this.translator.showPage(); + } else { + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + "Pausing translations and discarding the port" + ); + this.translator.hidePage(); + } + }; + + /** + * Remove any dangling event handlers. + */ + destroy() { + this.isDestroyed = true; + this.translator.destroy(); + this.stopMutationObserver(); + this.document.removeEventListener( + "visibilitychange", + this.handleVisibilityChange + ); + } + + /** + * Helper function for adding a new root to the mutation + * observer. + * @param {Node} root + */ + observeNewRoot(root) { + this.#rootNodes.add(root); + this.observer.observe(root, MUTATION_OBSERVER_OPTIONS); + } + + /** + * This function finds all sub shadow trees of node and + * add the ShadowRoot of those subtrees to the mutation + * observer. + */ + addShadowRootsToObserver(node) { + const nodeIterator = node.ownerDocument.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT, + function (node) { + return node.openOrClosedShadowRoot + ? NodeFilter.FILTER_ACCEPT + : NodeFilter.FILTER_SKIP; + } + ); + let currentNode; + while ((currentNode = nodeIterator.nextNode())) { + // Only shadow hosts are accepted nodes + const shadowRoot = currentNode.openOrClosedShadowRoot; + this.observeNewRoot(shadowRoot); + this.addShadowRootsToObserver(shadowRoot); + } + } + + /** + * Add a new element to start translating. This root is tracked for mutations and + * kept up to date with translations. This will be the body element and title tag + * for the document. + * + * @param {Element} [node] + */ + addRootElement(node) { + if (!node) { + return; + } + + if (node.nodeType !== Node.ELEMENT_NODE) { + // This node is not an element, do not add it. + return; + } + + if (this.#rootNodes.has(node)) { + // Exclude nodes that are already targetted. + return; + } + + this.#rootNodes.add(node); + + let viewportNodeTranslations = this.subdivideNodeForTranslations(node); + let viewportAttributeTranslations = this.translateAttributes(node); + + if (!this.viewportTranslated) { + this.viewportTranslated = Promise.allSettled([ + ...(viewportNodeTranslations ?? []), + ...(viewportAttributeTranslations ?? []), + ]); + } + + this.observer.observe(node, MUTATION_OBSERVER_OPTIONS); + this.addShadowRootsToObserver(node); + } + + /** + * Add qualified nodes to queueNodeForTranslation by recursively walk + * through the DOM tree of node, including elements in Shadow DOM. + * + * @param {Element} [node] + */ + processSubdivide(node) { + const nodeIterator = node.ownerDocument.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, + this.determineTranslationStatusForUnprocessedNodes + ); + + // This iterator will contain each node that has been subdivided enough to + // be translated. + let currentNode; + while ((currentNode = nodeIterator.nextNode())) { + const shadowRoot = currentNode.openOrClosedShadowRoot; + if (shadowRoot) { + this.processSubdivide(shadowRoot); + } else { + this.queueNodeForTranslation(currentNode); + } + } + } + + /** + * Start walking down through a node's subtree and decide which nodes to queue for + * translation. This first node could be the root nodes of the DOM, such as the + * document body, or the title element, or it could be a mutation target. + * + * The nodes go through a process of subdivision until an appropriate sized chunk + * of inline text can be found. + * + * @param {Node} node + */ + subdivideNodeForTranslations(node) { + if (!this.#rootNodes.has(node)) { + // This is a non-root node, which means it came from a mutation observer. + // This new node could be a host element for shadow tree + const shadowRoot = node.openOrClosedShadowRoot; + if (shadowRoot && !this.#rootNodes.has(shadowRoot)) { + this.observeNewRoot(shadowRoot); + } else { + // Ensure that it is a valid node to translate by checking all of its ancestors. + for (let parent of getAncestorsIterator(node)) { + // Parent is ShadowRoot. We can stop here since this is + // the top ancestor of the shadow tree. + if (parent.containingShadowRoot == parent) { + break; + } + if ( + this.determineTranslationStatus(parent) === + NodeStatus.NOT_TRANSLATABLE + ) { + return; + } + } + } + } + + switch (this.determineTranslationStatusForUnprocessedNodes(node)) { + case NodeStatus.NOT_TRANSLATABLE: + // This node is rejected as it shouldn't be translated. + return; + + // SHADOW_HOST and READY_TO_TRANSLATE both map to FILTER_ACCEPT + case NodeStatus.SHADOW_HOST: + case NodeStatus.READY_TO_TRANSLATE: + const shadowRoot = node.openOrClosedShadowRoot; + if (shadowRoot) { + this.processSubdivide(shadowRoot); + } else { + // This node is ready for translating, and doesn't need to be subdivided. There + // is no reason to run the TreeWalker, it can be directly submitted for + // translation. + this.queueNodeForTranslation(node); + } + break; + + case NodeStatus.SUBDIVIDE_FURTHER: + // This node may be translatable, but it needs to be subdivided into smaller + // pieces. Create a TreeWalker to walk the subtree, and find the subtrees/nodes + // that contain enough inline elements to send to be translated. + this.processSubdivide(node); + break; + } + + if (node.nodeName === "BODY") { + this.reportWordsInViewport(); + } + this.dispatchQueuedTranslations(); + } + + /** + * Get all the nodes which have selected attributes + * from the node/document and queue them. + * Call the translate function on these nodes + * @param {Node} node + * @returns {Array<Promise<void>> | null} + */ + translateAttributes(node) { + const attributeList = getTranslatableAttributes(node); + if (attributeList.length) { + // Queue the root node if it has any attributes + // Because querySelectorAll searches only child nodes. + this.queueAttributeNodeForTranslation(node, attributeList); + } + // Get all attributes in child nodes at once + const nodesWithTranslatableAttributes = node.querySelectorAll( + TRANSLATABLE_ATTRIBUTES_SELECTOR + ); + for (const node of nodesWithTranslatableAttributes) { + const attributeList = getTranslatableAttributes(node); + this.queueAttributeNodeForTranslation(node, attributeList); + } + return this.dispatchQueuedAttributeTranslations(); + } + + /** + * Test whether this is an element we do not want to translate. These are things like + * <code> elements, elements with a different "lang" attribute, and elements that + * have a `translate=no` attribute. + * + * @param {Node} node + */ + isExcludedNode(node) { + // Property access be expensive, so destructure required properties so they are + // not accessed multiple times. + const { nodeType } = node; + + if (nodeType === Node.TEXT_NODE) { + // Text nodes are never excluded. + return false; + } + if (nodeType !== Node.ELEMENT_NODE) { + // Only elements and and text nodes should be considered. + return true; + } + + const { nodeName } = node; + + if ( + EXCLUDED_TAGS.has( + // SVG tags can be lowercased, so ensure everything is uppercased. + nodeName.toUpperCase() + ) + ) { + // This is an excluded tag. + return true; + } + + if (!this.matchesDocumentLanguage(node)) { + // Exclude nodes that don't match the fromLanguage. + return true; + } + + if (node.getAttribute("translate") === "no") { + // This element has a translate="no" attribute. + // https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/translate + return true; + } + + if (node.classList.contains("notranslate")) { + // Google Translate skips translations if the classList contains "notranslate" + // https://cloud.google.com/translate/troubleshooting + return true; + } + + if (node.isContentEditable) { + // This field is editable, and so exclude it similar to the way that form input + // fields are excluded. + return true; + } + + return false; + } + + /** + * Runs `determineTranslationStatus`, but only on unprocessed nodes. + * + * @param {Node} node + * @return {number} - One of the NodeStatus values. + */ + determineTranslationStatusForUnprocessedNodes = node => { + if (this.#processedNodes.has(node)) { + // Skip nodes that have already been processed. + return NodeStatus.NOT_TRANSLATABLE; + } + + return this.determineTranslationStatus(node); + }; + + /** + * Determines if a node should be submitted for translation, not translatable, or if + * it should be subdivided further. It doesn't check if the node has already been + * processed. + * + * The return result works as a TreeWalker NodeFilter as well. + * + * @param {Node} node + * @returns {number} - One of the `NodeStatus` values. See that object + * for documentation. These values match the filters for the TreeWalker. + * These values also work as a `NodeFilter` value. + */ + determineTranslationStatus(node) { + if (node.openOrClosedShadowRoot) { + return NodeStatus.SHADOW_HOST; + } + + if (isNodeQueued(node, this.#queuedNodes)) { + // This node or its parent was already queued, reject it. + return NodeStatus.NOT_TRANSLATABLE; + } + + if (this.isExcludedNode(node)) { + // This is an explicitly excluded node. + return NodeStatus.NOT_TRANSLATABLE; + } + + if (node.textContent.trim().length === 0) { + // Do not use subtrees that are empty of text. This textContent call is fairly + // expensive. + return !node.hasChildNodes() + ? NodeStatus.NOT_TRANSLATABLE + : NodeStatus.SUBDIVIDE_FURTHER; + } + + if (nodeNeedsSubdividing(node)) { + // Skip this node, and dig deeper into its tree to cut off smaller pieces + // to translate. It is presumed to be a wrapper of block elements. + return NodeStatus.SUBDIVIDE_FURTHER; + } + + if ( + containsExcludedNode(node, this.excludedNodeSelector) && + !hasTextNodes(node) + ) { + // Skip this node, and dig deeper into its tree to cut off smaller pieces + // to translate. + return NodeStatus.SUBDIVIDE_FURTHER; + } + + // This node can be treated as entire block to submit for translation. + return NodeStatus.READY_TO_TRANSLATE; + } + + /** + * Queue a node for translation. + * @param {Node} node + */ + queueNodeForTranslation(node) { + /** @type {NodeVisibility} */ + let visibility = "out-of-viewport"; + if (isNodeHidden(node)) { + visibility = "hidden"; + } else if (isNodeInViewport(node)) { + visibility = "in-viewport"; + } + + this.#queuedNodes.set(node, visibility); + } + + /** + * Submit the translations giving priority to nodes in the viewport. + * @returns {Array<Promise<void>> | null} + */ + dispatchQueuedTranslations() { + let inViewportCounts = 0; + let outOfViewportCounts = 0; + let hiddenCounts = 0; + + let inViewportTranslations = null; + if (!this.viewportTranslated) { + inViewportTranslations = []; + } + + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "in-viewport") { + inViewportCounts++; + const promise = this.submitTranslation(node); + if (inViewportTranslations) { + inViewportTranslations.push(promise); + } + } + } + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "out-of-viewport") { + outOfViewportCounts++; + this.submitTranslation(node); + } + } + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "hidden") { + hiddenCounts++; + this.submitTranslation(node); + } + } + + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + `Translate ${this.#queuedNodes.size} nodes.\n\n` + + `In viewport: ${inViewportCounts}\n` + + `Out of viewport: ${outOfViewportCounts}\n` + + `Hidden: ${hiddenCounts}\n` + ); + + this.#queuedNodes.clear(); + return inViewportTranslations; + } + + /** + * Submit the Attribute translations giving priority to nodes in the viewport. + * @returns {Array<Promise<void>> | null} + */ + dispatchQueuedAttributeTranslations() { + let inViewportCounts = 0; + let outOfViewportCounts = 0; + let hiddenCounts = 0; + + let inViewportTranslations = null; + if (!this.viewportTranslated) { + inViewportTranslations = []; + } + // Submit the nodes with attrbutes to be translated. + for (const [node, { attributeList, visibility }] of this + .#queuedAttributeNodes) { + if (visibility === "in-viewport") { + inViewportCounts++; + const promise = this.submitAttributeTranslation(node, attributeList); + if (inViewportTranslations) { + inViewportTranslations.push(promise); + } + } + } + for (const [node, { attributeList, visibility }] of this + .#queuedAttributeNodes) { + if (visibility === "out-of-viewport") { + outOfViewportCounts++; + this.submitAttributeTranslation(node, attributeList); + } + } + for (const [node, { attributeList, visibility }] of this + .#queuedAttributeNodes) { + if (visibility === "hidden") { + hiddenCounts++; + this.submitAttributeTranslation(node, attributeList); + } + } + + ChromeUtils.addProfilerMarker( + "Attribute Translations", + { innerWindowId: this.innerWindowId }, + `Attribute Translate ${this.#queuedAttributeNodes.size} nodes.\n\n` + + `In viewport: ${inViewportCounts}\n` + + `Out of viewport: ${outOfViewportCounts}\n` + + `Hidden: ${hiddenCounts}\n` + ); + + this.#queuedAttributeNodes.clear(); + + return inViewportTranslations; + } + + /** + * Submit a node for Attribute translation to the translations engine. + * + * @param {Node} node + * @returns {Promise<void>} + */ + async submitAttributeTranslation(node, attributeList) { + if (node.nodeType === Node.ELEMENT_NODE) { + for (const attribute of attributeList) { + const text = node.getAttribute(attribute); + + if (text.trim().length === 0) { + continue; + } + const translation = await this.maybeTranslate( + node, + text, + false /*isHTML*/ + ); + if (translation != null) { + this.scheduleNodeUpdateWithTranslationAttribute( + node, + translation, + attribute + ); + } + } + } + } + + /** + * Schedule a node to be updated with a translation. + * + * @param {Node} node + * @param {string} translation + */ + scheduleNodeUpdateWithTranslationAttribute(node, translation, attribute) { + // Add the nodes to be populated with the next translation update. + this.#nodesWithTranslatedAttributes.add({ + node, + translation, + attribute, + }); + + if (this.#pendingTranslationsCount === 0) { + // No translations are pending, update the node. + this.updateNodesWithTranslationsAttributes(); + } else if (!this.#attributeUpdateTimeout) { + // Schedule an update. + this.#attributeUpdateTimeout = lazy.setTimeout( + this.updateNodesWithTranslationsAttributes.bind(this), + DOM_UPDATE_INTERVAL_MS + ); + } else { + // An update has been previously scheduled, do nothing here. + } + } + + /** + * This is called every `DOM_UPDATE_INTERVAL_MS` ms with translations + * for attributes in the nodes. + * + * This function is called asynchronously, so nodes may already be dead. Before + * accessing a node make sure and run `Cu.isDeadWrapper` to check that it is alive. + */ + updateNodesWithTranslationsAttributes() { + // Stop the mutations so that the updates won't trigger observations. + + this.pauseMutationObserverAndRun(() => { + for (const { node, translation, attribute } of this + .#nodesWithTranslatedAttributes) { + if (Cu.isDeadWrapper(node)) { + // The node is no longer alive. + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + "Node is no long alive." + ); + continue; + } + // Update the attribute of the node with translated attribute + if (attribute) { + node.setAttribute(attribute, translation); + } + } + this.#nodesWithTranslatedAttributes.clear(); + this.#attributeUpdateTimeout = null; + }); + } + + /** + * Record how many words were in the viewport, as this is the most important + * user-visible translation content. + */ + reportWordsInViewport() { + if ( + // This promise gets created for the first dispatchQueuedTranslations + this.viewportTranslated || + this.#queuedNodes.size === 0 + ) { + return; + } + + // TODO(Bug 1814195) - Add telemetry. + // TODO(Bug 1820618) - This whitespace regex will not work in CJK-like languages. + // This requires a segmenter for a proper implementation. + + const whitespace = /\s+/; + let wordCount = 0; + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "in-viewport") { + wordCount += node.textContent.trim().split(whitespace).length; + } + } + + const message = wordCount + " words are in the viewport."; + lazy.console.log(message); + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + message + ); + } + + /** + * Submit a node for translation to the translations engine. + * + * @param {Node} node + * @returns {Promise<void>} + */ + async submitTranslation(node) { + // Give each element an id that gets passed through the translation so it can be + // reunited later on. + if (node.nodeType === Node.ELEMENT_NODE) { + node.querySelectorAll("*").forEach((el, i) => { + el.dataset.mozTranslationsId = i; + }); + } + + /** @type {string} */ + let text; + /** @type {boolean} */ + let isHTML; + + if (node.nodeType === Node.ELEMENT_NODE) { + text = node.innerHTML; + isHTML = true; + } else { + text = node.textContent; + isHTML = false; + } + + if (text.trim().length === 0) { + return; + } + + // Mark this node as not to be translated again unless the contents are changed + // (which the observer will pick up on) + this.#processedNodes.add(node); + const translatedHTML = await this.maybeTranslate(node, text, isHTML); + if (translatedHTML != null) { + this.scheduleNodeUpdateWithTranslation(node, translatedHTML); + } + } + + /** + * A single function to update pendingTranslationsCount while + * calling the translate function + * @param {Node} node + * @param {string} text + * @prop {boolean} isHTML + * @returns {Promise<string | null>} + */ + async maybeTranslate(node, text, isHTML) { + this.#pendingTranslationsCount++; + try { + let translation = this.translationsCache.get(text, isHTML); + if (translation === undefined) { + translation = await this.translator.translate(node, text, isHTML); + this.translationsCache.set(text, translation, isHTML); + } + + return translation; + } catch (error) { + lazy.console.log("Translation failed", error); + } finally { + this.#pendingTranslationsCount--; + } + return null; + } + + /** + * Start the mutation observer, for instance after applying the translations to the DOM. + */ + startMutationObserver() { + if (Cu.isDeadWrapper(this.observer)) { + // This observer is no longer alive. + return; + } + for (const node of this.#rootNodes) { + if (Cu.isDeadWrapper(node)) { + // This node is no longer alive. + continue; + } + this.observer.observe(node, MUTATION_OBSERVER_OPTIONS); + } + } + + /** + * Stop the mutation observer, for instance to apply the translations to the DOM. + */ + stopMutationObserver() { + // Was the window already destroyed? + if (!Cu.isDeadWrapper(this.observer)) { + this.observer.disconnect(); + } + } + + /** + * This is called every `DOM_UPDATE_INTERVAL_MS` ms with translations for nodes. + * + * This function is called asynchronously, so nodes may already be dead. Before + * accessing a node make sure and run `Cu.isDeadWrapper` to check that it is alive. + */ + updateNodesWithTranslations() { + // Stop the mutations so that the updates won't trigger observations. + this.pauseMutationObserverAndRun(() => { + for (const { node, translatedHTML } of this.#nodesWithTranslatedHTML) { + if (Cu.isDeadWrapper(node)) { + // The node is no longer alive. + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + "Node is no long alive." + ); + continue; + } + switch (node.nodeType) { + case Node.TEXT_NODE: { + if (translatedHTML.trim().length !== 0) { + // Only update the node if there is new text. + node.textContent = translatedHTML; + } + break; + } + case Node.ELEMENT_NODE: { + // TODO (Bug 1820625) - This is slow compared to the original implementation + // in the addon which set the innerHTML directly. We can't set the innerHTML + // here, but perhaps there is another way to get back some of the performance. + const translationsDocument = this.domParser.parseFromString( + `<!DOCTYPE html><div>${translatedHTML}</div>`, + "text/html" + ); + updateElement(translationsDocument, node); + break; + } + } + } + + this.#nodesWithTranslatedHTML.clear(); + this.#updateTimeout = null; + }); + } + + /** + * Stop the mutations so that the updates of the translations + * in the nodes won't trigger observations. + * @param {Function} run The function to update translations + */ + pauseMutationObserverAndRun(run) { + this.stopMutationObserver(); + run(); + this.startMutationObserver(); + } + + /** + * Schedule a node to be updated with a translation. + * + * @param {Node} node + * @param {string} translatedHTML + */ + scheduleNodeUpdateWithTranslation(node, translatedHTML) { + // Add the nodes to be populated with the next translation update. + this.#nodesWithTranslatedHTML.add({ node, translatedHTML }); + + if (this.#pendingTranslationsCount === 0) { + // No translations are pending, update the node. + this.updateNodesWithTranslations(); + } else if (!this.#updateTimeout) { + // Schedule an update. + this.#updateTimeout = lazy.setTimeout( + this.updateNodesWithTranslations.bind(this), + DOM_UPDATE_INTERVAL_MS + ); + } else { + // An update has been previously scheduled, do nothing here. + } + } + + /** + * Check to see if a language matches the document language. + * + * @param {Node} node + */ + matchesDocumentLanguage(node) { + if (!node.lang) { + // No `lang` was present, so assume it matches the language. + return true; + } + + // First, cheaply check if language tags match, without canonicalizing. + if (langTagsMatch(this.documentLanguage, node.lang)) { + return true; + } + + try { + // Make sure the local is in the canonical form, and check again. This function + // throws, so don't trust that the language tags are formatting correctly. + const [language] = Intl.getCanonicalLocales(node.lang); + + return langTagsMatch(this.documentLanguage, language); + } catch (_error) { + return false; + } + } +} + +/** + * Get the list of attributes that need to be translated + * in a given node. + * @returns Array<string> + */ +function getTranslatableAttributes(node) { + if (node.nodeType !== Node.ELEMENT_NODE) { + return []; + } + return TRANSLATABLE_ATTRIBUTES.filter(attribute => + node.hasAttribute(attribute) + ); +} + +/** + * This function needs to be fairly fast since it's used on many nodes when iterating + * over the DOM to find nodes to translate. + * + * @param {Text | HTMLElement} node + */ +function isNodeHidden(node) { + /** @type {HTMLElement} */ + const element = getElementForStyle(node); + if (!element) { + throw new Error("Unable to find the Element to compute the style for node"); + } + + // This flushes the style, which is a performance cost. + const style = element.ownerGlobal.getComputedStyle(element); + return style.display === "none" || style.visibility === "hidden"; +} + +/** + * This function cheaply checks that language tags match. + * + * @param {string} knownLanguage + * @param {string} otherLanguage + */ +function langTagsMatch(knownLanguage, otherLanguage) { + if (knownLanguage === otherLanguage) { + // A simple direct match. + return true; + } + if (knownLanguage.length !== 2) { + throw new Error("Expected the knownLanguage to be of length 2."); + } + // Check if the language tags part match, e.g. "en" and "en-US". + return ( + knownLanguage[0] === otherLanguage[0] && + knownLanguage[1] === otherLanguage[1] && + otherLanguage[2] === "-" + ); +} + +/** + * This function returns the correct element to determine the + * style of node. + * + * @param {Node} node + * @returns {HTMLElement} */ +function getElementForStyle(node) { + if (node.nodeType != Node.TEXT_NODE) { + return node; + } + + if (node.parentElement) { + return node.parentElement; + } + + // For cases like text node where its parent is ShadowRoot, + // we'd like to use flattenedTreeParentNode + if (node.flattenedTreeParentNode) { + return node.flattenedTreeParentNode; + } + + // If the text node is not connected or doesn't have a frame. + return null; +} + +/** + * This function runs when walking the DOM, which means it is a hot function. It runs + * fairly fast even though it is computing the bounding box. This is all done in a tight + * loop, and it is done on mutations. Care should be taken with reflows caused by + * getBoundingClientRect, as this is a common performance issue. + * + * The following are the counts of how often this is run on a news site: + * + * Given: + * 1573 DOM nodes + * 504 Text nodes + * 1069 Elements + * + * There were: + * 209 calls to get this funcion. + * + * @param {Node} node + */ +function isNodeInViewport(node) { + const window = node.ownerGlobal; + const document = node.ownerDocument; + + /** @type {HTMLElement} */ + const element = getElementForStyle(node); + if (!element) { + throw new Error("Unable to find the Element to compute the style for node"); + } + + const rect = element.getBoundingClientRect(); + return ( + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= (window.innerWidth || document.documentElement.clientWidth) + ); +} + +/** + * Actually perform the update of the element with the translated node. This step + * will detach all of the "live" nodes, and match them up in the correct order as provided + * by the translations engine. + * + * @param {Document} translationsDocument + * @param {Element} element + * @returns {void} + */ +function updateElement(translationsDocument, element) { + // This text should have the same layout as the target, but it's not completely + // guaranteed since the content page could change at any time, and the translation process is async. + // + // The document has the following structure: + // + // <html> + // <head> + // <body>{translated content}</body> + // </html> + + const originalHTML = element.innerHTML; + + /** + * The Set of translation IDs for nodes that have been cloned. + * @type {Set<number>} + */ + const clonedNodes = new Set(); + + merge(element, translationsDocument.body.firstChild); + + /** + * Merge the live tree with the translated tree by re-using elements from the live tree. + * + * @param {Node} liveTree + * @param {Node} translatedTree + */ + function merge(liveTree, translatedTree) { + /** @type {Map<number, Element>} */ + const liveElementsById = new Map(); + + /** @type {Array<Text>} */ + const liveTextNodes = []; + + // Remove all the nodes from the liveTree, and categorize them by Text node or + // Element node. + let node; + while ((node = liveTree.firstChild)) { + node.remove(); + + if (node.nodeType === Node.ELEMENT_NODE) { + liveElementsById.set(node.dataset.mozTranslationsId, node); + } else if (node.nodeType === Node.TEXT_NODE) { + liveTextNodes.push(node); + } + } + + // The translated tree dictates the order. + const translatedNodes = [...translatedTree.childNodes]; + for ( + let translatedIndex = 0; + translatedIndex < translatedNodes.length; + translatedIndex++ + ) { + const translatedNode = translatedNodes[translatedIndex]; + + if (translatedNode.nodeType === Node.TEXT_NODE) { + // Copy the translated text to the original Text node and re-append it. + let liveTextNode = liveTextNodes.shift(); + + if (liveTextNode) { + liveTextNode.data = translatedNode.data; + } else { + liveTextNode = translatedNode; + } + + liveTree.appendChild(liveTextNode); + } else if (translatedNode.nodeType === Node.ELEMENT_NODE) { + const translationsId = translatedNode.dataset.mozTranslationsId; + // Element nodes try to use the already existing DOM nodes. + + // Find the element in the live tree that matches the one in the translated tree. + let liveElement = liveElementsById.get(translationsId); + + if (!liveElement) { + lazy.console.warn("Could not find a corresponding live element", { + path: createNodePath(translatedNode, translationsDocument.body), + translationsId, + liveElementsById, + translatedNode, + }); + continue; + } + + // Has this element already been added to the list? Then duplicate it and re-add + // it as a clone. The Translations Engine can sometimes duplicate HTML. + if (liveElement.parentNode) { + liveElement = liveElement.cloneNode(true /* deep clone */); + clonedNodes.add(translationsId); + lazy.console.warn( + "Cloning a node because it was already inserted earlier", + { + path: createNodePath(translatedNode, translationsDocument.body), + translatedNode, + liveElement, + } + ); + } + + if (isNodeTextEmpty(translatedNode)) { + // The original node had text, but the one that came out of translation + // didn't have any text. This scenario might be caused by one of two causes: + // + // 1) The element was duplicated by translation but then not given text + // content. This happens on Wikipedia articles for example. + // + // 2) The translator messed up and could not translate the text. This + // happens on YouTube in the language selector. In that case, having the + // original text is much better than no text at all. + // + // To make sure it is case 1 and not case 2 check whether this is the only occurrence. + for (let i = 0; i < translatedNodes.length; i++) { + if (translatedIndex === i) { + // This is the current node, not a sibling. + continue; + } + const sibling = translatedNodes[i]; + if ( + // Only consider other element nodes. + sibling.nodeType === Node.ELEMENT_NODE && + // If the sibling's translationsId matches, then use the sibling's + // node instead. + translationsId === sibling.dataset.mozTranslationsId + ) { + // This is case 1 from above. Remove this element's original text nodes, + // since a sibling text node now has all of the text nodes. + removeTextNodes(liveElement); + } + } + + // Report this issue to the console. + lazy.console.warn( + "The translated element has no text even though the original did.", + { + path: createNodePath(translatedNode, translationsDocument.body), + translatedNode, + liveElement, + } + ); + } else if (!isNodeTextEmpty(liveElement)) { + // There are still text nodes to find and update, recursively merge. + merge(liveElement, translatedNode); + } + + // Put the live node back in the live branch. But now t has been synced with the + // translated text and order. + liveTree.appendChild(liveElement); + } + } + + const unhandledElements = [...liveElementsById].filter( + ([, element]) => !element.parentNode + ); + + if (unhandledElements.length) { + lazy.console.warn( + `${createNodePath( + translatedTree, + translationsDocument.body + )} Not all nodes unified`, + { + unhandledElements, + clonedNodes, + originalHTML, + translatedHTML: translationsDocument.body.innerHTML, + liveTree: liveTree.outerHTML, + translatedTree: translatedTree.outerHTML, + } + ); + } + } +} + +/** + * For debug purposes, compute a string path to an element. + * + * e.g. "div/div#header/p.bold.string/a" + * + * @param {Node} node + * @param {Node | null} root + */ +function createNodePath(node, root) { + if (root === null) { + root = node.ownerDocument.body; + } + let path = + node.parentNode && node.parentNode !== root + ? createNodePath(node.parentNode) + : ""; + path += `/${node.nodeName}`; + if (node.id) { + path += `#${node.id}`; + } else if (node.className) { + for (const className of node.classList) { + path += "." + className; + } + } + return path; +} + +/** + * @param {Node} node + * @returns {boolean} + */ +function isNodeTextEmpty(node) { + if ("innerText" in node) { + return node.innerText.trim().length === 0; + } + if (node.nodeType === Node.TEXT_NODE && node.nodeValue) { + return node.nodeValue.trim().length === 0; + } + return true; +} + +/** + * @param {Node} node + */ +function removeTextNodes(node) { + for (const child of node.childNodes) { + switch (child.nodeType) { + case Node.TEXT_NODE: + node.removeChild(child); + break; + case Node.ELEMENT_NODE: + removeTextNodes(child); + break; + default: + break; + } + } +} + +/** + * Test whether any of the direct child text nodes of are non-whitespace + * text nodes. + * + * For example: + * - `<p>test</p>`: yes + * - `<p> </p>`: no + * - `<p><b>test</b></p>`: no + * @param {Node} node + * @returns {boolean} + */ +function hasTextNodes(node) { + if (node.nodeType !== Node.ELEMENT_NODE) { + // Only check element nodes. + return false; + } + + for (const child of node.childNodes) { + if (child.nodeType === Node.TEXT_NODE) { + if (child.textContent.trim() === "") { + // This is just whitespace. + continue; + } + // A text node with content was found. + return true; + } + } + + // No text nodes were found. + return false; +} + +/** + * Like `isExcludedNode` but looks at the full subtree. Used to see whether + * we can submit a subtree, or whether we should split it into smaller + * branches first to try to exclude more of the non-translatable content. + * + * @param {Node} node + * @param {string} excludedNodeSelector + * @returns {boolean} + */ +function containsExcludedNode(node, excludedNodeSelector) { + return ( + node.nodeType === Node.ELEMENT_NODE && + node.querySelector(excludedNodeSelector) + ); +} + +/** + * Check if this node has already been queued to be translated. This can be because + * the node is itself is queued, or its parent node is queued. + * + * @param {Node} node + * @param {Map<Node, any>} queuedNodes + * @returns {boolean} + */ +function isNodeQueued(node, queuedNodes) { + if (queuedNodes.has(node)) { + return true; + } + + // If the immediate parent is the body, it is allowed. + if (node.parentNode === node.ownerDocument.body) { + return false; + } + + // Accessing the parentNode is expensive here according to performance profilling. This + // is due to XrayWrappers. Minimize reading attributes by storing a reference to the + // `parentNode` in a named variable, rather than re-accessing it. + let parentNode; + let lastNode = node; + while ((parentNode = lastNode.parentNode)) { + if (queuedNodes.has(parentNode)) { + return parentNode; + } + lastNode = parentNode; + } + + return false; +} + +/** + * Reads the elements computed style and determines if the element is inline or not. + * + * @param {Element} element + */ +function getIsInline(element) { + const win = element.ownerGlobal; + if (element.namespaceURI === "http://www.w3.org/2000/svg") { + // SVG elements will report as inline, but there is no block layout in SVG. + // Treat every SVG element as being block so that every node will be subdivided. + return false; + } + return win.getComputedStyle(element).display === "inline"; +} + +/** + * Determine if this element is an inline element or a block element. Inline elements + * should be sent as a contiguous chunk of text, while block elements should be further + * subdivided before sending them in for translation. + * + * @param {Node} node + * @returns {boolean} + */ +function nodeNeedsSubdividing(node) { + if (node.nodeType === Node.TEXT_NODE) { + // Text nodes are fully subdivided. + return false; + } + + if (getIsInline(node)) { + return false; + } + + for (let child of node.childNodes) { + switch (child.nodeType) { + case Node.TEXT_NODE: + // Keep checking for more inline or text nodes. + continue; + case Node.ELEMENT_NODE: { + if (getIsInline(child)) { + // Keep checking for more inline or text nodes. + continue; + } + // A child element is not inline, so subdivide this node further. + return true; + } + default: + return true; + } + } + return false; +} + +/** + * Returns an iterator of a node's ancestors. + * + * @param {Node} node + * @returns {Generator<ParentNode>} + */ +function* getAncestorsIterator(node) { + const document = node.ownerDocument; + for ( + let parent = node.parentNode; + parent && parent !== document.documentElement; + parent = parent.parentNode + ) { + yield parent; + } +} + +/** + * This contains all of the information needed to perform a translation request. + * + * @typedef {Object} TranslationRequest + * @prop {Node} node + * @prop {string} sourceText + * @prop {boolean} isHTML + * @prop {Function} resolve + * @prop {Function} reject + */ + +/** + * When a page is hidden, mutations may occur in the DOM. It doesn't make sense to + * translate those elements while the page is hidden, especially as it may bring + * a translations engine back to life, which can be quite expensive. Queue those + * messages here. + */ +class QueuedTranslator { + /** + * @type {MessagePort | null} + */ + #port = null; + + /** + * @type {() => void} + */ + #actorRequestNewPort; + + /** + * An id for each message sent. This is used to match up the request and response. + */ + #nextMessageId = 0; + + /** + * Tie together a message id to a resolved response. + * @type {Map<number, TranslationRequest} + */ + #requests = new Map(); + + /** + * If the translations are paused, they are queued here. This Map is ordered by + * from oldest to newest requests with stale requests being removed. + * @type {Map<Node, TranslationRequest>} + */ + #queue = new Map(); + + /** + * @type {"uninitialized" | "ready" | "error" | "closed"} + */ + engineStatus = "uninitialized"; + + /** + * @param {MessagePort} port + * @param {Document} document + * @param {() => void} actorRequestNewPort + */ + constructor(port, actorRequestNewPort) { + this.#actorRequestNewPort = actorRequestNewPort; + + this.acquirePort(port); + } + + /** + * When an engine gets closed while still in use, a new one will need to be requested. + * + * @type {{ promise: Promise<void>, resolve: Function, reject: Function } | null} + */ + #portRequest = null; + + /** + * Keep track if the page is shown or hidden. When the page is hidden, no translations + * will be posted to the translations engine. + */ + #isPageShown = true; + + /** + * Note when a new port is being requested so we don't re-request it. + */ + showPage() { + this.#isPageShown = true; + if (this.#port) { + throw new Error( + "Attempting to show the page when there is already port available" + ); + } + if (this.#queue.size) { + // There are queued translations, request a new port. After the port is retrieved + // the pending queue will be processed. + this.#requestNewPort(); + } + } + + /** + * Hide the page, and move any outstanding translation requests to a queue. + */ + hidePage() { + this.#isPageShown = false; + this.discardPort(); + + if (this.#requests.size) { + lazy.console.log( + "Pausing translations with pending translation requests." + ); + } + this.#moveRequestsToQueue(); + } + + /** + * Request a new port. The port will come in via `acquirePort`, and then resolved + * through the `this.#portRequest.resolve`. + * @returns {Promise<void>} + */ + #requestNewPort() { + if (this.#portRequest) { + // A port was already requested. + return this.#portRequest.promise; + } + + const portRequest = { promise: null, resolve: null, reject: null }; + portRequest.promise = new Promise((resolve, reject) => { + portRequest.resolve = resolve; + portRequest.reject = reject; + }); + + this.#portRequest = portRequest; + + // Send a request through the actor for a new port. The request response will + // trigger the method `QueuedTranslator.prototype.acquirePort` + this.#actorRequestNewPort(); + + this.#portRequest.promise + .then( + () => { + this.#portRequest = null; + + // Resume the queued translations. + if (this.#queue.size) { + lazy.console.log( + `Resuming ${ + this.#queue.size + } translations from the pending translation queue.` + ); + + const oldQueue = this.#queue; + this.#queue = new Map(); + this.#repostTranslations(oldQueue); + } + }, + error => { + lazy.console.error(error); + } + ) + .finally(() => { + this.#portRequest = null; + }); + + return portRequest.promise; + } + + /** + * Send a request to translate text to the Translations Engine. If it returns `null` + * then the request is stale. A rejection means there was an error in the translation. + * This request may be queued. + * + * @param {node} Node + * @param {string} sourceText + * @param {boolean} isHTML + */ + async translate(node, sourceText, isHTML) { + if (this.#isPageShown && !this.#port) { + try { + await this.#requestNewPort(); + } catch {} + } + + // At this point we don't know if the page is still shown, or if the attempt + // to get a port was successful so check again. + + if (!this.#isPageShown || !this.#port) { + // Queue the request while the page isn't shown. + return new Promise((resolve, reject) => { + const previousRequest = this.#queue.get(node); + if (previousRequest) { + // Previous requests get resolved as null, as this new one will replace it. + previousRequest.resolve(null); + // Delete the entry so that the order of the queue is maintained. The + // new request will be put on the end. + this.#queue.delete(node); + } + + // This Promises's resolve and reject will be chained after the translation + // request. For now add it to the queue along with the other arguments. + this.#queue.set(node, { node, sourceText, isHTML, resolve, reject }); + }); + } + + return this.#postTranslationRequest(node, sourceText, isHTML); + } + + /** + * Posts the translation to the translations engine through the MessagePort. + * + * @param {Node} node + * @param {string} sourceText + * @param {boolean} isHTML + * @return {{ translateText: TranslationFunction, translateHTML: TranslationFunction}} + */ + #postTranslationRequest(node, sourceText, isHTML) { + return new Promise((resolve, reject) => { + const messageId = this.#nextMessageId++; + // Store the "resolve" for the promise. It will be matched back up with the + // `messageId` in #handlePortMessage. + this.#requests.set(messageId, { + node, + sourceText, + isHTML, + resolve, + reject, + }); + this.#port.postMessage({ + type: "TranslationsPort:TranslationRequest", + messageId, + sourceText, + isHTML, + }); + }); + } + + /** + * Close the port and move any pending translations onto a queue. + */ + discardPort() { + if (this.#port) { + this.#port.postMessage({ type: "TranslationsPort:DiscardTranslations" }); + this.#port.close(); + this.#port = null; + } + this.#moveRequestsToQueue(); + this.engineStatus = "uninitialized"; + } + + /** + * Move any unfulfilled requests to the queue so they can be sent again when + * the page is active again. + */ + #moveRequestsToQueue() { + if (this.#requests.size) { + for (const request of this.#requests.values()) { + this.#queue.set(request.node, request); + } + this.#requests = new Map(); + } + } + + /** + * Acquires a port, checks on the engine status, and then starts or resumes + * translations. + * @param {MessagePort} port + */ + acquirePort(port) { + if (this.#port) { + if (this.engineStatus === "ready") { + lazy.console.error( + "Received a new translation port while one already existed." + ); + } + this.discardPort(); + } + + this.#port = port; + + const portRequest = this.#portRequest; + + // Match up a response on the port to message that was sent. + port.onmessage = ({ data }) => { + switch (data.type) { + case "TranslationsPort:TranslationResponse": { + const { targetText, messageId } = data; + // A request may not match match a messageId if there is a race during the pausing + // and discarding of the queue. + this.#requests.get(messageId)?.resolve(targetText); + this.#requests.delete(messageId); + break; + } + case "TranslationsPort:GetEngineStatusResponse": { + if (portRequest) { + const { resolve, reject } = portRequest; + if (data.status === "ready") { + resolve(); + } else { + reject(new Error("The engine failed to load.")); + } + } + this.engineStatus = data.status; + break; + } + case "TranslationsPort:EngineTerminated": { + // The engine was terminated, and if a translation is needed a new port + // will need to be requested. + this.engineStatus = "closed"; + this.discardPort(); + if (this.#queue.size && this.#isPageShown) { + this.#requestNewPort(); + } + break; + } + default: + lazy.console.error("Unknown translations port message: " + data.type); + break; + } + }; + + port.postMessage({ type: "TranslationsPort:GetEngineStatusRequest" }); + } + + /** + * Re-send a list of translation requests. + * + * @param {Map<any, TranslationRequest>} mappedRequests + * This is either the this.#queue or this.#requests. + */ + #repostTranslations(mappedRequests) { + for (const value of mappedRequests.values()) { + const { node, sourceText, isHTML, resolve, reject } = value; + if (Cu.isDeadWrapper(node)) { + // If the node is dead, resolve without any text. Do not reject as that + // will be treated as an error. + resolve(null); + } else { + this.#postTranslationRequest(node, sourceText, isHTML).then( + resolve, + reject + ); + } + } + } + + /** + * Close the port and remove any pending or queued requests. + */ + destroy() { + this.#port.close(); + this.#requests = new Map(); + this.#queue = new Map(); + } +} diff --git a/toolkit/components/translations/content/translations-engine.html b/toolkit/components/translations/content/translations-engine.html new file mode 100644 index 0000000000..866a4d7f47 --- /dev/null +++ b/toolkit/components/translations/content/translations-engine.html @@ -0,0 +1,20 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8" /> + <meta + http-equiv="Content-Security-Policy" + content="default-src chrome: resource:; object-src 'none'" + /> + <!-- Run the translations engine in its own singleton unprivileged content process. --> + <script + type="module" + src="chrome://global/content/translations/translations-engine.sys.mjs" + ></script> + </head> + <body></body> +</html> diff --git a/toolkit/components/translations/content/translations-engine.sys.mjs b/toolkit/components/translations/content/translations-engine.sys.mjs new file mode 100644 index 0000000000..e9aeb8076b --- /dev/null +++ b/toolkit/components/translations/content/translations-engine.sys.mjs @@ -0,0 +1,557 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env browser */ +/* globals TE_addProfilerMarker, TE_getLogLevel, TE_log, TE_logError, TE_getLogLevel, + TE_destroyEngineProcess, TE_requestEnginePayload, TE_reportEngineStatus, + TE_resolveForceShutdown */ + +/** + * This file lives in the translation engine's process and is in charge of managing the + * lifecycle of the translations engines. This process is a singleton Web Content + * process that can be created and destroyed as needed. + * + * The goal of the code in this file is to be as unprivileged as possible, which should + * unlock Bug 1813789, which will make this file fully unprivileged. + * + * Each translation needs an engine for that specific translation pair. This engine is + * kept around as long as the CACHE_TIMEOUT_MS, after this if some keepAlive event does + * not happen, the engine is destroyed. An engine may be destroyed even when a page is + * still open and may need translations in the future. This is handled gracefully by + * creating new engines and MessagePorts on the fly. + * + * The engine communicates directly with the content page via a MessagePort. Each end + * of the port is transfered from the parent process to the content process, and this + * engine process. This port is transitory, and may be closed at any time. Only when a + * translation has been requested once (which is initiated by the parent process) can + * the content process re-request translation ports. This ensures a rogue content process + * only has the capabilities to perform tasks that the parent process has given it. + * + * The messaging flow can get a little convoluted to handle all of the correctness cases, + * but ideally communication passes through the message port as much as possible. There + * are many scenarios such as: + * + * - Translation pages becoming idle + * - Tab changing causing "pageshow" and "pagehide" visibility changes + * - Translation actor destruction (this can happen long after the page has been + * navigated away from, but is still alive in the + * page history) + * - Error states + * - Engine Process being graceful shut down (no engines left) + * - Engine Process being killed by the OS. + * + * The following is a diagram that attempts to illustrate the structure of the processes + * and the communication channels that exist between them. + * + * ┌─────────────────────────────────────────────────────────────┐ + * │ PARENT PROCESS │ + * │ │ + * │ [TranslationsParent] ←────→ [TranslationsEngineParent] │ + * │ ↑ ↑ │ + * └──────────────────│────────────────────────────────────│─────┘ + * │ JSWindowActor IPC calls │ JSWindowActor IPC calls + * │ │ + * ┌──────────────────│────────┐ ┌─────│─────────────────────────────┐ + * │ CONTENT PROCESS │ │ │ │ ENGINE PROCESS │ + * │ │ │ │ ↓ │ + * │ [french.html] │ │ │ [TranslationsEngineChild] │ + * │ ↕ ↓ │ │ ↕ │ + * │ [TranslationsChild] │ │ [translations-engine.html] │ + * │ └──TranslationsDocument │ │ ├── "fr to en" engine │ + * │ └──port1 « ═══════════ MessageChannel ════ » │ └── port2 │ + * │ │ │ └── "de to en" engine (idle) │ + * └───────────────────────────┘ └───────────────────────────────────┘ + */ + +// How long the cache remains alive between uses, in milliseconds. In automation the +// engine is manually created and destroyed to avoid timing issues. +const CACHE_TIMEOUT_MS = 15_000; + +/** + * @typedef {import("./translations-document.sys.mjs").TranslationsDocument} TranslationsDocument + * @typedef {import("../translations.js").TranslationsEnginePayload} TranslationsEnginePayload + */ + +/** + * The TranslationsEngine encapsulates the logic for translating messages. It can + * only be set up for a single language translation pair. In order to change languages + * a new engine should be constructed. + * + * The actual work for the translations happens in a worker. This class manages + * instantiating and messaging the worker. + * + * Keep unused engines around in the TranslationsEngine.#cachedEngine cache in case + * page navigation happens and we can re-use previous engines. The engines are very + * heavy-weight, so get rid of them after a timeout. Once all are destroyed the + * TranslationsEngineParent is notified that it can be destroyed. + */ +export class TranslationsEngine { + /** + * Maps a language pair key to a cached engine. Engines are kept around for a timeout + * before they are removed so that they can be re-used during navigation. + * + * @type {Map<string, Promise<TranslationsEngine>>} + */ + static #cachedEngines = new Map(); + + /** @type {TimeoutID | null} */ + #keepAliveTimeout = null; + + /** @type {Worker} */ + #worker; + + /** + * Multiple messages can be sent before a response is received. This ID is used to keep + * track of the messages. It is incremented on every use. + */ + #messageId = 0; + + /** + * Returns a getter function that will create a translations engine on the first + * call, and then return the cached one. After a timeout when the engine hasn't + * been used, it is destroyed. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {number} innerWindowId + * @returns {Promise<TranslationsEngine>} + */ + static getOrCreate(fromLanguage, toLanguage, innerWindowId) { + const languagePairKey = getLanguagePairKey(fromLanguage, toLanguage); + let enginePromise = TranslationsEngine.#cachedEngines.get(languagePairKey); + + if (enginePromise) { + return enginePromise; + } + + TE_log(`Creating a new engine for "${fromLanguage}" to "${toLanguage}".`); + + // A new engine needs to be created. + enginePromise = TranslationsEngine.create( + fromLanguage, + toLanguage, + innerWindowId + ); + + TranslationsEngine.#cachedEngines.set(languagePairKey, enginePromise); + + enginePromise.catch(error => { + TE_logError( + `The engine failed to load for translating "${fromLanguage}" to "${toLanguage}". Removing it from the cache.`, + error + ); + // Remove the engine if it fails to initialize. + TranslationsEngine.#removeEngineFromCache(languagePairKey); + }); + + return enginePromise; + } + + /** + * Removes the engine, and if it's the last, call the process to destroy itself. + * @param {string} languagePairKey + * @param {boolean} force - On forced shutdowns, it's not necessary to notify the + * parent process. + */ + static #removeEngineFromCache(languagePairKey, force) { + TranslationsEngine.#cachedEngines.delete(languagePairKey); + if (TranslationsEngine.#cachedEngines.size === 0 && !force) { + TE_log("The last engine was removed, destroying this process."); + TE_destroyEngineProcess(); + } + } + + /** + * Create a TranslationsEngine and bypass the cache. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {number} innerWindowId + * @returns {Promise<TranslationsEngine>} + */ + static async create(fromLanguage, toLanguage, innerWindowId) { + const startTime = performance.now(); + + const engine = new TranslationsEngine( + fromLanguage, + toLanguage, + await TE_requestEnginePayload(fromLanguage, toLanguage) + ); + + await engine.isReady; + + TE_addProfilerMarker({ + startTime, + message: `Translations engine loaded for "${fromLanguage}" to "${toLanguage}"`, + innerWindowId, + }); + + return engine; + } + + /** + * Signal to the engines that they are being forced to shutdown. + */ + static forceShutdown() { + return Promise.allSettled( + [...TranslationsEngine.#cachedEngines].map( + async ([langPair, enginePromise]) => { + TE_log(`Force shutdown of the engine "${langPair}"`); + const engine = await enginePromise; + engine.terminate(true /* force */); + } + ) + ); + } + + /** + * Terminates the engine and its worker after a timeout. + * @param {boolean} force + */ + terminate = (force = false) => { + const message = `Terminating translations engine "${this.languagePairKey}".`; + TE_addProfilerMarker({ message }); + TE_log(message); + this.#worker.terminate(); + this.#worker = null; + if (this.#keepAliveTimeout) { + clearTimeout(this.#keepAliveTimeout); + } + for (const [innerWindowId, data] of ports) { + const { fromLanguage, toLanguage, port } = data; + if ( + fromLanguage === this.fromLanguage && + toLanguage === this.toLanguage + ) { + // This port is still active but being closed. + ports.delete(innerWindowId); + port.postMessage({ type: "TranslationsPort:EngineTerminated" }); + port.close(); + } + } + TranslationsEngine.#removeEngineFromCache(this.languagePairKey, force); + }; + + /** + * The worker needs to be shutdown after some amount of time of not being used. + */ + keepAlive() { + if (this.#keepAliveTimeout) { + // Clear any previous timeout. + clearTimeout(this.#keepAliveTimeout); + } + // In automated tests, the engine is manually destroyed. + if (!Cu.isInAutomation) { + this.#keepAliveTimeout = setTimeout(this.terminate, CACHE_TIMEOUT_MS); + } + } + + /** + * Construct and initialize the worker. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {TranslationsEnginePayload} enginePayload - If there is no engine payload + * then the engine will be mocked. This allows this class to be used in tests. + */ + constructor(fromLanguage, toLanguage, enginePayload) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + this.languagePairKey = getLanguagePairKey(fromLanguage, toLanguage); + this.#worker = new Worker( + "chrome://global/content/translations/translations-engine.worker.js" + ); + + /** @type {Promise<void>} */ + this.isReady = new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + TE_log("Received initialization message", data); + if (data.type === "initialization-success") { + resolve(); + } else if (data.type === "initialization-error") { + reject(data.error); + } + this.#worker.removeEventListener("message", onMessage); + }; + this.#worker.addEventListener("message", onMessage); + + // Schedule the first timeout for keeping the engine alive. + this.keepAlive(); + }); + + // Make sure the ArrayBuffers are transferred, not cloned. + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects + const transferables = []; + if (enginePayload) { + transferables.push(enginePayload.bergamotWasmArrayBuffer); + for (const files of enginePayload.languageModelFiles) { + for (const { buffer } of Object.values(files)) { + transferables.push(buffer); + } + } + } + + this.#worker.postMessage( + { + type: "initialize", + fromLanguage, + toLanguage, + enginePayload, + messageId: this.#messageId++, + logLevel: TE_getLogLevel(), + }, + transferables + ); + } + + /** + * The implementation for translation. Use translateText or translateHTML for the + * public API. + * + * @param {string} sourceText + * @param {boolean} isHTML + * @param {number} innerWindowId + * @returns {Promise<string[]>} + */ + translate(sourceText, isHTML, innerWindowId) { + this.keepAlive(); + + const messageId = this.#messageId++; + + return new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if ( + data.type === "translations-discarded" && + data.innerWindowId === innerWindowId + ) { + // The page was unloaded, and we no longer need to listen for a response. + this.#worker.removeEventListener("message", onMessage); + return; + } + + if (data.messageId !== messageId) { + // Multiple translation requests can be sent before a response is received. + // Ensure that the response received here is the correct one. + return; + } + + if (data.type === "translation-response") { + // Also keep the translation alive after getting a result, as many translations + // can queue up at once, and then it can take minutes to resolve them all. + this.keepAlive(); + resolve(data.targetText); + } + if (data.type === "translation-error") { + reject(data.error); + } + this.#worker.removeEventListener("message", onMessage); + }; + + this.#worker.addEventListener("message", onMessage); + + this.#worker.postMessage({ + type: "translation-request", + isHTML, + sourceText, + messageId, + innerWindowId, + }); + }); + } + + /** + * Applies a function only if a cached engine exists. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {(engine: TranslationsEngine) => void} fn + */ + static withCachedEngine(fromLanguage, toLanguage, fn) { + const engine = TranslationsEngine.#cachedEngines.get( + getLanguagePairKey(fromLanguage, toLanguage) + ); + + if (engine) { + engine.then(fn).catch(() => {}); + } + } + + /** + * Stop processing the translation queue. All in-progress messages will be discarded. + * + * @param {number} innerWindowId + */ + discardTranslationQueue(innerWindowId) { + this.#worker.postMessage({ + type: "discard-translation-queue", + innerWindowId, + }); + } + + /** + * Pause or resume the translations from a cached engine. + * + * @param {boolean} pause + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {number} innerWindowId + */ + static pause(pause, fromLanguage, toLanguage, innerWindowId) { + TranslationsEngine.withCachedEngine(fromLanguage, toLanguage, engine => { + engine.pause(pause, innerWindowId); + }); + } +} + +/** + * Creates a lookup key that is unique to each fromLanguage-toLanguage pair. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {string} + */ +function getLanguagePairKey(fromLanguage, toLanguage) { + return `${fromLanguage},${toLanguage}`; +} + +/** + * Maps the innerWindowId to the port. + * @type {Map<number, { fromLanguage: string, toLanguage: string, port: MessagePort }} + */ +const ports = new Map(); + +/** + * Listen to the port to the content process for incoming messages, and pass + * them to the TranslationsEngine manager. The other end of the port is held + * in the content process by the TranslationsDocument. + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {number} innerWindowId + * @param {MessagePort} port + */ +function listenForPortMessages(fromLanguage, toLanguage, innerWindowId, port) { + async function handleMessage({ data }) { + switch (data.type) { + case "TranslationsPort:GetEngineStatusRequest": { + // This message gets sent first before the translation queue is processed. + // The engine is most likely to fail on the initial invocation. Any failure + // past the first one is not reported to the UI. + TranslationsEngine.getOrCreate( + fromLanguage, + toLanguage, + innerWindowId + ).then( + () => { + TE_log("The engine is ready for translations.", { + innerWindowId, + }); + TE_reportEngineStatus(innerWindowId, "ready"); + port.postMessage({ + type: "TranslationsPort:GetEngineStatusResponse", + status: "ready", + }); + }, + () => { + TE_reportEngineStatus(innerWindowId, "error"); + port.postMessage({ + type: "TranslationsPort:GetEngineStatusResponse", + status: "error", + }); + // After an error no more translation requests will be sent. Go ahead + // and close the port. + port.close(); + ports.delete(innerWindowId); + } + ); + break; + } + case "TranslationsPort:TranslationRequest": { + const { sourceText, isHTML, messageId } = data; + const engine = await TranslationsEngine.getOrCreate( + fromLanguage, + toLanguage, + innerWindowId + ); + const targetText = await engine.translate( + sourceText, + isHTML, + innerWindowId + ); + port.postMessage({ + type: "TranslationsPort:TranslationResponse", + messageId, + targetText, + }); + break; + } + case "TranslationsPort:DiscardTranslations": { + discardTranslations(innerWindowId); + break; + } + default: + TE_logError("Unknown translations port message: " + data.type); + break; + } + } + + if (port.onmessage) { + TE_logError( + new Error("The MessagePort onmessage handler was already present.") + ); + } + + port.onmessage = event => { + handleMessage(event).catch(error => TE_logError(error)); + }; +} + +/** + * Discards the queue and removes the port. + * + * @param {innerWindowId} number + */ +function discardTranslations(innerWindowId) { + TE_log("Discarding translations, innerWindowId:", innerWindowId); + + const portData = ports.get(innerWindowId); + if (portData) { + const { port, fromLanguage, toLanguage } = portData; + port.close(); + ports.delete(innerWindowId); + + TranslationsEngine.withCachedEngine(fromLanguage, toLanguage, engine => { + engine.discardTranslationQueue(innerWindowId); + }); + } +} + +/** + * Listen for events coming from the TranslationsEngine actor. + */ +window.addEventListener("message", ({ data }) => { + switch (data.type) { + case "StartTranslation": { + const { fromLanguage, toLanguage, innerWindowId, port } = data; + TE_log("Starting translation", innerWindowId); + listenForPortMessages(fromLanguage, toLanguage, innerWindowId, port); + ports.set(innerWindowId, { port, fromLanguage, toLanguage }); + break; + } + case "DiscardTranslations": { + const { innerWindowId } = data; + discardTranslations(innerWindowId); + break; + } + case "ForceShutdown": { + TranslationsEngine.forceShutdown().then(() => { + TE_resolveForceShutdown(); + }); + break; + } + default: + throw new Error("Unknown TranslationsEngineChromeToContent event."); + } +}); diff --git a/toolkit/components/translations/content/translations-engine.worker.js b/toolkit/components/translations/content/translations-engine.worker.js new file mode 100644 index 0000000000..f7eb23b61c --- /dev/null +++ b/toolkit/components/translations/content/translations-engine.worker.js @@ -0,0 +1,734 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/** + * @typedef {import("../translations").Bergamot} Bergamot + * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles + */ + +/* global loadBergamot */ +importScripts("chrome://global/content/translations/bergamot-translator.js"); + +// Respect the preference "browser.translations.logLevel". +let _loggingLevel = "Error"; +function log(...args) { + if (_loggingLevel !== "Error" && _loggingLevel !== "Warn") { + console.log("Translations:", ...args); + } +} +function trace(...args) { + if (_loggingLevel === "Trace" || _loggingLevel === "All") { + console.log("Translations:", ...args); + } +} + +// Throw Promise rejection errors so that they are visible in the console. +self.addEventListener("unhandledrejection", event => { + throw event.reason; +}); + +/** + * The alignment for each file type, file type strings should be same as in the + * model registry. + */ +const MODEL_FILE_ALIGNMENTS = { + model: 256, + lex: 64, + vocab: 64, + qualityModel: 64, + srcvocab: 64, + trgvocab: 64, +}; + +/** + * Initialize the engine, and get it ready to handle translation requests. + * The "initialize" message must be received before any other message handling + * requests will be processed. + */ +addEventListener("message", handleInitializationMessage); + +async function handleInitializationMessage({ data }) { + const startTime = performance.now(); + if (data.type !== "initialize") { + console.error( + "The TranslationEngine worker received a message before it was initialized." + ); + return; + } + + try { + const { fromLanguage, toLanguage, enginePayload, logLevel, innerWindowId } = + data; + + if (!fromLanguage) { + throw new Error('Worker initialization missing "fromLanguage"'); + } + if (!toLanguage) { + throw new Error('Worker initialization missing "toLanguage"'); + } + + if (logLevel) { + // Respect the "browser.translations.logLevel" preference. + _loggingLevel = logLevel; + } + + let engine; + if (enginePayload.isMocked) { + // The engine is testing mode, and no Bergamot wasm is available. + engine = new MockedEngine(fromLanguage, toLanguage); + } else { + const { bergamotWasmArrayBuffer, languageModelFiles } = enginePayload; + const bergamot = await BergamotUtils.initializeWasm( + bergamotWasmArrayBuffer + ); + engine = new Engine( + fromLanguage, + toLanguage, + bergamot, + languageModelFiles + ); + } + + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { startTime, innerWindowId }, + "Translations engine loaded." + ); + + handleMessages(engine); + postMessage({ type: "initialization-success" }); + } catch (error) { + console.error(error); + postMessage({ type: "initialization-error", error: error?.message }); + } + + removeEventListener("message", handleInitializationMessage); +} + +/** + * Sets up the message handling for the worker. + * + * @param {Engine | MockedEngine} engine + */ +function handleMessages(engine) { + let discardPromise; + addEventListener("message", async ({ data }) => { + try { + if (data.type === "initialize") { + throw new Error("The Translations engine must not be re-initialized."); + } + if (data.type === "translation-request") { + // Only show these messages when "All" logging is on, since there are so many + // of them. + trace("Received message", data); + } else { + log("Received message", data); + } + + switch (data.type) { + case "translation-request": { + const { sourceText, messageId, isHTML, innerWindowId } = data; + if (discardPromise) { + // Wait for messages to be discarded if there are any. + await discardPromise; + } + try { + // Add a translation to the work queue, and when it returns, post the message + // back. The translation may never return if the translations are discarded + // before it have time to be run. In this case this await is just never + // resolved, and the postMessage is never run. + const targetText = await engine.translate( + sourceText, + isHTML, + innerWindowId + ); + + // This logging level can be very verbose and slow, so only do it under the + // "Trace" level, which is the most verbose. Set the logging level to "Info" to avoid + // these, and get all of the other logs. + trace("Translation complete", { + sourceText, + targetText, + isHTML, + innerWindowId, + }); + + postMessage({ + type: "translation-response", + targetText, + messageId, + }); + } catch (error) { + console.error(error); + let message = "An error occurred in the engine worker."; + if (typeof error?.message === "string") { + message = error.message; + } + let stack = "(no stack)"; + if (typeof error?.stack === "string") { + stack = error.stack; + } + postMessage({ + type: "translation-error", + error: { message, stack }, + messageId, + innerWindowId, + }); + } + break; + } + case "discard-translation-queue": { + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { innerWindowId: data.innerWindowId }, + "Translations discard requested" + ); + + discardPromise = engine.discardTranslations(data.innerWindowId); + await discardPromise; + discardPromise = null; + + // Signal to the "message" listeners in the main thread to stop listening. + postMessage({ + type: "translations-discarded", + }); + break; + } + default: + console.warn("Unknown message type:", data.type); + } + } catch (error) { + // Ensure the unexpected errors are surfaced in the console. + console.error(error); + } + }); +} + +/** + * The Engine is created once for a language pair. The initialization process copies the + * ArrayBuffers for the language buffers from JS-managed ArrayBuffers, to aligned + * internal memory for the wasm heap. + * + * After this the ArrayBuffers are discarded and GC'd. This file should be managed + * from the TranslationsEngine class on the main thread. + * + * This class starts listening for messages only after the Bergamot engine has been + * fully initialized. + */ +class Engine { + /** + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {Bergamot} bergamot + * @param {Array<LanguageTranslationModelFiles>} languageTranslationModelFiles + */ + constructor( + fromLanguage, + toLanguage, + bergamot, + languageTranslationModelFiles + ) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + /** @type {Bergamot} */ + this.bergamot = bergamot; + /** @type {Bergamot["TranslationModel"][]} */ + this.languageTranslationModels = languageTranslationModelFiles.map( + languageTranslationModelFiles => + BergamotUtils.constructSingleTranslationModel( + bergamot, + languageTranslationModelFiles + ) + ); + + /** @type {Bergamot["BlockingService"]} */ + this.translationService = new bergamot.BlockingService({ + // Caching is disabled (see https://github.com/mozilla/firefox-translations/issues/288) + cacheSize: 0, + }); + } + + /** + * Run the translation models to perform a batch of message translations. The + * promise is rejected when the sync version of this function throws an error. + * This function creates an async interface over the synchronous translation + * mechanism. This allows other microtasks such as message handling to still work + * even though the translations are CPU-intensive. + * + * @param {string} sourceText + * @param {boolean} isHTML + * @param {number} innerWindowId - This is required + * + * @returns {Promise<string>}sourceText + */ + translate(sourceText, isHTML, innerWindowId) { + return this.#getWorkQueue(innerWindowId).runTask(() => + this.#syncTranslate(sourceText, isHTML, innerWindowId) + ); + } + + /** + * Map each innerWindowId to its own WorkQueue. This makes it easy to shut down + * an entire queue of work when the page is unloaded. + * + * @type {Map<number, WorkQueue>} + */ + #workQueues = new Map(); + + /** + * Get or create a `WorkQueue` that is unique to an `innerWindowId`. + * + * @param {number} innerWindowId + * @returns {WorkQueue} + */ + #getWorkQueue(innerWindowId) { + let workQueue = this.#workQueues.get(innerWindowId); + if (workQueue) { + return workQueue; + } + workQueue = new WorkQueue(innerWindowId); + this.#workQueues.set(innerWindowId, workQueue); + return workQueue; + } + + /** + * Cancels any in-progress translations by removing the work queue. + * + * @param {number} innerWindowId + */ + discardTranslations(innerWindowId) { + let workQueue = this.#workQueues.get(innerWindowId); + if (workQueue) { + workQueue.cancelWork(); + this.#workQueues.delete(innerWindowId); + } + } + + /** + * Run the translation models to perform a translation. This + * blocks the worker thread until it is completed. + * + * @param {string} sourceText + * @param {boolean} isHTML + * @param {number} innerWindowId + * @returns {string} + */ + #syncTranslate(sourceText, isHTML, innerWindowId) { + const startTime = performance.now(); + let response; + sourceText = sourceText.trim(); + const { messages, options } = BergamotUtils.getTranslationArgs( + this.bergamot, + sourceText, + isHTML + ); + try { + if (messages.size() === 0) { + return []; + } + + /** @type {Bergamot["VectorResponse"]} */ + let responses; + + if (this.languageTranslationModels.length === 1) { + responses = this.translationService.translate( + this.languageTranslationModels[0], + messages, + options + ); + } else if (this.languageTranslationModels.length === 2) { + responses = this.translationService.translateViaPivoting( + this.languageTranslationModels[0], + this.languageTranslationModels[1], + messages, + options + ); + } else { + throw new Error( + "Too many models were provided to the translation worker." + ); + } + + // Report on the time it took to do this translation. + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { startTime, innerWindowId }, + `Translated ${sourceText.length} code units.` + ); + + const targetText = responses.get(0).getTranslatedText(); + return targetText; + } finally { + // Free up any memory that was allocated. This will always run. + messages?.delete(); + options?.delete(); + response?.delete(); + } + } +} + +/** + * Static utilities to help work with the Bergamot wasm module. + */ +class BergamotUtils { + /** + * Construct a single translation model. + * + * @param {Bergamot} bergamot + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles + * @returns {Bergamot["TranslationModel"]} + */ + static constructSingleTranslationModel( + bergamot, + languageTranslationModelFiles + ) { + log(`Constructing translation model.`); + + const { model, lex, vocab, qualityModel, srcvocab, trgvocab } = + BergamotUtils.allocateModelMemory( + bergamot, + languageTranslationModelFiles + ); + + // Transform the bytes to mb, like "10.2mb" + const getMemory = memory => `${Math.floor(memory.size() / 100_000) / 10}mb`; + + let memoryLog = `Model memory sizes in wasm heap:`; + memoryLog += `\n Model: ${getMemory(model)}`; + memoryLog += `\n Shortlist: ${getMemory(lex)}`; + + // Set up the vocab list, which could either be a single "vocab" model, or a + // "srcvocab" and "trgvocab" pair. + const vocabList = new bergamot.AlignedMemoryList(); + + if (vocab) { + vocabList.push_back(vocab); + memoryLog += `\n Vocab: ${getMemory(vocab)}`; + } else if (srcvocab && trgvocab) { + vocabList.push_back(srcvocab); + vocabList.push_back(trgvocab); + memoryLog += `\n Src Vocab: ${getMemory(srcvocab)}`; + memoryLog += `\n Trg Vocab: ${getMemory(trgvocab)}`; + } else { + throw new Error("Vocabulary key is not found."); + } + + if (qualityModel) { + memoryLog += `\n QualityModel: ${getMemory(qualityModel)}\n`; + } + + const config = BergamotUtils.generateTextConfig({ + "beam-size": "1", + normalize: "1.0", + "word-penalty": "0", + "max-length-break": "128", + "mini-batch-words": "1024", + workspace: "128", + "max-length-factor": "2.0", + "skip-cost": (!qualityModel).toString(), + "cpu-threads": "0", + quiet: "true", + "quiet-translation": "true", + "gemm-precision": + languageTranslationModelFiles.model.record.name.endsWith("intgemm8.bin") + ? "int8shiftAll" + : "int8shiftAlphaAll", + alignment: "soft", + }); + + log(`Bergamot translation model config: ${config}`); + log(memoryLog); + + return new bergamot.TranslationModel( + config, + model, + lex, + vocabList, + qualityModel ?? null + ); + } + + /** + * The models must be placed in aligned memory that the Bergamot wasm module has access + * to. This function copies over the model blobs into this memory space. + * + * @param {Bergamot} bergamot + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles + * @returns {LanguageTranslationModelFilesAligned} + */ + static allocateModelMemory(bergamot, languageTranslationModelFiles) { + /** @type {LanguageTranslationModelFilesAligned} */ + const results = {}; + + for (const [fileType, file] of Object.entries( + languageTranslationModelFiles + )) { + const alignment = MODEL_FILE_ALIGNMENTS[fileType]; + if (!alignment) { + throw new Error(`Unknown file type: "${fileType}"`); + } + + const alignedMemory = new bergamot.AlignedMemory( + file.buffer.byteLength, + alignment + ); + + alignedMemory.getByteArrayView().set(new Uint8Array(file.buffer)); + + results[fileType] = alignedMemory; + } + + return results; + } + + /** + * Initialize the Bergamot translation engine. It is a wasm compiled version of the + * Marian translation software. The wasm is delivered remotely to cut down on binary size. + * + * https://github.com/mozilla/bergamot-translator/ + * + * @param {ArrayBuffer} wasmBinary + * @returns {Promise<Bergamot>} + */ + static initializeWasm(wasmBinary) { + return new Promise((resolve, reject) => { + /** @type {number} */ + let start = performance.now(); + + /** @type {Bergamot} */ + const bergamot = loadBergamot({ + // This is the amount of memory that a simple run of Bergamot uses, in bytes. + INITIAL_MEMORY: 234_291_200, + print: log, + onAbort() { + reject(new Error("Error loading Bergamot wasm module.")); + }, + onRuntimeInitialized: async () => { + const duration = performance.now() - start; + log( + `Bergamot wasm runtime initialized in ${duration / 1000} seconds.` + ); + // Await at least one microtask so that the captured `bergamot` variable is + // fully initialized. + await Promise.resolve(); + resolve(bergamot); + }, + wasmBinary, + }); + }); + } + + /** + * Maps the Bergamot Vector to a JS array + * + * @param {Bergamot["Vector"]} vector + * @param {Function} fn + * @returns {Array} + */ + static mapVector(vector, fn) { + const result = []; + for (let index = 0; index < vector.size(); index++) { + result.push(fn(vector.get(index), index)); + } + return result; + } + + /** + * Generate a config for the Marian translation service. It requires specific whitespace. + * + * https://marian-nmt.github.io/docs/cmd/marian-decoder/ + * + * @param {Record<string, string>} config + * @returns {string} + */ + static generateTextConfig(config) { + const indent = " "; + let result = "\n"; + + for (const [key, value] of Object.entries(config)) { + result += `${indent}${key}: ${value}\n`; + } + + return result + indent; + } + + /** + * JS objects need to be translated into wasm objects to configure the translation engine. + * + * @param {Bergamot} bergamot + * @param {string[]} sourceText + * @returns {{ messages: Bergamot["VectorString"], options: Bergamot["VectorResponseOptions"] }} + */ + static getTranslationArgs(bergamot, sourceText, isHTML) { + const messages = new bergamot.VectorString(); + const options = new bergamot.VectorResponseOptions(); + + sourceText = sourceText.trim(); + // Empty paragraphs break the translation. + if (sourceText) { + messages.push_back(sourceText); + options.push_back({ + qualityScores: false, + alignment: true, + html: isHTML, + }); + } + + return { messages, options }; + } +} + +/** + * For testing purposes, provide a fully mocked engine. This allows for easy integration + * testing of the UI, without having to rely on downloading remote models and remote + * wasm binaries. + */ +class MockedEngine { + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + constructor(fromLanguage, toLanguage) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + } + + /** + * Create a fake translation of the text. + * + * @param {string} sourceText + * @param {bool} isHTML + * @returns {string} + */ + translate(sourceText, isHTML) { + // Note when an HTML translations is requested. + let html = isHTML ? ", html" : ""; + const targetText = sourceText.toUpperCase(); + return `${targetText} [${this.fromLanguage} to ${this.toLanguage}${html}]`; + } + + discardTranslations() {} +} + +/** + * This class takes tasks that may block the thread's event loop, and has them yield + * after a time budget via setTimeout calls to allow other code to execute. + */ +class WorkQueue { + #TIME_BUDGET = 100; // ms + #RUN_IMMEDIATELY_COUNT = 20; + + /** @type {Array<{task: Function, resolve: Function}>} */ + #tasks = []; + #isRunning = false; + #isWorkCancelled = false; + #runImmediately = this.#RUN_IMMEDIATELY_COUNT; + + /** + * @param {number} innerWindowId + */ + constructor(innerWindowId) { + this.innerWindowId = innerWindowId; + } + + /** + * Run the task and return the result. + * + * @template {T} + * @param {() => T} task + * @returns {Promise<T>} + */ + runTask(task) { + if (this.#runImmediately > 0) { + // Run the first N translations immediately, most likely these are the user-visible + // translations on the page, as they are sent in first. The setTimeout of 0 can + // still delay the translations noticeably. + this.#runImmediately--; + return Promise.resolve(task()); + } + return new Promise((resolve, reject) => { + this.#tasks.push({ task, resolve, reject }); + this.#run().catch(error => console.error(error)); + }); + } + + /** + * The internal run function. + */ + async #run() { + if (this.#isRunning) { + // The work queue is already running. + return; + } + + this.#isRunning = true; + + // Measure the timeout + let lastTimeout = null; + + let tasksInBatch = 0; + const addProfilerMarker = () => { + ChromeUtils.addProfilerMarker( + "TranslationsWorker WorkQueue", + { startTime: lastTimeout, innerWindowId: this.innerWindowId }, + `WorkQueue processed ${tasksInBatch} tasks` + ); + }; + + while (this.#tasks.length !== 0) { + if (this.#isWorkCancelled) { + // The work was already cancelled. + break; + } + const now = performance.now(); + + if (lastTimeout === null) { + lastTimeout = now; + // Allow other work to get on the queue. + await new Promise(resolve => setTimeout(resolve, 0)); + } else if (now - lastTimeout > this.#TIME_BUDGET) { + // Perform a timeout with no effective wait. This clears the current + // promise queue from the event loop. + await new Promise(resolve => setTimeout(resolve, 0)); + addProfilerMarker(); + lastTimeout = performance.now(); + } + + // Check this between every `await`. + if (this.#isWorkCancelled || !this.#tasks.length) { + break; + } + + tasksInBatch++; + const { task, resolve, reject } = this.#tasks.shift(); + try { + const result = await task(); + + // Check this between every `await`. + if (this.#isWorkCancelled) { + break; + } + // The work is done, resolve the original task. + resolve(result); + } catch (error) { + reject(error); + } + } + addProfilerMarker(); + this.#isRunning = false; + } + + async cancelWork() { + this.#isWorkCancelled = true; + this.#tasks = []; + await new Promise(resolve => setTimeout(resolve, 0)); + this.#isWorkCancelled = false; + } +} diff --git a/toolkit/components/translations/content/translations.css b/toolkit/components/translations/content/translations.css new file mode 100644 index 0000000000..ee3c0ba8ec --- /dev/null +++ b/toolkit/components/translations/content/translations.css @@ -0,0 +1,169 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +:root { + /* Provide defaults for when this page is viewed in "toolkit". */ + background-color: var(--in-content-page-background, #fff); + color: var(--in-content-page-color, #15141a); + + /* Provide backup values for some of the variables used in "browser" so that the styles + look nice by default in "toolkit". */ + --AT-box-background: var(--in-content-box-background, #fff); + --AT-box-border-color: var(--in-content-box-border-color, #9e9ea0); + --AT-box-info-background: var(--in-content-box-info-background, #f0f0f4); + + /* Variables used in the page layout */ + --AT-page-margin: 20px; + --AT-input-padding: 20px; + /* This is somewhat arbitrary, but works well for the current design. If the computed + header height changes, this will need to be adjusted. */ + --AT-header-height: 156px; + --AT-input-height: calc(min(400px, calc(100vh - var(--AT-header-height)))); + --AT-select-arrow-inset: 5px; +} + +body { + display: flex; + justify-content: center; + align-items: center; + inset: 0; + position: absolute; + visibility: hidden; + flex-direction: column; +} + +.about-translations-header { + display: flex; +} + +.about-translations-header > * { + flex: 1; + display: flex; + max-width: 50%; +} + +.about-translations-header-start { + justify-content: start; +} + +.about-translations-header-end { + justify-content: end; +} + +/* Increase the selector specificity to override the base `select` styles. */ +select.about-translations-select { + position: relative; + padding-inline: 10px 20px; + padding-block: 0px; + min-width: 50%; + margin: 5px; + background-position: right var(--AT-select-arrow-inset) center; +} + +select.about-translations-select:dir(rtl) { + background-position-x: left var(--AT-select-arrow-inset); +} + +.about-translations-contents { + display: flex; + flex-direction: column; + box-sizing: border-box; + width: calc(100% - var(--AT-page-margin) * 2); + max-width: 1200px; + background-color: var(--AT-box-background); + border: 1px solid var(--AT-box-border-color); + border-radius: 4px; +} + +.about-translations-input { + display: flex; + width: 100%; + border-top: 1px solid var(--AT-box-border-color); +} + +.about-translations-input-start { + border-inline-end: 1px solid var(--AT-box-border-color); +} + +.about-translations-input > * { + position: relative; + width: 50%; +} + +.about-translations-input-textarea { + /* Override user's dragging of the textarea width. */ + width: 100% !important; + height: var(--AT-input-height); + box-sizing: border-box; + margin: 0; + padding: var(--AT-input-padding); + border: 0; +} + +.about-translations-input-results-blank { + opacity: 0.7; +} + +.about-translations-input-results { + position: absolute; + inset: 0; + padding: var(--AT-input-padding); + box-sizing: border-box; + overflow-y: scroll; +} + +.about-translations-info { + display: none; + padding: 10px; + background-color: var(--AT-box-info-background); + border-radius: 4px; + margin-bottom: var(--AT-input-padding); +} + +.about-translations-info-message { + flex: 1; + align-self: center; +} + +.about-translations-info-icon { + width: 16px; + height: 16px; + margin: 10px; + background-image: url('chrome://global/skin/icons/info.svg'); + -moz-context-properties: fill; + fill: currentColor; +} + +@media (max-width: 700px) { + :root { + --AT-page-margin: 10px; + } + h1 { + margin-top: 15px; + } + body { + padding-bottom: var(--AT-page-margin); + } + .about-translations-input { + flex-direction: column; + flex: 1; + } + .about-translations-input-textarea, + .about-translations-input { + font-size: 16px; + } + .about-translations-input > * { + width: 100%; + flex: 1; + } + .about-translations-input-end { + border-top: 1px solid var(--AT-box-border-color); + } + .about-translations-input-textarea { + height: 100%; + } + .about-translations-contents { + flex: 1; + } +} diff --git a/toolkit/components/translations/content/translations.html b/toolkit/components/translations/content/translations.html new file mode 100644 index 0000000000..bd2c114a0a --- /dev/null +++ b/toolkit/components/translations/content/translations.html @@ -0,0 +1,70 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Security-Policy" content="default-src chrome:; object-src 'none'"> + <meta name="color-scheme" content="light dark"> + <meta name="viewport" content="width=device-width" /> + <title data-l10n-id="about-translations-title"></title> + <link rel="stylesheet" href="chrome://global/skin/global.css"> + <link rel="stylesheet" href="chrome://global/skin/in-content/common.css"> + <link rel="stylesheet" href="chrome://global/content/translations/translations.css"> + <link rel="localization" href="toolkit/branding/brandings.ftl"/> + <link rel="localization" href="locales-preview/aboutTranslations.ftl"/> + <script type="module" src="chrome://global/content/translations/translations.mjs"></script> + </head> + <body> + <h1 data-l10n-id="about-translations-header"></h1> + <main class="about-translations-contents"> + + <header class="about-translations-header"> + <div class="about-translations-header-start"> + <select + class="about-translations-select" + id="language-from" + disabled> + <option data-l10n-id="about-translations-detect" value="detect"></option> + </select> + </div> + <div class="about-translations-header-end"> + <select + class="about-translations-select" + id="language-to" + disabled> + <option data-l10n-id="about-translations-select" value=""></option> + </select> + </div> + </header> + + <main class="about-translations-input"> + <div class="about-translations-input-start"> + <textarea + class="about-translations-input-textarea" + data-l10n-id="about-translations-textarea" + id="translation-from" + ></textarea> + </div> + <div class="about-translations-input-end"> + <div + class="about-translations-input-results about-translations-input-results-blank" + id="translation-to-blank"> + <div class="about-translations-info" id="translation-info"> + <div class="about-translations-info-icon"></div> + <div class="about-translations-info-message" id="translation-info-message"></div> + </div> + <div data-l10n-id="about-translations-results-placeholder"></div> + </div> + <div + class="about-translations-input-results" + id="translation-to"> + </div> + </div> + </main> + + </div> + </body> +</html> diff --git a/toolkit/components/translations/content/translations.mjs b/toolkit/components/translations/content/translations.mjs new file mode 100644 index 0000000000..0ec8b2d475 --- /dev/null +++ b/toolkit/components/translations/content/translations.mjs @@ -0,0 +1,788 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The following globals are injected via the AboutTranslationsChild actor. +// translations.mjs is running in an unprivileged context, and these injected functions +// allow for the page to get access to additional privileged features. + +/* global AT_getSupportedLanguages, AT_log, AT_getScriptDirection, + AT_logError, AT_createTranslationsPort, AT_isHtmlTranslation, + AT_isTranslationEngineSupported, AT_identifyLanguage */ + +// Allow tests to override this value so that they can run faster. +// This is the delay in milliseconds. +window.DEBOUNCE_DELAY = 200; +// Allow tests to test the debounce behavior by counting debounce runs. +window.DEBOUNCE_RUN_COUNT = 0; + +/** + * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + */ + +/** + * The model and controller for initializing about:translations. + */ +class TranslationsState { + /** + * This class is responsible for all UI updated. + * + * @type {TranslationsUI} + */ + ui; + + /** + * The language to translate from, in the form of a BCP 47 language tag, + * e.g. "en" or "fr". + * + * @type {string} + */ + fromLanguage = ""; + + /** + * The language to translate to, in the form of a BCP 47 language tag, + * e.g. "en" or "fr". + * + * @type {string} + */ + toLanguage = ""; + + /** + * The message to translate, cached so that it can be determined if the text + * needs to be re-translated. + * + * @type {string} + */ + messageToTranslate = ""; + + /** + * Only send one translation in at a time to the worker. + * @type {Promise<string[]>} + */ + translationRequest = Promise.resolve([]); + + /** + * The translator is only valid for a single language pair, and needs + * to be recreated if the language pair changes. + * + * @type {null | Promise<Translator>} + */ + translator = null; + + /** + * @param {boolean} isSupported + */ + constructor(isSupported) { + /** + * Is the engine supported by the device? + * @type {boolean} + */ + this.isTranslationEngineSupported = isSupported; + + /** + * @type {SupportedLanguages} + */ + this.supportedLanguages = isSupported + ? AT_getSupportedLanguages() + : Promise.resolve([]); + + this.ui = new TranslationsUI(this); + this.ui.setup(); + + // Set the UI as ready after all of the state promises have settled. + this.supportedLanguages + .then(() => { + this.ui.setAsReady(); + }) + .catch(error => { + AT_logError("Failed to load the supported languages", error); + }); + } + + /** + * Identifies the human language in which the message is written and returns + * the BCP 47 language tag of the language it is determined to be. + * + * e.g. "en" for English. + * + * @param {string} message + */ + async identifyLanguage(message) { + const start = performance.now(); + const { langTag, confidence } = await AT_identifyLanguage(message); + const duration = performance.now() - start; + AT_log( + `[ ${langTag}(${(confidence * 100).toFixed(2)}%) ]`, + `Source language identified in ${duration / 1000} seconds` + ); + return langTag; + } + + /** + * Only request a translation when it's ready. + */ + maybeRequestTranslation = debounce({ + /** + * Debounce the translation requests so that the worker doesn't fire for every + * single keyboard input, but instead the keyboard events are ignored until + * there is a short break, or enough events have happened that it's worth sending + * in a new translation request. + */ + onDebounce: async () => { + // The contents of "this" can change between async steps, store a local variable + // binding of these values. + const { + fromLanguage, + toLanguage, + messageToTranslate, + translator: translatorPromise, + } = this; + + if (!this.isTranslationEngineSupported) { + // Never translate when the engine isn't supported. + return; + } + + if ( + !fromLanguage || + !toLanguage || + !messageToTranslate || + !translatorPromise + ) { + // Not everything is set for translation. + this.ui.updateTranslation(""); + return; + } + + const [translator] = await Promise.all([ + // Ensure the engine is ready to go. + translatorPromise, + // Ensure the previous translation has finished so that only the latest + // translation goes through. + this.translationRequest, + ]); + + if ( + // Check if the current configuration has changed and if this is stale. If so + // then skip this request, as there is already a newer request with more up to + // date information. + this.translator !== translatorPromise || + this.fromLanguage !== fromLanguage || + this.toLanguage !== toLanguage || + this.messageToTranslate !== messageToTranslate + ) { + return; + } + + const start = performance.now(); + + this.translationRequest = translator.translate(messageToTranslate); + const translation = await this.translationRequest; + + // The measure events will show up in the Firefox Profiler. + performance.measure( + `Translations: Translate "${this.fromLanguage}" to "${this.toLanguage}" with ${messageToTranslate.length} characters.`, + { + start, + end: performance.now(), + } + ); + + this.ui.updateTranslation(translation); + const duration = performance.now() - start; + AT_log(`Translation done in ${duration / 1000} seconds`); + }, + + // Mark the events so that they show up in the Firefox Profiler. This makes it handy + // to visualize the debouncing behavior. + doEveryTime: () => { + performance.mark( + `Translations: input changed to ${this.messageToTranslate.length} characters` + ); + }, + }); + + /** + * Any time a language pair is changed, a new Translator needs to be created. + */ + async maybeCreateNewTranslator() { + // If we may need to re-building the worker, the old translation is no longer valid. + this.ui.updateTranslation(""); + + // These are cases in which it wouldn't make sense or be possible to load any translations models. + if ( + // If fromLanguage or toLanguage are unpopulated we cannot load anything. + !this.fromLanguage || + !this.toLanguage || + // If fromLanguage's value is "detect", rather than a BCP 47 language tag, then no language + // has been detected yet. + this.fromLanguage === "detect" || + // If fromLanguage and toLanguage are the same, this means that the detected language + // is the same as the toLanguage, and we do not want to translate from one language to itself. + this.fromLanguage === this.toLanguage + ) { + if (this.translator) { + // The engine is no longer needed. + this.translator.then(translator => translator.destroy()); + this.translator = null; + } + return; + } + + const start = performance.now(); + AT_log( + `Creating a new translator for "${this.fromLanguage}" to "${this.toLanguage}"` + ); + + this.translator = Translator.create(this.fromLanguage, this.toLanguage); + this.maybeRequestTranslation(); + + try { + await this.translator; + const duration = performance.now() - start; + // Signal to tests that the translator was created so they can exit. + window.postMessage("translator-ready"); + AT_log(`Created a new Translator in ${duration / 1000} seconds`); + } catch (error) { + this.ui.showInfo("about-translations-engine-error"); + AT_logError("Failed to get the Translations worker", error); + } + } + + /** + * Updates the fromLanguage to match the detected language only if the + * about-translations-detect option is selected in the language-from dropdown. + * + * If the new fromLanguage is different than the previous fromLanguage this + * may update the UI to display the new language and may rebuild the translations + * worker if there is a valid selected target language. + */ + async maybeUpdateDetectedLanguage() { + if (!this.ui.detectOptionIsSelected() || this.messageToTranslate === "") { + // If we are not detecting languages or if the message has been cleared + // we should ensure that the UI is not displaying a detected language + // and there is no need to run any language detection. + this.ui.setDetectOptionTextContent(""); + return; + } + + const [langTag, supportedLanguages] = await Promise.all([ + this.identifyLanguage(this.messageToTranslate), + this.supportedLanguages, + ]); + + // Only update the language if the detected language matches + // one of our supported languages. + const entry = supportedLanguages.fromLanguages.find( + ({ langTag: existingTag }) => existingTag === langTag + ); + if (entry) { + const { displayName } = entry; + await this.setFromLanguage(langTag); + this.ui.setDetectOptionTextContent(displayName); + } + } + + /** + * @param {string} lang + */ + async setFromLanguage(lang) { + if (lang !== this.fromLanguage) { + this.fromLanguage = lang; + await this.maybeCreateNewTranslator(); + } + } + + /** + * @param {string} lang + */ + setToLanguage(lang) { + if (lang !== this.toLanguage) { + this.toLanguage = lang; + this.maybeCreateNewTranslator(); + } + } + + /** + * @param {string} message + */ + async setMessageToTranslate(message) { + if (message !== this.messageToTranslate) { + this.messageToTranslate = message; + await this.maybeUpdateDetectedLanguage(); + this.maybeRequestTranslation(); + } + } +} + +/** + * + */ +class TranslationsUI { + /** @type {HTMLSelectElement} */ + languageFrom = document.getElementById("language-from"); + /** @type {HTMLSelectElement} */ + languageTo = document.getElementById("language-to"); + /** @type {HTMLTextAreaElement} */ + translationFrom = document.getElementById("translation-from"); + /** @type {HTMLDivElement} */ + translationTo = document.getElementById("translation-to"); + /** @type {HTMLDivElement} */ + translationToBlank = document.getElementById("translation-to-blank"); + /** @type {HTMLDivElement} */ + translationInfo = document.getElementById("translation-info"); + /** @type {HTMLDivElement} */ + translationInfoMessage = document.getElementById("translation-info-message"); + /** @type {TranslationsState} */ + state; + + /** + * The detect-language option element. We want to maintain a handle to this so that + * we can dynamically update its display text to include the detected language. + * + * @type {HTMLOptionElement} + */ + #detectOption; + + /** + * @param {TranslationsState} state + */ + constructor(state) { + this.state = state; + this.translationTo.style.visibility = "visible"; + this.#detectOption = document.querySelector('option[value="detect"]'); + } + + /** + * Do the initial setup. + */ + setup() { + if (!this.state.isTranslationEngineSupported) { + this.showInfo("about-translations-no-support"); + this.disableUI(); + return; + } + this.setupDropdowns(); + this.setupTextarea(); + } + + /** + * Signals that the UI is ready, for tests. + */ + setAsReady() { + document.body.setAttribute("ready", ""); + } + + /** + * Once the models have been synced from remote settings, populate them with the display + * names of the languages. + */ + async setupDropdowns() { + const supportedLanguages = await this.state.supportedLanguages; + + // Update the DOM elements with the display names. + for (const { langTag, displayName } of supportedLanguages.toLanguages) { + const option = document.createElement("option"); + option.value = langTag; + option.text = displayName; + this.languageTo.add(option); + } + + for (const { langTag, displayName } of supportedLanguages.fromLanguages) { + const option = document.createElement("option"); + option.value = langTag; + option.text = displayName; + this.languageFrom.add(option); + } + + // Enable the controls. + this.languageFrom.disabled = false; + this.languageTo.disabled = false; + + // Focus the language dropdowns if they are empty. + if (this.languageFrom.value == "") { + this.languageFrom.focus(); + } else if (this.languageTo.value == "") { + this.languageTo.focus(); + } + + this.state.setFromLanguage(this.languageFrom.value); + this.state.setToLanguage(this.languageTo.value); + this.updateOnLanguageChange(); + + this.languageFrom.addEventListener("input", () => { + this.state.setFromLanguage(this.languageFrom.value); + this.updateOnLanguageChange(); + }); + + this.languageTo.addEventListener("input", () => { + this.state.setToLanguage(this.languageTo.value); + this.updateOnLanguageChange(); + this.translationTo.setAttribute("lang", this.languageTo.value); + }); + } + + /** + * Show an info message to the user. + * + * @param {string} l10nId + */ + showInfo(l10nId) { + document.l10n.setAttributes(this.translationInfoMessage, l10nId); + this.translationInfo.style.display = "flex"; + } + + /** + * Hides the info UI. + */ + hideInfo() { + this.translationInfo.style.display = "none"; + } + + /** + * Returns true if about-translations-detect is the currently + * selected option in the language-from dropdown, otherwise false. + * + * @returns {boolean} + */ + detectOptionIsSelected() { + return this.languageFrom.value === "detect"; + } + + /** + * Sets the textContent of the about-translations-detect option in the + * language-from dropdown to include the detected language's display name. + * + * @param {string} displayName + */ + setDetectOptionTextContent(displayName) { + // Set the text to the fluent value that takes an arg to display the language name. + if (displayName) { + document.l10n.setAttributes( + this.#detectOption, + "about-translations-detect-lang", + { language: displayName } + ); + } else { + // Reset the text to the fluent value that does not display any language name. + document.l10n.setAttributes( + this.#detectOption, + "about-translations-detect" + ); + } + } + + /** + * React to language changes. + */ + updateOnLanguageChange() { + this.#updateDropdownLanguages(); + this.#updateMessageDirections(); + } + + /** + * You cant translate from one language to another language. Hide the options + * if this is the case. + */ + #updateDropdownLanguages() { + for (const option of this.languageFrom.options) { + option.hidden = false; + } + for (const option of this.languageTo.options) { + option.hidden = false; + } + if (this.state.toLanguage) { + const option = this.languageFrom.querySelector( + `[value=${this.state.toLanguage}]` + ); + if (option) { + option.hidden = true; + } + } + if (this.state.fromLanguage) { + const option = this.languageTo.querySelector( + `[value=${this.state.fromLanguage}]` + ); + if (option) { + option.hidden = true; + } + } + this.state.maybeUpdateDetectedLanguage(); + } + + /** + * Define the direction of the language message text, otherwise it might not display + * correctly. For instance English in an RTL UI would display incorrectly like so: + * + * LTR text in LTR UI: + * + * ┌──────────────────────────────────────────────┐ + * │ This is in English. │ + * └──────────────────────────────────────────────┘ + * + * LTR text in RTL UI: + * ┌──────────────────────────────────────────────┐ + * │ .This is in English │ + * └──────────────────────────────────────────────┘ + * + * LTR text in RTL UI, but in an LTR container: + * ┌──────────────────────────────────────────────┐ + * │ This is in English. │ + * └──────────────────────────────────────────────┘ + * + * The effects are similar, but reversed for RTL text in an LTR UI. + */ + #updateMessageDirections() { + if (this.state.toLanguage) { + this.translationTo.setAttribute( + "dir", + AT_getScriptDirection(this.state.toLanguage) + ); + } else { + this.translationTo.removeAttribute("dir"); + } + if (this.state.fromLanguage) { + this.translationFrom.setAttribute( + "dir", + AT_getScriptDirection(this.state.fromLanguage) + ); + } else { + this.translationFrom.removeAttribute("dir"); + } + } + + setupTextarea() { + this.state.setMessageToTranslate(this.translationFrom.value); + this.translationFrom.addEventListener("input", () => { + this.state.setMessageToTranslate(this.translationFrom.value); + }); + } + + disableUI() { + this.translationFrom.disabled = true; + this.languageFrom.disabled = true; + this.languageTo.disabled = true; + } + + /** + * @param {string} message + */ + updateTranslation(message) { + this.translationTo.innerText = message; + if (message) { + this.translationTo.style.visibility = "visible"; + this.translationToBlank.style.visibility = "hidden"; + this.hideInfo(); + } else { + this.translationTo.style.visibility = "hidden"; + this.translationToBlank.style.visibility = "visible"; + } + } +} + +/** + * Listen for events coming from the AboutTranslations actor. + */ +window.addEventListener("AboutTranslationsChromeToContent", ({ detail }) => { + switch (detail.type) { + case "enable": { + // While the feature is in development, hide the feature behind a pref. See the + // "browser.translations.enable" pref in modules/libpref/init/all.js and Bug 971044 + // for the status of enabling this project. + if (window.translationsState) { + throw new Error("about:translations was already initialized."); + } + AT_isTranslationEngineSupported().then(isSupported => { + window.translationsState = new TranslationsState(isSupported); + }); + document.body.style.visibility = "visible"; + break; + } + default: + throw new Error("Unknown AboutTranslationsChromeToContent event."); + } +}); + +/** + * Debounce a function so that it is only called after some wait time with no activity. + * This is good for grouping text entry via keyboard. + * + * @param {Object} settings + * @param {Function} settings.onDebounce + * @param {Function} settings.doEveryTime + * @returns {Function} + */ +function debounce({ onDebounce, doEveryTime }) { + /** @type {number | null} */ + let timeoutId = null; + let lastDispatch = null; + + return (...args) => { + doEveryTime(...args); + + const now = Date.now(); + if (lastDispatch === null) { + // This is the first call to the function. + lastDispatch = now; + } + + const timeLeft = lastDispatch + window.DEBOUNCE_DELAY - now; + + // Always discard the old timeout, either the function will run, or a new + // timer will be scheduled. + clearTimeout(timeoutId); + + if (timeLeft <= 0) { + // It's been long enough to go ahead and call the function. + timeoutId = null; + lastDispatch = null; + window.DEBOUNCE_RUN_COUNT += 1; + onDebounce(...args); + return; + } + + // Re-set the timeout with the current time left. + clearTimeout(timeoutId); + + timeoutId = setTimeout(() => { + // Timeout ended, call the function. + timeoutId = null; + lastDispatch = null; + window.DEBOUNCE_RUN_COUNT += 1; + onDebounce(...args); + }, timeLeft); + }; +} + +/** + * Perform transalations over a `MessagePort`. This class manages the communications to + * the translations engine. + */ +class Translator { + /** + * @type {MessagePort} + */ + #port; + + /** + * An id for each message sent. This is used to match up the request and response. + */ + #nextMessageId = 0; + + /** + * Tie together a message id to a resolved response. + * @type {Map<number, TranslationRequest} + */ + #requests = new Map(); + + engineStatus = "initializing"; + + /** + * @param {MessagePort} port + */ + constructor(port) { + this.#port = port; + + // Create a promise that will be resolved when the engine is ready. + let engineLoaded; + let engineFailed; + this.ready = new Promise((resolve, reject) => { + engineLoaded = resolve; + engineFailed = reject; + }); + + // Match up a response on the port to message that was sent. + port.onmessage = ({ data }) => { + switch (data.type) { + case "TranslationsPort:TranslationResponse": { + const { targetText, messageId } = data; + // A request may not match match a messageId if there is a race during the pausing + // and discarding of the queue. + this.#requests.get(messageId)?.resolve(targetText); + break; + } + case "TranslationsPort:GetEngineStatusResponse": { + if (data.status === "ready") { + engineLoaded(); + } else { + engineFailed(); + } + break; + } + default: + AT_logError("Unknown translations port message: " + data.type); + break; + } + }; + + port.postMessage({ type: "TranslationsPort:GetEngineStatusRequest" }); + } + + /** + * Opens up a port and creates a new translator. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {Promise<Translator>} + */ + static create(fromLanguage, toLanguage) { + return new Promise((resolve, reject) => { + AT_createTranslationsPort(fromLanguage, toLanguage); + + function getResponse({ data }) { + if ( + data.type == "GetTranslationsPort" && + fromLanguage === data.fromLanguage && + toLanguage === data.toLanguage + ) { + // The response matches, resolve the port. + const translator = new Translator(data.port); + + // Resolve the translator once it is ready, or propagate the rejection + // if it failed. + translator.ready.then(() => resolve(translator), reject); + window.removeEventListener("message", getResponse); + } + } + + // Listen for a response for the message port. + window.addEventListener("message", getResponse); + }); + } + + /** + * Send a request to translate text to the Translations Engine. If it returns `null` + * then the request is stale. A rejection means there was an error in the translation. + * This request may be queued. + * + * @param {string} sourceText + * @returns {Promise<string>} + */ + translate(sourceText) { + return new Promise((resolve, reject) => { + const messageId = this.#nextMessageId++; + // Store the "resolve" for the promise. It will be matched back up with the + // `messageId` in #handlePortMessage. + const isHTML = AT_isHtmlTranslation(); + this.#requests.set(messageId, { + sourceText, + isHTML, + resolve, + reject, + }); + this.#port.postMessage({ + type: "TranslationsPort:TranslationRequest", + messageId, + sourceText, + isHTML, + }); + }); + } + + /** + * Close the port and remove any pending or queued requests. + */ + destroy() { + this.#port.close(); + } +} diff --git a/toolkit/components/translations/docs/img/about-translations.png b/toolkit/components/translations/docs/img/about-translations.png Binary files differnew file mode 100644 index 0000000000..51c4d12042 --- /dev/null +++ b/toolkit/components/translations/docs/img/about-translations.png diff --git a/toolkit/components/translations/docs/index.md b/toolkit/components/translations/docs/index.md new file mode 100644 index 0000000000..7a13d5cc1c --- /dev/null +++ b/toolkit/components/translations/docs/index.md @@ -0,0 +1,17 @@ +# Firefox Translations + +Firefox Translations is a project initiative to give Firefox the ability to translate web content from one language to another language as first-class functionality in the browser. + +This project is based initially on the [Firefox Translations WebExtension]. + +## Resources + +```{toctree} +:titlesonly: +:maxdepth: 1 +:glob: + +resources/* +``` + +[Firefox Translations WebExtension]: https://github.com/mozilla/firefox-translations diff --git a/toolkit/components/translations/docs/resources/01_overview.md b/toolkit/components/translations/docs/resources/01_overview.md new file mode 100644 index 0000000000..632b3002df --- /dev/null +++ b/toolkit/components/translations/docs/resources/01_overview.md @@ -0,0 +1,145 @@ +# Overview + +The following is a high-level overview of the technologies associated with Firefox Translations. + +- [Supported Platforms](#supported-platforms) +- [Language Translations](#language-translation) + - [Technology](#technology) + - [Models](#models) + - [Pivot Translations](#pivot-translations) +- [Language Identification](#language-identification) + - [Technology](#technology-1) + - [Models](#models-1) +- [Remote Settings](#remote-settings) + - [Enabling Firefox Translations](#enabling-firefox-translations) + - [Translating Web Pages](#translating-web-pages) + - [about:translations](#abouttranslations) + +--- +## Supported Platforms + +<input type="checkbox" style="pointer-events: none;" checked><b>Desktop</b><br> +<input type="checkbox" style="pointer-events: none;" checked><b>Android</b><br> +<input type="checkbox" style="pointer-events: none;"><b>iOS</b><br> + +```{note} +- Firefox Translations is available only on devices support [SSE4.1] due to required [SIMD] calculations in [WASM]. +``` + + +--- +## Language Translation + +Firefox Translations utilizes trained machine-learning models that run locally on client +architecture to translate web content from one language to another. + +### Technology + +Firefox Translations utilizes a [WASM] version of the [Bergamot] library to translate from +one language to another. [Bergamot] is powered by [Marian]. + +### Models + +[Bergamot] translation models are single-direction, one-to-one models trained to translate from one language +to one other language (e.g. **`en ⟶ es`**). When Firefox Translations determines a source language and a target language +it utilizes a model specific to this language pair to translate from one to the other. + +### Pivot Translations + +In the event that there is no model to translate directly from a source language and a target language, +Firefox Translations will attempt to satisfy a transitive translation path and will perform a multi-step +translation from the source language to the target language. + + + +```{admonition} Example +> **_No direct translation model exists_**<br> +> <input type="checkbox" style="pointer-events: none;"><b>`es ⟶ fr`</b><br> +> +> **_Transitive dependency satisfied_**<br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`es ⟶ en`</b><br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`en ⟶ fr`</b><br> +> +> **_Pivot translation_**<br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`es ⟶ en ⟶ fr`</b><br> + +In this example, no direct model exists for **`es ⟶ fr`**, but a transitive dependency is satisfied by the two +models for **`es ⟶ en`** and **`en ⟶ fr`**. Firefox Translations will pivot on the **`en`** language by first +translating from **`es`** to **`en`** and then from **`en`** to **`fr`**. +``` +```{note} +- Firefox Translations will not pivot more than once. +- At present, only **`en`** is used as a pivot language. +``` + +--- +## Language Identification + +Firefox Translations utilizes trained machine-learning models that run locally on client +architecture to identify content as being written in a detected language. + +### Technology + +Firefox Translations utilizes a [CLD2] language detector to identify in which language content is written. + +### Models + +No models are currently used for language identification, since [CLD2] exists in the Firefox source tree. + +--- +## Remote Settings + +Remote Settings is not currently used for language identification, since [CLD2] exists in the Firefox source tree. + +--- +## Using Firefox Translations + +The following documentation describes a high-level overview of using Firefox Translations. + +```{note} +- Firefox Translations is actively under development and is currently available only in [Firefox Nightly]. +``` + +### Enabling Firefox Translations + +Firefox Translations functionality can be enabled by modifying the [translations preferences] in **`about:config`**. + +These configurations are likely to change as the project develops which is why this documentation links to them +in the source code rather than defining them. + +At a time when the preferences are more stable, they can be documented here more clearly. + +### Translating Web Pages + +Once Firefox Translations is enabled, Firefox will analyze each web page to determine if it is translatable +via the available translations models. + +If the web page is translatable, then a translations icon will appear in the URL bar of the browser, allowing +the user to initiate the available translation process. + +### about:translations + +When Firefox Translations is enabled, a page called **`about:translations`** becomes available in the browser. + +This is a test page where there user can select a source language and a target language by typing content into +the source-language text box and seeing the translated text in the target-language text box. + +```{note} +**`about:translations`** is a developer-focused UI that is useful for testing the state, performance, and quality of the language models in an interactive environment. It is fairly unpolished and not intended to be shipped as a product at this time. + +It is, however, useful and fun, so it is documented here. +``` + + + + +<!-- Hyperlinks --> +[Bergamot]: https://browser.mt/ +[CLD2]: https://github.com/CLD2Owners/cld2 +[Firefox Nightly]: https://www.mozilla.org/en-US/firefox/channel/desktop/ +[Marian]: https://aclanthology.org/P18-4020/ +[Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ +[SIMD]: https://en.wikipedia.org/wiki/Single_instruction,_multiple_data +[SSE4.1]: https://en.wikipedia.org/wiki/SSE4#SSE4.1 +[translations preferences]: https://searchfox.org/mozilla-central/search?q=browser.translations&path=all.js&case=true®exp=false +[WASM]: https://webassembly.org/ diff --git a/toolkit/components/translations/docs/resources/02_contributing.md b/toolkit/components/translations/docs/resources/02_contributing.md new file mode 100644 index 0000000000..c093ee550d --- /dev/null +++ b/toolkit/components/translations/docs/resources/02_contributing.md @@ -0,0 +1,157 @@ +# Contributing + +The following content goes more in-depth than the [Overview](./01_overview.md) section +to provide helpful information regarding contributing to Firefox Translations. + +- [Source Code](#source-code) +- [Architecture](#architecture) + - [JSActors](#jsactors) +- [Remote Settings](#remote-settings) + - [Admin Dashboards](#admin-dashboards) + - [Prod Admin Access](#prod-admin-access) + - [Pulling From Different Sources](#pulling-from-different-sources) + - [Versioning](#versioning) + - [Non-Breaking Changes](#non-breaking-changes) + - [Breaking Changes](#breaking-changes) +- [Language Identification](#language-identification) + +--- +## Source Code + +The primary source code for Firefox Translations lives in the following directory: + +> **[toolkit/components/translations]** + +--- +## Architecture + +### JSActors + +Translations functionality is divided into different classes based on which access privileges are needed. +Generally, this split is between [Parent] and [Child] versions of [JSWindowActors]. + +The [Parent] actors have access to privileged content and is responsible for things like downloading models +from [Remote Settings](#remote-settings), modifying privileged UI components etc. + +The [Child] actors are responsible for interacting with content on the page itself, requesting content from +the [Parent] actors, and creating [Workers] to carry out tasks. + +--- +## Remote Settings + +The machine-learning models and [WASM] binaries are all hosted in Remote Settings and are downloaded/cached when needed. + +### Admin Dashboards + +In order to get access to Firefox Translations content in the Remote Settings admin dashboards, you will need to request +access in the Remote Settings component on [Bugzilla]. + +Once you have access to Firefox Translations content in Remote Settings, you will be able to view it in the admin dashboards: + +**Dev**<br> +> [https://settings.dev.mozaws.net/v0/admin](https://settings.dev.mozaws.net/v1/admin) + +**Stage**<br> +> [https://remote-settings.allizom.org/v0/admin](https://settings-writer.stage.mozaws.net/v1/admin) + +### Prod Admin Access + +In order to access the prod admin dashboard, you must also have access to a VPN that is authorized to view the dashboard. +To gain access to the VPN, follow [Step 3] on this page in the Remote Settings documentation. + +**Prod**<br> +> [https://remote-settings.mozilla.org/v1/admin](https://settings-writer.prod.mozaws.net/v1/admin) + + +### Pulling From Different Sources + +When you are running Firefox, you can choose to pull data from **Dev**, **Stage**, or **Prod** by downloading and installing +the latest [remote-settings-devtools] Firefox extension. + +### Versioning + +Firefox Translations uses semantic versioning for all of its records via the **`version`** property. + +#### Non-breaking Changes + +Firefox Translations code will always retrieve the maximum compatible version of each record from Remote Settings. +If two records exist with different versions, (e.g. **`1.0`** and **`1.1`**) then only the version **`1.1`** record +will be considered. + +This allows us to update and ship new versions of models that are compatible with the current source code and wasm runtimes +in both backward-compatible and forward-compatible ways. These can be released through remote settings independent of the +[Firefox Release Schedule]. + +#### Breaking Changes + +Breaking changes for Firefox Translations are a bit more tricky. These are changes that make older-version records +incompatible with the current Firefox source code and/or [WASM] runtimes. + +While a breaking change will result in a change of the semver number (e.g. **`1.1 ⟶ 2.0`**), this alone is not +sufficient. Since Firefox Translations always attempts to use the maximum compatible version, only bumping this number +would result in older versions of Firefox attempting to use a newer-version record that is no longer compatible with the +Firefox source code or [WASM] runtimes. + +To handle these changes, Firefox Translations utilizes Remote Settings [Filter Expressions] to make certain records +available to only particular releases of Firefox. This will allow Firefox Translations to make different sets of Remote Settings records available to different versions +of Firefox. + +```{admonition} Example + +Let's say that Firefox 108 through Firefox 120 is compatible with translations model records in the **`1.*`** major-version range, however Firefox 121 and onward is compatible with only model records in the **`2.*`** major-version range. + +This will allow us to mark the **`1.*`** major-version records with the following filter expression: + +**` +"filter_expression": "env.version|versionCompare('108.a0') >= 0 && env.version|versionCompare('121.a0') < 0" +`** + +This means that these records will only be available in Firefox versions greater than or equal to 108, and less than 121. + +Similarly, we will be able to mark all of the **`2.*`** major-version records with this filter expression: + +**` +"filter_expression": "env.version|versionCompare('121.a0') >= 0" +`** + +This means that these records will only be available in Firefox versions greater than or equal to Firefox 121. + +``` + +Tying breaking changes to releases in this way frees up Firefox Translations to make changes as large as entirely +switching one third-party library for another in the compiled source code, while allowing older versions of Firefox to continue utilizing the old library and allowing newer versions of Firefox to utilize the new library. + +--- +## Language Identification + +Translations currently uses the [CLD2] language detector. + +We have previously experimented with using the [fastText] language detector, but we opted to use [CLD2] due to complications with [fastText] [WASM] runtime performance. The benefit of the [CLD2] language detector is that it already exists in the Firefox source tree. In the future, we would still like to explore moving to a more modern language detector such as [CLD3], or perhaps something else. + + +<!-- Hyperlinks --> +[Bugzilla]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Cloud%20Services&component=Server%3A%20Remote%20Settings +[Child]: https://searchfox.org/mozilla-central/search?q=TranslationsChild +[CLD2]: https://github.com/CLD2Owners/cld2 +[CLD3]: https://github.com/google/cld3 +[Download and Install]: https://emscripten.org/docs/getting_started/downloads.html#download-and-install +[emscripten (2.0.3)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#203-09102020 +[emscripten (2.0.18)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#2018-04232021 +[emscripten (3.1.35)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3135---040323 +[Environments]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#environments +[eval()]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval +[fastText]: https://fasttext.cc/ +[Filter Expressions]: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions +[Firefox Release Schedule]: https://wiki.mozilla.org/Release_Management/Calendar +[generate functions]: https://emscripten.org/docs/api_reference/emscripten.h.html?highlight=dynamic_execution#functions +[Getting Set Up To Work On The Firefox Codebase]: https://firefox-source-docs.mozilla.org/setup/index.html +[importScripts()]: https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/importScripts +[JSWindowActors]: https://firefox-source-docs.mozilla.org/dom/ipc/jsactors.html#jswindowactor +[minify]: https://github.com/tdewolff/minify +[Parent]: https://searchfox.org/mozilla-central/search?q=TranslationsParent +[Step 3]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#create-a-new-official-type-of-remote-settings +[remote-settings-devtools]: https://github.com/mozilla-extensions/remote-settings-devtools/releases +[Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ +[toolkit/components/translations]: https://searchfox.org/mozilla-central/search?q=toolkit%2Fcomponents%2Ftranslations +[WASM]: https://webassembly.org/ +[Workers]: https://searchfox.org/mozilla-central/search?q=%2Ftranslations.*worker&path=&case=false®exp=true diff --git a/toolkit/components/translations/docs/resources/03_bergamot.md b/toolkit/components/translations/docs/resources/03_bergamot.md new file mode 100644 index 0000000000..a73eaa8393 --- /dev/null +++ b/toolkit/components/translations/docs/resources/03_bergamot.md @@ -0,0 +1,119 @@ +# The Bergamot Translator + +The [Bergamot Translator](https://github.com/browsermt/bergamot-translator) is the translations engine used to power Firefox translations. The project configures a fork of [Marian NMT](https://marian-nmt.github.io/) that enables translations through a Wasm API. + +Bergamot adds a few additional pieces of code on top of the Marian code, which includes HTML alignments (matching up source and target tags in a translation) and sentence iteration. It provides the [Wasm API](https://github.com/browsermt/bergamot-translator/tree/main/wasm) that Firefox uses in its own translation implementation. The Bergamot Translator uses a forked copy of the Marian NMT package in order to provide support for quantized translation models. + +--- +## Building Bergamot + +The Wasm and the JS file that integrate with Firefox can be generated using the `build-bergamot.py` script. + +```sh +cd toolkit/components/translations/bergamot-translator +./build-bergamot.py +``` + +There are a few additional options and up to date documentation for building which are documented by: + +```sh +./build-bergamot.py --help +``` + +After building, the Wasm can be loaded locally for testing by uncommenting the lines at the bottom of `toolkit/components/translations/jar.mn`. In addition, debug symbols can be built with the `--debug` option. This is useful for using the Firefox Profiler. + +--- +## Uploading to Remote Settings + +The Wasm artifact is uploaded and distributed via [Remote Settings](https://remote-settings.readthedocs.io/en/latest/index.html). An upload script is available for updating the Wasm in Remote Setting via: + +```sh +cd toolkit/components/translations/bergamot-translator +./upload-bergamot.py --help +``` + +The help flag will output up to date documentation on how to run the script. In order to do a full release: + +### Breaking changes + +If the Bergamot Translator has a breaking change, then the `BERGAMOT_MAJOR_VERSION` in `toolkit/components/translations/actors/TranslationsParent.sys.mjs` will need to be incremented by one. Any given release of Firefox will pull in minor changes when the records are updated, but major changes will need to ride the release trains. + +### Releasing + +1. Run the `./build-bergamot.py` script + +1. Bump the `remote_settings.version` in `toolkit/components/translations/bergamot-translator/moz.yaml`. + - A minor release would be `"1.0"` ➡️ `"1.1"`. + - A major release would be `"1.1"` ➡️ `"2.0"`. + +1. Run the `./upload-bergamot.py --server prod` + - Follow the instructions for adding the Bearer Token. + - By default new updates use JEXL filters and are filtered to just Nightly and local builds. + +1. Request review on the changes. + - Log in to the [Mozilla Corporate VPN](https://mozilla-hub.atlassian.net/wiki/spaces/IT/pages/15761733/Mozilla+Corporate+VPN) + - Log into the [Remote Settings admin](https://remote-settings.mozilla.org/v1/admin) + - If this is a major change, then the `filter_expression` can be removed, as the change will ride the trains. + - Request review on the changes. + +1. Verify the changes on Nightly. + - Install the [Remote Settings Devtool](https://github.com/mozilla-extensions/remote-settings-devtools/releases). + - Open the Remote Settings Devtool. + - Switch the environment to `Prod (preview)`. + - Clear all local data. + - Restart Nightly. + - Verify that it is working in Nightly by trigging different translations. + +1. Publish to Nightly + - Notify release drivers (<release-drivers@mozilla.org>) that a new translation engine release is hitting Nightly (see example emails below). This is optional for a major release, since it will ride the trains. + - Have another team member approve the release from Remote Settings. + +1. Prepare to publish to Beta / Release + - (Do not do this step if it's a major release.) + - Wait a few days to verify there are no issues on Nightly. + - Log into the [Remote Settings admin](https://remote-settings.mozilla.org/v1/admin) + - Remove the "filter_expression" text from the `bergamot-translator` version. + - Request review. + - Repeat Step 5 to verify for Beta and Release. + +1. Publish to Beta / Release + - (Do not do this step if it's a major release.) + - Notify release drivers (<release-drivers@mozilla.org>) that a new translation engine release is hitting Beta / Release (see example emails below). + - Publish the changes + - Monitor for any increased breakage via [telemetry](https://sql.telemetry.mozilla.org/dashboard/translations?p_date=d_last_7_days). + + +### Example Nightly release email + +``` +Hello Release Drivers, + +The Translations team is releasing a new version of the translations engine via remote +settings. We are releasing a test update on Nightly [Fx123], and plan to follow-up on +[DATE] with a release to both Beta [Fx123] and Release [Fx123] if we've found there are +no issues. We can roll back the release if any unexpected issues are found. + +The plan for this release is available: + +https://firefox-source-docs.mozilla.org/toolkit/components/translations/resources/03_bergamot.html#release + +Thank you, +[NAME] +``` + +### Example Beta / Release release email + +``` +Hello Release Drivers, + +The Translations team is moving forward with a release of a new translations engine +to both Beta [Fx123] and Release [Fx123]. It has been in Nightly [Fx123] with no issues +found. We can roll back the release if any unexpected issues are found. + +The plan for this release is available: + +https://firefox-source-docs.mozilla.org/toolkit/components/translations/resources/03_bergamot.html#release + +Thank you, +[NAME] +``` diff --git a/toolkit/components/translations/jar.mn b/toolkit/components/translations/jar.mn new file mode 100644 index 0000000000..b31f17718f --- /dev/null +++ b/toolkit/components/translations/jar.mn @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +toolkit.jar: + content/global/translations/bergamot-translator.js (bergamot-translator/bergamot-translator.js) + content/global/translations/translations-document.sys.mjs (content/translations-document.sys.mjs) + content/global/translations/translations-engine.html (content/translations-engine.html) + content/global/translations/translations-engine.sys.mjs (content/translations-engine.sys.mjs) + content/global/translations/translations-engine.worker.js (content/translations-engine.worker.js) + content/global/translations/translations.html (content/translations.html) + content/global/translations/translations.css (content/translations.css) + content/global/translations/translations.mjs (content/translations.mjs) + content/global/translations/TranslationsTelemetry.sys.mjs (TranslationsTelemetry.sys.mjs) + + # Uncomment this line to test a local build of Bergamot. It will automatically be loaded in. + # content/global/translations/bergamot-translator-worker.wasm (bergamot-translator/thirdparty/build-wasm/bergamot-translator-worker.wasm) diff --git a/toolkit/components/translations/metrics.yaml b/toolkit/components/translations/metrics.yaml new file mode 100644 index 0000000000..01e77c46bd --- /dev/null +++ b/toolkit/components/translations/metrics.yaml @@ -0,0 +1,608 @@ +# This Source Code Form is subject to the terms of the Mozilla Public↩ +# License, v. 2.0. If a copy of the MPL was not distributed with this↩ +# file, You can obtain one at http://mozilla.org/MPL/2.0/.↩ + +# Adding a new metric? We have docs for that!↩ +# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html↩ + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 +$tags: + - 'Firefox :: Translation' + +translations: + requests_count: + type: counter + description: > + The count of translation requests. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + error_rate: + type: rate + description: > + The rate of failed translations requests. + denominator_metric: translations.requests_count + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + error: + type: event + description: > + The specific translations error that caused a full-page translation failure. + extra_keys: + flow_id: &flow_id + type: string + description: A uuid to relate events associated with the current panel session. + first_interaction: &first_interaction + type: boolean + description: > + True if this event occurred in the user's first interaction with translations, + otherwise false. + reason: + type: string + description: The reason for the error. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436#c4 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366#c11 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + translation_request: + type: event + description: > + Triggers when a full-page translation request is sent. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + from_language: + type: string + description: The language being translated from. + to_language: + type: string + description: The language being translated to. + document_language: + type: string + description: The detected BCP-47 language tag of the document. + top_preferred_language: + type: string + description: The top preferred language for this user. + auto_translate: + type: boolean + description: > + True if this translation was triggered automatically + due to configured always-translate-language settings. + False if this translation was triggered manually. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836381 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1847150 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836381#c4 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366#c11 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428#c16 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1847150#c4 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + restore_page: + type: event + description: > + Triggers when the a restore-page event is triggered. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + +translations.panel: + open: + type: event + description: > + Triggers when the translations panel is opened. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + auto_show: + type: boolean + description: Whether the panel was opened automatically or manually by the user. + view_name: + type: string + description: The type of view shown in the panel. + opened_from: + type: string + description: The method by which the translations panel was opened. + document_language: + type: string + description: The detected BCP-47 language tag of the document. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1835502 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1847150 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1835502#c7 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1841366#c11 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428#c16 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1847150#c4 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + close: + type: event + description: > + Triggers when translations panel is closed. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + open_from_language_menu: + type: event + description: > + Triggers when the from-language dropdown is opened in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + change_from_language: + type: event + description: > + Triggers when the from-language selected value is changed. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + language: + type: string + description: > + - The BCP47 language tag. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + close_from_language_menu: + type: event + description: > + Triggers when the from-language dropdown is closed in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + open_to_language_menu: + type: event + description: > + Triggers when the from-language dropdown is opened in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + change_to_language: + type: event + description: > + Triggers when the to-language selected value is changed. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + language: + type: string + description: > + - The BCP47 language tag. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + close_to_language_menu: + type: event + description: > + Triggers when the from-language dropdown is closed in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + open_settings_menu: + type: event + description: > + Triggers when the settings menu is opened in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + close_settings_menu: + type: event + description: > + Triggers when the settings menu is closed in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + cancel_button: + type: event + description: > + Triggers when the cancel button is invoked in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + change_source_language_button: + type: event + description: > + Triggers when the change-source-language button is invoked in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + dismiss_error_button: + type: event + description: > + Triggers when the dismiss-error button is invoked in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + restore_page_button: + type: event + description: > + Triggers when the restore-page button is invoked in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + translate_button: + type: event + description: > + Triggers when the translate button is invoked in the translations panel. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + always_offer_translations: + type: event + description: > + Triggers when the always-offer-translations menuitem is invoked in the translations panel seeings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + toggled_on: + type: boolean + description: > + - Whether the setting was toggled on or off. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1842646 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1842646 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + always_translate_language: + type: event + description: > + Triggers when the always-translate-language menuitem is invoked in the translations panel settings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + language: + type: string + description: > + - The BCP47 language tag. + toggled_on: + type: boolean + description: > + - Whether the setting was toggled on or off for this language. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + never_translate_language: + type: event + description: > + Triggers when the never-translate-language menuitem is invoked in the translations panel settings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + language: + type: string + description: > + - The BCP47 language tag. + toggled_on: + type: boolean + description: > + - Whether the setting was toggled on or off for this language. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + never_translate_site: + type: event + description: > + Triggers when the never-translate-site menuitem is invoked in the translations panel settings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + toggled_on: + type: boolean + description: > + - Whether the setting was toggled on or off for this site. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + manage_languages: + type: event + description: > + Triggers when the manage-languages menuitem is invoked in the translations panel settings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + about_translations: + type: event + description: > + Triggers when the about-translations menuitem is invoked in the translations panel settings. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never + + learn_more: + type: event + description: > + Triggers when the learn-more link is invoked in the translations panel first interaction. + extra_keys: + flow_id: *flow_id + first_interaction: *first_interaction + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1845428 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1861319 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: never diff --git a/toolkit/components/translations/moz.build b/toolkit/components/translations/moz.build new file mode 100644 index 0000000000..cdb0ca4e75 --- /dev/null +++ b/toolkit/components/translations/moz.build @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SPHINX_TREES["/toolkit/components/translations"] = "docs" + +JAR_MANIFESTS += ["jar.mn"] + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Translation") + +DIRS += ["actors"] + +BROWSER_CHROME_MANIFESTS += ["tests/browser/browser.toml"] + +with Files("docs/**"): + SCHEDULES.exclusive = ["docs"] diff --git a/toolkit/components/translations/tests/browser/browser.toml b/toolkit/components/translations/tests/browser/browser.toml new file mode 100644 index 0000000000..50a1be7150 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser.toml @@ -0,0 +1,53 @@ +[DEFAULT] +support-files = [ + "head.js", + "shared-head.js", + "translations-test.mjs", + "translations-tester-empty-pdf-file.pdf", + "translations-tester-en.html", + "translations-tester-es.html", + "translations-tester-es-2.html", + "translations-tester-fr.html", + "translations-tester-no-tag.html", + "translations-tester-shadow-dom-es.html", + "translations-tester-shadow-dom-mutation-es.html", + "translations-tester-shadow-dom-mutation-es-2.html", + "translations-tester-shadow-dom-slot-es.html", +] + +["browser_about_translations_debounce.js"] +skip-if = ["os == 'linux'"] # Bug 1821461 + +["browser_about_translations_directions.js"] + +["browser_about_translations_dropdowns.js"] + +["browser_about_translations_enabling.js"] + +["browser_about_translations_translations.js"] + +["browser_translations_actor.js"] + +["browser_translations_actor_detected_langs.js"] + +["browser_translations_actor_empty_langs.js"] + +["browser_translations_actor_preferred_language.js"] + +["browser_translations_actor_sync_models.js"] + +["browser_translations_actor_versioning.js"] + +["browser_translations_full_page.js"] + +["browser_translations_lang_tags.js"] + +["browser_translations_pdf_is_disabled.js"] + +["browser_translations_remote_settings.js"] + +["browser_translations_shadow_dom.js"] + +["browser_translations_shadow_dom_mutation.js"] + +["browser_translations_translation_document.js"] diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js b/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js new file mode 100644 index 0000000000..c1a5a3ae2c --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js @@ -0,0 +1,91 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Test the debounce behavior. + */ +add_task(async function test_about_translations_debounce() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr" }, + { fromLang: "fr", toLang: "en" }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + // Do not allow the debounce to come to completion. + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"`, + 100, + 200 + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + function setInput(element, value) { + element.value = value; + element.dispatchEvent(new Event("input")); + } + setInput(fromSelect, "en"); + setInput(toSelect, "fr"); + setInput(translationTextarea, "T"); + + info("Get the translations into a stable translationed state"); + await assertTranslationResult("T [en to fr]"); + + info("Reset and pause the debounce state."); + Cu.waiveXrays(window).DEBOUNCE_DELAY = 1_000_000_000; + Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT = 0; + + info("Input text which will be debounced."); + setInput(translationTextarea, "T"); + setInput(translationTextarea, "Te"); + setInput(translationTextarea, "Tex"); + is(Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT, 0, "Debounce has not run."); + + info("Allow the debounce to actually come to completion."); + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; + setInput(translationTextarea, "Text"); + + await assertTranslationResult("TEXT [en to fr]"); + is(Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT, 1, "Debounce ran once."); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_directions.js b/toolkit/components/translations/tests/browser/browser_about_translations_directions.js new file mode 100644 index 0000000000..1bd02256b1 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_directions.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_about_translations_language_directions() { + await openAboutTranslations({ + languagePairs: [ + // English (en) is LTR and Arabic (ar) is RTL. + { fromLang: "en", toLang: "ar" }, + { fromLang: "ar", toLang: "en" }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "ar"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This text starts as LTR."; + translationTextarea.dispatchEvent(new Event("input")); + + is( + window.getComputedStyle(translationTextarea).direction, + "ltr", + "The English input is LTR" + ); + is( + window.getComputedStyle(translationResult).direction, + "rtl", + "The Arabic results are RTL" + ); + + toSelect.value = ""; + toSelect.dispatchEvent(new Event("input")); + fromSelect.value = "ar"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "en"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This text starts as RTL."; + translationTextarea.dispatchEvent(new Event("input")); + + is( + window.getComputedStyle(translationTextarea).direction, + "rtl", + "The Arabic input is RTL" + ); + is( + window.getComputedStyle(translationResult).direction, + "ltr", + "The English results are LTR" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js b/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js new file mode 100644 index 0000000000..6aed512952 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js @@ -0,0 +1,112 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_about_translations_dropdowns() { + let languagePairs = [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en" }, + ]; + await openAboutTranslations({ + languagePairs, + dataForContent: languagePairs, + runInPage: async ({ dataForContent: languagePairs, selectors }) => { + const { document } = content; + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** + * Some languages can be marked as hidden in the dropbdown. This function + * asserts the configuration of the options. + * + * @param {object} args + * @param {string} args.message + * @param {HTMLSelectElement} args.select + * @param {string[]} args.availableOptions + * @param {string} args.selectedValue + */ + function assertOptions({ + message, + select, + availableOptions, + selectedValue, + }) { + const options = [...select.options]; + info(message); + Assert.deepEqual( + options.filter(option => !option.hidden).map(option => option.value), + availableOptions, + "The available options match." + ); + + is(selectedValue, select.value, "The selected value matches."); + } + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + + assertOptions({ + message: 'From languages have "detect" already selected.', + select: fromSelect, + availableOptions: ["detect", "en", "is", "es"], + selectedValue: "detect", + }); + + assertOptions({ + message: + 'The "to" options do not have "detect" in the list, and nothing is selected.', + select: toSelect, + availableOptions: ["", "en", "es"], + selectedValue: "", + }); + + info('Switch the "to" language to "es".'); + toSelect.value = "es"; + toSelect.dispatchEvent(new Event("input")); + + assertOptions({ + message: 'The "from" languages no longer suggest "es".', + select: fromSelect, + availableOptions: ["detect", "en", "is"], + selectedValue: "detect", + }); + + assertOptions({ + message: 'The "to" options remain the same, but "es" is selected.', + select: toSelect, + availableOptions: ["", "en", "es"], + selectedValue: "es", + }); + + info('Switch the "from" language to English.'); + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + + assertOptions({ + message: 'The "to" languages no longer suggest "en".', + select: toSelect, + availableOptions: ["", "es"], + selectedValue: "es", + }); + + assertOptions({ + message: 'The "from" options remain the same, but "en" is selected.', + select: fromSelect, + availableOptions: ["detect", "en", "is"], + selectedValue: "en", + }); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js b/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js new file mode 100644 index 0000000000..cde5fd03b4 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js @@ -0,0 +1,137 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Checks that the page renders without issue, and that the expected elements + * are there. + */ +add_task(async function test_about_translations_enabled() { + await openAboutTranslations({ + runInPage: async ({ selectors }) => { + const { document, window } = content; + + await ContentTaskUtils.waitForCondition( + () => { + const trElement = document.querySelector(selectors.translationResult); + const trBlankElement = document.querySelector( + selectors.translationResultBlank + ); + const { visibility: trVisibility } = + window.getComputedStyle(trElement); + const { visibility: trBlankVisibility } = + window.getComputedStyle(trBlankElement); + return trVisibility === "hidden" && trBlankVisibility === "visible"; + }, + `Waiting for placeholder text to be visible."`, + 100, + 200 + ); + + function checkElementIsVisible(expectVisible, name) { + const expected = expectVisible ? "visible" : "hidden"; + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is( + visibility, + expected, + `Element ${name} was not ${expected} but should be.` + ); + } + + checkElementIsVisible(true, "pageHeader"); + checkElementIsVisible(true, "fromLanguageSelect"); + checkElementIsVisible(true, "toLanguageSelect"); + checkElementIsVisible(true, "translationTextarea"); + checkElementIsVisible(true, "translationResultBlank"); + + checkElementIsVisible(false, "translationResult"); + }, + }); +}); + +/** + * Checks that the page does not show the content when disabled. + */ +add_task(async function test_about_translations_disabled() { + await openAboutTranslations({ + disabled: true, + runInPage: async ({ selectors }) => { + const { document, window } = content; + + await ContentTaskUtils.waitForCondition( + () => { + const element = document.querySelector(selectors.translationResult); + const { visibility } = window.getComputedStyle(element); + return visibility === "hidden"; + }, + `Waiting for translated text to be hidden."`, + 100, + 200 + ); + + function checkElementIsInvisible(name) { + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is(visibility, "hidden", `Element ${name} was invisible.`); + } + + checkElementIsInvisible("pageHeader"); + checkElementIsInvisible("fromLanguageSelect"); + checkElementIsInvisible("toLanguageSelect"); + checkElementIsInvisible("translationTextarea"); + checkElementIsInvisible("translationResult"); + checkElementIsInvisible("translationResultBlank"); + }, + }); +}); + +/** + * Test that the page is properly disabled when the engine isn't supported. + */ +add_task(async function test_about_translations_disabling() { + await openAboutTranslations({ + prefs: [["browser.translations.simulateUnsupportedEngine", true]], + runInPage: async ({ selectors }) => { + const { document, window } = content; + + info('Checking for the "no support" message.'); + await ContentTaskUtils.waitForCondition( + () => document.querySelector(selectors.noSupportMessage), + 'Waiting for the "no support" message.', + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + + ok(fromSelect.disabled, "The from select is disabled"); + ok(toSelect.disabled, "The to select is disabled"); + ok(translationTextarea.disabled, "The textarea is disabled"); + + function checkElementIsVisible(expectVisible, name) { + const expected = expectVisible ? "visible" : "hidden"; + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is( + visibility, + expected, + `Element ${name} was not ${expected} but should be.` + ); + } + + checkElementIsVisible(true, "translationInfo"); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_translations.js b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js new file mode 100644 index 0000000000..033adf5ad8 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js @@ -0,0 +1,272 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Test that the UI actually translates text, but use a mocked translations engine. + * The results of the "translation" will be modifying the text to be full width latin + * characters, so that the results visually appear modified. + */ +add_task(async function test_about_translations_translations() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr" }, + { fromLang: "fr", toLang: "en" }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en" }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"`, + 100, + 200 + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr]"); + is( + translationResult.getAttribute("lang"), + "fr", + "The result is listed as in French." + ); + + // Blank out the "to" select so it doesn't try to translate between is to fr. + toSelect.value = ""; + toSelect.dispatchEvent(new Event("input")); + + fromSelect.value = "is"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "en"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This is the second translation."; + translationTextarea.dispatchEvent(new Event("input")); + + await assertTranslationResult( + "THIS IS THE SECOND TRANSLATION. [is to en]" + ); + is( + translationResult.getAttribute("lang"), + "en", + "The result is listed as in English." + ); + }, + }); +}); + +/** + * Test the useHTML pref. + */ +add_task(async function test_about_translations_html() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr" }, + { fromLang: "fr", toLang: "en" }, + ], + prefs: [["browser.translations.useHTML", true]], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"`, + 100, + 200 + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr, html]"); + }, + }); +}); + +add_task(async function test_about_translations_language_identification() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr" }, + { fromLang: "fr", toLang: "en" }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition( + () => { + return document.body.hasAttribute("ready"); + }, + "Waiting for the document to be ready.", + 100, + 200 + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"`, + 100, + 200 + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + is( + translation, + translationResult.innerText, + "The language identification correctly informs the translation." + ); + } + + const fromSelectStartValue = fromSelect.value; + const detectStartText = fromSelect.options[0].textContent; + + is( + fromSelectStartValue, + "detect", + 'The fromSelect starting value is "detect"' + ); + + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + + await ContentTaskUtils.waitForCondition( + () => { + const element = document.querySelector( + selectors.translationResultBlank + ); + const { visibility } = window.getComputedStyle(element); + return visibility === "hidden"; + }, + `Waiting for placeholder text to be visible."`, + 100, + 200 + ); + + const fromSelectFinalValue = fromSelect.value; + is( + fromSelectFinalValue, + fromSelectStartValue, + "The fromSelect value has not changed" + ); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr]"); + + const detectFinalText = fromSelect.options[0].textContent; + is( + true, + detectFinalText.startsWith(detectStartText) && + detectFinalText.length > detectStartText.length, + `fromSelect starting display text (${detectStartText}) should be a substring of the final text (${detectFinalText})` + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor.js b/toolkit/components/translations/tests/browser/browser_translations_actor.js new file mode 100644 index 0000000000..457d032ea9 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor.js @@ -0,0 +1,234 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This file contains unit tests for the translations actor. Generally it's preferable + * to test behavior in a full integration test, but occasionally it's useful to test + * specific implementation behavior. + */ + +/** + * Enforce the pivot language behavior in ensureLanguagePairsHavePivots. + */ +add_task(async function test_pivot_language_behavior() { + info( + "Expect 4 console.error messages notifying of the lack of a pivot language." + ); + + const fromLanguagePairs = [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "yue" }, + { fromLang: "yue", toLang: "en" }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en" }, + // These are non-pivot languages. + { fromLang: "zh", toLang: "ja" }, + { fromLang: "ja", toLang: "zh" }, + ]; + + // Sort the language pairs, as the order is not guaranteed. + function sort(list) { + return list.sort((a, b) => + `${a.fromLang}-${a.toLang}`.localeCompare(`${b.fromLang}-${b.toLang}`) + ); + } + + const { cleanup } = await setupActorTest({ + languagePairs: fromLanguagePairs, + }); + + const { languagePairs } = await TranslationsParent.getSupportedLanguages(); + + // The pairs aren't guaranteed to be sorted. + languagePairs.sort((a, b) => + TranslationsParent.languagePairKey(a.fromLang, a.toLang).localeCompare( + TranslationsParent.languagePairKey(b.fromLang, b.toLang) + ) + ); + + if (SpecialPowers.isDebugBuild) { + Assert.deepEqual( + sort(languagePairs), + sort([ + { fromLang: "en", toLang: "es" }, + { fromLang: "en", toLang: "yue" }, + { fromLang: "es", toLang: "en" }, + { fromLang: "is", toLang: "en" }, + { fromLang: "yue", toLang: "en" }, + ]), + "Non-pivot languages were removed on debug builds." + ); + } else { + Assert.deepEqual( + sort(languagePairs), + sort(fromLanguagePairs), + "Non-pivot languages are retained on non-debug builds." + ); + } + + return cleanup(); +}); + +async function usingAppLocale(locale, callback) { + info(`Mocking the locale "${locale}", expect missing resource errors.`); + const { availableLocales, requestedLocales } = Services.locale; + Services.locale.availableLocales = [locale]; + Services.locale.requestedLocales = [locale]; + + if (Services.locale.appLocaleAsBCP47 !== locale) { + throw new Error("Unable to change the app locale."); + } + await callback(); + + // Reset back to the originals. + Services.locale.availableLocales = availableLocales; + Services.locale.requestedLocales = requestedLocales; +} + +add_task(async function test_translating_to_and_from_app_language() { + const PIVOT_LANGUAGE = "en"; + + const { cleanup } = await setupActorTest({ + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + /** + * Each language pair has multiple models. De-duplicate the language pairs and + * return a sorted list. + */ + function getUniqueLanguagePairs(records) { + const langPairs = new Set(); + for (const { fromLang, toLang } of records) { + langPairs.add(TranslationsParent.languagePairKey(fromLang, toLang)); + } + return Array.from(langPairs) + .sort() + .map(langPair => { + const [fromLang, toLang] = langPair.split(","); + return { + fromLang, + toLang, + }; + }); + } + + function assertLanguagePairs({ + app, + requested, + message, + languagePairs, + isForDeletion, + }) { + return usingAppLocale(app, async () => { + Assert.deepEqual( + getUniqueLanguagePairs( + await TranslationsParent.getRecordsForTranslatingToAndFromAppLanguage( + requested, + isForDeletion + ) + ), + languagePairs, + message + ); + }); + } + + await assertLanguagePairs({ + message: + "When the app locale is the pivot language, download another language.", + app: PIVOT_LANGUAGE, + requested: "fr", + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "When a pivot language is required, they are both downloaded.", + app: "fr", + requested: "pl", + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "When downloading the pivot language, only download the one for the app's locale.", + app: "es", + requested: PIVOT_LANGUAGE, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "Delete just the requested language when the app locale is the pivot language", + app: PIVOT_LANGUAGE, + requested: "fr", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the requested language, and not the pivot.", + app: "fr", + requested: "pl", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the requested language, and not the pivot.", + app: "fr", + requested: "pl", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the pivot → app and app → pivot.", + app: "es", + requested: PIVOT_LANGUAGE, + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "If the app and request language are the same, nothing is returned.", + app: "fr", + requested: "fr", + languagePairs: [], + }); + + return cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js b/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js new file mode 100644 index 0000000000..65479a968e --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_detected_langs.js @@ -0,0 +1,86 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_detected_language() { + const { cleanup, tab } = await loadTestPage({ + // This page will get its language changed by the test. + page: ENGLISH_PAGE_URL, + autoDownloadFromRemoteSettings: true, + languagePairs: [ + // Spanish + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + + // Norwegian Bokmål + { fromLang: "nb", toLang: "en" }, + { fromLang: "en", toLang: "nb" }, + ], + }); + + async function getDetectedLanguagesFor(docLangTag) { + await ContentTask.spawn( + tab.linkedBrowser, + { docLangTag }, + function changeLanguage({ docLangTag }) { + content.document.body.parentNode.setAttribute("lang", docLangTag); + } + ); + // Clear out the cached values. + getTranslationsParent().languageState.detectedLanguages = null; + return getTranslationsParent().getDetectedLanguages(docLangTag); + } + + Assert.deepEqual( + await getDetectedLanguagesFor("es"), + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + }, + "Spanish is detected as a supported language." + ); + + Assert.deepEqual( + await getDetectedLanguagesFor("chr"), + { + docLangTag: "chr", + userLangTag: "en", + isDocLangTagSupported: false, + }, + "Cherokee is detected, but is not a supported language." + ); + + Assert.deepEqual( + await getDetectedLanguagesFor("no"), + { + docLangTag: "nb", + userLangTag: "en", + isDocLangTagSupported: true, + }, + "The Norwegian macro language is detected, but it defaults to Norwegian Bokmål." + ); + + Assert.deepEqual( + await getDetectedLanguagesFor("spa"), + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + }, + 'The three letter "spa" locale is canonicalized to "es".' + ); + + Assert.deepEqual( + await getDetectedLanguagesFor("gibberish"), + { + docLangTag: "en", + userLangTag: null, + isDocLangTagSupported: true, + }, + "A gibberish locale is discarded, and the language is detected." + ); + + return cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js b/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js new file mode 100644 index 0000000000..132bb9a1f2 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_empty_langs.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Test some corner cases from Bug 1849815 where empty web languages were causing + * issues. + */ +add_task(async function test_detected_language() { + const { cleanup, tab } = await loadTestPage({ + // This page will get its language changed by the test. + page: ENGLISH_PAGE_URL, + autoDownloadFromRemoteSettings: true, + // Empty out the accept languages. + languagePairs: [ + // Spanish + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + // French + { fromLang: "fr", toLang: "en" }, + { fromLang: "en", toLang: "fr" }, + ], + }); + + async function getDetectedLanguagesFor(docLangTag) { + await ContentTask.spawn( + tab.linkedBrowser, + { docLangTag }, + function changeLanguage({ docLangTag }) { + content.document.body.parentNode.setAttribute("lang", docLangTag); + } + ); + // Clear out the cached values. + getTranslationsParent().languageState.detectedLanguages = null; + return getTranslationsParent().getDetectedLanguages(docLangTag); + } + + { + const cleanupLocales = await mockLocales({ + systemLocales: ["en"], + appLocales: ["en"], + webLanguages: [""], + }); + + Assert.deepEqual( + await getDetectedLanguagesFor("en"), + { + docLangTag: "en", + userLangTag: null, + isDocLangTagSupported: true, + }, + "If the web languages are empty, do not offer a language matching the app locale." + ); + + await cleanupLocales(); + } + + { + const cleanupLocales = await mockLocales({ + systemLocales: ["en", "es"], + appLocales: ["en"], + webLanguages: [""], + }); + + Assert.deepEqual( + await getDetectedLanguagesFor("en"), + { + docLangTag: "en", + userLangTag: null, + isDocLangTagSupported: true, + }, + "When there are multiple system locales, the app locale is used." + ); + + Assert.deepEqual( + await getDetectedLanguagesFor("es"), + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + }, + "When there are multiple system locales, the app locale is used." + ); + + await cleanupLocales(); + } + + return cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js b/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js new file mode 100644 index 0000000000..0960e05b6e --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js @@ -0,0 +1,81 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +function waitForAppLocaleChanged() { + new Promise(resolve => { + function onChange() { + Services.obs.removeObserver(onChange, "intl:app-locales-changed"); + resolve(); + } + Services.obs.addObserver(onChange, "intl:app-locales-changed"); + }); +} + +async function testWithLocales({ + systemLocales, + appLocales, + webLanguages, + test, +}) { + const cleanup = await mockLocales({ + systemLocales, + appLocales, + webLanguages, + }); + test(); + return cleanup(); +} + +add_task(async function test_preferred_language() { + await testWithLocales({ + systemLocales: ["en-US"], + appLocales: ["en-US"], + webLanguages: ["en-US"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + ["en"], + "When all locales are English, only English is preferred." + ); + }, + }); + + await testWithLocales({ + systemLocales: ["es-ES"], + appLocales: ["en-US"], + webLanguages: ["en-US"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + ["en", "es"], + "When the operating system differs, it is added to the end of the preferred languages." + ); + }, + }); + + await testWithLocales({ + systemLocales: ["zh-TW", "zh-CN", "de"], + appLocales: ["pt-BR", "pl"], + webLanguages: ["cs", "hu"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + [ + // webLanguages + "cs", + "hu", + // appLocales, notice that "en" is the last fallback. + "pt", + "pl", + "en", + // systemLocales + "zh", + "de", + ], + "Demonstrate an unrealistic but complicated locale situation." + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_sync_models.js b/toolkit/components/translations/tests/browser/browser_translations_actor_sync_models.js new file mode 100644 index 0000000000..1d1877fc38 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_sync_models.js @@ -0,0 +1,235 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * An actor unit test for testing RemoteSettings update behavior. This uses the + * recommendations from: + * + * https://firefox-source-docs.mozilla.org/services/settings/index.html#unit-tests + */ +add_task(async function test_translations_actor_sync_update() { + const { remoteClients, cleanup } = await setupActorTest({ + autoDownloadFromRemoteSettings: true, + languagePairs: [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + ], + }); + + const decoder = new TextDecoder(); + const modelsPromise = TranslationsParent.getLanguageTranslationModelFiles( + "en", + "es" + ); + + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR + ); + + const oldModels = await modelsPromise; + + is( + decoder.decode(oldModels.model.buffer), + "Mocked download: test-translation-models model.enes.intgemm.alphas.bin 1.0", + "The version 1.0 model is downloaded." + ); + + const newModelRecords = createRecordsForLanguagePair("en", "es"); + for (const newModelRecord of newModelRecords) { + newModelRecord.version = "1.1"; + } + + info('Emitting a remote client "sync" event with an updated record.'); + await remoteClients.translationModels.client.emit("sync", { + data: { + created: [], + updated: newModelRecords.map(newRecord => ({ + old: oldModels[newRecord.fileType].record, + new: newRecord, + })), + deleted: [], + }, + }); + + const updatedModelsPromise = + TranslationsParent.getLanguageTranslationModelFiles("en", "es"); + + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR + ); + + const { model: updatedModel } = await updatedModelsPromise; + + is( + decoder.decode(updatedModel.buffer), + "Mocked download: test-translation-models model.enes.intgemm.alphas.bin 1.1", + "The version 1.1 model is downloaded." + ); + + return cleanup(); +}); + +/** + * An actor unit test for testing RemoteSettings delete behavior. + */ +add_task(async function test_translations_actor_sync_delete() { + const { remoteClients, cleanup } = await setupActorTest({ + autoDownloadFromRemoteSettings: true, + languagePairs: [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + ], + }); + + const decoder = new TextDecoder(); + const modelsPromise = TranslationsParent.getLanguageTranslationModelFiles( + "en", + "es" + ); + + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR + ); + + const { model } = await modelsPromise; + + is( + decoder.decode(model.buffer), + "Mocked download: test-translation-models model.enes.intgemm.alphas.bin 1.0", + "The version 1.0 model is downloaded." + ); + + info('Emitting a remote client "sync" event with a deleted record.'); + await remoteClients.translationModels.client.emit("sync", { + data: { + created: [], + updated: [], + deleted: [model.record], + }, + }); + + let errorMessage; + await TranslationsParent.getLanguageTranslationModelFiles("en", "es").catch( + error => { + errorMessage = error?.message; + } + ); + + is( + errorMessage, + 'No model file was found for "en" to "es."', + "The model was successfully removed." + ); + + return cleanup(); +}); + +/** + * An actor unit test for testing RemoteSettings creation behavior. + */ +add_task(async function test_translations_actor_sync_create() { + const { remoteClients, cleanup } = await setupActorTest({ + autoDownloadFromRemoteSettings: true, + languagePairs: [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + ], + }); + + const decoder = new TextDecoder(); + const modelsPromise = TranslationsParent.getLanguageTranslationModelFiles( + "en", + "es" + ); + + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR + ); + + is( + decoder.decode((await modelsPromise).model.buffer), + "Mocked download: test-translation-models model.enes.intgemm.alphas.bin 1.0", + "The version 1.0 model is downloaded." + ); + + info('Emitting a remote client "sync" event with new records.'); + await remoteClients.translationModels.client.emit("sync", { + data: { + created: createRecordsForLanguagePair("en", "fr"), + updated: [], + deleted: [], + }, + }); + + const updatedModelsPromise = + TranslationsParent.getLanguageTranslationModelFiles("en", "fr"); + + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR + ); + + const { vocab, lex, model } = await updatedModelsPromise; + + is( + decoder.decode(vocab.buffer), + "Mocked download: test-translation-models vocab.enfr.spm 1.0", + "The en to fr vocab is downloaded." + ); + is( + decoder.decode(lex.buffer), + "Mocked download: test-translation-models lex.50.50.enfr.s2t.bin 1.0", + "The en to fr lex is downloaded." + ); + is( + decoder.decode(model.buffer), + "Mocked download: test-translation-models model.enfr.intgemm.alphas.bin 1.0", + "The en to fr model is downloaded." + ); + + return cleanup(); +}); + +add_task(async function test_translations_parent_download_size() { + const { cleanup } = await setupActorTest({ + languagePairs: [ + { fromLang: "en", toLang: "es" }, + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "de" }, + { fromLang: "de", toLang: "en" }, + ], + }); + + const directSize = + await TranslationsParent.getExpectedTranslationDownloadSize("en", "es"); + // Includes model, lex, and vocab files (x3), each mocked at 123 bytes. + is( + directSize, + 3 * 123, + "Returned the expected download size for a direct translation." + ); + + const pivotSize = await TranslationsParent.getExpectedTranslationDownloadSize( + "es", + "de" + ); + // Includes a pivot (x2), model, lex, and vocab files (x3), each mocked at 123 bytes. + is( + pivotSize, + 2 * 3 * 123, + "Returned the expected download size for a pivot." + ); + + const notApplicableSize = + await TranslationsParent.getExpectedTranslationDownloadSize( + "unknown", + "unknown" + ); + is( + notApplicableSize, + 0, + "Returned the expected download size for an unknown or not applicable model." + ); + return cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_versioning.js b/toolkit/components/translations/tests/browser/browser_translations_actor_versioning.js new file mode 100644 index 0000000000..7e96f62fbf --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_versioning.js @@ -0,0 +1,81 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_remote_settings_versioning() { + const tests = [ + { + majorVersion: 1, + existingVersion: "1.0", + nextVersion: "1.1", + expectation: true, + }, + { + majorVersion: 1, + existingVersion: null, + nextVersion: "1.1", + expectation: true, + }, + { + majorVersion: 1, + existingVersion: null, + nextVersion: "1.0beta", + expectation: true, + }, + { + majorVersion: 1, + existingVersion: null, + nextVersion: "1.0a", + expectation: true, + }, + { + majorVersion: 2, + existingVersion: null, + nextVersion: "1.0a", + expectation: false, + }, + { + majorVersion: 2, + existingVersion: "2.0", + nextVersion: "1.0a", + expectation: false, + }, + { + majorVersion: 2, + existingVersion: "2.1", + nextVersion: "3.2", + expectation: false, + }, + { + majorVersion: 2, + existingVersion: null, + nextVersion: "3.2", + expectation: false, + }, + { + majorVersion: 1, + nextVersion: "1.0", + existingVersion: undefined, + expectation: true, + }, + ]; + for (const { + majorVersion, + existingVersion, + nextVersion, + expectation, + } of tests) { + is( + TranslationsParent.isBetterRecordVersion( + majorVersion, + nextVersion, + existingVersion + ), + expectation, + `Given a major version of ${majorVersion}, an existing version ${existingVersion} ` + + `and a next version of ${nextVersion}, is the next version is ` + + `${expectation ? "" : "not "}best.` + ); + } +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_full_page.js b/toolkit/components/translations/tests/browser/browser_translations_full_page.js new file mode 100644 index 0000000000..7ee15902f8 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_full_page.js @@ -0,0 +1,135 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Check that the full page translation feature works. + */ +add_task(async function test_full_page_translation() { + await autoTranslatePage({ + page: SPANISH_PAGE_URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + const selectors = TranslationsTest.getSelectors(); + + await TranslationsTest.assertTranslationResult( + "The main title gets translated.", + selectors.getH1, + "DON QUIJOTE DE LA MANCHA [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "The last paragraph gets translated. It is out of the viewport.", + selectors.getLastParagraph, + "— PUES, AUNQUE MOVÁIS MÁS BRAZOS QUE LOS DEL GIGANTE BRIAREO, ME LO HABÉIS DE PAGAR. [es to en, html]" + ); + + selectors.getH1().innerText = "Este es un titulo"; + + await TranslationsTest.assertTranslationResult( + "Mutations get tracked", + selectors.getH1, + "ESTE ES UN TITULO [es to en]" + ); + + await TranslationsTest.assertTranslationResult( + "Other languages do not get translated.", + selectors.getHeader, + "The following is an excerpt from Don Quijote de la Mancha, which is in the public domain" + ); + }, + }); +}); + +/** + * Check that the full page translation feature doesn't translate pages in the app's + * locale. + */ +add_task(async function test_about_translations_enabled() { + const { appLocaleAsBCP47 } = Services.locale; + if (!appLocaleAsBCP47.startsWith("en")) { + console.warn( + "This test assumes to be running in an 'en' app locale, however the app locale " + + `is set to ${appLocaleAsBCP47}. Skipping the test.` + ); + ok(true, "Skipping test."); + return; + } + + await autoTranslatePage({ + page: ENGLISH_PAGE_URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async () => { + const { document } = content; + + for (let i = 0; i < 5; i++) { + // There is no way to directly check the non-existence of a translation, as + // the translations engine works async, and you can't dispatch a CustomEvent + // to listen for translations, as this script runs after the initial translations + // check. So resort to a setTimeout and check a few times. This relies on timing, + // but _cannot fail_ if it's working correctly. It _will most likely fail_ if + // this page accidentally gets translated. + + const timeout = 10; + + info("Waiting for the timeout."); + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + await new Promise(resolve => setTimeout(resolve, timeout)); + is( + document.querySelector("h1").innerText, + `"The Wonderful Wizard of Oz" by L. Frank Baum`, + `The page remains untranslated after ${(i + 1) * timeout}ms.` + ); + } + }, + }); +}); + +/** + * Check that the full page translation feature works. + */ +add_task(async function test_language_identification_for_page_translation() { + await autoTranslatePage({ + page: NO_LANGUAGE_URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + const selectors = TranslationsTest.getSelectors(); + + await TranslationsTest.assertTranslationResult( + "The main title gets translated.", + selectors.getH1, + "DON QUIJOTE DE LA MANCHA [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "The last paragraph gets translated. It is out of the viewport.", + selectors.getLastParagraph, + "— PUES, AUNQUE MOVÁIS MÁS BRAZOS QUE LOS DEL GIGANTE BRIAREO, ME LO HABÉIS DE PAGAR. [es to en, html]" + ); + + selectors.getH1().innerText = "Este es un titulo"; + + await TranslationsTest.assertTranslationResult( + "Mutations get tracked", + selectors.getH1, + "ESTE ES UN TITULO [es to en]" + ); + + await TranslationsTest.assertTranslationResult( + "Other languages do not get translated.", + selectors.getHeader, + "The following is an excerpt from Don Quijote de la Mancha, which is in the public domain" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js b/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js new file mode 100644 index 0000000000..3540df0beb --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js @@ -0,0 +1,114 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +async function runLangTagsTest( + { + systemLocales, + appLocales, + webLanguages, + page, + languagePairs = LANGUAGE_PAIRS, + }, + langTags +) { + const cleanupLocales = await mockLocales({ + systemLocales, + appLocales, + webLanguages, + }); + + const { cleanup: cleanupTestPage } = await loadTestPage({ + page, + languagePairs, + }); + const actor = getTranslationsParent(); + + await waitForCondition( + async () => actor.languageState.detectedLanguages?.docLangTag, + "Waiting for a document language tag to be found." + ); + + Assert.deepEqual(actor.languageState.detectedLanguages, langTags); + + await cleanupLocales(); + await cleanupTestPage(); +} + +add_task(async function test_lang_tags_direct_translations() { + info( + "Test the detected languages for translations when a translation pair is available" + ); + await runLangTagsTest( + { + systemLocales: ["en"], + appLocales: ["en"], + webLanguages: ["en"], + page: SPANISH_PAGE_URL, + }, + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_pivots() { + info("Test the detected languages for translations when a pivot is needed."); + await runLangTagsTest( + { + systemLocales: ["fr"], + appLocales: ["fr", "en"], + webLanguages: ["fr", "en"], + page: SPANISH_PAGE_URL, + }, + { + docLangTag: "es", + userLangTag: "fr", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_pivots_second_preferred() { + info( + "Test using a pivot language when the first preferred lang tag doesn't match" + ); + await runLangTagsTest( + { + systemLocales: ["it"], + appLocales: ["it", "en"], + webLanguages: ["it", "en"], + page: SPANISH_PAGE_URL, + }, + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_non_supported_doc_language() { + info("Test using a pivot language when the doc language isn't supported"); + await runLangTagsTest( + { + systemLocales: ["fr"], + appLocales: ["fr", "en"], + webLanguages: ["fr", "en"], + page: SPANISH_PAGE_URL, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + // No Spanish support. + ], + }, + { + docLangTag: "es", + userLangTag: "fr", + isDocLangTagSupported: false, + } + ); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_pdf_is_disabled.js b/toolkit/components/translations/tests/browser/browser_translations_pdf_is_disabled.js new file mode 100644 index 0000000000..b3ac09169f --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_pdf_is_disabled.js @@ -0,0 +1,31 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests that the translations button becomes disabled when entering pdf. + */ +add_task(async function test_translations_button_disabled_in_pdf() { + const { cleanup } = await loadTestPage({ + page: EMPTY_PDF_URL, + }); + + const appMenuButton = document.getElementById("PanelUI-menu-button"); + + click(appMenuButton, "Opening the app menu"); + await BrowserTestUtils.waitForEvent(window.PanelUI.mainView, "ViewShown"); + + const translateSiteButton = document.getElementById( + "appMenu-translate-button" + ); + is( + translateSiteButton.disabled, + true, + "The app-menu translate button should be disabled because PDFs are restricted" + ); + + click(appMenuButton, "Closing the app menu"); + + await cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_remote_settings.js b/toolkit/components/translations/tests/browser/browser_translations_remote_settings.js new file mode 100644 index 0000000000..50987babdf --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_remote_settings.js @@ -0,0 +1,303 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient + * @typedef {import("../../translations").TranslationModelRecord} TranslationModelRecord + */ + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +// The full Firefox version string. +const firefoxFullVersion = AppConstants.MOZ_APP_VERSION_DISPLAY; + +// The Firefox major version string (i.e. the first set of digits). +const firefoxMajorVersion = firefoxFullVersion.match(/\d+/); + +// The Firefox "AlphaZero" version string. +// This is a version that is less than even the latest Nightly +// which is of the form `${firefoxMajorVersion}.a1`. +const firefoxAlphaZeroVersion = `${firefoxMajorVersion}.a0`; + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {string} mockedKey + * @returns {RemoteSettingsClient} + */ +async function createRemoteSettingsClient(mockedKey) { + const client = RemoteSettings(mockedKey); + await client.db.clear(); + await client.db.importChanges({}, Date.now()); + return client; +} + +// The following test ensures the capabilities of `filter_expression` in remote settings +// to successfully discriminate against the Firefox version when retrieving records. +// +// This is used when making major breaking changes that would require particular records +// to only show up in certain versions of Firefox, such as actual code changes that no +// longer allow compatibility with given records. +// +// Some examples might be: +// +// - Modifying a wasm.js file that is no longer compatible with a previous wasm binary record. +// In such a case, the old binary record would need to be shipped in versions with the old +// wasm.js file, and the new binary record would need to be shipped in version with the new +// wasm.js file. +// +// - Switching to a different library for translation or language detection. +// Using a different library would not only mean major changes to the code, but it would +// certainly mean that the records for the old libraries are no longer compatible. +// We will need to ship those records only in older versions of Firefox that utilize the old +// libraries, and ship new records in the new versions of Firefox. +add_task(async function test_filter_current_firefox_version() { + // Create a new RemoteSettingsClient just for this test. + let client = await createRemoteSettingsClient( + "test_filter_current_firefox_version" + ); + + // Create a list of records that are expected to pass the filter_expression and be + // successfully retrieved from the remote settings client. + const expectedPresentRecords = [ + { + name: `undefined filter expression`, + filter_expression: undefined, + }, + { + name: `null filter expression`, + filter_expression: null, + }, + { + name: `empty filter expression`, + filter_expression: ``, + }, + { + name: `env.version == ${firefoxFullVersion}`, + filter_expression: `env.version|versionCompare('${firefoxFullVersion}') == 0`, + }, + { + name: `env.version > ${firefoxAlphaZeroVersion}`, + filter_expression: `env.version|versionCompare('${firefoxAlphaZeroVersion}') > 0`, + }, + ]; + for (let record of expectedPresentRecords) { + client.db.create(record); + } + + // Create a list of records that are expected fo fail the filter_expression and be + // absent when we retrieve the records from the remote settings client. + const expectedAbsentRecords = [ + { + name: `env.version < 1`, + filter_expression: `env.version|versionCompare('1') < 0`, + }, + ]; + for (let record of expectedAbsentRecords) { + client.db.create(record); + } + + const retrievedRecords = await client.get(); + + // Ensure that each record that is expected to be present exists in the retrieved records. + for (let expectedPresentRecord of expectedPresentRecords) { + is( + retrievedRecords.some( + record => record.name == expectedPresentRecord.name + ), + true, + `The following record was expected to be present but was not found: ${expectedPresentRecord.name}\n` + ); + } + + // Ensure that each record that is expected to be absent does not exist in the retrieved records. + for (let expectedAbsentRecord of expectedAbsentRecords) { + is( + retrievedRecords.some(record => record.name == expectedAbsentRecord.name), + false, + `The following record was expected to be absent but was found: ${expectedAbsentRecord.name}\n` + ); + } + + // Ensure that the length of the retrieved records is exactly the length of the records expected to be present. + is( + retrievedRecords.length, + expectedPresentRecords.length, + `Expected ${expectedPresentRecords.length} items but got ${retrievedRecords.length} items\n` + ); +}); + +// The following test ensures that we are able to always retrieve the maximum +// compatible version of records. These are for version changes that do not +// require shipping different records based on a particular Firefox version, +// but rather for changes to model content or wasm runtimes that are fully +// compatible with the existing source code. +add_task(async function test_get_records_with_multiple_versions() { + // Create a new RemoteSettingsClient just for this test. + let client = await createRemoteSettingsClient( + "test_get_translation_model_records" + ); + + const lookupKey = record => + `${record.name}${TranslationsParent.languagePairKey( + record.fromLang, + record.toLang + )}`; + + // A mapping of each record name to its max version. + const maxVersionMap = {}; + + // Create a list of records that are all version 1.0 + /** @type {TranslationModelRecord[]} */ + const versionOneRecords = [ + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "en", + toLang: "es", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "es", + toLang: "en", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "lex.50.50.enes.s2t.bin", + fromLang: "en", + toLang: "es", + fileType: "lex", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "model.enes.intgemm.alphas.bin", + fromLang: "en", + toLang: "es", + fileType: "model", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.deen.spm", + fromLang: "en", + toLang: "de", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "lex.50.50.ende.s2t.bin", + fromLang: "en", + toLang: "de", + fileType: "lex", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "model.ende.intgemm.alphas.bin", + fromLang: "en", + toLang: "de", + fileType: "model", + version: "1.0", + }, + ]; + versionOneRecords.reduce((map, record) => { + map[lookupKey(record)] = record.version; + return map; + }, maxVersionMap); + for (const record of versionOneRecords) { + client.db.create(record); + } + + // Create a list of records that are identical to some of the above, but with higher version numbers. + const higherVersionRecords = [ + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.1", + }, + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.2", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "en", + toLang: "es", + fileType: "vocab", + version: "1.1", + }, + ]; + higherVersionRecords.reduce((map, record) => { + const key = lookupKey(record); + if (record.version > map[key]) { + map[key] = record.version; + } + return map; + }, maxVersionMap); + for (const record of higherVersionRecords) { + client.db.create(record); + } + + TranslationsParent.mockTranslationsEngine( + client, + await createTranslationsWasmRemoteClient() + ); + + const retrievedRecords = await TranslationsParent.getMaxVersionRecords( + client, + { lookupKey, majorVersion: 1 } + ); + + for (const record of retrievedRecords) { + is( + lookupKey(record) in maxVersionMap, + true, + `Expected record ${record.name} to be contained in the nameToVersionMap, but found none\n` + ); + is( + record.version, + maxVersionMap[lookupKey(record)], + `Expected record ${record.name} to be version ${ + maxVersionMap[lookupKey(record)] + }, but found version ${record.version}\n` + ); + } + + const expectedSize = Object.keys(maxVersionMap).length; + is( + retrievedRecords.length, + expectedSize, + `Expected retrieved records to be the same size as the name-to-version map ( + ${expectedSize} + ), but found ${retrievedRecords.length}\n` + ); + + TranslationsParent.unmockTranslationsEngine(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_shadow_dom.js b/toolkit/components/translations/tests/browser/browser_translations_shadow_dom.js new file mode 100644 index 0000000000..86c8d8b33f --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_shadow_dom.js @@ -0,0 +1,82 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const URL = + "https://example.com/browser/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-es.html"; + +const URL_SLOT = + "https://example.com/browser/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-slot-es.html"; + +/** + * Check that the translation feature works with ShadowDOM. + */ +add_task(async function test_shadow_dom_translation() { + await autoTranslatePage({ + page: URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + await TranslationsTest.assertTranslationResult( + "Text outside of the Shadow DOM is translated", + function () { + return content.document.querySelector("h1"); + }, + "ESTO SE CONTENTA EN LUZ DOM [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "The content in the Shadow DOM is translated.", + function () { + const root = content.document.getElementById("host").shadowRoot; + return root.querySelector("p"); + }, + "ESTO SE CONTENTO EN SHADOW DOM [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "Content in the interior root of a Shadow DOM is translated.", + function () { + const outerRoot = content.document.getElementById("host").shadowRoot; + const innerRoot = outerRoot.querySelector("div").shadowRoot; + return innerRoot.querySelector("p"); + }, + "ESTO SE CONTENTA EN RAÍZ INTERIOR [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "Content in the Shaodw DOM where the host element is inside an empty textContent element is translated.", + function () { + const root = content.document.getElementById("host2").shadowRoot; + return root.querySelector("p"); + }, + "ESTO SE CONTENTO EN SHADOW DOM 2 [es to en, html]" + ); + }, + }); +}); + +/** + * Check that the translation feature works with ShadowDOM with slotted text node. + */ +add_task(async function test_shadow_dom_translation_slotted() { + await autoTranslatePage({ + page: URL_SLOT, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + await TranslationsTest.assertTranslationResult( + "Slotted text node is translated", + function () { + return content.document.getElementById("host"); + }, + "ESTO SE CONTENTA EN LUZ DOM [es to en]" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_shadow_dom_mutation.js b/toolkit/components/translations/tests/browser/browser_translations_shadow_dom_mutation.js new file mode 100644 index 0000000000..c5a7891ee2 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_shadow_dom_mutation.js @@ -0,0 +1,164 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const URL = + "https://example.com/browser/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es.html"; + +const URL_2 = + "https://example.com/browser/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es-2.html"; + +/** + * Check that the translation feature works with mutations around ShadowDOM + */ +add_task(async function test_shadow_dom_mutation() { + await autoTranslatePage({ + page: URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + await TranslationsTest.assertTranslationResult( + "Basic translation works", + function () { + return content.document.querySelector("h1"); + }, + "THIS IS CONTENT IN LIGHT DOM [es to en, html]" + ); + + info("Test 1: Mutation on existing shadow tree"); + const root1 = content.document.getElementById("host1").shadowRoot; + root1.innerHTML = "<p>This is mutated content in the Shadow DOM</p>"; + await TranslationsTest.assertTranslationResult( + "The content is translated when shadow tree is modified", + function () { + const root1 = content.document.getElementById("host1").shadowRoot; + return root1.querySelector("p"); + }, + "THIS IS MUTATED CONTENT IN THE SHADOW DOM [es to en, html]" + ); + + info("Test 2: Shadow host added later"); + const host2 = content.document.createElement("div"); + host2.id = "host2"; + const root2 = host2.attachShadow({ mode: "open" }); + root2.innerHTML = "<p>This is content in a shadow DOM</p>"; + content.document.body.appendChild(host2); + await TranslationsTest.assertTranslationResult( + "The content is translated when the host element is added later", + function () { + const root2 = content.document.getElementById("host2").shadowRoot; + return root2.querySelector("p"); + }, + "THIS IS CONTENT IN A SHADOW DOM [es to en, html]" + ); + + info("Test 3: Mutation Works on newly added shadow tree"); + const newNode = content.document.createElement("p"); + newNode.innerHTML = + "<p>This is mutated content in newly added shadow DOM</p>"; + newNode.id = "newNode"; + root2.appendChild(newNode); + await TranslationsTest.assertTranslationResult( + "The content is translated when a new node is added to the newly added shadow tree", + function () { + const root2 = content.document.getElementById("host2").shadowRoot; + return root2.getElementById("newNode"); + }, + "THIS IS MUTATED CONTENT IN NEWLY ADDED SHADOW DOM [es to en, html]" + ); + }, + }); +}); + +add_task(async function test_shadow_dom_mutation_nested_1() { + await autoTranslatePage({ + page: URL, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + await TranslationsTest.assertTranslationResult( + "Basic translation works", + function () { + return content.document.querySelector("h1"); + }, + "THIS IS CONTENT IN LIGHT DOM [es to en, html]" + ); + + info("Test 1: Nested shadow host added later"); + const root1 = content.document.getElementById("host1").shadowRoot; + root1.innerHTML = "<div id='innerHost'></div>"; + + const innerHost = root1.getElementById("innerHost"); + innerHost.id = "innerHost"; + const innerRoot = innerHost.attachShadow({ mode: "open" }); + innerRoot.innerHTML = "<p>This is content in nested shadow DOM</p>"; + + await TranslationsTest.assertTranslationResult( + "The content is translated when a inner host element is added later", + function () { + const root1 = content.document.getElementById("host1").shadowRoot; + const innerRoot = root1.getElementById("innerHost").shadowRoot; + return innerRoot.querySelector("p"); + }, + "THIS IS CONTENT IN NESTED SHADOW DOM [es to en, html]" + ); + + info("Test 2: Mutation on inner shadow tree works"); + const newInnerNode = content.document.createElement("p"); + newInnerNode.innerHTML = + "<p>This is mutated content in nested shadow DOM</p>"; + newInnerNode.id = "newInnerNode"; + innerRoot.appendChild(newInnerNode); + await TranslationsTest.assertTranslationResult( + "The content is translated when inner shadow tree is mutated", + function () { + const root = content.document.getElementById("host1").shadowRoot; + const innerRoot = root.getElementById("innerHost").shadowRoot; + return innerRoot.getElementById("newInnerNode"); + }, + "THIS IS MUTATED CONTENT IN NESTED SHADOW DOM [es to en, html]" + ); + }, + }); +}); + +// Test to ensure mutations on a nested shadow tree that is +// added before pageload works. +add_task(async function test_shadow_dom_mutation_nested_2() { + await autoTranslatePage({ + page: URL_2, + languagePairs: [ + { fromLang: "es", toLang: "en" }, + { fromLang: "en", toLang: "es" }, + ], + runInPage: async TranslationsTest => { + await TranslationsTest.assertTranslationResult( + "Basic translation works", + function () { + return content.document.querySelector("h1"); + }, + "THIS IS CONTENT IN LIGHT DOM [es to en, html]" + ); + + const root1 = content.document.getElementById("host1").shadowRoot; + const innerRoot = root1.getElementById("innerHost").shadowRoot; + innerRoot.innerHTML = + "<p>This is mutated content in inner shadow DOM</p>"; + + await TranslationsTest.assertTranslationResult( + "The content is translated when the inner shadow tree is modified", + function () { + const root1 = content.document.getElementById("host1").shadowRoot; + const innerRoot = root1.getElementById("innerHost").shadowRoot; + return innerRoot.querySelector("p"); + }, + "THIS IS MUTATED CONTENT IN INNER SHADOW DOM [es to en, html]" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_translation_document.js b/toolkit/components/translations/tests/browser/browser_translations_translation_document.js new file mode 100644 index 0000000000..9a00da9ccf --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_translation_document.js @@ -0,0 +1,1446 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * @type {typeof import("../../content/translations-document.sys.mjs")} + */ +const { TranslationsDocument, LRUCache } = ChromeUtils.importESModule( + "chrome://global/content/translations/translations-document.sys.mjs" +); +/** + * @param {string} html + * @param {{ + * mockedTranslatorPort?: (message: string) => Promise<string> + * }} [options] + */ +async function createDoc(html, options) { + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ], + }); + + const parser = new DOMParser(); + const document = parser.parseFromString(html, "text/html"); + + // For some reason, the document <body> here from the DOMParser is "display: flex" by + // default. Ensure that it is "display: block" instead, otherwise the children of the + // <body> will not be "display: inline". + document.body.style.display = "block"; + + const translate = () => { + info("Creating the TranslationsDocument."); + return new TranslationsDocument( + document, + "en", + "EN", + 0, // This is a fake innerWindowID + options?.mockedTranslatorPort ?? createMockedTranslatorPort(), + () => { + throw new Error("Cannot request a new port"); + }, + performance.now(), + () => performance.now(), + new LRUCache() + ); + }; + + /** + * Test utility to check that the document matches the expected markup + * + * @param {string} html + */ + async function htmlMatches(message, html) { + const expected = naivelyPrettify(html); + try { + await waitForCondition( + () => naivelyPrettify(document.body.innerHTML) === expected, + "Waiting for HTML to match." + ); + ok(true, message); + } catch (error) { + console.error(error); + + // Provide a nice error message. + const actual = naivelyPrettify(document.body.innerHTML); + ok( + false, + `${message}\n\nExpected HTML:\n\n${expected}\n\nActual HTML:\n\n${actual}\n\n` + ); + } + } + + function cleanup() { + SpecialPowers.popPrefEnv(); + } + + return { htmlMatches, cleanup, translate, document }; +} + +add_task(async function test_translated_div_element() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "A single element with a single text node is translated into uppercase.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_translated_textnode() { + const { translate, htmlMatches, cleanup } = await createDoc( + "This is a simple text translation." + ); + + translate(); + + await htmlMatches( + "A Text node at the root is translated into all caps", + "THIS IS A SIMPLE TEXT TRANSLATION." + ); + + cleanup(); +}); + +add_task(async function test_no_text_trees() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <div></div> + <span></span> + </div> + `); + + translate(); + + await htmlMatches( + "Trees with no text are not affected", + /* html */ ` + <div> + <div></div> + <span></span> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_no_text_trees() { + const { translate, htmlMatches, cleanup } = await createDoc(""); + translate(); + await htmlMatches("No text is still no text", ""); + cleanup(); +}); + +add_task(async function test_translated_title() { + const { cleanup, document, translate } = await createDoc(/* html */ ` + <!DOCTYPE html> + <html> + <head> + <meta charset="utf-8" /> + <title>This is an actual full page.</title> + </head> + <body> + + </body> + </html> + `); + + translate(); + + const translatedTitle = "THIS IS AN ACTUAL FULL PAGE."; + try { + await waitForCondition(() => document.title === translatedTitle); + } catch (error) {} + is(document.title, translatedTitle, "The title was changed."); + + cleanup(); +}); + +add_task(async function test_translated_nested_elements() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div class="menu-main-menu-container"> + <ul class="menu-list"> + <li class="menu-item menu-item-top-level"> + <a href="/">Latest Work</a> + </li> + <li class="menu-item menu-item-top-level"> + <a href="/category/interactive/">Creative Coding</a> + </li> + <li id="menu-id-categories" class="menu-item menu-item-top-level"> + <a href="#"><span class='category-arrow'>Categories</span></a> + </li> + </ul> + </div> + `); + + translate(); + + await htmlMatches( + "The nested elements are translated into all caps.", + /* html */ ` + <div class="menu-main-menu-container"> + <ul class="menu-list"> + <li class="menu-item menu-item-top-level"> + <a href="/" data-moz-translations-id="0"> + LATEST WORK + </a> + </li> + <li class="menu-item menu-item-top-level"> + <a href="/category/interactive/" data-moz-translations-id="0"> + CREATIVE CODING + </a> + </li> + <li id="menu-id-categories" class="menu-item menu-item-top-level"> + <a href="#" data-moz-translations-id="0"> + <span class="category-arrow" data-moz-translations-id="1"> + CATEGORIES + </span> + </a> + </li> + </ul> + </div> + ` + ); + + cleanup(); +}); + +/** + * Only translate elements with a matching "from" language. + */ +add_task(async function test_translated_language() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <div> + No lang property + </div> + <div lang="en"> + Language matches + </div> + <div lang="fr"> + Language mismatch is ignored. + </div> + <div lang="en-US"> + Language match with region. + </div> + <div lang="fr"> + <div> + Language mismatch with + </div> + <div> + nested elements. + </div> + </div> + </div> + `); + + translate(); + + await htmlMatches( + "Language matching of elements behaves as expected.", + /* html */ ` + <div> + <div> + NO LANG PROPERTY + </div> + <div lang="en"> + LANGUAGE MATCHES + </div> + <div lang="fr"> + Language mismatch is ignored. + </div> + <div lang="en-US"> + LANGUAGE MATCH WITH REGION. + </div> + <div lang="fr"> + <div> + Language mismatch with + </div> + <div> + nested elements. + </div> + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test elements that have been marked as ignored. + */ +add_task(async function test_ignored_translations() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div translate="yes"> + This is translated. + </div> + <div translate="no"> + This is not translated. + </div> + <div class="notranslate"> + This is not translated. + </div> + <div class="class-before notranslate class-after"> + This is not translated. + </div> + <div contenteditable> + This is not translated. + </div> + <div contenteditable="true"> + This is not translated. + </div> + <div contenteditable="false"> + This is translated. + </div> + `); + + translate(); + + await htmlMatches( + "Language matching of elements behaves as expected.", + /* html */ ` + <div translate="yes"> + THIS IS TRANSLATED. + </div> + <div translate="no"> + This is not translated. + </div> + <div class="notranslate"> + This is not translated. + </div> + <div class="class-before notranslate class-after"> + This is not translated. + </div> + <div contenteditable=""> + This is not translated. + </div> + <div contenteditable="true"> + This is not translated. + </div> + <div contenteditable="false"> + THIS IS TRANSLATED. + </div> + ` + ); + + cleanup(); +}); + +/** + * Test excluded tags. + */ +add_task(async function test_excluded_tags() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is translated. + </div> + <code> + This is ignored + </code> + <script> + This is ignored + </script> + <textarea> + This is ignored + </textarea> + `); + + translate(); + + await htmlMatches( + "EXCLUDED_TAGS are not translated", + /* html */ ` + <div> + THIS IS TRANSLATED. + </div> + <code> + This is ignored + </code> + <script> + This is ignored + </script> + <textarea> + This is ignored + </textarea> + ` + ); + + cleanup(); +}); + +add_task(async function test_comments() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <!-- Comments don't make it to the DOM --> + <div> + <!-- These will be ignored in the translation. --> + This is translated. + </div> + `); + + translate(); + + await htmlMatches( + "Comments do not affect things.", + /* html */ ` + <div> + <!-- These will be ignored in the translation. --> + THIS IS TRANSLATED. + </div> + ` + ); + + cleanup(); +}); + +/** + * Test the batching behavior on what is sent in for a translation. + */ +add_task(async function test_translation_batching() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + This is a simple section. + </div> + <div> + <span>This entire</span> section continues in a <b>batch</b>. + </div> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + aaaa aa a aaaaaa aaaaaaa. + </div> + <div> + <span data-moz-translations-id="0"> + bbbb bbbbbb + </span> + bbbbbbb bbbbbbbbb bb b + <b data-moz-translations-id="1"> + bbbbb + </b> + . + </div> + ` + ); + + cleanup(); +}); + +/** + * Test the inline/block behavior on what is sent in for a translation. + */ +add_task(async function test_translation_inline_styling() { + const { document, translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + Bare text is sent in a batch. + <span> + Inline text is sent in a <b>batch</b>. + </span> + <span id="spanAsBlock"> + Display "block" overrides the inline designation. + </span> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + info("Setting a span as display: block."); + const span = document.getElementById("spanAsBlock"); + span.style.display = "block"; + is(span.ownerGlobal.getComputedStyle(span).display, "block"); + + translate(); + + await htmlMatches( + "Span as a display: block", + /* html */ ` + aaaa aaaa aa aaaa aa a aaaaa. + <span> + bbbbbb bbbb bb bbbb bb b + <b data-moz-translations-id="0"> + bbbbb + </b> + . + </span> + <span id="spanAsBlock" style="display: block;"> + ccccccc "ccccc" ccccccccc ccc cccccc ccccccccccc. + </span> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_many_inlines() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + <span> + This is a + </span> + <span> + much longer + </span> + <span> + section that includes + </span> + <span> + many span elements + </span> + <span> + to test what happens + </span> + <span> + in cases like this. + </span> + </div> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + <span data-moz-translations-id="0"> + aaaa aa a + </span> + <span data-moz-translations-id="1"> + aaaa aaaaaa + </span> + <span data-moz-translations-id="2"> + aaaaaaa aaaa aaaaaaaa + </span> + <span data-moz-translations-id="3"> + aaaa aaaa aaaaaaaa + </span> + <span data-moz-translations-id="4"> + aa aaaa aaaa aaaaaaa + </span> + <span data-moz-translations-id="5"> + aa aaaaa aaaa aaaa. + </span> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_many_inlines() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + <div> + This is a + </div> + <div> + much longer + </div> + <div> + section that includes + </div> + <div> + many div elements + </div> + <div> + to test what happens + </div> + <div> + in cases like this. + </div> + </div> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + <div> + aaaa aa a + </div> + <div> + bbbb bbbbbb + </div> + <div> + ccccccc cccc cccccccc + </div> + <div> + dddd ddd dddddddd + </div> + <div> + ee eeee eeee eeeeeee + </div> + <div> + ff fffff ffff ffff. + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test a mix of inline text and block elements. + */ +add_task(async function test_presumed_inlines1() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <div>Block element</div> + </div> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Mixing a text node with block elements will send in two batches.", + /* html */ ` + <div> + aaaa aaaa + <div> + bbbbb bbbbbbb + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_presumed_inlines2() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <span>Inline</span> + <div>Block Element</div> + </div> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "A mix of inline and blocks will be sent in separately.", + /* html */ ` + <div> + aaaa aaaa + <span> + bbbbbb + </span> + <div> + ccccc ccccccc + </div> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_presumed_inlines3() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <span>Inline</span> + <div>Block Element</div> + <div>Block Element</div> + <div>Block Element</div> + </span> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Conflicting inlines will be sent in as separate blocks if there are more block elements", + /* html */ ` + <div> + aaaa aaaa + <span> + bbbbbb + </span> + <div> + ccccc ccccccc + </div> + <div> + ddddd ddddddd + </div> + <div> + eeeee eeeeeee + </div> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_chunking_large_text() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <pre> + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque fermentum est ante, ut porttitor enim molestie et. Nam mattis ullamcorper justo a ultrices. Ut ac sodales lorem. Sed feugiat ultricies lacus. Proin dapibus sit amet nunc a ullamcorper. Donec leo purus, convallis quis urna non, semper pulvinar augue. Nulla placerat turpis arcu, sit amet imperdiet sapien tincidunt ut. Donec sit amet luctus lorem, sed consectetur lectus. Pellentesque est nisi, feugiat et ipsum quis, vestibulum blandit nulla. + + Proin accumsan sapien ut nibh mattis tincidunt. Donec facilisis nibh sodales, mattis risus et, malesuada lorem. Nam suscipit lacinia venenatis. Praesent ac consectetur ante. Vestibulum pulvinar ut massa in viverra. Nunc tincidunt tortor nunc. Vivamus sit amet hendrerit mi. Aliquam posuere velit non ante facilisis euismod. In ullamcorper, lacus vel hendrerit tincidunt, dui justo iaculis nulla, sit amet tincidunt nisl magna et urna. Sed varius tincidunt ligula. Interdum et malesuada fames ac ante ipsum primis in faucibus. Nam sed gravida ligula. Donec tincidunt arcu eros, ac maximus magna auctor eu. Vivamus suscipit neque velit, in ullamcorper elit pulvinar et. Morbi auctor tempor risus, imperdiet placerat velit gravida vel. Duis ultricies accumsan libero quis molestie. + + Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam nec arcu dapibus enim vulputate vulputate aliquet a libero. Nam hendrerit pulvinar libero, eget posuere quam porta eu. Pellentesque dignissim justo eu leo accumsan, sit amet suscipit ante gravida. Vivamus eu faucibus orci. Quisque sagittis tortor eget orci venenatis porttitor. Quisque mollis ipsum a dignissim dignissim. + + Aenean sagittis nisi lectus, non lacinia orci dapibus viverra. Donec diam lorem, tincidunt sed massa vel, vulputate tincidunt metus. In quam felis, egestas et faucibus faucibus, vestibulum quis tortor. Morbi odio mi, suscipit vitae leo in, consequat interdum augue. Quisque purus velit, dictum ac ante eget, volutpat dapibus ante. Suspendisse quis augue vitae velit elementum dictum nec aliquet nisl. Maecenas vestibulum quam augue, eu maximus urna blandit eu. Donec nunc risus, elementum id ligula nec, ultrices venenatis libero. Suspendisse ullamcorper ex ante, malesuada pulvinar sem placerat vel. + + In hac habitasse platea dictumst. Duis vulputate tellus arcu, at posuere ligula viverra luctus. Fusce ultrices malesuada neque vitae vehicula. Aliquam blandit nisi sed nibh facilisis, non varius turpis venenatis. Vestibulum ut velit laoreet, sagittis leo ac, pharetra ex. Aenean mollis risus sed nibh auctor, et feugiat neque iaculis. Fusce fermentum libero metus, at consectetur massa euismod sed. Mauris ut metus sit amet leo porttitor mollis. Vivamus tincidunt lorem non purus suscipit sollicitudin. Maecenas ut tristique elit. Ut eu volutpat turpis. Suspendisse nec tristique augue. Nullam faucibus egestas volutpat. Sed tempor eros et mi ultrices, nec feugiat eros egestas. + </pre> + `, + { mockedTranslatorPort: createBatchedMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Large chunks of text can be still sent in for translation in one big pass, " + + "this could be slow bad behavior for the user.", + /* html */ ` + <pre> + aaaaa aaaaa aaaaa aaa aaaa, aaaaaaaaaaa aaaaaaaaaa aaaa. aaaaaaa aaaaaaaaa aaa aaaa, aa aaaaaaaaa aaaa aaaaaaaa aa. aaa aaaaaa aaaaaaaaaaa aaaaa a aaaaaaaa. aa aa aaaaaaa aaaaa. aaa aaaaaaa aaaaaaaaa aaaaa. aaaaa aaaaaaa aaa aaaa aaaa a aaaaaaaaaaa. aaaaa aaa aaaaa, aaaaaaaaa aaaa aaaa aaa, aaaaaa aaaaaaaa aaaaa. aaaaa aaaaaaaa aaaaaa aaaa, aaa aaaa aaaaaaaaa aaaaaa aaaaaaaaa aa. aaaaa aaa aaaa aaaaaa aaaaa, aaa aaaaaaaaaaa aaaaaa. aaaaaaaaaaaa aaa aaaa, aaaaaaa aa aaaaa aaaa, aaaaaaaaaa aaaaaaa aaaaa. + + aaaaa aaaaaaaa aaaaaa aa aaaa aaaaaa aaaaaaaaa. aaaaa aaaaaaaaa aaaa aaaaaaa, aaaaaa aaaaa aa, aaaaaaaaa aaaaa. aaa aaaaaaaa aaaaaaa aaaaaaaaa. aaaaaaaa aa aaaaaaaaaaa aaaa. aaaaaaaaaa aaaaaaaa aa aaaaa aa aaaaaaa. aaaa aaaaaaaaa aaaaaa aaaa. aaaaaaa aaa aaaa aaaaaaaaa aa. aaaaaaa aaaaaaa aaaaa aaa aaaa aaaaaaaaa aaaaaaa. aa aaaaaaaaaaa, aaaaa aaa aaaaaaaaa aaaaaaaaa, aaa aaaaa aaaaaaa aaaaa, aaa aaaa aaaaaaaaa aaaa aaaaa aa aaaa. aaa aaaaaa aaaaaaaaa aaaaaa. aaaaaaaa aa aaaaaaaaa aaaaa aa aaaa aaaaa aaaaaa aa aaaaaaaa. aaa aaa aaaaaaa aaaaaa. aaaaa aaaaaaaaa aaaa aaaa, aa aaaaaaa aaaaa aaaaaa aa. aaaaaaa aaaaaaaa aaaaa aaaaa, aa aaaaaaaaaaa aaaa aaaaaaaa aa. aaaaa aaaaaa aaaaaa aaaaa, aaaaaaaaa aaaaaaaa aaaaa aaaaaaa aaa. aaaa aaaaaaaaa aaaaaaaa aaaaaa aaaa aaaaaaaa. + + aaaaaaaaaa aaaa aaaaa aaaaaa aa aaaaaaaa aaaa aaaaaa aa aaaaaaaa aaaaaaa aaaaaaa aaaaa; aaaaa aaa aaaa aaaaaaa aaaa aaaaaaaaa aaaaaaaaa aaaaaaa a aaaaaa. aaa aaaaaaaaa aaaaaaaa aaaaaa, aaaa aaaaaaa aaaa aaaaa aa. aaaaaaaaaaaa aaaaaaaaa aaaaa aa aaa aaaaaaaa, aaa aaaa aaaaaaaa aaaa aaaaaaa. aaaaaaa aa aaaaaaaa aaaa. aaaaaaa aaaaaaaa aaaaaa aaaa aaaa aaaaaaaaa aaaaaaaaa. aaaaaaa aaaaaa aaaaa a aaaaaaaaa aaaaaaaaa. + + aaaaaa aaaaaaaa aaaa aaaaaa, aaa aaaaaaa aaaa aaaaaaa aaaaaaa. aaaaa aaaa aaaaa, aaaaaaaaa aaa aaaaa aaa, aaaaaaaaa aaaaaaaaa aaaaa. aa aaaa aaaaa, aaaaaaa aa aaaaaaaa aaaaaaaa, aaaaaaaaaa aaaa aaaaaa. aaaaa aaaa aa, aaaaaaaa aaaaa aaa aa, aaaaaaaaa aaaaaaaa aaaaa. aaaaaaa aaaaa aaaaa, aaaaaa aa aaaa aaaa, aaaaaaaa aaaaaaa aaaa. aaaaaaaaaaa aaaa aaaaa aaaaa aaaaa aaaaaaaaa aaaaaa aaa aaaaaaa aaaa. aaaaaaaa aaaaaaaaaa aaaa aaaaa, aa aaaaaaa aaaa aaaaaaa aa. aaaaa aaaa aaaaa, aaaaaaaaa aa aaaaaa aaa, aaaaaaaa aaaaaaaaa aaaaaa. aaaaaaaaaaa aaaaaaaaaaa aa aaaa, aaaaaaaaa aaaaaaaa aaa aaaaaaaa aaa. + + aa aaa aaaaaaaaa aaaaaa aaaaaaaa. aaaa aaaaaaaaa aaaaaa aaaa, aa aaaaaaa aaaaaa aaaaaaa aaaaaa. aaaaa aaaaaaaa aaaaaaaaa aaaaa aaaaa aaaaaaaa. aaaaaaa aaaaaaa aaaa aaa aaaa aaaaaaaaa, aaa aaaaaa aaaaaa aaaaaaaaa. aaaaaaaaaa aa aaaaa aaaaaaa, aaaaaaaa aaa aa, aaaaaaaa aa. aaaaaa aaaaaa aaaaa aaa aaaa aaaaaa, aa aaaaaaa aaaaa aaaaaaa. aaaaa aaaaaaaaa aaaaaa aaaaa, aa aaaaaaaaaaa aaaaa aaaaaaa aaa. aaaaaa aa aaaaa aaa aaaa aaa aaaaaaaaa aaaaaa. aaaaaaa aaaaaaaaa aaaaa aaa aaaaa aaaaaaaa aaaaaaaaaaaa. aaaaaaaa aa aaaaaaaaa aaaa. aa aa aaaaaaaa aaaaaa. aaaaaaaaaaa aaa aaaaaaaaa aaaaa. aaaaaa aaaaaaaa aaaaaaa aaaaaaaa. aaa aaaaaa aaaa aa aa aaaaaaaa, aaa aaaaaaa aaaa aaaaaaa. + </pre> + ` + ); + + cleanup(); +}); + +add_task(async function test_reordering() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <span> + B - This was first. + </span> + <span> + A - This was second. + </span> + <span> + C - This was third. + </span> + `, + { mockedTranslatorPort: createdReorderingMockedTranslatorPort() } + ); + + translate(); + + await htmlMatches( + "Nodes can be re-ordered by the translator", + /* html */ ` + <span data-moz-translations-id="1"> + A - THIS WAS SECOND. + </span> + <span data-moz-translations-id="0"> + B - THIS WAS FIRST. + </span> + <span data-moz-translations-id="2"> + C - THIS WAS THIRD. + </span> + ` + ); + + cleanup(); +}); + +add_task(async function test_reordering2() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + B - This was first. + <span> + A - This was second. + </span> + C - This was third. + `, + { mockedTranslatorPort: createdReorderingMockedTranslatorPort() } + ); + + translate(); + + // Note: ${" "} is used below to ensure that the whitespace is not stripped from + // the test. + await htmlMatches( + "Text nodes can be re-ordered.", + /* html */ ` + <span data-moz-translations-id="0"> + A - THIS WAS SECOND. + </span> + B - THIS WAS FIRST. +${" "} + C - THIS WAS THIRD. + ` + ); + + cleanup(); +}); + +add_task(async function test_mutations() { + const { translate, htmlMatches, cleanup, document } = + await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "It translates.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + info('Trigger the "childList" mutation.'); + const div = document.createElement("div"); + div.innerText = "This is an added node."; + document.body.appendChild(div); + + await htmlMatches( + "The added node gets translated.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + <div> + THIS IS AN ADDED NODE. + </div> + ` + ); + + info('Trigger the "characterData" mutation.'); + document.querySelector("div").firstChild.nodeValue = + "This is a changed node."; + + await htmlMatches( + "The changed node gets translated", + /* html */ ` + <div> + THIS IS A CHANGED NODE. + </div> + <div> + THIS IS AN ADDED NODE. + </div> + ` + ); + cleanup(); +}); + +add_task(async function test_svgs() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <div>Text before is translated</div> + <svg width="200" height="200" xmlns="http://www.w3.org/2000/svg"> + <style>.myText { font-family: sans-serif; }</style> + <rect x="10" y="10" width="80" height="60" class="myRect" /> + <circle cx="150" cy="50" r="30" class="myCircle" /> + <text x="50%" y="50%" text-anchor="middle" alignment-baseline="middle" class="myText"> + Text inside of the SVG is untranslated. + </text> + </svg> + <div>Text after is translated</div> + </div> + `); + + translate(); + + await htmlMatches( + "SVG text gets translated, and style elements are left alone.", + /* html */ ` + <div> + <div> + TEXT BEFORE IS TRANSLATED + </div> + <svg width="200" height="200" xmlns="http://www.w3.org/2000/svg"> + <style> + .myText { font-family: sans-serif; } + </style> + <rect x="10" y="10" width="80" height="60" class="myRect"> + </rect> + <circle cx="150" cy="50" r="30" class="myCircle"> + </circle> + <text x="50%" y="50%" text-anchor="middle" alignment-baseline="middle" class="myText"> + TEXT INSIDE OF THE SVG IS UNTRANSLATED. + </text> + </svg> + <div> + TEXT AFTER IS TRANSLATED + </div> + </div> + ` + ); + + await cleanup(); +}); + +add_task(async function test_svgs_more() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <svg viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg"> + <foreignObject x="20" y="20" width="160" height="160"> + <div xmlns="http://www.w3.org/1999/xhtml"> + This is a div inside of an SVG. + </div> + </foreignObject> + </svg> + `); + + translate(); + + await htmlMatches( + "Foreign objects get translated", + /* html */ ` + <svg viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg"> + <foreignObject x="20" y="20" width="160" height="160"> + <div xmlns="http://www.w3.org/1999/xhtml"> + THIS IS A DIV INSIDE OF AN SVG. + </div> + </foreignObject> + </svg> + ` + ); + + await cleanup(); +}); + +add_task(async function test_tables() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <table> + <tr> + <th>Table header 1</th> + <th>Table header 2</th> + </tr> + <tr> + <td>Table data 1</td> + <td>Table data 2</td> + </tr> + </table> + `); + + translate(); + + await htmlMatches( + "Tables are correctly translated.", + /* html */ ` + <table> + <tbody> + <tr> + <th> + TABLE HEADER 1 + </th> + <th> + TABLE HEADER 2 + </th> + </tr> + <tr> + <td> + TABLE DATA 1 + </td> + <td> + TABLE DATA 2 + </td> + </tr> + </tbody> + </table> + ` + ); + + cleanup(); +}); + +// Attribute translation for title and placeholder +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <label title="Titles are user visible">Enter information:</label> + <input type="text" placeholder="This is a placeholder"> + `); + + translate(); + + // This is what this test should assert: + // eslint-disable-next-line no-unused-vars + const actualExpected = /* html */ ` + <label title="TITLES ARE USER VISIBLE"> + ENTER INFORMATION: + </label> + <input type="text" placeholder="THIS IS A PLACEHOLDER" > + `; + + await htmlMatches( + "Placeholders support added", + /* html */ ` + <label title="TITLES ARE USER VISIBLE"> + ENTER INFORMATION: + </label> + <input type="text" placeholder="THIS IS A PLACEHOLDER"> + ` + ); + + cleanup(); +}); + +add_task(async function test_html_attributes() { + const { translate, document, cleanup } = await createDoc(/* html */ ` + <!DOCTYPE html> + <html lang="en" > + <head> + <meta charset="utf-8" /> + </head> + <body> + </body> + </html> + `); + + translate(); + + try { + await waitForCondition(() => document.documentElement.lang === "EN"); + } catch (error) {} + is(document.documentElement.lang, "EN", "The lang attribute was changed"); + + cleanup(); +}); + +// Attribute translation for title +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div title="Titles are user visible"> + </div> + `); + + translate(); + + await htmlMatches( + "Attribute translation for title", + /* html */ ` + <div title="TITLES ARE USER VISIBLE"> + </div> + ` + ); + + cleanup(); +}); + +// Attribute translation for title with innerHTML +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div title="Titles are user visible"> + Simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "translation for title with innerHTML", + /* html */ ` + <div title="TITLES ARE USER VISIBLE"> + SIMPLE TRANSLATION. + </div> + ` + ); + + cleanup(); +}); + +// Attribute translation for title and placeholder in same element +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <input type="text" placeholder="This is a placeholder" title="Titles are user visible"> + `); + + translate(); + + await htmlMatches( + "title and placeholder together", + /* html */ ` + <input type="text" placeholder="THIS IS A PLACEHOLDER" title="TITLES ARE USER VISIBLE"> + ` + ); + cleanup(); +}); + +// Attribute translation for placeholder +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <input type="text" placeholder="This is a placeholder"> + `); + + translate(); + + await htmlMatches( + "Attribute translation for placeholder", + /* html */ ` + <input type="text" placeholder="THIS IS A PLACEHOLDER"> + ` + ); + cleanup(); +}); + +add_task(async function test_translated_title() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div title="The title is translated" class="do-not-translate-this"> + Inner text is translated. + </div> + `); + + translate(); + + await htmlMatches( + "Language matching of elements behaves as expected.", + /* html */ ` + <div title="THE TITLE IS TRANSLATED" class="do-not-translate-this"> + INNER TEXT IS TRANSLATED. + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_title_attribute_subnodes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <span>Span text 1</span> + <span>Span text 2</span> + <span>Span text 3</span> + <span>Span text 4</span> + <span>Span text 5</span> + This is text. + </div> + `); + + translate(); + + await htmlMatches( + "Titles are translated", + /* html */ ` + <div> + <span data-moz-translations-id="0">SPAN TEXT 1</span> + <span data-moz-translations-id="1">SPAN TEXT 2</span> + <span data-moz-translations-id="2">SPAN TEXT 3</span> + <span data-moz-translations-id="3">SPAN TEXT 4</span> + <span data-moz-translations-id="4">SPAN TEXT 5</span> + THIS IS TEXT. + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_title_attribute_subnodes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div title="Title in div"> + <span title="Title 1">Span text 1</span> + <span title="Title 2">Span text 2</span> + <span title="Title 3">Span text 3</span> + <span title="Title 4">Span text 4</span> + <span title="Title 5">Span text 5</span> + This is text. + </div> + `); + + translate(); + + await htmlMatches( + "Titles are translated", + /* html */ ` + <div title="TITLE IN DIV"> + <span title="TITLE 1" data-moz-translations-id="0">SPAN TEXT 1</span> + <span title="TITLE 2" data-moz-translations-id="1">SPAN TEXT 2</span> + <span title="TITLE 3" data-moz-translations-id="2">SPAN TEXT 3</span> + <span title="TITLE 4" data-moz-translations-id="3">SPAN TEXT 4</span> + <span title="TITLE 5" data-moz-translations-id="4">SPAN TEXT 5</span> + THIS IS TEXT. + </div> + ` + ); + + cleanup(); +}); + +// Attribute translation for nested text +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is the outer div + <label> + Enter information: + <input type="text"> + </label> + </div> + `); + + translate(); + + await htmlMatches( + "translation for Nested with text", + /* html */ ` + <div> + THIS IS THE OUTER DIV + <label data-moz-translations-id="0"> + ENTER INFORMATION: + <input type="text" data-moz-translations-id="1"> + </label> + </div> + ` + ); + + cleanup(); +}); + +// Attribute translation Nested Attributes +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div title="Titles are user visible"> + This is the outer div + <label> + Enter information: + <input type="text" placeholder="This is a placeholder"> + </label> + </div> + `); + + translate(); + + await htmlMatches( + "Translations: Nested Attributes", + /* html */ ` + <div title="TITLES ARE USER VISIBLE"> + THIS IS THE OUTER DIV + <label data-moz-translations-id="0"> + ENTER INFORMATION: + <input type="text" placeholder="THIS IS A PLACEHOLDER" data-moz-translations-id="1"> + </label> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is the outer div + <label> + Enter information 1: + <label> + Enter information 2: + </label> + </label> + </div> + `); + + translate(); + + await htmlMatches( + "Translations: Nested elements", + /* html */ ` + <div> + THIS IS THE OUTER DIV + <label data-moz-translations-id="0"> + ENTER INFORMATION 1: + <label data-moz-translations-id="1"> + ENTER INFORMATION 2: + </label> + </label> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_mutations_with_attributes() { + const { translate, htmlMatches, cleanup, document } = + await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "It translates.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + info('Trigger the "childList" mutation.'); + const div = document.createElement("div"); + div.innerText = "This is an added node."; + div.setAttribute("title", "title is added"); + document.body.appendChild(div); + + await htmlMatches( + "The added node gets translated.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + <div title="TITLE IS ADDED"> + THIS IS AN ADDED NODE. + </div> + ` + ); + + info('Trigger the "characterData" mutation.'); + document.querySelector("div").firstChild.nodeValue = + "This is a changed node."; + + await htmlMatches( + "The changed node gets translated", + /* html */ ` + <div> + THIS IS A CHANGED NODE. + </div> + <div title="TITLE IS ADDED"> + THIS IS AN ADDED NODE. + </div> + ` + ); + + info('Trigger the "childList" mutation.'); + const inp = document.createElement("input"); + inp.setAttribute("placeholder", "input placeholder is added"); + document.body.appendChild(inp); + + await htmlMatches( + "The placeholder in input node gets translated.", + /* html */ ` + <div> + THIS IS A CHANGED NODE. + </div> + <div title="TITLE IS ADDED"> + THIS IS AN ADDED NODE. + </div> + <input placeholder="INPUT PLACEHOLDER IS ADDED"> + ` + ); + + info("Trigger attribute mutation."); + // adding attribute to first div + document.querySelector("div").setAttribute("title", "New attribute"); + document.querySelector("input").setAttribute("title", "New attribute input"); + + await htmlMatches( + "The new attribute gets translated.", + /* html */ ` + <div title="NEW ATTRIBUTE"> + THIS IS A CHANGED NODE. + </div> + <div title="TITLE IS ADDED"> + THIS IS AN ADDED NODE. + </div> + <input placeholder="INPUT PLACEHOLDER IS ADDED" title="NEW ATTRIBUTE INPUT"> + ` + ); + + cleanup(); +}); + +add_task(async function test_mutations_subtree_attributes() { + const { translate, htmlMatches, cleanup, document } = + await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "It translates.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + info('Trigger the "childList" mutation.'); + const div = document.createElement("div"); + div.innerHTML = /* html */ ` + <div title="This is an outer node"> + This is some inner text. + <input placeholder="This is a placeholder" /> + </div> + `; + document.body.appendChild(div.children[0]); + + await htmlMatches( + "The added node gets translated.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + <div title="THIS IS AN OUTER NODE"> + THIS IS SOME INNER TEXT. + <input placeholder="THIS IS A PLACEHOLDER"> + </div> + ` + ); + + cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/head.js b/toolkit/components/translations/tests/browser/head.js new file mode 100644 index 0000000000..08c5793a46 --- /dev/null +++ b/toolkit/components/translations/tests/browser/head.js @@ -0,0 +1,9 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/toolkit/components/translations/tests/browser/shared-head.js", + this +); diff --git a/toolkit/components/translations/tests/browser/shared-head.js b/toolkit/components/translations/tests/browser/shared-head.js new file mode 100644 index 0000000000..bad8e48a1b --- /dev/null +++ b/toolkit/components/translations/tests/browser/shared-head.js @@ -0,0 +1,1442 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Avoid about:blank's non-standard behavior. +const BLANK_PAGE = + "data:text/html;charset=utf-8,<!DOCTYPE html><title>Blank</title>Blank page"; + +const URL_COM_PREFIX = "https://example.com/browser/"; +const URL_ORG_PREFIX = "https://example.org/browser/"; +const CHROME_URL_PREFIX = "chrome://mochitests/content/browser/"; +const DIR_PATH = "toolkit/components/translations/tests/browser/"; +const ENGLISH_PAGE_URL = + URL_COM_PREFIX + DIR_PATH + "translations-tester-en.html"; +const SPANISH_PAGE_URL = + URL_COM_PREFIX + DIR_PATH + "translations-tester-es.html"; +const FRENCH_PAGE_URL = + URL_COM_PREFIX + DIR_PATH + "translations-tester-fr.html"; +const SPANISH_PAGE_URL_2 = + URL_COM_PREFIX + DIR_PATH + "translations-tester-es-2.html"; +const SPANISH_PAGE_URL_DOT_ORG = + URL_ORG_PREFIX + DIR_PATH + "translations-tester-es.html"; +const NO_LANGUAGE_URL = + URL_COM_PREFIX + DIR_PATH + "translations-tester-no-tag.html"; +const EMPTY_PDF_URL = + URL_COM_PREFIX + DIR_PATH + "translations-tester-empty-pdf-file.pdf"; + +const PIVOT_LANGUAGE = "en"; +const LANGUAGE_PAIRS = [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "uk" }, + { fromLang: "uk", toLang: PIVOT_LANGUAGE }, +]; + +const TRANSLATIONS_PERMISSION = "translations"; +const ALWAYS_TRANSLATE_LANGS_PREF = + "browser.translations.alwaysTranslateLanguages"; +const NEVER_TRANSLATE_LANGS_PREF = + "browser.translations.neverTranslateLanguages"; + +/** + * The mochitest runs in the parent process. This function opens up a new tab, + * opens up about:translations, and passes the test requirements into the content process. + * + * @template T + * + * @param {object} options + * + * @param {T} options.dataForContent + * The data must support structural cloning and will be passed into the + * content process. + * + * @param {(args: { dataForContent: T, selectors: Record<string, string> }) => Promise<void>} options.runInPage + * This function must not capture any values, as it will be cloned in the content process. + * Any required data should be passed in using the "dataForContent" parameter. The + * "selectors" property contains any useful selectors for the content. + * + * @param {boolean} [options.disabled] + * Disable the panel through a pref. + * + * @param {Array<{ fromLang: string, toLang: string }>} options.languagePairs + * The translation languages pairs to mock for the test. + * + * @param {Array<[string, string]>} options.prefs + * Prefs to push on for the test. + */ +async function openAboutTranslations({ + dataForContent, + disabled, + runInPage, + languagePairs = LANGUAGE_PAIRS, + prefs, +}) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", !disabled], + ["browser.translations.logLevel", "All"], + ...(prefs ?? []), + ], + }); + + /** + * Collect any relevant selectors for the page here. + */ + const selectors = { + pageHeader: '[data-l10n-id="about-translations-header"]', + fromLanguageSelect: "select#language-from", + toLanguageSelect: "select#language-to", + translationTextarea: "textarea#translation-from", + translationResult: "#translation-to", + translationResultBlank: "#translation-to-blank", + translationInfo: "#translation-info", + noSupportMessage: "[data-l10n-id='about-translations-no-support']", + }; + + // Start the tab at a blank page. + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { removeMocks, remoteClients } = await createAndMockRemoteSettings({ + languagePairs, + // TODO(Bug 1814168) - Do not test download behavior as this is not robustly + // handled for about:translations yet. + autoDownloadFromRemoteSettings: true, + }); + + // Now load the about:translations page, since the actor could be mocked. + BrowserTestUtils.startLoadingURIString( + tab.linkedBrowser, + "about:translations" + ); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + await remoteClients.translationsWasm.resolvePendingDownloads(1); + await remoteClients.translationModels.resolvePendingDownloads( + languagePairs.length * FILES_PER_LANGUAGE_PAIR + ); + + await ContentTask.spawn( + tab.linkedBrowser, + { dataForContent, selectors }, + runInPage + ); + + await loadBlankPage(); + BrowserTestUtils.removeTab(tab); + + await removeMocks(); + await TranslationsParent.destroyEngineProcess(); + + await SpecialPowers.popPrefEnv(); +} + +/** + * Naively prettify's html based on the opening and closing tags. This is not robust + * for general usage, but should be adequate for these tests. + * @param {string} html + * @returns {string} + */ +function naivelyPrettify(html) { + let result = ""; + let indent = 0; + + function addText(actualEndIndex) { + const text = html.slice(startIndex, actualEndIndex).trim(); + if (text) { + for (let i = 0; i < indent; i++) { + result += " "; + } + result += text + "\n"; + } + startIndex = actualEndIndex; + } + + let startIndex = 0; + let endIndex = 0; + for (; endIndex < html.length; endIndex++) { + if ( + html[endIndex] === " " || + html[endIndex] === "\t" || + html[endIndex] === "n" + ) { + // Skip whitespace. + // " <div>foobar</div>" + // ^^^ + startIndex = endIndex; + continue; + } + + // Find all of the text. + // "<div>foobar</div>" + // ^^^^^^ + while (endIndex < html.length && html[endIndex] !== "<") { + endIndex++; + } + + addText(endIndex); + + if (html[endIndex] === "<") { + if (html[endIndex + 1] === "/") { + // "<div>foobar</div>" + // ^ + while (endIndex < html.length && html[endIndex] !== ">") { + endIndex++; + } + indent--; + addText(endIndex + 1); + } else { + // "<div>foobar</div>" + // ^ + while (endIndex < html.length && html[endIndex] !== ">") { + endIndex++; + } + // "<div>foobar</div>" + // ^ + addText(endIndex + 1); + indent++; + } + } + } + + return result.trim(); +} + +/** + * Recursively transforms all child nodes to have uppercased text. + * + * @param {Node} node + */ +function upperCaseNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.toUpperCase(); + } + for (const childNode of node.childNodes) { + upperCaseNode(childNode); + } +} + +/** + * Creates a mocked message port for translations. + * + * @returns {MessagePort} This is mocked + */ +function createMockedTranslatorPort(transformNode = upperCaseNode) { + const parser = new DOMParser(); + const mockedPort = { + async postMessage(message) { + // Make this response async. + await TestUtils.waitForTick(); + + switch (message.type) { + case "TranslationsPort:GetEngineStatusRequest": + mockedPort.onmessage({ + data: { + type: "TranslationsPort:GetEngineStatusResponse", + status: "ready", + }, + }); + break; + case "TranslationsPort:TranslationRequest": { + const { messageId, sourceText } = message; + + const translatedDoc = parser.parseFromString(sourceText, "text/html"); + transformNode(translatedDoc.body); + mockedPort.onmessage({ + data: { + type: "TranslationsPort:TranslationResponse", + targetText: translatedDoc.body.innerHTML, + messageId, + }, + }); + } + } + }, + }; + return mockedPort; +} + +/** + * This mocked translator reports on the batching of calls by replacing the text + * with a letter. Each call of the function moves the letter forward alphabetically. + * + * So consecutive calls would transform things like: + * "First translation" -> "aaaa aaaaaaaaa" + * "Second translation" -> "bbbbb bbbbbbbbb" + * "Third translation" -> "cccc ccccccccc" + * + * This can visually show what the translation batching behavior looks like. + * + * @returns {MessagePort} A mocked port. + */ +function createBatchedMockedTranslatorPort() { + let letter = "a"; + + /** + * @param {Node} node + */ + function transformNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.replace(/\w/g, letter); + } + for (const childNode of node.childNodes) { + transformNode(childNode); + } + } + + return createMockedTranslatorPort(node => { + transformNode(node); + letter = String.fromCodePoint(letter.codePointAt(0) + 1); + }); +} + +/** + * This mocked translator reorders Nodes to be in alphabetical order, and then + * uppercases the text. This allows for testing the reordering behavior of the + * translation engine. + * + * @returns {MessagePort} A mocked port. + */ +function createdReorderingMockedTranslatorPort() { + /** + * @param {Node} node + */ + function transformNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.toUpperCase(); + } + const nodes = [...node.childNodes]; + nodes.sort((a, b) => + (a.textContent?.trim() ?? "").localeCompare(b.textContent?.trim() ?? "") + ); + for (const childNode of nodes) { + childNode.remove(); + } + for (const childNode of nodes) { + // Re-append in sorted order. + node.appendChild(childNode); + transformNode(childNode); + } + } + + return createMockedTranslatorPort(transformNode); +} + +/** + * @returns {import("../../actors/TranslationsParent.sys.mjs").TranslationsParent} + */ +function getTranslationsParent() { + return gBrowser.selectedBrowser.browsingContext.currentWindowGlobal.getActor( + "Translations" + ); +} + +/** + * Closes the context menu if it is open. + */ +function closeContextMenuIfOpen() { + return waitForCondition(async () => { + const contextMenu = document.getElementById("contentAreaContextMenu"); + if (!contextMenu) { + return true; + } + if (contextMenu.state === "closed") { + return true; + } + let popuphiddenPromise = BrowserTestUtils.waitForEvent( + contextMenu, + "popuphidden" + ); + PanelMultiView.hidePopup(contextMenu); + await popuphiddenPromise; + return false; + }); +} + +/** + * Closes the translations panel settings menu if it is open. + */ +function closeSettingsMenuIfOpen() { + return waitForCondition(async () => { + const settings = document.getElementById( + "translations-panel-settings-menupopup" + ); + if (!settings) { + return true; + } + if (settings.state === "closed") { + return true; + } + let popuphiddenPromise = BrowserTestUtils.waitForEvent( + settings, + "popuphidden" + ); + PanelMultiView.hidePopup(settings); + await popuphiddenPromise; + return false; + }); +} + +/** + * Closes the translations panel if it is open. + */ +async function closeTranslationsPanelIfOpen() { + await closeSettingsMenuIfOpen(); + return waitForCondition(async () => { + const panel = document.getElementById("translations-panel"); + if (!panel) { + return true; + } + if (panel.state === "closed") { + return true; + } + let popuphiddenPromise = BrowserTestUtils.waitForEvent( + panel, + "popuphidden" + ); + PanelMultiView.hidePopup(panel); + await popuphiddenPromise; + return false; + }); +} + +/** + * This is for tests that don't need a browser page to run. + */ +async function setupActorTest({ + languagePairs, + prefs, + autoDownloadFromRemoteSettings = false, +}) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ...(prefs ?? []), + ], + }); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + autoDownloadFromRemoteSettings, + }); + + // Create a new tab so each test gets a new actor, and doesn't re-use the old one. + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + ENGLISH_PAGE_URL, + true // waitForLoad + ); + + const actor = getTranslationsParent(); + return { + actor, + remoteClients, + async cleanup() { + await loadBlankPage(); + await TranslationsParent.destroyEngineProcess(); + await closeTranslationsPanelIfOpen(); + await closeContextMenuIfOpen(); + BrowserTestUtils.removeTab(tab); + await removeMocks(); + TestTranslationsTelemetry.reset(); + return SpecialPowers.popPrefEnv(); + }, + }; +} + +async function createAndMockRemoteSettings({ + languagePairs = LANGUAGE_PAIRS, + autoDownloadFromRemoteSettings = false, +}) { + const remoteClients = { + translationModels: await createTranslationModelsRemoteClient( + autoDownloadFromRemoteSettings, + languagePairs + ), + translationsWasm: await createTranslationsWasmRemoteClient( + autoDownloadFromRemoteSettings + ), + }; + + // The TranslationsParent will pull the language pair values from the JSON dump + // of Remote Settings. Clear these before mocking the translations engine. + TranslationsParent.clearCache(); + + TranslationsParent.mockTranslationsEngine( + remoteClients.translationModels.client, + remoteClients.translationsWasm.client + ); + + return { + async removeMocks() { + await remoteClients.translationModels.client.attachments.deleteAll(); + await remoteClients.translationModels.client.db.clear(); + await remoteClients.translationsWasm.client.db.clear(); + + TranslationsParent.unmockTranslationsEngine(); + TranslationsParent.clearCache(); + }, + remoteClients, + }; +} + +async function loadTestPage({ + languagePairs, + autoDownloadFromRemoteSettings = false, + page, + prefs, + autoOffer, + permissionsUrls, +}) { + info(`Loading test page starting at url: ${page}`); + // Ensure no engine is being carried over from a previous test. + await TranslationsParent.destroyEngineProcess(); + Services.fog.testResetFOG(); + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ["browser.translations.panelShown", true], + ["browser.translations.automaticallyPopup", true], + ["browser.translations.alwaysTranslateLanguages", ""], + ["browser.translations.neverTranslateLanguages", ""], + ...(prefs ?? []), + ], + }); + await SpecialPowers.pushPermissions( + [ + ENGLISH_PAGE_URL, + FRENCH_PAGE_URL, + NO_LANGUAGE_URL, + SPANISH_PAGE_URL, + SPANISH_PAGE_URL_2, + SPANISH_PAGE_URL_DOT_ORG, + ...(permissionsUrls || []), + ].map(url => ({ + type: TRANSLATIONS_PERMISSION, + allow: true, + context: url, + })) + ); + + if (autoOffer) { + TranslationsParent.testAutomaticPopup = true; + } + + // Start the tab at a blank page. + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + autoDownloadFromRemoteSettings, + }); + + BrowserTestUtils.startLoadingURIString(tab.linkedBrowser, page); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + if (autoOffer && TranslationsParent.shouldAlwaysOfferTranslations()) { + info("Waiting for the popup to be automatically shown."); + await waitForCondition(() => { + const panel = document.getElementById("translations-panel"); + return panel && panel.state === "open"; + }); + } + + return { + tab, + remoteClients, + + /** + * @param {number} count - Count of the language pairs expected. + */ + async resolveDownloads(count) { + await remoteClients.translationsWasm.resolvePendingDownloads(1); + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR * count + ); + }, + + /** + * @param {number} count - Count of the language pairs expected. + */ + async rejectDownloads(count) { + await remoteClients.translationsWasm.rejectPendingDownloads(1); + await remoteClients.translationModels.rejectPendingDownloads( + FILES_PER_LANGUAGE_PAIR * count + ); + }, + + /** + * @returns {Promise<void>} + */ + async cleanup() { + await loadBlankPage(); + await TranslationsParent.destroyEngineProcess(); + await closeTranslationsPanelIfOpen(); + await closeContextMenuIfOpen(); + await removeMocks(); + Services.fog.testResetFOG(); + TranslationsParent.testAutomaticPopup = false; + TranslationsParent.resetHostsOffered(); + BrowserTestUtils.removeTab(tab); + TestTranslationsTelemetry.reset(); + return Promise.all([ + SpecialPowers.popPrefEnv(), + SpecialPowers.popPermissions(), + ]); + }, + + /** + * Runs a callback in the content page. The function's contents are serialized as + * a string, and run in the page. The `translations-test.mjs` module is made + * available to the page. + * + * @param {(TranslationsTest: import("./translations-test.mjs")) => any} callback + * @returns {Promise<void>} + */ + runInPage(callback, data = {}) { + // ContentTask.spawn runs the `Function.prototype.toString` on this function in + // order to send it into the content process. The following function is doing its + // own string manipulation in order to load in the TranslationsTest module. + const fn = new Function(/* js */ ` + const TranslationsTest = ChromeUtils.importESModule( + "chrome://mochitests/content/browser/toolkit/components/translations/tests/browser/translations-test.mjs" + ); + + // Pass in the values that get injected by the task runner. + TranslationsTest.setup({Assert, ContentTaskUtils, content}); + + const data = ${JSON.stringify(data)}; + + return (${callback.toString()})(TranslationsTest, data); + `); + + return ContentTask.spawn( + tab.linkedBrowser, + {}, // Data to inject. + fn + ); + }, + }; +} + +/** + * Captures any reported errors in the TranslationsParent. + * + * @param {Function} callback + * @returns {Array<{ error: Error, args: any[] }>} + */ +async function captureTranslationsError(callback) { + const { reportError } = TranslationsParent; + + let errors = []; + TranslationsParent.reportError = (error, ...args) => { + errors.push({ error, args }); + }; + + await callback(); + + // Restore the original function. + TranslationsParent.reportError = reportError; + return errors; +} + +/** + * Load a test page and run + * @param {Object} options - The options for `loadTestPage` plus a `runInPage` function. + */ +async function autoTranslatePage(options) { + const { prefs, languagePairs, ...otherOptions } = options; + const fromLangs = languagePairs.map(language => language.fromLang).join(","); + const { cleanup, runInPage } = await loadTestPage({ + autoDownloadFromRemoteSettings: true, + prefs: [ + ["browser.translations.alwaysTranslateLanguages", fromLangs], + ...(prefs ?? []), + ], + ...otherOptions, + }); + await runInPage(options.runInPage); + await cleanup(); +} + +/** + * @param {RemoteSettingsClient} client + * @param {string} mockedCollectionName - The name of the mocked collection without + * the incrementing "id" part. This is provided so that attachments can be asserted + * as being of a certain version. + * @param {boolean} autoDownloadFromRemoteSettings - Skip the manual download process, + * and automatically download the files. Normally it's preferrable to manually trigger + * the downloads to trigger the download behavior, but this flag lets you bypass this + * and automatically download the files. + */ +function createAttachmentMock( + client, + mockedCollectionName, + autoDownloadFromRemoteSettings +) { + const pendingDownloads = []; + client.attachments.download = record => + new Promise((resolve, reject) => { + console.log("Download requested:", client.collectionName, record.name); + if (autoDownloadFromRemoteSettings) { + const encoder = new TextEncoder(); + const { buffer } = encoder.encode( + `Mocked download: ${mockedCollectionName} ${record.name} ${record.version}` + ); + + resolve({ buffer }); + } else { + pendingDownloads.push({ record, resolve, reject }); + } + }); + + function resolvePendingDownloads(expectedDownloadCount) { + info( + `Resolving ${expectedDownloadCount} mocked downloads for "${client.collectionName}"` + ); + return downloadHandler(expectedDownloadCount, download => + download.resolve({ buffer: new ArrayBuffer() }) + ); + } + + async function rejectPendingDownloads(expectedDownloadCount) { + info( + `Intentionally rejecting ${expectedDownloadCount} mocked downloads for "${client.collectionName}"` + ); + + // Add 1 to account for the original attempt. + const attempts = TranslationsParent.MAX_DOWNLOAD_RETRIES + 1; + return downloadHandler(expectedDownloadCount * attempts, download => + download.reject(new Error("Intentionally rejecting downloads.")) + ); + } + + async function downloadHandler(expectedDownloadCount, action) { + const names = []; + let maxTries = 100; + while (names.length < expectedDownloadCount && maxTries-- > 0) { + await new Promise(resolve => setTimeout(resolve, 0)); + let download = pendingDownloads.shift(); + if (!download) { + // Uncomment the following to debug download issues: + // console.log(`No pending download:`, client.collectionName, names.length); + continue; + } + console.log(`Handling download:`, client.collectionName); + action(download); + names.push(download.record.name); + } + + // This next check is not guaranteed to catch an unexpected download, but wait + // at least one event loop tick to see if any more downloads were added. + await new Promise(resolve => setTimeout(resolve, 0)); + + if (pendingDownloads.length) { + throw new Error( + `An unexpected download was found, only expected ${expectedDownloadCount} downloads` + ); + } + + return names.sort((a, b) => a.localeCompare(b)); + } + + async function assertNoNewDownloads() { + await new Promise(resolve => setTimeout(resolve, 0)); + is( + pendingDownloads.length, + 0, + `No downloads happened for "${client.collectionName}"` + ); + } + + return { + client, + pendingDownloads, + resolvePendingDownloads, + rejectPendingDownloads, + assertNoNewDownloads, + }; +} + +/** + * The amount of files that are generated per mocked language pair. + */ +const FILES_PER_LANGUAGE_PAIR = 3; + +function createRecordsForLanguagePair(fromLang, toLang) { + const records = []; + const lang = fromLang + toLang; + const models = [ + { fileType: "model", name: `model.${lang}.intgemm.alphas.bin` }, + { fileType: "lex", name: `lex.50.50.${lang}.s2t.bin` }, + { fileType: "vocab", name: `vocab.${lang}.spm` }, + ]; + + const attachment = { + hash: `${crypto.randomUUID()}`, + size: `123`, + filename: `model.${lang}.intgemm.alphas.bin`, + location: `main-workspace/translations-models/${crypto.randomUUID()}.bin`, + mimetype: "application/octet-stream", + }; + + if (models.length !== FILES_PER_LANGUAGE_PAIR) { + throw new Error("Files per language pair was wrong."); + } + + for (const { fileType, name } of models) { + records.push({ + id: crypto.randomUUID(), + name, + fromLang, + toLang, + fileType, + version: TranslationsParent.LANGUAGE_MODEL_MAJOR_VERSION + ".0", + last_modified: Date.now(), + schema: Date.now(), + attachment, + }); + } + return records; +} + +/** + * Increments each time a remote settings client is created to ensure a unique client + * name for each test run. + */ +let _remoteSettingsMockId = 0; + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @param {Object[]} langPairs + * @returns {RemoteSettingsClient} + */ +async function createTranslationModelsRemoteClient( + autoDownloadFromRemoteSettings, + langPairs +) { + const records = []; + for (const { fromLang, toLang } of langPairs) { + records.push(...createRecordsForLanguagePair(fromLang, toLang)); + } + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const mockedCollectionName = "test-translation-models"; + const client = RemoteSettings( + `${mockedCollectionName}-${_remoteSettingsMockId++}` + ); + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock( + client, + mockedCollectionName, + autoDownloadFromRemoteSettings + ); +} + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @returns {RemoteSettingsClient} + */ +async function createTranslationsWasmRemoteClient( + autoDownloadFromRemoteSettings +) { + const records = ["bergamot-translator"].map(name => ({ + id: crypto.randomUUID(), + name, + version: TranslationsParent.BERGAMOT_MAJOR_VERSION + ".0", + last_modified: Date.now(), + schema: Date.now(), + })); + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const mockedCollectionName = "test-translation-wasm"; + const client = RemoteSettings( + `${mockedCollectionName}-${_remoteSettingsMockId++}` + ); + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock( + client, + mockedCollectionName, + autoDownloadFromRemoteSettings + ); +} + +async function selectAboutPreferencesElements() { + const document = gBrowser.selectedBrowser.contentDocument; + + const settingsButton = document.getElementById( + "translations-manage-settings-button" + ); + + const rows = await waitForCondition(() => { + const elements = document.querySelectorAll(".translations-manage-language"); + if (elements.length !== 4) { + return false; + } + return elements; + }, "Waiting for manage language rows."); + + const [downloadAllRow, frenchRow, spanishRow, ukrainianRow] = rows; + + const downloadAllLabel = downloadAllRow.querySelector("label"); + const downloadAll = downloadAllRow.querySelector( + "#translations-manage-install-all" + ); + const deleteAll = downloadAllRow.querySelector( + "#translations-manage-delete-all" + ); + const frenchLabel = frenchRow.querySelector("label"); + const frenchDownload = frenchRow.querySelector( + `[data-l10n-id="translations-manage-language-install-button"]` + ); + const frenchDelete = frenchRow.querySelector( + `[data-l10n-id="translations-manage-language-remove-button"]` + ); + const spanishLabel = spanishRow.querySelector("label"); + const spanishDownload = spanishRow.querySelector( + `[data-l10n-id="translations-manage-language-install-button"]` + ); + const spanishDelete = spanishRow.querySelector( + `[data-l10n-id="translations-manage-language-remove-button"]` + ); + const ukrainianLabel = ukrainianRow.querySelector("label"); + const ukrainianDownload = ukrainianRow.querySelector( + `[data-l10n-id="translations-manage-language-install-button"]` + ); + const ukrainianDelete = ukrainianRow.querySelector( + `[data-l10n-id="translations-manage-language-remove-button"]` + ); + + return { + document, + downloadAllLabel, + downloadAll, + deleteAll, + frenchLabel, + frenchDownload, + frenchDelete, + ukrainianLabel, + ukrainianDownload, + ukrainianDelete, + settingsButton, + spanishLabel, + spanishDownload, + spanishDelete, + }; +} + +function click(button, message) { + info(message); + if (button.hidden) { + throw new Error("The button was hidden when trying to click it."); + } + button.click(); +} + +function hitEnterKey(button, message) { + info(message); + button.dispatchEvent( + new KeyboardEvent("keypress", { + key: "Enter", + keyCode: KeyboardEvent.DOM_VK_RETURN, + }) + ); +} + +/** + * Similar to assertVisibility, but is asynchronous and attempts + * to wait for the elements to match the expected states if they + * do not already. + * + * @see assertVisibility + * + * @param {Object} options + * @param {string} options.message + * @param {Record<string, Element[]>} options.visible + * @param {Record<string, Element[]>} options.hidden + */ +async function ensureVisibility({ message = null, visible = {}, hidden = {} }) { + try { + // First wait for the condition to be met. + await waitForCondition(() => { + for (const element of Object.values(visible)) { + if (BrowserTestUtils.isHidden(element)) { + return false; + } + } + for (const element of Object.values(hidden)) { + if (BrowserTestUtils.isVisible(element)) { + return false; + } + } + return true; + }); + } catch (error) { + // Ignore, this will get caught below. + } + // Now report the conditions. + assertVisibility({ message, visible, hidden }); +} + +/** + * Asserts that the provided elements are either visible or hidden. + * + * @param {Object} options + * @param {string} options.message + * @param {Record<string, Element[]>} options.visible + * @param {Record<string, Element[]>} options.hidden + */ +function assertVisibility({ message = null, visible = {}, hidden = {} }) { + if (message) { + info(message); + } + for (const [name, element] of Object.entries(visible)) { + ok(BrowserTestUtils.isVisible(element), `${name} is visible.`); + } + for (const [name, element] of Object.entries(hidden)) { + ok(BrowserTestUtils.isHidden(element), `${name} is hidden.`); + } +} + +async function setupAboutPreferences( + languagePairs, + { prefs = [], permissionsUrls = [] } = {} +) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ...prefs, + ], + }); + await SpecialPowers.pushPermissions( + permissionsUrls.map(url => ({ + type: TRANSLATIONS_PERMISSION, + allow: true, + context: url, + })) + ); + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + }); + + BrowserTestUtils.startLoadingURIString( + tab.linkedBrowser, + "about:preferences" + ); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + const elements = await selectAboutPreferencesElements(); + + async function cleanup() { + await loadBlankPage(); + await TranslationsParent.destroyEngineProcess(); + await closeTranslationsPanelIfOpen(); + await closeContextMenuIfOpen(); + BrowserTestUtils.removeTab(tab); + await removeMocks(); + await SpecialPowers.popPrefEnv(); + TestTranslationsTelemetry.reset(); + } + + return { + cleanup, + remoteClients, + elements, + }; +} + +function waitForAppLocaleChanged() { + new Promise(resolve => { + function onChange() { + Services.obs.removeObserver(onChange, "intl:app-locales-changed"); + resolve(); + } + Services.obs.addObserver(onChange, "intl:app-locales-changed"); + }); +} + +async function mockLocales({ systemLocales, appLocales, webLanguages }) { + const appLocaleChanged1 = waitForAppLocaleChanged(); + + TranslationsParent.mockedSystemLocales = systemLocales; + const { availableLocales, requestedLocales } = Services.locale; + + info("Mocking locales, so expect potential .ftl resource errors."); + Services.locale.availableLocales = appLocales; + Services.locale.requestedLocales = appLocales; + + await appLocaleChanged1; + + await SpecialPowers.pushPrefEnv({ + set: [["intl.accept_languages", webLanguages.join(",")]], + }); + + return async () => { + const appLocaleChanged2 = waitForAppLocaleChanged(); + + // Reset back to the originals. + TranslationsParent.mockedSystemLocales = null; + Services.locale.availableLocales = availableLocales; + Services.locale.requestedLocales = requestedLocales; + + await appLocaleChanged2; + + await SpecialPowers.popPrefEnv(); + }; +} + +/** + * Helpful test functions for translations telemetry + */ +class TestTranslationsTelemetry { + static #previousFlowId = null; + + static reset() { + TestTranslationsTelemetry.#previousFlowId = null; + } + + /** + * Asserts qualities about a counter telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} counter - The Glean counter object. + * @param {Object} expectedCount - The expected value of the counter. + */ + static async assertCounter(name, counter, expectedCount) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const count = counter.testGetValue() ?? 0; + is( + count, + expectedCount, + `Telemetry counter ${name} should have expected count` + ); + } + + /** + * Asserts qualities about an event telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} event - The Glean event object. + * @param {Object} expectations - The test expectations. + * @param {number} expectations.expectedEventCount - The expected count of events. + * @param {boolean} expectations.expectNewFlowId + * - Expects the flowId to be different than the previous flowId if true, + * and expects it to be the same if false. + * @param {Array<function>} [expectations.allValuePredicates=[]] + * - An array of function predicates to assert for all event values. + * @param {Array<function>} [expectations.finalValuePredicates=[]] + * - An array of function predicates to assert for only the final event value. + */ + static async assertEvent( + event, + { + expectedEventCount, + expectNewFlowId = null, + expectFirstInteraction = null, + allValuePredicates = [], + finalValuePredicates = [], + } + ) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const events = event.testGetValue() ?? []; + const eventCount = events.length; + const name = + eventCount > 0 ? `${events[0].category}.${events[0].name}` : null; + + if (eventCount > 0 && expectFirstInteraction !== null) { + is( + events[eventCount - 1].extra.first_interaction, + expectFirstInteraction ? "true" : "false", + "The newest event should be match the given first-interaction expectation" + ); + } + + if (eventCount > 0 && expectNewFlowId !== null) { + const flowId = events[eventCount - 1].extra.flow_id; + if (expectNewFlowId) { + is( + events[eventCount - 1].extra.flow_id !== + TestTranslationsTelemetry.#previousFlowId, + true, + `The newest flowId ${flowId} should be different than the previous flowId ${ + TestTranslationsTelemetry.#previousFlowId + }` + ); + } else { + is( + events[eventCount - 1].extra.flow_id === + TestTranslationsTelemetry.#previousFlowId, + true, + `The newest flowId ${flowId} should be equal to the previous flowId ${ + TestTranslationsTelemetry.#previousFlowId + }` + ); + } + TestTranslationsTelemetry.#previousFlowId = flowId; + } + + if (eventCount !== expectedEventCount) { + console.error("Actual events:", events); + } + + is( + eventCount, + expectedEventCount, + `There should be ${expectedEventCount} telemetry events of type ${name}` + ); + + if (allValuePredicates.length !== 0) { + is( + eventCount > 0, + true, + `Telemetry event ${name} should contain values if allPredicates are specified` + ); + for (const value of events) { + for (const predicate of allValuePredicates) { + is( + predicate(value), + true, + `Telemetry event ${name} allPredicate { ${predicate.toString()} } should pass for each value` + ); + } + } + } + + if (finalValuePredicates.length !== 0) { + is( + eventCount > 0, + true, + `Telemetry event ${name} should contain values if finalPredicates are specified` + ); + for (const predicate of finalValuePredicates) { + is( + predicate(events[eventCount - 1]), + true, + `Telemetry event ${name} finalPredicate { ${predicate.toString()} } should pass for final value` + ); + } + } + } + + /** + * Asserts qualities about a rate telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} rate - The Glean rate object. + * @param {Object} expectations - The test expectations. + * @param {number} expectations.expectedNumerator - The expected value of the numerator. + * @param {number} expectations.expectedDenominator - The expected value of the denominator. + */ + static async assertRate( + name, + rate, + { expectedNumerator, expectedDenominator } + ) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const { numerator = 0, denominator = 0 } = rate.testGetValue() ?? {}; + is( + numerator, + expectedNumerator, + `Telemetry rate ${name} should have expected numerator` + ); + is( + denominator, + expectedDenominator, + `Telemetry rate ${name} should have expected denominator` + ); + } +} + +/** + * Provide longer defaults for the waitForCondition. + * + * @param {Function} callback + * @param {string} messages + */ +function waitForCondition(callback, message) { + const interval = 100; + // Use 4 times the defaults to guard against intermittents. Many of the tests rely on + // communication between the parent and child process, which is inherently async. + const maxTries = 50 * 4; + return TestUtils.waitForCondition(callback, message, interval, maxTries); +} + +/** + * Retrieves the always-translate language list as an array. + * + * @returns {Array<string>} + */ +function getAlwaysTranslateLanguagesFromPref() { + let langs = Services.prefs.getCharPref(ALWAYS_TRANSLATE_LANGS_PREF); + return langs ? langs.split(",") : []; +} + +/** + * Retrieves the never-translate language list as an array. + * + * @returns {Array<string>} + */ +function getNeverTranslateLanguagesFromPref() { + let langs = Services.prefs.getCharPref(NEVER_TRANSLATE_LANGS_PREF); + return langs ? langs.split(",") : []; +} + +/** + * Retrieves the never-translate site list as an array. + * + * @returns {Array<string>} + */ +function getNeverTranslateSitesFromPerms() { + let results = []; + for (let perm of Services.perms.all) { + if ( + perm.type == TRANSLATIONS_PERMISSION && + perm.capability == Services.perms.DENY_ACTION + ) { + results.push(perm.principal); + } + } + + return results; +} + +/** + * Opens a dialog window for about:preferences + * @param {string} dialogUrl - The URL of the dialog window + * @param {Function} callback - The function to open the dialog via UI + * @returns {Object} The dialog window object + */ +async function waitForOpenDialogWindow(dialogUrl, callback) { + const dialogLoaded = promiseLoadSubDialog(dialogUrl); + await callback(); + const dialogWindow = await dialogLoaded; + return dialogWindow; +} + +/** + * Closes an open dialog window and waits for it to close. + * + * @param {Object} dialogWindow + */ +async function waitForCloseDialogWindow(dialogWindow) { + const closePromise = BrowserTestUtils.waitForEvent( + content.gSubDialog._dialogStack, + "dialogclose" + ); + dialogWindow.close(); + await closePromise; +} + +// Extracted from https://searchfox.org/mozilla-central/rev/40ef22080910c2e2c27d9e2120642376b1d8b8b2/browser/components/preferences/in-content/tests/head.js#41 +function promiseLoadSubDialog(aURL) { + return new Promise((resolve, reject) => { + content.gSubDialog._dialogStack.addEventListener( + "dialogopen", + function dialogopen(aEvent) { + if ( + aEvent.detail.dialog._frame.contentWindow.location == "about:blank" + ) { + return; + } + content.gSubDialog._dialogStack.removeEventListener( + "dialogopen", + dialogopen + ); + + Assert.equal( + aEvent.detail.dialog._frame.contentWindow.location.toString(), + aURL, + "Check the proper URL is loaded" + ); + + // Check visibility + isnot( + aEvent.detail.dialog._overlay, + null, + "Element should not be null, when checking visibility" + ); + Assert.ok( + !BrowserTestUtils.isHidden(aEvent.detail.dialog._overlay), + "The element is visible" + ); + + // Check that stylesheets were injected + let expectedStyleSheetURLs = + aEvent.detail.dialog._injectedStyleSheets.slice(0); + for (let styleSheet of aEvent.detail.dialog._frame.contentDocument + .styleSheets) { + let i = expectedStyleSheetURLs.indexOf(styleSheet.href); + if (i >= 0) { + info("found " + styleSheet.href); + expectedStyleSheetURLs.splice(i, 1); + } + } + Assert.equal( + expectedStyleSheetURLs.length, + 0, + "All expectedStyleSheetURLs should have been found" + ); + + // Wait for the next event tick to make sure the remaining part of the + // testcase runs after the dialog gets ready for input. + executeSoon(() => resolve(aEvent.detail.dialog._frame.contentWindow)); + } + ); + }); +} + +/** + * Loads the blank-page URL. + * + * This is useful for resetting the state during cleanup, and also + * before starting a test, to further help ensure that there is no + * unintentional state left over from test case. + */ +async function loadBlankPage() { + BrowserTestUtils.startLoadingURIString(gBrowser.selectedBrowser, BLANK_PAGE); + await BrowserTestUtils.browserLoaded(gBrowser.selectedBrowser); +} diff --git a/toolkit/components/translations/tests/browser/translations-test.mjs b/toolkit/components/translations/tests/browser/translations-test.mjs new file mode 100644 index 0000000000..3e16be57e9 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-test.mjs @@ -0,0 +1,129 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// eslint-disable-next-line no-unused-vars +let ok; +let is; +// eslint-disable-next-line no-unused-vars +let isnot; +let ContentTaskUtils; + +/** @type {{ document: Document, window: Window }} */ +let content; + +/** + * Inject the global variables from the test scope into the ES module scope. + */ +export function setup(config) { + // When a function is provided to `ContentTask.spawn`, that function is provided the + // Assert library through variable capture. In this case, this code is an ESM module, + // and does not have access to that scope. To work around this issue, pass any any + // relevant variables to that can be bound to the module scope. + // + // See: https://searchfox.org/mozilla-central/rev/cdddec7fd690700efa4d6b48532cf70155e0386b/testing/mochitest/BrowserTestUtils/content/content-task.js#78 + const { Assert } = config; + ok = Assert.ok.bind(Assert); + is = Assert.equal.bind(Assert); + isnot = Assert.notEqual.bind(Assert); + + ContentTaskUtils = config.ContentTaskUtils; + content = config.content; +} + +export function getSelectors() { + return { + getH1() { + return content.document.querySelector("h1"); + }, + getHeader() { + return content.document.querySelector("header"); + }, + getFirstParagraph() { + return content.document.querySelector("p:first-of-type"); + }, + getLastParagraph() { + return content.document.querySelector("p:last-of-type"); + }, + getSpanishParagraph() { + return content.document.getElementById("spanish-paragraph"); + }, + getSpanishHyperlink() { + return content.document.getElementById("spanish-hyperlink"); + }, + getEnglishHyperlink() { + return content.document.getElementById("english-hyperlink"); + }, + }; +} + +/** + * Provide longer defaults for the waitForCondition. + * + * @param {Function} callback + * @param {string} messages + */ +function waitForCondition(callback, message) { + const interval = 100; + // Use 4 times the defaults to guard against intermittents. Many of the tests rely on + // communication between the parent and child process, which is inherently async. + const maxTries = 50 * 4; + return ContentTaskUtils.waitForCondition( + callback, + message, + interval, + maxTries + ); +} + +/** + * Asserts that a page was translated with a specific result. + * + * @param {string} message The assertion message. + * @param {Function} getNode A function to get the node. + * @param {string} translation The translated message. + */ +export async function assertTranslationResult(message, getNode, translation) { + try { + await waitForCondition( + () => translation === getNode()?.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is(translation, getNode()?.innerText, message); +} + +/** + * Simulates right-clicking an element with the mouse. + * + * @param {element} element - The element to right-click. + */ +export function rightClickContentElement(element) { + return new Promise(resolve => { + element.addEventListener( + "contextmenu", + function () { + resolve(); + }, + { once: true } + ); + + const EventUtils = ContentTaskUtils.getEventUtils(content); + EventUtils.sendMouseEvent({ type: "contextmenu" }, element, content.window); + }); +} + +/** + * Selects all the content within a specified element. + * + * @param {Element} element - The element containing the content to be selected. + * @returns {string} - The text content of the selection. + */ +export function selectContentElement(element) { + content.focus(); + content.getSelection().selectAllChildren(element); + return element.textContent; +} diff --git a/toolkit/components/translations/tests/browser/translations-tester-empty-pdf-file.pdf b/toolkit/components/translations/tests/browser/translations-tester-empty-pdf-file.pdf new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-empty-pdf-file.pdf diff --git a/toolkit/components/translations/tests/browser/translations-tester-en.html b/toolkit/components/translations/tests/browser/translations-tester-en.html new file mode 100644 index 0000000000..f32a7486ed --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-en.html @@ -0,0 +1,34 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <!-- The following is an excerpt from The Wondeful Wizard of Oz, which is in the public domain --> + <h1>"The Wonderful Wizard of Oz" by L. Frank Baum</h1> + <p>The little girl, seeing she had lost one of her pretty shoes, grew angry, and said to the Witch, “Give me back my shoe!”</p> + <p>“I will not,” retorted the Witch, “for it is now my shoe, and not yours.”</p> + <p>“You are a wicked creature!” cried Dorothy. “You have no right to take my shoe from me.”</p> + <p>“I shall keep it, just the same,” said the Witch, laughing at her, “and someday I shall get the other one from you, too.”</p> + <p>This made Dorothy so very angry that she picked up the bucket of water that stood near and dashed it over the Witch, wetting her from head to foot.</p> + <p>Instantly the wicked woman gave a loud cry of fear, and then, as Dorothy looked at her in wonder, the Witch began to shrink and fall away.</p> + <p>“See what you have done!” she screamed. “In a minute I shall melt away.”</p> + <p>“I’m very sorry, indeed,” said Dorothy, who was truly frightened to see the Witch actually melting away like brown sugar before her very eyes.</p> + <p>“Didn’t you know water would be the end of me?” asked the Witch, in a wailing, despairing voice.</p> + <p>“Of course not,” answered Dorothy. “How should I?”</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-es-2.html b/toolkit/components/translations/tests/browser/translations-tester-es-2.html new file mode 100644 index 0000000000..abf2d42c62 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-es-2.html @@ -0,0 +1,36 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p>Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-es.html b/toolkit/components/translations/tests/browser/translations-tester-es.html new file mode 100644 index 0000000000..f589df0649 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-es.html @@ -0,0 +1,44 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p id="spanish-paragraph">Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> + <div> + <header lang="en">The following is a link to another test page in Spanish.</header> + <p><a id="spanish-hyperlink" href="https://example.org/browser/translations-tester-es.html">Otra pagina en español.</a></p> + </div> + <div> + <header lang="en">The following is a link to another test page in English.</header> + <p lang="en"><a id="english-hyperlink" href="https://example.org/browser/translations-tester-en.html">Another page in English.</a></p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-fr.html b/toolkit/components/translations/tests/browser/translations-tester-fr.html new file mode 100644 index 0000000000..de8a158d25 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-fr.html @@ -0,0 +1,9 @@ +<!DOCTYPE html> +<html lang="fr"> +<head> + <meta charset="utf-8" /> + <title>Translations Test (fr)</title> +</head> +<body> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-no-tag.html b/toolkit/components/translations/tests/browser/translations-tester-no-tag.html new file mode 100644 index 0000000000..7ef29cb70b --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-no-tag.html @@ -0,0 +1,35 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p>Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-es.html b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-es.html new file mode 100644 index 0000000000..ed4b1ceec2 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-es.html @@ -0,0 +1,38 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <h1>Esto se contenta en Luz DOM</h1> + <div id="host"></div> + <div><div><div id="host2"></div></div></div> + <script> + const host = document.getElementById("host"); + const root = host.attachShadow({mode: "open"}); + root.innerHTML = "<p>Esto se contento en Shadow DOM</p><div id='innerHost'></div>"; + + // Nested shadow tree + const innerHost = root.querySelector("div"); + const innerRoot = innerHost.attachShadow({mode: "open"}); + innerRoot.innerHTML = "<p>Esto se contenta en raíz interior </p>"; + + // Host within an empty textContent element + const host2 = document.getElementById("host2"); + const root2 = host2.attachShadow({mode: "open"}); + root2.innerHTML = "<p>Esto se contento en Shadow DOM 2</p>"; + </script> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es-2.html b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es-2.html new file mode 100644 index 0000000000..b63d05cee8 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es-2.html @@ -0,0 +1,30 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <h1>This is content in light DOM</h1> + <div id="host1"></div> + <script> + const host1 = document.getElementById("host1"); + const root1 = host1.attachShadow({mode: "open"}); + root1.innerHTML = "<div id='innerHost'></div>"; + + const innerHost = root1.querySelector("div"); + const innerRoot = innerHost.attachShadow({mode: "open"}); + </script> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es.html b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es.html new file mode 100644 index 0000000000..45add59a18 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-mutation-es.html @@ -0,0 +1,26 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <h1>This is content in light DOM</h1> + <div id="host1"></div> + <script> + const host1 = document.getElementById("host1"); + host1.attachShadow({mode: "open"}); + </script> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-slot-es.html b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-slot-es.html new file mode 100644 index 0000000000..a46e76b14c --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-shadow-dom-slot-es.html @@ -0,0 +1,28 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div id="host"> + Esto se contenta en luz dom + <div> + <script> + const host = document.getElementById("host"); + const root = host.attachShadow({mode: "open"}); + root.innerHTML = "<slot></slot>"; + </script> +</body> +</html> diff --git a/toolkit/components/translations/translations.d.ts b/toolkit/components/translations/translations.d.ts new file mode 100644 index 0000000000..ec1e899af4 --- /dev/null +++ b/toolkit/components/translations/translations.d.ts @@ -0,0 +1,271 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains the shared types for the translations component. The intended use + * is for defining types to be used in JSDoc. They are used in a form that the TypeScript + * language server can read them, and provide code hints. + */ + +/** + * For Remote Settings, the JSON details about the attachment. + */ +export interface Attachment { + // e.g. "2f7c0f7bbc...ca79f0850c4de", + hash: string; + // e.g. 5047568, + size: string; + // e.g. "lex.50.50.deen.s2t.bin", + filename: string; + // e.g. "main-workspace/translations-models/316ebb3a-0682-42cc-8e73-a3ba4bbb280f.bin", + location: string; + // e.g. "application/octet-stream" + mimetype: string; +} + +/** + * The JSON that is synced from Remote Settings for the translation models. + */ +export interface TranslationModelRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The full model name, e.g. "lex.50.50.deen.s2t.bin" + name: string; + // The BCP 47 language tag, e.g. "de" + fromLang: string; + // The BCP 47 language tag, e.g. "en" + toLang: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // e.g. "lex" + fileType: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673023100578 + schema: number; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + +/** + * The JSON that is synced from Remote Settings for the wasm binaries. + */ +export interface WasmRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The name of the project, e.g. "bergamot-translator" + name: string; + // The human readable identifier for the release. e.g. "v0.4.4" + release: string; + // The commit hash for the project that generated the wasm. + revision: string; + // The license of the wasm, as a https://spdx.org/licenses/ + license: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + +/** + * The following are the types that are provided by the Bergamot wasm library. + * + * See: https://github.com/mozilla/bergamot-translator/tree/main/wasm/bindings + */ +export namespace Bergamot { + /** + * The main module that is returned from bergamot-translator.js. + */ + export interface ModuleExport { + BlockingService: typeof BlockingService; + AlignedMemoryList: typeof AlignedMemoryList; + TranslationModel: typeof TranslationModel; + AlignedMemory: typeof AlignedMemory; + VectorResponseOptions: typeof VectorResponseOptions; + VectorString: typeof VectorString; + } + + /** + * This class represents a C++ std::vector. The implementations will extend from it. + */ + export class Vector<T> { + size(): number; + get(index: number): T; + push_back(item: T); + } + + export class VectorResponse extends Vector<Response> {} + export class VectorString extends Vector<string> {} + export class VectorResponseOptions extends Vector<ResponseOptions> {} + export class AlignedMemoryList extends Vector<AlignedMemory> {} + + /** + * A blocking (e.g. non-threaded) translation service, via Bergamot. + */ + export class BlockingService { + /** + * Translate multiple messages in a single synchronous API call using a single model. + */ + translate( + translationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + + /** + * Translate by pivoting between two models + * + * For example to translate "fr" to "es", pivot using "en": + * "fr" to "en" + * "en" to "es" + * + * See https://github.com/mozilla/bergamot-translator/blob/5ae1b1ebb3fa9a3eabed8a64ca6798154bd486eb/src/translator/service.h#L80 + */ + translateViaPivoting( + first: TranslationModel, + second: TranslationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + } + + /** + * The actual translation model, which is passed into the `BlockingService` methods. + */ + export class TranslationModel {} + + /** + * The models need to be placed in the wasm memory space. This object represents + * aligned memory that was allocated on the wasm side of things. The memory contents + * can be set via the getByteArrayView method and the Uint8Array.prototype.set method. + */ + export class AlignedMemory { + constructor(size: number, alignment: number); + size(): number; + getByteArrayView(): Uint8Array; + } + + /** + * The response from the translation. This definition isn't complete, but just + * contains a subset of the available methods. + * + * See https://github.com/mozilla/bergamot-translator/blob/main/src/translator/response.h + */ + export class Response { + getOriginalText(): string; + getTranslatedText(): string; + } + + /** + * The options to configure a translation response. + * + * See https://github.com/mozilla/bergamot-translator/blob/main/src/translator/response_options.h + */ + export class ResponseOptions { + // Include the quality estimations. + qualityScores: boolean; + // Include the alignments. + alignment: boolean; + // Remove HTML tags from text and insert it back into the output. + html: boolean; + // Whether to include sentenceMappings or not. Alignments require + // sentenceMappings and are available irrespective of this option if + // `alignment=true`. + sentenceMappings: boolean + } +} + + +/** + * The client to interact with RemoteSettings. + * See services/settings/RemoteSettingsClient.jsm + */ +interface RemoteSettingsClient { + on: Function, + get: Function, + attachments: any, +} + +/** + * A single language model file. + */ +interface LanguageTranslationModelFile { + buffer: ArrayBuffer, + record: TranslationModelRecord, +} + +/** + * The files necessary to run the translations, these will be sent to the Bergamot + * translation engine. + */ +interface LanguageTranslationModelFiles { + // The machine learning language model. + model: LanguageTranslationModelFile, + // The lexical shortlist that limits possible output of the decoder and makes + // inference faster. + lex: LanguageTranslationModelFile, + // A model that can generate a translation quality estimation. + qualityModel?: LanguageTranslationModelFile, + + // There is either a single vocab file: + vocab?: LanguageTranslationModelFile, + + // Or there are two: + srcvocab?: LanguageTranslationModelFile, + trgvocab?: LanguageTranslationModelFile, +}; + +/** + * This is the type that is generated when the models are loaded into wasm aligned memory. + */ +type LanguageTranslationModelFilesAligned = { + [K in keyof LanguageTranslationModelFiles]: AlignedMemory +}; + +/** + * These are the files that are downloaded from Remote Settings that are necessary + * to start the translations engine. These may not be available if running in tests, + * and so the engine will be mocked. + */ +interface TranslationsEnginePayload { + bergamotWasmArrayBuffer: ArrayBuffer, + languageModelFiles: LanguageTranslationModelFiles[] + isMocked: boolean, +} + +/** + * Nodes that are being translated are given priority according to their visibility. + */ +export type NodeVisibility = "in-viewport" | "out-of-viewport" | "hidden"; + +/** + * Used to decide how to translate a page for full page translations. + */ +export interface LangTags { + isDocLangTagSupported: boolean, + docLangTag: string | null, + userLangTag: string | null, +} + +export interface LanguagePair { fromLang: string, toLang: string }; + +/** + * A structure that contains all of the information needed to render dropdowns + * for translation language selection. + */ +export interface SupportedLanguages { + languagePairs: LanguagePair[], + fromLanguages: Array<{ langTag: string, displayName: string, }>, + toLanguages: Array<{ langTag: string, displayName: string }>, +} + +export type TranslationErrors = "engine-load-error"; |
