diff options
Diffstat (limited to 'toolkit/components/translations')
48 files changed, 16041 insertions, 0 deletions
diff --git a/toolkit/components/translations/README.md b/toolkit/components/translations/README.md new file mode 100644 index 0000000000..0ebabee6cf --- /dev/null +++ b/toolkit/components/translations/README.md @@ -0,0 +1,3 @@ +# Firefox Translations (Work in Progress) + +This folder contains the work in progress integration of Firefox Translations directly into Firefox. See Bug 971044. diff --git a/toolkit/components/translations/TranslationsTelemetry.sys.mjs b/toolkit/components/translations/TranslationsTelemetry.sys.mjs new file mode 100644 index 0000000000..abd3267510 --- /dev/null +++ b/toolkit/components/translations/TranslationsTelemetry.sys.mjs @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Telemetry functions for Translations actors + */ +export class TranslationsTelemetry { + /** + * Records a telemetry event when full page translation fails. + * + * @param {Error} error + */ + static onError(error) { + Glean.translations.errorRate.addToNumerator(1); + Glean.translations.error.record({ + reason: String(error), + }); + } + + /** + * Records a telemetry event when a translation request is sent. + * + * @param {object} data + * @param {string} data.fromLanguage + * @param {string} data.toLanguage + * @param {boolean} data.autoTranslate + */ + static onTranslate(data) { + Glean.translations.requestsCount.add(1); + Glean.translations.translationRequest.record({ + from_language: data.fromLanguage, + to_language: data.toLanguage, + auto_translate: data.autoTranslate, + }); + } +} diff --git a/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs new file mode 100644 index 0000000000..112ac3c444 --- /dev/null +++ b/toolkit/components/translations/actors/AboutTranslationsChild.sys.mjs @@ -0,0 +1,312 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +XPCOMUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +/** + * @typedef {import("./TranslationsChild.sys.mjs").LanguageIdEngine} LanguageIdEngine + * @typedef {import("./TranslationsChild.sys.mjs").TranslationsEngine} TranslationsEngine + * @typedef {import("./TranslationsChild.sys.mjs").SupportedLanguages} SupportedLanguages + */ + +/** + * The AboutTranslationsChild is responsible for coordinating what privileged APIs + * are exposed to the un-privileged scope of the about:translations page. + */ +export class AboutTranslationsChild extends JSWindowActorChild { + /** @type {LanguageIdEngine | null} */ + languageIdEngine = null; + + /** @type {TranslationsEngine | null} */ + translationsEngine = null; + + /** + * The translations engine uses text translations by default in about:translations, + * but it can be changed to translate HTML by setting this pref to true. This is + * useful for manually testing HTML translation behavior, but is not useful to surface + * as a user-facing feature. + * + * @type {bool} + */ + #isHtmlTranslation = Services.prefs.getBoolPref( + "browser.translations.useHTML" + ); + + handleEvent(event) { + if (event.type === "DOMDocElementInserted") { + this.#exportFunctions(); + } + + if ( + event.type === "DOMContentLoaded" && + Services.prefs.getBoolPref("browser.translations.enable") + ) { + this.#sendEventToContent({ type: "enable" }); + } + } + + /** + * @param {object} detail + */ + #sendEventToContent(detail) { + this.contentWindow.dispatchEvent( + new this.contentWindow.CustomEvent("AboutTranslationsChromeToContent", { + detail: Cu.cloneInto(detail, this.contentWindow), + }) + ); + } + + /** + * @returns {TranslationsChild} + */ + #getTranslationsChild() { + const child = this.contentWindow.windowGlobalChild.getActor("Translations"); + if (!child) { + throw new Error("Unable to find the TranslationsChild"); + } + return child; + } + + /** + * A privileged promise can't be used in the content page, so convert a privileged + * promise into a content one. + * + * @param {Promise<any>} promise + * @returns {Promise<any>} + */ + #convertToContentPromise(promise) { + return new this.contentWindow.Promise((resolve, reject) => + promise.then(resolve, error => { + let contentWindow; + try { + contentWindow = this.contentWindow; + } catch (error) { + // The content window is no longer available. + reject(); + return; + } + // Create an error in the content window, if the content window is still around. + let message = "An error occured in the AboutTranslations actor."; + if (typeof error === "string") { + message = error; + } + if (typeof error?.message === "string") { + message = error.message; + } + if (typeof error?.stack === "string") { + message += `\n\nOriginal stack:\n\n${error.stack}\n`; + } + + reject(new contentWindow.Error(message)); + }) + ); + } + + /** + * Export any of the child functions that start with "AT_" to the unprivileged content + * page. This restricts the security capabilities of the the content page. + */ + #exportFunctions() { + const window = this.contentWindow; + + const fns = [ + "AT_log", + "AT_logError", + "AT_getAppLocale", + "AT_getSupportedLanguages", + "AT_isTranslationEngineSupported", + "AT_createLanguageIdEngine", + "AT_createTranslationsEngine", + "AT_identifyLanguage", + "AT_translate", + "AT_destroyTranslationsEngine", + "AT_getScriptDirection", + ]; + for (const name of fns) { + Cu.exportFunction(this[name].bind(this), window, { defineAs: name }); + } + } + + /** + * Log messages if "browser.translations.logLevel" is set to "All". + * + * @param {...any} args + */ + AT_log(...args) { + lazy.console.log(...args); + } + + /** + * Report an error to the console. + * + * @param {...any} args + */ + AT_logError(...args) { + lazy.console.error(...args); + } + + /** + * Returns the app's locale. + * + * @returns {Intl.Locale} + */ + AT_getAppLocale() { + return Services.locale.appLocaleAsBCP47; + } + + /** + * Wire this function to the TranslationsChild. + * + * @returns {Promise<SupportedLanguages>} + */ + AT_getSupportedLanguages() { + return this.#convertToContentPromise( + this.#getTranslationsChild() + .getSupportedLanguages() + .then(data => Cu.cloneInto(data, this.contentWindow)) + ); + } + + /** + * Does this device support the translation engine? + * @returns {Promise<boolean>} + */ + AT_isTranslationEngineSupported() { + return this.#convertToContentPromise( + this.#getTranslationsChild().isTranslationsEngineSupported + ); + } + + /** + * Creates the LanguageIdEngine which attempts to identify in which + * human language a string is written. + * + * Unlike TranslationsEngine, which handles only a single language pair + * and must be rebuilt to handle a new language pair, the LanguageIdEngine + * is a one-to-many engine that can recognize all of its supported languages. + * + * Subsequent calls to this function after the engine is initialized will do nothing + * instead of rebuilding the engine. + * + * @returns {Promise<void>} + */ + AT_createLanguageIdEngine() { + if (this.languageIdEngine) { + return this.#convertToContentPromise(Promise.resolve()); + } + return this.#convertToContentPromise( + this.#getTranslationsChild() + .createLanguageIdEngine() + .then(engine => { + this.languageIdEngine = engine; + }) + ); + } + + /** + * Creates the TranslationsEngine which is responsible for translating + * from one language to the other. + * + * The instantiated TranslationsEngine is unique to its language pair. + * In order to translate a different language pair, a new engine must be + * created for that pair. + * + * Subsequent calls to this function will destroy the existing engine and + * rebuild a new engine for the new language pair. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {Promise<void>} + */ + AT_createTranslationsEngine(fromLanguage, toLanguage) { + if (this.translationsEngine) { + this.translationsEngine.terminate(); + this.translationsEngine = null; + } + return this.#convertToContentPromise( + this.#getTranslationsChild() + .createTranslationsEngine(fromLanguage, toLanguage) + .then(engine => { + this.translationsEngine = engine; + }) + ); + } + + /** + * Attempts to identify the human language in which the message is written. + * @see LanguageIdEngine#identifyLanguage for more detailed documentation. + * + * @param {string} message + * @returns {Promise<{ langTag: string, confidence: number }>} + */ + AT_identifyLanguage(message) { + if (!this.languageIdEngine) { + const { Promise, Error } = this.contentWindow; + return Promise.reject( + new Error("The language identification was not created.") + ); + } + + return this.#convertToContentPromise( + this.languageIdEngine + .identifyLanguage(message) + .then(data => Cu.cloneInto(data, this.contentWindow)) + ); + } + + /** + * @param {string[]} messageBatch + * @param {number} innerWindowId + * @returns {Promise<string[]>} + */ + AT_translate(messageBatch, innerWindowId) { + if (!this.translationsEngine) { + throw new this.contentWindow.Error( + "The translations engine was not created." + ); + } + const promise = this.#isHtmlTranslation + ? this.translationsEngine.translateHTML(messageBatch, innerWindowId) + : this.translationsEngine.translateText(messageBatch, innerWindowId); + + return this.#convertToContentPromise( + promise.then(translations => + Cu.cloneInto(translations, this.contentWindow) + ) + ); + } + + /** + * This is not strictly necessary, but could free up resources quicker. + */ + AT_destroyTranslationsEngine() { + if (this.translationsEngine) { + this.translationsEngine.terminate(); + this.translationsEngine = null; + } + } + + /** + * TODO - Remove this when Intl.Locale.prototype.textInfo is available to + * content scripts. + * + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Locale/textInfo + * https://bugzilla.mozilla.org/show_bug.cgi?id=1693576 + * + * @param {string} locale + * @returns {string} + */ + AT_getScriptDirection(locale) { + return Services.intl.getScriptDirection(locale); + } +} diff --git a/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs b/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs new file mode 100644 index 0000000000..097a0a1d92 --- /dev/null +++ b/toolkit/components/translations/actors/AboutTranslationsParent.sys.mjs @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This parent is blank because the Translations actor handles most of the features + * needed in AboutTranslations. + */ +export class AboutTranslationsParent extends JSWindowActorParent { + constructor() { + super(); + } +} diff --git a/toolkit/components/translations/actors/TranslationsChild.sys.mjs b/toolkit/components/translations/actors/TranslationsChild.sys.mjs new file mode 100644 index 0000000000..4a5388ec88 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsChild.sys.mjs @@ -0,0 +1,1106 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * @typedef {import("../content/translations-document.sys.mjs").TranslationsDocument} TranslationsDocument + * @typedef {import("../translations").LanguageIdEnginePayload} LanguageIdEnginePayload + * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles + * @typedef {import("../translations").TranslationsEnginePayload} TranslationsEnginePayload + * @typedef {import("../translations").LanguagePair} LanguagePair + * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + * @typedef {import("../translations").LangTags} LangTags + */ + +/** + * @type {{ + * TranslationsDocument: typeof TranslationsDocument + * console: typeof console + * }} + */ +const lazy = {}; + +/** + * The threshold that the language-identification confidence + * value must be greater than in order to provide the detected language + * tag for translations. + * + * This value should ideally be one that does not allow false positives + * while also not being too restrictive. + * + * At this time, this value is not driven by statistical data or analysis. + */ +const DOC_LANGUAGE_DETECTION_THRESHOLD = 0.65; + +/** + * The length of the substring to pull from the document's text for language + * identification. + * + * This value should ideally be one that is large enough to yield a confident + * identification result without being too large or expensive to extract. + * + * At this time, this value is not driven by statistical data or analysis. + */ +const DOC_TEXT_TO_IDENTIFY_LENGTH = 1024; + +const PIVOT_LANGUAGE = "en"; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +ChromeUtils.defineESModuleGetters(lazy, { + setTimeout: "resource://gre/modules/Timer.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", + TranslationsDocument: + "chrome://global/content/translations/translations-document.sys.mjs", + TranslationsTelemetry: + "chrome://global/content/translations/TranslationsTelemetry.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "logLevel", + "browser.translations.logLevel" +); + +export class LanguageIdEngine { + /** @type {Worker} */ + #languageIdWorker; + // Multiple messages can be sent before a response is received. This ID is used to keep + // track of the messages. It is incremented on every use. + #messageId = 0; + + /** + * Construct and initialize the language-id worker. + * + * @param {Object} data + * @param {string} data.type - The message type, expects "initialize". + * @param {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + * @param {null | string} data.mockedLangTag - The mocked language tag value (only present when mocking). + * @param {null | number} data.mockedConfidence - The mocked confidence value (only present when mocking). + * @param {boolean} data.isLoggingEnabled + */ + constructor(data) { + this.#languageIdWorker = new Worker( + "chrome://global/content/translations/language-id-engine-worker.js" + ); + + this.isReady = new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if (data.type === "initialization-success") { + resolve(); + } else if (data.type === "initialization-error") { + reject(data.error); + } + this.#languageIdWorker.removeEventListener("message", onMessage); + }; + this.#languageIdWorker.addEventListener("message", onMessage); + }); + + const transferables = []; + // Make sure the ArrayBuffers are transferred, not cloned. + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects + transferables.push(data.wasmBuffer, data.modelBuffer); + + this.#languageIdWorker.postMessage(data, transferables); + } + + /** + * Attempts to identify the human language in which the message is written. + * Generally, the longer a message is, the higher the likelihood that the + * identified language will be correct. Shorter messages increase the chance + * of false identification. + * + * The returned confidence is a number between 0.0 and 1.0 of how confident + * the language identification model was that it identified the correct language. + * + * @param {string} message + * @returns {Promise<{ langTag: string, confidence: number }>} + */ + identifyLanguage(message) { + const messageId = this.#messageId++; + return new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if (data.messageId !== messageId) { + // Multiple translation requests can be sent before a response is received. + // Ensure that the response received here is the correct one. + return; + } + if (data.type === "language-id-response") { + let { langTag, confidence } = data; + resolve({ langTag, confidence }); + } + if (data.type === "language-id-error") { + reject(data.error); + } + this.#languageIdWorker.removeEventListener("message", onMessage); + }; + this.#languageIdWorker.addEventListener("message", onMessage); + this.#languageIdWorker.postMessage({ + type: "language-id-request", + message, + messageId, + }); + }); + } +} + +// How long the cache remains alive between uses, in milliseconds. +const CACHE_TIMEOUT_MS = 10_000; + +class TranslationsEngineCache { + /** @type {Record<string, Promise<TranslationsEngine>>} */ + #engines = {}; + + /** @type {Record<string, TimeoutID>} */ + #timeouts = {}; + + /** + * Returns a getter function that will create a translations engine on the first + * call, and then return the cached one. After a timeout when the engine hasn't + * been used, it is destroyed. + * + * @param {TranslationsChild} actor + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {(() => Promise<TranslationsEngine>) | ((onlyFromCache: true) => Promise<TranslationsEngine | null>)} + */ + createGetter(actor, fromLanguage, toLanguage) { + return async (onlyFromCache = false) => { + let enginePromise = + this.#engines[ + TranslationsChild.languagePairKey(fromLanguage, toLanguage) + ]; + if (enginePromise) { + return enginePromise; + } + if (onlyFromCache) { + return null; + } + + // A new engine needs to be created. + enginePromise = actor.createTranslationsEngine(fromLanguage, toLanguage); + + const key = TranslationsChild.languagePairKey(fromLanguage, toLanguage); + this.#engines[key] = enginePromise; + + // Remove the engine if it fails to initialize. + enginePromise.catch(error => { + lazy.console.log( + `The engine failed to load for translating "${fromLanguage}" to "${toLanguage}". Removing it from the cache.`, + error + ); + this.#engines[key] = null; + }); + + const engine = await enginePromise; + + // These methods will be spied on, and when called they will keep the engine alive. + this.spyOn(engine, "translateText"); + this.spyOn(engine, "translateHTML"); + this.spyOn(engine, "discardTranslationQueue"); + this.keepAlive(fromLanguage, toLanguage); + + return engine; + }; + } + + /** + * Spies on a method, so that when it is called, the engine is kept alive. + * + * @param {TranslationsEngine} engine + * @param {string} methodName + */ + spyOn(engine, methodName) { + const method = engine[methodName].bind(engine); + engine[methodName] = (...args) => { + this.keepAlive(engine.fromLanguage, engine.toLanguage); + return method(...args); + }; + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + keepAlive(fromLanguage, toLanguage) { + const key = TranslationsChild.languagePairKey(fromLanguage); + const timeoutId = this.#timeouts[key]; + if (timeoutId) { + lazy.clearTimeout(timeoutId); + } + const enginePromise = this.#engines[key]; + if (!enginePromise) { + // It appears that the engine is already dead. + return; + } + this.#timeouts[key] = lazy.setTimeout(() => { + // Delete the caches. + delete this.#engines[key]; + delete this.#timeouts[key]; + + // Terminate the engine worker. + enginePromise.then(engine => engine.terminate()); + }, CACHE_TIMEOUT_MS); + } + + /** + * Sees if an engine is still in the cache. + */ + isInCache(fromLanguage, toLanguage) { + this.keepAlive(fromLanguage, toLanguage); + return Boolean( + this.#engines[TranslationsChild.languagePairKey(fromLanguage, toLanguage)] + ); + } +} + +const translationsEngineCache = new TranslationsEngineCache(); + +/** + * The TranslationsEngine encapsulates the logic for translating messages. It can + * only be set up for a single language translation pair. In order to change languages + * a new engine should be constructed. + * + * The actual work for the translations happens in a worker. This class manages + * instantiating and messaging the worker. + */ +export class TranslationsEngine { + /** @type {Worker} */ + #translationsWorker; + // Multiple messages can be sent before a response is received. This ID is used to keep + // track of the messages. It is incremented on every use. + #messageId = 0; + + /** + * Construct and initialize the worker. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {TranslationsEnginePayload} enginePayload - If there is no engine payload + * then the engine will be mocked. This allows this class to be used in tests. + * @param {number} innerWindowId - This only used for creating profiler markers in + * the initial creation of the engine. + */ + constructor(fromLanguage, toLanguage, enginePayload, innerWindowId) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + this.#translationsWorker = new Worker( + "chrome://global/content/translations/translations-engine-worker.js" + ); + + /** @type {Promise<void>} */ + this.isReady = new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + lazy.console.log("Received initialization message", data); + if (data.type === "initialization-success") { + resolve(); + } else if (data.type === "initialization-error") { + reject(data.error); + } + this.#translationsWorker.removeEventListener("message", onMessage); + }; + this.#translationsWorker.addEventListener("message", onMessage); + }); + + // Make sure the ArrayBuffers are transferred, not cloned. + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Transferable_objects + const transferables = []; + if (enginePayload) { + transferables.push(enginePayload.bergamotWasmArrayBuffer); + for (const files of enginePayload.languageModelFiles) { + for (const { buffer } of Object.values(files)) { + transferables.push(buffer); + } + } + } + + this.#translationsWorker.postMessage( + { + type: "initialize", + fromLanguage, + toLanguage, + enginePayload, + innerWindowId, + messageId: this.#messageId++, + logLevel: lazy.logLevel, + }, + transferables + ); + } + + /** + * Translate text without any HTML. + * + * @param {string[]} messageBatch + * @param {number} innerWindowId + * @returns {Promise<string[]>} + */ + translateText(messageBatch, innerWindowId) { + return this.#translate(messageBatch, false, innerWindowId); + } + + /** + * Translate valid HTML. Note that this method throws if invalid markup is provided. + * + * @param {string[]} messageBatch + * @param {number} innerWindowId + * @returns {Promise<string[]>} + */ + translateHTML(messageBatch, innerWindowId) { + return this.#translate(messageBatch, true, innerWindowId); + } + + /** + * The implementation for translation. Use translateText or translateHTML for the + * public API. + * + * @param {string[]} messageBatch + * @param {boolean} isHTML + * @param {number} innerWindowId + * @returns {Promise<string[]>} + */ + #translate(messageBatch, isHTML, innerWindowId) { + const messageId = this.#messageId++; + + return new Promise((resolve, reject) => { + const onMessage = ({ data }) => { + if ( + data.type === "translations-discarded" && + data.innerWindowId === innerWindowId + ) { + // The page was unloaded, and we no longer need to listen for a response. + this.#translationsWorker.removeEventListener("message", onMessage); + return; + } + + if (data.messageId !== messageId) { + // Multiple translation requests can be sent before a response is received. + // Ensure that the response received here is the correct one. + return; + } + + if (data.type === "translation-response") { + resolve(data.translations); + } + if (data.type === "translation-error") { + reject(data.error); + } + this.#translationsWorker.removeEventListener("message", onMessage); + }; + + this.#translationsWorker.addEventListener("message", onMessage); + + this.#translationsWorker.postMessage({ + type: "translation-request", + isHTML, + messageBatch, + messageId, + innerWindowId, + }); + }); + } + + /** + * The worker should be GCed just fine on its own, but go ahead and signal to + * the worker that it's no longer needed. This will immediately cancel any in-progress + * translations. + */ + terminate() { + this.#translationsWorker.terminate(); + } + + /** + * Stop processing the translation queue. All in-progress messages will be discarded. + * + * @param {number} innerWindowId + */ + discardTranslationQueue(innerWindowId) { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + null, + "Request to discard translation queue" + ); + this.#translationsWorker.postMessage({ + type: "discard-translation-queue", + innerWindowId, + }); + } +} + +/** + * See the TranslationsParent for documentation. + */ +export class TranslationsChild extends JSWindowActorChild { + constructor() { + super(); + ChromeUtils.addProfilerMarker( + "TranslationsChild", + null, + "TranslationsChild constructor" + ); + } + + /** + * The getter for the TranslationsEngine, managed by the EngineCache. + * + * @type {null | (() => Promise<TranslationsEngine>) | ((fromCache: true) => Promise<TranslationsEngine | null>)} + */ + #getTranslationsEngine = null; + + /** + * The actor can be destroyed leaving dangling references to dead objects. + */ + #isDestroyed = false; + + /** + * Store this at the beginning so that there is no risk of access a dead object + * to read it. + * @type {number | null} + */ + innerWindowId = null; + + /** + * @type {TranslationsDocument | null} + */ + translatedDoc = null; + + /** + * The matched language tags for the page. Used to find a default language pair for + * translations. + * + * @type {null | LangTags} + * */ + #langTags = null; + + /** + * Creates a lookup key that is unique to each fromLanguage-toLanguage pair. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {string} + */ + static languagePairKey(fromLanguage, toLanguage) { + return `${fromLanguage},${toLanguage}`; + } + + /** + * Retrieve a substring of text from the document body to be + * analyzed by the LanguageIdEngine to determine the page's language. + * + * @returns {string} + */ + #getTextToIdentify() { + let encoder = Cu.createDocumentEncoder("text/plain"); + encoder.init(this.document, "text/plain", encoder.SkipInvisibleContent); + return encoder + .encodeToStringWithMaxLength(DOC_TEXT_TO_IDENTIFY_LENGTH) + .replaceAll("\r", "") + .replaceAll("\n", " "); + } + + /** + * @overrides JSWindowActorChild.prototype.handleEvent + * @param {{ type: string }} event + */ + handleEvent(event) { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + null, + "Event: " + event.type + ); + switch (event.type) { + case "DOMContentLoaded": + this.innerWindowId = this.contentWindow.windowGlobalChild.innerWindowId; + this.maybeOfferTranslation().catch(error => lazy.console.log(error)); + break; + case "pagehide": + lazy.console.log( + "pagehide", + this.contentWindow.location, + this.#langTags + ); + this.reportDetectedLangTagsToParent(null); + break; + } + return undefined; + } + + /** + * This is used to conditionally add the translations button. + * @param {null | LangTags} langTags + */ + reportDetectedLangTagsToParent(langTags) { + this.sendAsyncMessage("Translations:ReportDetectedLangTags", { + langTags, + }); + } + + /** + * Returns the principal from the content window's origin. + * @returns {nsIPrincipal} + */ + getContentWindowPrincipal() { + return Services.scriptSecurityManager.createContentPrincipalFromOrigin( + this.contentWindow.location.origin + ); + } + + /** + * Only translate pages that match certain protocols, that way internal pages like + * about:* pages will not be translated. + */ + #isRestrictedPage() { + const { href } = this.contentWindow.location; + // Keep this logic up to date with TranslationsParent.isRestrictedPage. + return !( + href.startsWith("http://") || + href.startsWith("https://") || + href.startsWith("file:///") + ); + } + + /** + * Determine if the page should be translated by checking the App's languages and + * comparing it to the reported language of the page. Return the best translation fit + * (if available). + * + * @param {number} [translationsStart] + * @returns {Promise<LangTags>} + */ + async getLangTagsForTranslation(translationsStart = this.docShell.now()) { + if (this.#langTags) { + return this.#langTags; + } + + const langTags = { + docLangTag: null, + userLangTag: null, + isDocLangTagSupported: false, + }; + this.#langTags = langTags; + + if (this.#isRestrictedPage()) { + // The langTags are still blank here. + return langTags; + } + let languagePairs = await this.getLanguagePairs(); + + const determineIsDocLangTagSupported = docLangTag => + Boolean( + languagePairs.find(({ fromLang }) => fromLang === langTags.docLangTag) + ); + + // First try to get the langTag from the document's markup. + try { + const docLocale = new Intl.Locale(this.document.documentElement.lang); + langTags.docLangTag = docLocale.language; + langTags.isDocLangTagSupported = determineIsDocLangTagSupported( + docLocale.language + ); + } catch (error) {} + + // If the document's markup had no specified langTag, attempt + // to identify the page's language using the LanguageIdEngine. + if (!langTags.docLangTag) { + let languageIdEngine = await this.createLanguageIdEngine(); + let { langTag, confidence } = await languageIdEngine.identifyLanguage( + this.#getTextToIdentify() + ); + lazy.console.log( + `${langTag}(${confidence.toFixed(2)}) Detected Page Language` + ); + if (confidence >= DOC_LANGUAGE_DETECTION_THRESHOLD) { + langTags.docLangTag = langTag; + langTags.isDocLangTagSupported = + determineIsDocLangTagSupported(langTag); + } + } + + const preferredLanguages = await this.getPreferredLanguages(); + + if (!langTags.docLangTag) { + const message = "No valid language detected."; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, this.contentWindow.location.href); + + const languagePairs = await this.getLanguagePairs(); + + // Attempt to find a good language to select for the user. + langTags.userLangTag = + preferredLanguages.find(langTag => langTag === languagePairs.toLang) ?? + null; + + return langTags; + } + + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId, startTime: translationsStart }, + "Time to determine langTags" + ); + + // This is a special case where we do not offer a translation if the main app language + // and the doc language match. The main app language should be the first preferred + // language. + if (preferredLanguages[0] === langTags.docLangTag) { + // The doc language and the main language match. + const message = + "The app and document languages match, so not translating."; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, this.contentWindow.location.href); + // The docLangTag will be set, while the userLangTag will be null. + return langTags; + } + + // Attempt to find a matching language pair for a preferred language. + for (const preferredLangTag of preferredLanguages) { + if ( + translationsEngineCache.isInCache(langTags.docLangTag, preferredLangTag) + ) { + // There is no reason to look at the language pairs if the engine is already in + // the cache. + langTags.userLangTag = preferredLangTag; + break; + } + + if (!langTags.isDocLangTagSupported) { + if (languagePairs.some(({ toLang }) => toLang === preferredLangTag)) { + // Only match the "to" language, since the "from" is not supported. + langTags.userLangTag = preferredLangTag; + } + break; + } + + // Is there a direct language pair match? + if ( + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === langTags.docLangTag && toLang === preferredLangTag + ) + ) { + // A match was found in one of the preferred languages. + langTags.userLangTag = preferredLangTag; + break; + } + + // Is there a pivot language match? + if ( + // Match doc -> pivot + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === langTags.docLangTag && toLang === PIVOT_LANGUAGE + ) && + // Match pivot -> preferred language + languagePairs.some( + ({ fromLang, toLang }) => + fromLang === PIVOT_LANGUAGE && toLang === preferredLangTag + ) + ) { + langTags.userLangTag = preferredLangTag; + break; + } + } + + if (!langTags.userLangTag) { + // No language pairs match. + const message = `No matching translation pairs were found for translating from "${langTags.docLangTag}".`; + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId }, + message + ); + lazy.console.log(message, languagePairs); + } + + return langTags; + } + + /** + * Deduce the language tags on the page, and either: + * 1. Show an offer to translate. + * 2. Auto-translate. + * 3. Do nothing. + */ + async maybeOfferTranslation() { + const translationsStart = this.docShell.now(); + + const isSupported = await this.isTranslationsEngineSupported; + if (!isSupported) { + return; + } + + const langTags = await this.getLangTagsForTranslation(translationsStart); + + this.#langTags = langTags; + this.reportDetectedLangTagsToParent(langTags); + + if (langTags.docLangTag && langTags.userLangTag) { + const { maybeAutoTranslate, maybeNeverTranslate } = await this.sendQuery( + "Translations:GetTranslationConditions", + langTags + ); + if (maybeAutoTranslate && !maybeNeverTranslate) { + lazy.TranslationsTelemetry.onTranslate({ + fromLanguage: langTags.docLangTag, + toLanguage: langTags.userLangTag, + autoTranslate: maybeAutoTranslate, + }); + this.translatePage( + langTags.docLangTag, + langTags.userLangTag, + translationsStart + ); + } + } + } + + /** + * Lazily initialize this value. It doesn't change after being set. + * + * @type {Promise<boolean>} + */ + get isTranslationsEngineSupported() { + // Delete the getter and set the real value directly onto the TranslationsChild's + // prototype. This value never changes while a browser is open. + delete TranslationsChild.isTranslationsEngineSupported; + return (TranslationsChild.isTranslationsEngineSupported = this.sendQuery( + "Translations:GetIsTranslationsEngineSupported" + )); + } + + /** + * Load the translation engine and translate the page. + * + * @param {{fromLanguage: string, toLanguage: string}} langTags + * @param {number} [translationsStart] + * @returns {Promise<void>} + */ + async translatePage( + fromLanguage, + toLanguage, + translationsStart = this.docShell.now() + ) { + if (this.translatedDoc) { + lazy.console.warn("This page was already translated."); + return; + } + if (this.#isRestrictedPage()) { + lazy.console.warn("Attempting to translate a restricted page."); + return; + } + + try { + const engineLoadStart = this.docShell.now(); + // Create a function to get an engine. These engines are pretty heavy in terms + // of memory usage, so they will be destroyed when not in use, and attempt to + // be re-used when loading a new page. + this.#getTranslationsEngine = await translationsEngineCache.createGetter( + this, + fromLanguage, + toLanguage + ); + if (this.#isDestroyed) { + return; + } + + // Start loading the engine if it doesn't exist. + this.#getTranslationsEngine().then( + () => { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId, startTime: engineLoadStart }, + "Load Translations Engine" + ); + }, + error => { + lazy.console.log("Failed to load the translations engine.", error); + } + ); + } catch (error) { + lazy.TranslationsTelemetry.onError(error); + lazy.console.log( + "Failed to load the translations engine", + error, + this.contentWindow.location.href + ); + this.sendAsyncMessage("Translations:FullPageTranslationFailed", { + reason: "engine-load-failure", + }); + return; + } + + // Ensure the translation engine loads correctly at least once before instantiating + // the TranslationsDocument. + try { + await this.#getTranslationsEngine(); + } catch (error) { + lazy.TranslationsTelemetry.onError(error); + this.sendAsyncMessage("Translations:FullPageTranslationFailed", { + reason: "engine-load-failure", + }); + return; + } + + this.translatedDoc = new lazy.TranslationsDocument( + this.document, + fromLanguage, + this.innerWindowId, + html => + this.#getTranslationsEngine().then(engine => + engine.translateHTML([html], this.innerWindowId) + ), + text => + this.#getTranslationsEngine().then(engine => + engine.translateText([text], this.innerWindowId) + ), + () => this.docShell.now() + ); + + lazy.console.log( + "Beginning to translate.", + this.contentWindow.location.href + ); + + this.sendAsyncMessage("Translations:EngineIsReady"); + + this.translatedDoc.addRootElement(this.document.querySelector("title")); + this.translatedDoc.addRootElement( + this.document.body, + true /* reportWordsInViewport */ + ); + + { + const startTime = this.docShell.now(); + this.translatedDoc.viewportTranslated.then(() => { + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId, startTime }, + "Viewport translations" + ); + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId, startTime: translationsStart }, + "Time to first translation" + ); + }); + } + } + + /** + * Receive a message from the parent. + * + * @param {{ name: string, data: any }} message + */ + receiveMessage({ name, data }) { + switch (name) { + case "Translations:TranslatePage": + const langTags = data ?? this.#langTags; + if (!langTags) { + lazy.console.warn( + "Attempting to translate a page, but no language tags were given." + ); + break; + } + lazy.TranslationsTelemetry.onTranslate({ + fromLanguage: langTags.fromLanguage, + toLanguage: langTags.toLanguage, + autoTranslate: false, + }); + this.translatePage(langTags.fromLanguage, langTags.toLanguage); + break; + case "Translations:GetLangTagsForTranslation": + return this.getLangTagsForTranslation(); + case "Translations:GetContentWindowPrincipal": + return this.getContentWindowPrincipal(); + default: + lazy.console.warn("Unknown message.", name); + } + return undefined; + } + + /** + * Get the list of languages and their display names, sorted by their display names. + * This is more expensive of a call than getLanguagePairs since the display names + * are looked up. + * + * @returns {Promise<Array<SupportedLanguages>>} + */ + getSupportedLanguages() { + return this.sendQuery("Translations:GetSupportedLanguages"); + } + + /** + * @param {string} language The BCP 47 language tag. + */ + hasAllFilesForLanguage(language) { + return this.sendQuery("Translations:HasAllFilesForLanguage", { + language, + }); + } + + /** + * @param {string} language The BCP 47 language tag. + */ + deleteLanguageFiles(language) { + return this.sendQuery("Translations:DeleteLanguageFiles", { + language, + }); + } + + /** + * @param {string} language The BCP 47 language tag. + */ + downloadLanguageFiles(language) { + return this.sendQuery("Translations:DownloadLanguageFiles", { + language, + }); + } + + /** + * Download all files from Remote Settings. + */ + downloadAllFiles() { + return this.sendQuery("Translations:DownloadAllFiles"); + } + + /** + * Delete all language files. + * @returns {Promise<string[]>} Returns a list of deleted record ids. + */ + deleteAllLanguageFiles() { + return this.sendQuery("Translations:DeleteAllLanguageFiles"); + } + + /** + * Get the language pairs that can be used for translations. This is cheaper than + * the getSupportedLanguages call, since the localized display names of the languages + * are not needed. + * + * @returns {Promise<Array<LanguagePair>>} + */ + getLanguagePairs() { + return this.sendQuery("Translations:GetLanguagePairs"); + } + + /** + * The ordered list of preferred BCP 47 language tags. + * + * 1. App languages + * 2. Web requested languages + * 3. OS languages + * + * @returns {Promise<string[]>} + */ + getPreferredLanguages() { + return this.sendQuery("Translations:GetPreferredLanguages"); + } + + /** + * Retrieve the payload for creating a LanguageIdEngine. + * + * @returns {Promise<LanguageIdEnginePayload>} + */ + async #getLanguageIdEnginePayload() { + return this.sendQuery("Translations:GetLanguageIdEnginePayload"); + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {TranslationsEnginePayload} + */ + async #getTranslationsEnginePayload(fromLanguage, toLanguage) { + return this.sendQuery("Translations:GetTranslationsEnginePayload", { + fromLanguage, + toLanguage, + }); + } + + /** + * Construct and initialize the LanguageIdEngine. + * + * @returns {LanguageIdEngine} + */ + async createLanguageIdEngine() { + const payload = await this.#getLanguageIdEnginePayload(); + const engine = new LanguageIdEngine({ + type: "initialize", + isLoggingEnabled: lazy.logLevel === "All", + ...payload, + }); + await engine.isReady; + return engine; + } + + /** + * Construct and initialize the Translations Engine. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {TranslationsEngine | null} + */ + async createTranslationsEngine(fromLanguage, toLanguage) { + const startTime = this.docShell.now(); + const enginePayload = await this.#getTranslationsEnginePayload( + fromLanguage, + toLanguage + ); + + const engine = new TranslationsEngine( + fromLanguage, + toLanguage, + enginePayload, + this.innerWindowId + ); + + await engine.isReady; + + ChromeUtils.addProfilerMarker( + "TranslationsChild", + { innerWindowId: this.innerWindowId, startTime }, + `Translations engine loaded for "${fromLanguage}" to "${toLanguage}"` + ); + return engine; + } + + /** + * Override JSWindowActorChild.prototype.didDestroy. This is called by the actor + * manager when the actor was destroyed. + */ + async didDestroy() { + this.#isDestroyed = true; + const getTranslationsEngine = this.#getTranslationsEngine; + if (!getTranslationsEngine) { + return; + } + const engine = await getTranslationsEngine( + // Just get it from cache, don't create a new one. + true + ); + if (engine) { + // Discard the queue otherwise the worker will continue to translate. + engine.discardTranslationQueue(this.innerWindowId); + + // Keep it alive long enough for another page load. + translationsEngineCache.keepAlive(engine.fromLanguage, engine.toLanguage); + } + } +} diff --git a/toolkit/components/translations/actors/TranslationsParent.sys.mjs b/toolkit/components/translations/actors/TranslationsParent.sys.mjs new file mode 100644 index 0000000000..5b3953be50 --- /dev/null +++ b/toolkit/components/translations/actors/TranslationsParent.sys.mjs @@ -0,0 +1,1953 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The pivot language is used to pivot between two different language translations + * when there is not a model available to translate directly between the two. In this + * case "en" is common between the various supported models. + * + * For instance given the following two models: + * "fr" -> "en" + * "en" -> "it" + * + * You can accomplish: + * "fr" -> "it" + * + * By doing: + * "fr" -> "en" -> "it" + */ +const PIVOT_LANGUAGE = "en"; + +const TRANSLATIONS_PERMISSION = "translations"; +const ALWAYS_TRANSLATE_LANGS_PREF = + "browser.translations.alwaysTranslateLanguages"; +const NEVER_TRANSLATE_LANGS_PREF = + "browser.translations.neverTranslateLanguages"; + +const lazy = {}; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +ChromeUtils.defineESModuleGetters(lazy, { + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "translationsEnabledPref", + "browser.translations.enable" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "autoTranslatePagePref", + "browser.translations.autoTranslate" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "chaosErrorsPref", + "browser.translations.chaos.errors" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "chaosTimeoutMSPref", + "browser.translations.chaos.timeoutMS" +); + +/** + * Returns the always-translate language tags as an array. + */ +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "alwaysTranslateLangTags", + ALWAYS_TRANSLATE_LANGS_PREF, + /* aDefaultPrefValue */ "", + /* onUpdate */ null, + /* aTransform */ rawLangTags => (rawLangTags ? rawLangTags.split(",") : []) +); + +/** + * Returns the never-translate language tags as an array. + */ +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "neverTranslateLangTags", + NEVER_TRANSLATE_LANGS_PREF, + /* aDefaultPrefValue */ "", + /* onUpdate */ null, + /* aTransform */ rawLangTags => (rawLangTags ? rawLangTags.split(",") : []) +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "simulateUnsupportedEnginePref", + "browser.translations.simulateUnsupportedEngine" +); + +// At this time the signatures of the files are not being checked when they are being +// loaded from disk. This signature check involves hitting the network, and translations +// are explicitly an offline-capable feature. See Bug 1827265 for re-enabling this +// check. +const VERIFY_SIGNATURES_FROM_FS = false; + +/** + * @typedef {import("../translations").TranslationModelRecord} TranslationModelRecord + * @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient + * @typedef {import("../translations").LanguageIdEngineMockedPayload} LanguageIdEngineMockedPayload + * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles + * @typedef {import("../translations").WasmRecord} WasmRecord + * @typedef {import("../translations").LangTags} LangTags + * @typedef {import("../translations").LanguagePair} LanguagePair + * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + * @typedef {import("../translations").LanguageIdModelRecord} LanguageIdModelRecord + * @typedef {import("../translations").TranslationErrors} TranslationErrors + */ + +/** + * @typedef {Object} TranslationPair + * @prop {string} fromLanguage + * @prop {string} toLanguage + * @prop {string} [fromDisplayLanguage] + * @prop {string} [toDisplayLanguage] + */ + +/** + * The translations parent is used to orchestrate translations in Firefox. It can + * download the wasm translation engines, and the machine learning language models. + * + * See Bug 971044 for more details of planned work. + */ +export class TranslationsParent extends JSWindowActorParent { + /** + * Contains the state that would affect UI. Anytime this state is changed, a dispatch + * event is sent so that UI can react to it. The actor is inside of /toolkit and + * needs a way of notifying /browser code (or other users) of when the state changes. + * + * @type {TranslationsLanguageState} + */ + languageState; + + actorCreated() { + this.languageState = new TranslationsLanguageState(this); + + if (TranslationsParent.#translateOnPageReload) { + // The actor was recreated after a page reload, start the translation. + const { fromLanguage, toLanguage } = + TranslationsParent.#translateOnPageReload; + TranslationsParent.#translateOnPageReload = null; + + lazy.console.log( + `Translating on a page reload from "${fromLanguage}" to "${toLanguage}".` + ); + + this.translate(fromLanguage, toLanguage); + } + } + + /** + * The remote settings client that retrieves the language-identification model binary. + * + * @type {RemoteSettingsClient | null} + */ + static #languageIdModelsRemoteClient = null; + + /** + * A map of the TranslationModelRecord["id"] to the record of the model in Remote Settings. + * Used to coordinate the downloads. + * + * @type {Map<string, TranslationModelRecord>} + */ + #translationModelRecords = new Map(); + + /** + * The RemoteSettingsClient that downloads the translation models. + * + * @type {RemoteSettingsClient | null} + */ + static #translationModelsRemoteClient = null; + + /** + * The RemoteSettingsClient that downloads the wasm binaries. + * + * @type {RemoteSettingsClient | null} + */ + static #translationsWasmRemoteClient = null; + + /** + * If "browser.translations.autoTranslate" is set to "true" then the page will + * auto-translate. A user can restore the page to the original UI. This flag indicates + * that an auto-translate should be skipped. + */ + static #isPageRestoredForAutoTranslate = false; + + /** + * Allows the actor's behavior to be changed when the translations engine is mocked via + * a dummy RemoteSettingsClient. + * + * @type {bool} + */ + static #isTranslationsEngineMocked = false; + + /** + * The language identification engine can be mocked for testing + * by pre-defining this value. + * + * @type {string | null} + */ + static #mockedLangTag = null; + + /** + * The language identification engine can be mocked for testing + * by pre-defining this value. + * + * @type {number | null} + */ + static #mockedLanguageIdConfidence = null; + + /** + * @type {null | Promise<boolean>} + */ + static #isTranslationsEngineSupported = null; + + /** + * When reloading the page, store the translation pair that needs translating. + * + * @type {null | TranslationPair} + */ + static #translateOnPageReload = null; + + /** + * An ordered list of preferred languages based on: + * 1. App languages + * 2. Web requested languages + * 3. OS language + * + * @type {null | string[]} + */ + static #preferredLanguages = null; + static #observingLanguages = false; + + // On a fast connection, 10 concurrent downloads were measured to be the fastest when + // downloading all of the language files. + static MAX_CONCURRENT_DOWNLOADS = 10; + static MAX_DOWNLOAD_RETRIES = 3; + + /** + * Detect if Wasm SIMD is supported, and cache the value. It's better to check + * for support before downloading large binary blobs to a user who can't even + * use the feature. This function also respects mocks and simulating unsupported + * engines. + * + * @type {Promise<boolean>} + */ + static getIsTranslationsEngineSupported() { + if (lazy.simulateUnsupportedEnginePref) { + // Use the non-lazy console.log so that the user is always informed as to why + // the translations engine is not working. + console.log( + "Translations: The translations engine is disabled through the pref " + + '"browser.translations.simulateUnsupportedEngine".' + ); + + // The user is manually testing unsupported engines. + return Promise.resolve(false); + } + + if (TranslationsParent.#isTranslationsEngineMocked) { + // A mocked translations engine is always supported. + return Promise.resolve(true); + } + + if (TranslationsParent.#isTranslationsEngineSupported === null) { + TranslationsParent.#isTranslationsEngineSupported = detectSimdSupport(); + + TranslationsParent.#isTranslationsEngineSupported.then( + isSupported => () => { + // Use the non-lazy console.log so that the user is always informed as to why + // the translations engine is not working. + if (!isSupported) { + console.log( + "Translations: The translations engine is not supported on your device as " + + "it does not support Wasm SIMD operations." + ); + } + } + ); + } + + return TranslationsParent.#isTranslationsEngineSupported; + } + + /** + * Only translate pages that match certain protocols, that way internal pages like + * about:* pages will not be translated. + * @param {string} url + */ + static isRestrictedPage(url) { + // Keep this logic up to date with TranslationsChild.prototype.#isRestrictedPage. + return !( + url.startsWith("http://") || + url.startsWith("https://") || + url.startsWith("file:///") + ); + } + + static #resetPreferredLanguages() { + TranslationsParent.#preferredLanguages = null; + TranslationsParent.getPreferredLanguages(); + } + + static async observe(_subject, topic, _data) { + switch (topic) { + case "nsPref:changed": + case "intl:app-locales-changed": { + this.#resetPreferredLanguages(); + break; + } + default: + throw new Error("Unknown observer event", topic); + } + } + + /** + * Provide a way for tests to override the system locales. + * @type {null | string[]} + */ + mockedSystemLocales = null; + + /** + * An ordered list of preferred languages based on: + * + * 1. App languages + * 2. Web requested languages + * 3. OS language + * + * @returns {string[]} + */ + static getPreferredLanguages() { + if (TranslationsParent.#preferredLanguages) { + return TranslationsParent.#preferredLanguages; + } + + if (!TranslationsParent.#observingLanguages) { + Services.obs.addObserver( + TranslationsParent.#resetPreferredLanguages, + "intl:app-locales-changed" + ); + Services.prefs.addObserver( + "intl.accept_languages", + TranslationsParent.#resetPreferredLanguages + ); + TranslationsParent.#observingLanguages = true; + } + + // The "Accept-Language" values that the localizer or user has indicated for + // the preferences for the web. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language + // Note that this preference often falls back ultimately to English, even if the + // user doesn't actually speak English, or to other languages they do not speak. + // However, this preference will be used as an indication that a user may prefer + // this language. + const webLanguages = Services.prefs + .getComplexValue("intl.accept_languages", Ci.nsIPrefLocalizedString) + .data.split(/\s*,\s*/g); + + // The system language could also be a good option for a language to offer the user. + const osPrefs = Cc["@mozilla.org/intl/ospreferences;1"].getService( + Ci.mozIOSPreferences + ); + const systemLocales = this.mockedSystemLocales ?? osPrefs.systemLocales; + + // Combine the locales together. + const preferredLocales = new Set([ + ...Services.locale.appLocalesAsBCP47, + ...webLanguages, + ...systemLocales, + ]); + + // Attempt to convert the locales to lang tags. Do not completely trust the + // values coming from preferences and the OS to have been validated as correct + // BCP 47 locale identifiers. + const langTags = new Set(); + for (const locale of preferredLocales) { + try { + langTags.add(new Intl.Locale(locale).language); + } catch (_) { + // The locale was invalid, discard it. + } + } + + // Convert the Set to an array to indicate that it is an ordered listing of languages. + TranslationsParent.#preferredLanguages = [...langTags]; + + return TranslationsParent.#preferredLanguages; + } + + async receiveMessage({ name, data }) { + switch (name) { + case "Translations:GetTranslationsEnginePayload": { + const { fromLanguage, toLanguage } = data; + const bergamotWasmArrayBuffer = this.#getBergamotWasmArrayBuffer(); + + let files = await this.getLanguageTranslationModelFiles( + fromLanguage, + toLanguage + ); + + let languageModelFiles; + if (files) { + languageModelFiles = [files]; + } else { + // No matching model was found, try to pivot between English. + const [files1, files2] = await Promise.all([ + this.getLanguageTranslationModelFiles(fromLanguage, PIVOT_LANGUAGE), + this.getLanguageTranslationModelFiles(PIVOT_LANGUAGE, toLanguage), + ]); + if (!files1 || !files2) { + throw new Error( + `No language models were found for ${fromLanguage} to ${toLanguage}` + ); + } + languageModelFiles = [files1, files2]; + } + + return { + bergamotWasmArrayBuffer: await bergamotWasmArrayBuffer, + languageModelFiles, + isMocked: TranslationsParent.#isTranslationsEngineMocked, + }; + } + case "Translations:GetLanguageIdEnginePayload": { + const [modelBuffer, wasmBuffer] = await Promise.all([ + this.#getLanguageIdModelArrayBuffer(), + this.#getLanguageIdWasmArrayBuffer(), + ]); + return { + modelBuffer, + wasmBuffer, + mockedConfidence: TranslationsParent.#mockedLanguageIdConfidence, + mockedLangTag: TranslationsParent.#mockedLangTag, + }; + } + case "Translations:GetIsTranslationsEngineMocked": { + return TranslationsParent.#isTranslationsEngineMocked; + } + case "Translations:GetIsTranslationsEngineSupported": { + return TranslationsParent.getIsTranslationsEngineSupported(); + } + case "Translations:FullPageTranslationFailed": { + this.languageState.error = data.reason; + break; + } + case "Translations:GetSupportedLanguages": { + return this.getSupportedLanguages(); + } + case "Translations:HasAllFilesForLanguage": { + return this.hasAllFilesForLanguage(data.language); + } + case "Translations:DownloadLanguageFiles": { + return this.downloadLanguageFiles(data.language); + } + case "Translations:DownloadAllFiles": { + return this.downloadAllFiles(); + } + case "Translations:DeleteAllLanguageFiles": { + return this.deleteAllLanguageFiles(); + } + case "Translations:DeleteLanguageFiles": { + return this.deleteLanguageFiles(data.language); + } + case "Translations:GetLanguagePairs": { + return this.getLanguagePairs(); + } + case "Translations:GetPreferredLanguages": { + return TranslationsParent.getPreferredLanguages(); + } + case "Translations:EngineIsReady": { + this.isEngineReady = true; + this.languageState.isEngineReady = true; + break; + } + case "Translations:GetTranslationConditions": { + const maybeAutoTranslate = TranslationsParent.#maybeAutoTranslate( + data.docLangTag + ); + const maybeNeverTranslate = + TranslationsParent.shouldNeverTranslateLanguage(data.docLangTag) || + (await this.shouldNeverTranslateSite()); + + if (maybeAutoTranslate && !maybeNeverTranslate) { + this.languageState.requestedTranslationPair = { + fromLanguage: data.docLangTag, + toLanguage: data.userLangTag, + }; + } + + return { maybeAutoTranslate, maybeNeverTranslate }; + } + case "Translations:ReportDetectedLangTags": { + this.languageState.detectedLanguages = data.langTags; + return undefined; + } + } + return undefined; + } + + /** + * Returns true if translations should auto-translate from the given + * language, otherwise returns false. + * + * @param {string} langTag - A BCP-47 language tag + * @returns {boolean} + */ + static #maybeAutoTranslate(langTag) { + if ( + // The user has not marked this language as always translate. + !TranslationsParent.shouldAlwaysTranslateLanguage(langTag) && + // The pref to always auto-translate is off. + !lazy.autoTranslatePagePref + ) { + return false; + } + + if (TranslationsParent.#isPageRestoredForAutoTranslate) { + // The user clicked the restore button. Respect it for one page load. + TranslationsParent.#isPageRestoredForAutoTranslate = false; + + // Skip this auto-translation. + return false; + } + + // The page can be auto-translated + return true; + } + + /** + * Retrieves the language-identification model binary from remote settings. + * + * @returns {Promise<ArrayBuffer>} + */ + async #getLanguageIdModelArrayBuffer() { + lazy.console.log("Getting language-identification model array buffer."); + const now = Date.now(); + const client = this.#getLanguageIdModelRemoteClient(); + + /** @type {LanguageIdModelRecord[]} */ + let modelRecords = await TranslationsParent.getMaxVersionRecords(client); + + if (modelRecords.length === 0) { + throw new Error( + "Unable to get language-identification model record from remote settings" + ); + } + + if (modelRecords.length > 1) { + TranslationsParent.reportError( + new Error( + "Expected the language-identification model collection to have only 1 record." + ), + modelRecords + ); + } + const [modelRecord] = modelRecords; + + await chaosMode(1 / 3); + + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download(modelRecord); + + const duration = (Date.now() - now) / 1000; + lazy.console.log( + `Remote language-identification model loaded in ${duration} seconds.` + ); + + return buffer; + } + + /** + * Initializes the RemoteSettingsClient for the language-identification model binary. + * + * @returns {RemoteSettingsClient} + */ + #getLanguageIdModelRemoteClient() { + if (TranslationsParent.#languageIdModelsRemoteClient) { + return TranslationsParent.#languageIdModelsRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-identification-models"); + + TranslationsParent.#languageIdModelsRemoteClient = client; + return client; + } + + /** + * Retrieves the language-identification wasm binary from remote settings. + * + * @returns {Promise<ArrayBuffer>} + */ + async #getLanguageIdWasmArrayBuffer() { + const start = Date.now(); + const client = this.#getTranslationsWasmRemoteClient(); + + // Load the wasm binary from remote settings, if it hasn't been already. + lazy.console.log(`Getting remote language-identification wasm binary.`); + + /** @type {WasmRecord[]} */ + let wasmRecords = await TranslationsParent.getMaxVersionRecords(client, { + filters: { name: "fasttext-wasm" }, + }); + + if (wasmRecords.length === 0) { + // The remote settings client provides an empty list of records when there is + // an error. + throw new Error( + 'Unable to get "fasttext-wasm" language-identification wasm binary from Remote Settings.' + ); + } + + if (wasmRecords.length > 1) { + TranslationsParent.reportError( + new Error( + 'Expected the "fasttext-wasm" language-identification wasm collection to only have 1 record.' + ), + wasmRecords + ); + } + + // Unlike the models, greedily download the wasm. It will pull it from a locale + // cache on disk if it's already been downloaded. Do not retain a copy, as + // this will be running in the parent process. It's not worth holding onto + // this much memory, so reload it every time it is needed. + + await chaosMode(1 / 3); + + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download(wasmRecords[0]); + + const duration = (Date.now() - start) / 1000; + lazy.console.log( + `Remote language-identification wasm binary loaded in ${duration} seconds.` + ); + + return buffer; + } + + /** + * Creates a lookup key that is unique to each fromLanguage-toLanguage pair. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @returns {string} + */ + static languagePairKey(fromLanguage, toLanguage) { + return `${fromLanguage},${toLanguage}`; + } + + /** + * Get the list of translation pairs supported by the translations engine. + * + * @returns {Promise<Array<LanguagePair>>} + */ + async getLanguagePairs() { + const records = await this.#getTranslationModelRecords(); + const languagePairMap = new Map(); + + for (const { fromLang, toLang, version } of records.values()) { + const isBeta = Services.vc.compare(version, "1.0") < 0; + const key = TranslationsParent.languagePairKey(fromLang, toLang); + if (!languagePairMap.has(key)) { + languagePairMap.set(key, { fromLang, toLang, isBeta }); + } + } + + return Array.from(languagePairMap.values()); + } + + /** + * Returns all of the information needed to render dropdowns for translation + * language selection. + * + * @returns {Promise<SupportedLanguages>} + */ + async getSupportedLanguages() { + const languagePairs = await this.getLanguagePairs(); + + /** @type {Map<string, boolean>} */ + const fromLanguages = new Map(); + /** @type {Map<string, boolean>} */ + const toLanguages = new Map(); + + for (const { fromLang, toLang, isBeta } of languagePairs) { + // [BetaLanguage, BetaLanguage] => isBeta == true, + // [BetaLanguage, NonBetaLanguage] => isBeta == true, + // [NonBetaLanguage, BetaLanguage] => isBeta == true, + // [NonBetaLanguage, NonBetaLanguage] => isBeta == false, + if (isBeta) { + // If these languages are part of a beta languagePair, at least one of them is a beta language + // but the other may not be, so only tentatively mark them as beta if there is no entry. + if (!fromLanguages.has(fromLang)) { + fromLanguages.set(fromLang, isBeta); + } + if (!toLanguages.has(toLang)) { + toLanguages.set(toLang, isBeta); + } + } else { + // If these languages are part of a non-beta languagePair, then they are both + // guaranteed to be non-beta languages. Idempotently overwrite any previous entry. + fromLanguages.set(fromLang, isBeta); + toLanguages.set(toLang, isBeta); + } + } + + // Build a map of the langTag to the display name. + /** @type {Map<string, string>} */ + const displayNames = new Map(); + { + const dn = new Services.intl.DisplayNames(undefined, { + type: "language", + }); + + for (const langTagSet of [fromLanguages, toLanguages]) { + for (const langTag of langTagSet.keys()) { + if (displayNames.has(langTag)) { + continue; + } + displayNames.set(langTag, dn.of(langTag)); + } + } + } + + const addDisplayName = ([langTag, isBeta]) => ({ + langTag, + isBeta, + displayName: displayNames.get(langTag), + }); + + const sort = (a, b) => a.displayName.localeCompare(b.displayName); + + return { + languagePairs, + fromLanguages: Array.from(fromLanguages.entries()) + .map(addDisplayName) + .sort(sort), + toLanguages: Array.from(toLanguages.entries()) + .map(addDisplayName) + .sort(sort), + }; + } + + /** + * Lazily initializes the RemoteSettingsClient for the language models. + * + * @returns {RemoteSettingsClient} + */ + #getTranslationModelsRemoteClient() { + if (TranslationsParent.#translationModelsRemoteClient) { + return TranslationsParent.#translationModelsRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-models"); + TranslationsParent.#translationModelsRemoteClient = client; + + client.on("sync", async ({ data: { created, updated, deleted } }) => { + // Language model attachments will only be downloaded when they are used. + lazy.console.log( + `Remote Settings "sync" event for remote language models `, + { + created, + updated, + deleted, + } + ); + + // Remove all the deleted records. + for (const record of deleted) { + await client.attachments.deleteDownloaded(record); + this.#translationModelRecords.delete(record.id); + } + + // Pre-emptively remove the old downloads, and set the new updated record. + for (const { old: oldRecord, new: newRecord } of updated) { + await client.attachments.deleteDownloaded(oldRecord); + // The language pairs should be the same on the update, but use the old + // record just in case. + this.#translationModelRecords.delete(oldRecord.id); + this.#translationModelRecords.set(newRecord.id, newRecord); + } + + // Add the new records, but don't download any attachments. + for (const record of created) { + this.#translationModelRecords.set(record.id, record); + } + }); + + return client; + } + + /** + * Retrieves the maximum version of each record in the RemoteSettingsClient. + * + * If the client contains two different-version copies of the same record (e.g. 1.0 and 1.1) + * then only the 1.1-version record will be returned in the resulting collection. + * + * @param {RemoteSettingsClient} remoteSettingsClient + * @param {Object} [options] + * @param {Object} [options.filters={}] + * The filters to apply when retrieving the records from RemoteSettings. + * Filters should correspond to properties on the RemoteSettings records themselves. + * For example, A filter to retrieve only records with a `fromLang` value of "en" and a `toLang` value of "es": + * { filters: { fromLang: "en", toLang: "es" } } + * @param {Function} [options.lookupKey=(record => record.name)] + * The function to use to extract a lookup key from each record. + * This function should take a record as input and return a string that represents the lookup key for the record. + * For most record types, the name (default) is sufficient, however if a collection contains records with + * non-unique name values, it may be necessary to provide an alternative function here. + * @returns {Array<TranslationModelRecord | LanguageIdModelRecord | WasmRecord>} + */ + static async getMaxVersionRecords( + remoteSettingsClient, + { filters = {}, lookupKey = record => record.name } = {} + ) { + try { + await chaosMode(1 / 4); + } catch (_error) { + // Simulate an error by providing empty records. + return []; + } + const retrievedRecords = await remoteSettingsClient.get({ + // Pull the records from the network. + syncIfEmpty: true, + // Don't verify the signature if the client is mocked. + verifySignature: VERIFY_SIGNATURES_FROM_FS, + // Apply any filters for retrieving the records. + filters, + }); + + // Create a mapping to only the max version of each record discriminated by + // the result of the lookupKey() function. + const maxVersionRecordMap = retrievedRecords.reduce((records, record) => { + const key = lookupKey(record); + const existing = records.get(key); + if ( + !existing || + // existing version less than record version + Services.vc.compare(existing.version, record.version) < 0 + ) { + records.set(key, record); + } + return records; + }, new Map()); + + return Array.from(maxVersionRecordMap.values()); + } + + /** + * Lazily initializes the model records, and returns the cached ones if they + * were already retrieved. The key of the returned `Map` is the record id. + * + * @returns {Promise<Map<string, TranslationModelRecord>>} + */ + async #getTranslationModelRecords() { + if (this.#translationModelRecords.size > 0) { + return this.#translationModelRecords; + } + + const now = Date.now(); + const client = this.#getTranslationModelsRemoteClient(); + + // Load the models. If no data is present, then there will be an initial sync. + // Rely on Remote Settings for the syncing strategy for receiving updates. + lazy.console.log(`Getting remote language models.`); + + /** @type {TranslationModelRecord[]} */ + const translationModelRecords = + await TranslationsParent.getMaxVersionRecords(client, { + // Names in this collection are not unique, so we are appending the languagePairKey + // to guarantee uniqueness. + lookupKey: record => + `${record.name}${TranslationsParent.languagePairKey( + record.fromLang, + record.toLang + )}`, + }); + + if (translationModelRecords.length === 0) { + throw new Error("Unable to retrieve the translation models."); + } + + for (const record of TranslationsParent.ensureLanguagePairsHavePivots( + translationModelRecords + )) { + this.#translationModelRecords.set(record.id, record); + } + + const duration = (Date.now() - now) / 1000; + lazy.console.log( + `Remote language models loaded in ${duration} seconds.`, + this.#translationModelRecords + ); + + return this.#translationModelRecords; + } + + /** + * This implementation assumes that every language pair has access to the + * pivot language. If any languages are added without a pivot language, or the + * pivot language is changed, then this implementation will need a more complicated + * language solver. This means that any UI pickers would need to be updated, and + * the pivot language selection would need a solver. + * + * @param {TranslationModelRecord[] | LanguagePair[]} records + */ + static ensureLanguagePairsHavePivots(records) { + // lang -> pivot + const hasToPivot = new Set(); + // pivot -> en + const hasFromPivot = new Set(); + + const fromLangs = new Set(); + const toLangs = new Set(); + + for (const { fromLang, toLang } of records) { + fromLangs.add(fromLang); + toLangs.add(toLang); + + if (toLang === PIVOT_LANGUAGE) { + // lang -> pivot + hasToPivot.add(fromLang); + } + if (fromLang === PIVOT_LANGUAGE) { + // pivot -> en + hasFromPivot.add(toLang); + } + } + + const fromLangsToRemove = new Set(); + const toLangsToRemove = new Set(); + + for (const lang of fromLangs) { + if (lang === PIVOT_LANGUAGE) { + continue; + } + // Check for "lang -> pivot" + if (!hasToPivot.has(lang)) { + TranslationsParent.reportError( + new Error( + `The "from" language model "${lang}" is being discarded as it doesn't have a pivot language.` + ) + ); + fromLangsToRemove.add(lang); + } + } + + for (const lang of toLangs) { + if (lang === PIVOT_LANGUAGE) { + continue; + } + // Check for "pivot -> lang" + if (!hasFromPivot.has(lang)) { + TranslationsParent.reportError( + new Error( + `The "to" language model "${lang}" is being discarded as it doesn't have a pivot language.` + ) + ); + toLangsToRemove.add(lang); + } + } + + const after = records.filter(record => { + if (fromLangsToRemove.has(record.fromLang)) { + return false; + } + if (toLangsToRemove.has(record.toLang)) { + return false; + } + return true; + }); + return after; + } + + /** + * Lazily initializes the RemoteSettingsClient for the downloaded wasm binary data. + * + * @returns {RemoteSettingsClient} + */ + #getTranslationsWasmRemoteClient() { + if (TranslationsParent.#translationsWasmRemoteClient) { + return TranslationsParent.#translationsWasmRemoteClient; + } + + /** @type {RemoteSettingsClient} */ + const client = lazy.RemoteSettings("translations-wasm"); + + TranslationsParent.#translationsWasmRemoteClient = client; + + client.on("sync", async ({ data: { created, updated, deleted } }) => { + lazy.console.log(`"sync" event for remote bergamot wasm `, { + created, + updated, + deleted, + }); + + // Remove all the deleted records. + for (const record of deleted) { + await client.attachments.deleteDownloaded(record); + } + + // Remove any updated records, and download the new ones. + for (const { old: oldRecord } of updated) { + await client.attachments.deleteDownloaded(oldRecord); + } + + // Do nothing for the created records. + }); + + return client; + } + + /** + * Bergamot is the translation engine that has been compiled to wasm. It is shipped + * to the user via Remote Settings. + * + * https://github.com/mozilla/bergamot-translator/ + */ + /** + * @returns {Promise<ArrayBuffer>} + */ + async #getBergamotWasmArrayBuffer() { + const start = Date.now(); + const client = this.#getTranslationsWasmRemoteClient(); + + // Load the wasm binary from remote settings, if it hasn't been already. + lazy.console.log(`Getting remote bergamot-translator wasm records.`); + + /** @type {WasmRecord[]} */ + const wasmRecords = await TranslationsParent.getMaxVersionRecords(client, { + filters: { name: "bergamot-translator" }, + }); + + if (wasmRecords.length === 0) { + // The remote settings client provides an empty list of records when there is + // an error. + throw new Error( + "Unable to get the bergamot translator from Remote Settings." + ); + } + + if (wasmRecords.length > 1) { + TranslationsParent.reportError( + new Error("Expected the bergamot-translator to only have 1 record."), + wasmRecords + ); + } + + // Unlike the models, greedily download the wasm. It will pull it from a locale + // cache on disk if it's already been downloaded. Do not retain a copy, as + // this will be running in the parent process. It's not worth holding onto + // this much memory, so reload it every time it is needed. + + await chaosModeError(1 / 3); + + /** @type {{buffer: ArrayBuffer}} */ + const { buffer } = await client.attachments.download(wasmRecords[0]); + + const duration = Date.now() - start; + lazy.console.log( + `"bergamot-translator" wasm binary loaded in ${duration / 1000} seconds` + ); + + return buffer; + } + + /** + * Deletes language files that match a language. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + async deleteLanguageFiles(language) { + const client = this.#getTranslationModelsRemoteClient(); + const isForDeletion = true; + return Promise.all( + Array.from( + await this.getRecordsForTranslatingToAndFromAppLanguage( + language, + isForDeletion + ) + ).map(record => { + lazy.console.log("Deleting record", record); + return client.attachments.deleteDownloaded(record); + }) + ); + } + + /** + * Download language files that match a language. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + async downloadLanguageFiles(language) { + const client = this.#getTranslationModelsRemoteClient(); + + const queue = []; + + for (const record of await this.getRecordsForTranslatingToAndFromAppLanguage( + language + )) { + const download = () => { + lazy.console.log("Downloading record", record.name, record.id); + return client.attachments.download(record); + }; + queue.push({ download }); + } + + return downloadManager(queue); + } + + /** + * Download all files used for translations. + */ + async downloadAllFiles() { + const client = this.#getTranslationModelsRemoteClient(); + + const queue = []; + + for (const [recordId, record] of await this.#getTranslationModelRecords()) { + queue.push({ + onSuccess: () => { + this.sendQuery("Translations:DownloadedLanguageFile", { recordId }); + }, + // The download may be attempted multiple times. + onFailure: () => { + this.sendQuery("Translations:DownloadLanguageFileError", { + recordId, + }); + }, + download: () => client.attachments.download(record), + }); + } + + queue.push({ download: () => this.#getBergamotWasmArrayBuffer() }); + queue.push({ download: () => this.#getLanguageIdModelArrayBuffer() }); + queue.push({ download: () => this.#getLanguageIdWasmArrayBuffer() }); + + return downloadManager(queue); + } + + /** + * Delete all language model files. + * @returns {Promise<string[]>} A list of record IDs. + */ + async deleteAllLanguageFiles() { + const client = this.#getTranslationModelsRemoteClient(); + await chaosMode(); + await client.attachments.deleteAll(); + return [...(await this.#getTranslationModelRecords()).keys()]; + } + + /** + * Only returns true if all language files are present for a requested language. + * It's possible only half the files exist for a pivot translation into another + * language, or there was a download error, and we're still missing some files. + * + * @param {string} requestedLanguage The BCP 47 language tag. + */ + async hasAllFilesForLanguage(requestedLanguage) { + const client = this.#getTranslationModelsRemoteClient(); + for (const record of await this.getRecordsForTranslatingToAndFromAppLanguage( + requestedLanguage, + true + )) { + if (!(await client.attachments.isDownloaded(record))) { + return false; + } + } + + return true; + } + + /** + * Get the necessary files for translating to and from the app language and a + * requested language. This may require the files for a pivot language translation + * if there is no language model for a direct translation. + * + * @param {string} requestedLanguage The BCP 47 language tag. + * @param {boolean} isForDeletion - Return a more restrictive set of languages, as + * these files are marked for deletion. We don't want to remove + * files that are needed for some other language's pivot translation. + * @returns {Set<TranslationModelRecord>} + */ + async getRecordsForTranslatingToAndFromAppLanguage( + requestedLanguage, + isForDeletion = false + ) { + const records = await this.#getTranslationModelRecords(); + const appLanguage = new Intl.Locale(Services.locale.appLocaleAsBCP47) + .language; + + let matchedRecords = new Set(); + + if (requestedLanguage === appLanguage) { + // There are no records if the requested language and app language are the same. + return matchedRecords; + } + + const addLanguagePair = (fromLang, toLang) => { + let matchFound = false; + for (const record of records.values()) { + if (record.fromLang === fromLang && record.toLang === toLang) { + matchedRecords.add(record); + matchFound = true; + } + } + return matchFound; + }; + + if ( + // Is there a direct translation? + !addLanguagePair(requestedLanguage, appLanguage) + ) { + // This is no direct translation, get the pivot files. + addLanguagePair(requestedLanguage, PIVOT_LANGUAGE); + // These files may be required for other pivot translations, so don't remove + // them if we are deleting records. + if (!isForDeletion) { + addLanguagePair(PIVOT_LANGUAGE, appLanguage); + } + } + + if ( + // Is there a direct translation? + !addLanguagePair(appLanguage, requestedLanguage) + ) { + // This is no direct translation, get the pivot files. + addLanguagePair(PIVOT_LANGUAGE, requestedLanguage); + // These files may be required for other pivot translations, so don't remove + // them if we are deleting records. + if (!isForDeletion) { + addLanguagePair(appLanguage, PIVOT_LANGUAGE); + } + } + + return matchedRecords; + } + + /** + * Gets the language model files in an array buffer by downloading attachments from + * Remote Settings, or retrieving them from the local cache. Each translation + * requires multiple files. + * + * Results are only returned if the model is found. + * + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {boolean} withQualityEstimation + * @returns {null | LanguageTranslationModelFiles} + */ + async getLanguageTranslationModelFiles( + fromLanguage, + toLanguage, + withQualityEstimation = false + ) { + const client = this.#getTranslationModelsRemoteClient(); + + lazy.console.log( + `Beginning model downloads: "${fromLanguage}" to "${toLanguage}"` + ); + + const records = [...(await this.#getTranslationModelRecords()).values()]; + + /** @type {LanguageTranslationModelFiles} */ + let results; + + // Use Promise.all to download (or retrieve from cache) the model files in parallel. + await Promise.all( + records.map(async record => { + if (record.fileType === "qualityModel" && !withQualityEstimation) { + // Do not include the quality models if they aren't needed. + return; + } + + if (record.fromLang !== fromLanguage || record.toLang !== toLanguage) { + // Only use models that match. + return; + } + + if (!results) { + results = {}; + } + + const start = Date.now(); + + // Download or retrieve from the local cache: + + await chaosMode(1 / 3); + + /** @type {{buffer: ArrayBuffer }} */ + const { buffer } = await client.attachments.download(record); + + results[record.fileType] = { + buffer, + record, + }; + + const duration = Date.now() - start; + lazy.console.log( + `Translation model fetched in ${duration / 1000} seconds:`, + record.fromLang, + record.toLang, + record.fileType + ); + }) + ); + + if (!results) { + // No model files were found, pivoting will be required. + return null; + } + + // Validate that all of the files we expected were actually available and + // downloaded. + + if (!results.model) { + throw new Error( + `No model file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (!results.lex) { + throw new Error( + `No lex file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (withQualityEstimation && !results.qualityModel) { + throw new Error( + `No quality file was found for "${fromLanguage}" to "${toLanguage}."` + ); + } + + if (results.vocab) { + if (results.srcvocab) { + throw new Error( + `A srcvocab and vocab file were both included for "${fromLanguage}" to "${toLanguage}." Only one is needed.` + ); + } + if (results.trgvocab) { + throw new Error( + `A trgvocab and vocab file were both included for "${fromLanguage}" to "${toLanguage}." Only one is needed.` + ); + } + } else if (!results.srcvocab || !results.srcvocab) { + throw new Error( + `No vocab files were provided for "${fromLanguage}" to "${toLanguage}."` + ); + } + + return results; + } + + /** + * For testing purposes, allow the Translations Engine to be mocked. If called + * with `null` the mock is removed. + * + * @param {null | RemoteSettingsClient} [translationModelsRemoteClient] + * @param {null | RemoteSettingsClient} [translationsWasmRemoteClient] + */ + static mockTranslationsEngine( + translationModelsRemoteClient, + translationsWasmRemoteClient + ) { + lazy.console.log("Mocking RemoteSettings for the translations engine."); + TranslationsParent.#translationModelsRemoteClient = + translationModelsRemoteClient; + TranslationsParent.#translationsWasmRemoteClient = + translationsWasmRemoteClient; + TranslationsParent.#isTranslationsEngineMocked = true; + } + + /** + * Remove the mocks. + */ + static unmockTranslationsEngine() { + lazy.console.log( + "Removing RemoteSettings mock for the translations engine." + ); + TranslationsParent.#translationModelsRemoteClient = null; + TranslationsParent.#translationsWasmRemoteClient = null; + TranslationsParent.#isTranslationsEngineMocked = false; + } + + /** + * For testing purposes, allow the LanguageIdEngine to be mocked. If called + * with `null` in each argument, the mock is removed. + * + * @param {string} langTag - The BCP 47 language tag. + * @param {number} confidence - The confidence score of the detected language. + * @param {RemoteSettingsClient} client + */ + static mockLanguageIdentification(langTag, confidence, client) { + lazy.console.log("Mocking language identification.", { + langTag, + confidence, + }); + TranslationsParent.#mockedLangTag = langTag; + TranslationsParent.#mockedLanguageIdConfidence = confidence; + TranslationsParent.#languageIdModelsRemoteClient = client; + } + + /** + * Remove the mocks + */ + static unmockLanguageIdentification() { + lazy.console.log("Removing language identification mock."); + TranslationsParent.#mockedLangTag = null; + TranslationsParent.#mockedLanguageIdConfidence = null; + TranslationsParent.#languageIdModelsRemoteClient = null; + } + /** + * Report an error. Having this as a method allows tests to check that an error + * was properly reported. + * @param {Error} error - Providing an Error object makes sure the stack is properly + * reported. + * @param {any[]} args - Any args to pass on to console.error. + */ + static reportError(error, ...args) { + lazy.console.log(error, ...args); + } + + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + translate(fromLanguage, toLanguage) { + if (this.languageState.requestedTranslationPair) { + // This page has already been translated, restore it and translate it + // again once the actor has been recreated. + TranslationsParent.#translateOnPageReload = { fromLanguage, toLanguage }; + this.restorePage(fromLanguage); + } else { + this.languageState.requestedTranslationPair = { + fromLanguage, + toLanguage, + }; + this.sendAsyncMessage("Translations:TranslatePage", { + fromLanguage, + toLanguage, + }); + } + } + + /** + * Restore the page to the original language by doing a hard reload. + * + * @param {string} fromLanguage A BCP-47 language tag + */ + restorePage(fromLanguage) { + if ( + lazy.autoTranslatePagePref || + TranslationsParent.shouldAlwaysTranslateLanguage(fromLanguage) + ) { + // Skip auto-translate for one page load. + TranslationsParent.#isPageRestoredForAutoTranslate = true; + } + this.languageState.requestedTranslationPair = null; + + const browser = this.browsingContext.embedderElement; + browser.reload(); + } + + /** + * Keep track of when the location changes. + */ + static #locationChangeId = 0; + + static onLocationChange(browser) { + if (!lazy.translationsEnabledPref) { + // The pref isn't enabled, so don't attempt to get the actor. + return; + } + let windowGlobal = browser.browsingContext.currentWindowGlobal; + let actor = windowGlobal.getActor("Translations"); + TranslationsParent.#locationChangeId++; + actor.languageState.locationChangeId = TranslationsParent.#locationChangeId; + } + + /** + * Is this actor active for the current location change? + * + * @param {number} locationChangeId - The id sent by the "TranslationsParent:LanguageState" event. + * @returns {boolean} + */ + static isActiveLocation(locationChangeId) { + return locationChangeId === TranslationsParent.#locationChangeId; + } + + /** + * Returns the lang tags that should be offered for translation. + * + * @returns {Promise<LangTags>} + */ + getLangTagsForTranslation() { + return this.sendQuery("Translations:GetLangTagsForTranslation"); + } + + /** + * Returns the principal from the content window's origin. + * @returns {nsIPrincipal} + */ + getContentWindowPrincipal() { + return this.sendQuery("Translations:GetContentWindowPrincipal"); + } + + /** + * Returns true if the given language tag is present in the always-translate + * languages preference, otherwise false. + * + * @param {string} langTag - A BCP-47 language tag + * @returns {boolean} + */ + static shouldAlwaysTranslateLanguage(langTag) { + return lazy.alwaysTranslateLangTags.includes(langTag); + } + + /** + * Returns true if the given language tag is present in the never-translate + * languages preference, otherwise false. + * + * @param {string} langTag - A BCP-47 language tag + * @returns {boolean} + */ + static shouldNeverTranslateLanguage(langTag) { + return lazy.neverTranslateLangTags.includes(langTag); + } + + /** + * Returns true if the current site is denied permissions to translate, + * otherwise returns false. + * + * @returns {Promise<boolean>} + */ + async shouldNeverTranslateSite() { + let principal; + try { + principal = await this.getContentWindowPrincipal(); + } catch { + // Unable to get content window principal. + return false; + } + const perms = Services.perms; + const permission = perms.getPermissionObject( + principal, + TRANSLATIONS_PERMISSION, + /* exactHost */ false + ); + return permission?.capability === perms.DENY_ACTION; + } + + /** + * Removes the given language tag from the given preference. + * + * @param {string} langTag - A BCP-47 language tag + * @param {string} prefName - The pref name + */ + static #removeLangTagFromPref(langTag, prefName) { + const langTags = + prefName === ALWAYS_TRANSLATE_LANGS_PREF + ? lazy.alwaysTranslateLangTags + : lazy.neverTranslateLangTags; + const newLangTags = langTags.filter(tag => tag !== langTag); + Services.prefs.setCharPref(prefName, newLangTags.join(",")); + } + + /** + * Adds the given language tag to the given preference. + * + * @param {string} langTag - A BCP-47 language tag + * @param {string} prefName - The pref name + */ + static #addLangTagToPref(langTag, prefName) { + const langTags = + prefName === ALWAYS_TRANSLATE_LANGS_PREF + ? lazy.alwaysTranslateLangTags + : lazy.neverTranslateLangTags; + if (!langTags.includes(langTag)) { + langTags.push(langTag); + } + Services.prefs.setCharPref(prefName, langTags.join(",")); + } + + /** + * Toggles the always-translate language preference by adding the language + * to the pref list if it is not present, or removing it if it is present. + * + * @param {string} langTag - A BCP-47 language tag + */ + static toggleAlwaysTranslateLanguagePref(langTag) { + if (TranslationsParent.shouldAlwaysTranslateLanguage(langTag)) { + // The pref was toggled off for this langTag + this.#removeLangTagFromPref(langTag, ALWAYS_TRANSLATE_LANGS_PREF); + return; + } + + // The pref was toggled on for this langTag + this.#addLangTagToPref(langTag, ALWAYS_TRANSLATE_LANGS_PREF); + this.#removeLangTagFromPref(langTag, NEVER_TRANSLATE_LANGS_PREF); + } + + /** + * Toggles the never-translate language preference by adding the language + * to the pref list if it is not present, or removing it if it is present. + * + * @param {string} langTag - A BCP-47 language tag + */ + static toggleNeverTranslateLanguagePref(langTag) { + if (TranslationsParent.shouldNeverTranslateLanguage(langTag)) { + // The pref was toggled off for this langTag + this.#removeLangTagFromPref(langTag, NEVER_TRANSLATE_LANGS_PREF); + return; + } + + // The pref was toggled on for this langTag + this.#addLangTagToPref(langTag, NEVER_TRANSLATE_LANGS_PREF); + this.#removeLangTagFromPref(langTag, ALWAYS_TRANSLATE_LANGS_PREF); + } + + /** + * Toggles the never-translate site permissions by adding DENY_ACTION to + * the site principal if it is not present, or removing it if it is present. + */ + async toggleNeverTranslateSitePermissions() { + const perms = Services.perms; + const principal = await this.getContentWindowPrincipal(); + const shouldNeverTranslateSite = await this.shouldNeverTranslateSite(); + if (shouldNeverTranslateSite) { + perms.removeFromPrincipal(principal, TRANSLATIONS_PERMISSION); + } else { + perms.addFromPrincipal( + principal, + TRANSLATIONS_PERMISSION, + perms.DENY_ACTION + ); + } + } +} + +/** + * WebAssembly modules must be instantiated from a Worker, since it's considered + * unsafe eval. + */ +function detectSimdSupport() { + return new Promise(resolve => { + lazy.console.log("Loading wasm simd detector worker."); + + const worker = new Worker( + "chrome://global/content/translations/simd-detect-worker.js" + ); + + // This should pretty much immediately resolve, so it does not need Firefox shutdown + // detection. + worker.addEventListener("message", ({ data }) => { + resolve(data.isSimdSupported); + worker.terminate(); + }); + }); +} + +/** + * State that affects the UI. Any of the state that gets set triggers a dispatch to update + * the UI. + */ +class TranslationsLanguageState { + /** + * @param {TranslationsParent} actor + */ + constructor(actor) { + this.#actor = actor; + this.dispatch(); + } + + /** + * The data members for TranslationsLanguageState, see the getters for their + * documentation. + */ + + /** @type {TranslationsParent} */ + #actor; + + /** @type {TranslationPair | null} */ + #requestedTranslationPair = null; + + /** @type {LangTags | null} */ + #detectedLanguages = null; + + /** @type {number} */ + #locationChangeId = -1; + + /** @type {null | TranslationErrors} */ + #error = null; + + #isEngineReady = false; + + /** + * Dispatch anytime the language details change, so that any UI can react to it. + */ + dispatch() { + if (!TranslationsParent.isActiveLocation(this.#locationChangeId)) { + // Do not dispatch as this location is not active. + return; + } + + const browser = this.#actor.browsingContext.top.embedderElement; + if (!browser) { + return; + } + const { CustomEvent } = browser.ownerGlobal; + browser.dispatchEvent( + new CustomEvent("TranslationsParent:LanguageState", { + bubbles: true, + detail: { + detectedLanguages: this.#detectedLanguages, + requestedTranslationPair: this.#requestedTranslationPair, + error: this.#error, + isEngineReady: this.#isEngineReady, + }, + }) + ); + } + + /** + * When a translation is requested, this contains the translation pair. This means + * that the TranslationsChild should be creating a TranslationsDocument and keep + * the page updated with the target language. + * + * @returns {TranslationPair | null} + */ + get requestedTranslationPair() { + return this.#requestedTranslationPair; + } + + set requestedTranslationPair(requestedTranslationPair) { + this.#error = null; + this.#isEngineReady = false; + this.#requestedTranslationPair = requestedTranslationPair; + this.dispatch(); + } + + /** + * The TranslationsChild will detect languages and offer them up for translation. + * The results are stored here. + * + * @returns {LangTags | null} + */ + get detectedLanguages() { + return this.#detectedLanguages; + } + + set detectedLanguages(detectedLanguages) { + this.#detectedLanguages = detectedLanguages; + this.dispatch(); + } + + /** + * This id represents the last location change that happened for this actor. This + * allows the UI to disambiguate when there are races and out of order events that + * are dispatched. Only the most up to date `locationChangeId` is used. + * + * @returns {number} + */ + get locationChangeId() { + return this.#locationChangeId; + } + + set locationChangeId(locationChangeId) { + this.#locationChangeId = locationChangeId; + + // When the location changes remove the previous error. + this.#error = null; + + this.dispatch(); + } + + /** + * The last error that occured during translation. + */ + get error() { + return this.#error; + } + + set error(error) { + this.#error = error; + // Setting an error invalidates the requested translation pair. + this.#requestedTranslationPair = null; + this.#isEngineReady = false; + this.dispatch(); + } + + /** + * Stores when the translations engine is ready. The wasm and language files must + * be downloaded, which can take some time. + */ + get isEngineReady() { + return this.#isEngineReady; + } + + set isEngineReady(isEngineReady) { + this.#isEngineReady = isEngineReady; + this.dispatch(); + } +} + +/** + * @typedef {Object} QueueItem + * @prop {Function} download + * @prop {Function} [onSuccess] + * @prop {Function} [onFailure] + * @prop {number} [retriesLeft] + */ + +/** + * Manage the download of the files by providing a maximum number of concurrent files + * and the ability to retry a file download in case of an error. + * + * @param {QueueItem[]} queue + */ +async function downloadManager(queue) { + const NOOP = () => {}; + + const pendingDownloadAttempts = new Set(); + let failCount = 0; + let index = 0; + const start = Date.now(); + const originalQueueLength = queue.length; + + while (index < queue.length || pendingDownloadAttempts.size > 0) { + // Start new downloads up to the maximum limit + while ( + index < queue.length && + pendingDownloadAttempts.size < TranslationsParent.MAX_CONCURRENT_DOWNLOADS + ) { + lazy.console.log(`Starting download ${index + 1} of ${queue.length}`); + + const { + download, + onSuccess = NOOP, + onFailure = NOOP, + retriesLeft = TranslationsParent.MAX_DOWNLOAD_RETRIES, + } = queue[index]; + + const handleFailedDownload = error => { + // The download failed. Either retry it, or report the failure. + TranslationsParent.reportError( + new Error("Failed to download file."), + error + ); + + const newRetriesLeft = retriesLeft - 1; + + if (retriesLeft > 0) { + lazy.console.log( + `Queueing another attempt. ${newRetriesLeft} attempts left.` + ); + queue.push({ + download, + retriesLeft: newRetriesLeft, + onSuccess, + onFailure, + }); + } else { + // Give up on this download. + failCount++; + onFailure(); + } + }; + + const afterDownloadAttempt = () => { + pendingDownloadAttempts.delete(downloadAttempt); + }; + + // Kick off the download. If it fails, retry it a certain number of attempts. + // This is done asynchronously from the rest of the for loop. + const downloadAttempt = download() + .then(onSuccess, handleFailedDownload) + .then(afterDownloadAttempt); + + pendingDownloadAttempts.add(downloadAttempt); + index++; + } + + // Wait for any active downloads to complete. + await Promise.race(pendingDownloadAttempts); + } + + const duration = ((Date.now() - start) / 1000).toFixed(3); + + if (failCount > 0) { + const message = `Finished downloads in ${duration} seconds, but ${failCount} download(s) failed.`; + lazy.console.log( + `Finished downloads in ${duration} seconds, but ${failCount} download(s) failed.` + ); + throw new Error(message); + } + + lazy.console.log( + `Finished ${originalQueueLength} downloads in ${duration} seconds.` + ); +} + +/** + * The translations code has lots of async code and fallible network requests. To test + * this manually while using the feature, enable chaos mode by setting "errors" to true + * and "timeoutMS" to a positive number of milliseconds. + * prefs to true: + * + * - browser.translations.chaos.timeoutMS + * - browser.translations.chaos.errors + */ +async function chaosMode(probability = 0.5) { + await chaosModeTimer(); + await chaosModeError(probability); +} + +/** + * The translations code has lots of async code that relies on the network. To test + * this manually while using the feature, enable chaos mode by setting the following pref + * to a positive number of milliseconds. + * + * - browser.translations.chaos.timeoutMS + */ +async function chaosModeTimer() { + if (lazy.chaosTimeoutMSPref) { + const timeout = Math.random() * lazy.chaosTimeoutMSPref; + lazy.console.log( + `Chaos mode timer started for ${(timeout / 1000).toFixed(1)} seconds.` + ); + await new Promise(resolve => lazy.setTimeout(resolve, timeout)); + } +} + +/** + * The translations code has lots of async code that is fallible. To test this manually + * while using the feature, enable chaos mode by setting the following pref to true. + * + * - browser.translations.chaos.errors + */ +async function chaosModeError(probability = 0.5) { + if (lazy.chaosErrorsPref && Math.random() < probability) { + lazy.console.trace(`Chaos mode error generated.`); + throw new Error( + `Chaos Mode error from the pref "browser.translations.chaos.errors".` + ); + } +} diff --git a/toolkit/components/translations/actors/moz.build b/toolkit/components/translations/actors/moz.build new file mode 100644 index 0000000000..b05acabab5 --- /dev/null +++ b/toolkit/components/translations/actors/moz.build @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FINAL_TARGET_FILES.actors += [ + "AboutTranslationsChild.sys.mjs", + "AboutTranslationsParent.sys.mjs", + "TranslationsChild.sys.mjs", + "TranslationsParent.sys.mjs", +] diff --git a/toolkit/components/translations/bergamot-translator/LICENSE b/toolkit/components/translations/bergamot-translator/LICENSE new file mode 100644 index 0000000000..a612ad9813 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/toolkit/components/translations/bergamot-translator/bergamot-translator.js b/toolkit/components/translations/bergamot-translator/bergamot-translator.js new file mode 100644 index 0000000000..6667121a47 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/bergamot-translator.js @@ -0,0 +1,3456 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function loadBergamot(Module) { + var BERGAMOT_VERSION_FULL = "v0.4.4+5ae1b1e"; + null; + + var Module = typeof Module != "undefined" ? Module : {}; + + var moduleOverrides = Object.assign({}, Module); + + var arguments_ = []; + + var thisProgram = "./this.program"; + + var quit_ = (status, toThrow) => { + throw toThrow; + }; + + var ENVIRONMENT_IS_WEB = typeof window == "object"; + + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + + var ENVIRONMENT_IS_NODE = + typeof process == "object" && + typeof process.versions == "object" && + typeof process.versions.node == "string"; + + var scriptDirectory = ""; + + function locateFile(path) { + if (Module["locateFile"]) { + return Module["locateFile"](path, scriptDirectory); + } + return scriptDirectory + path; + } + + var read_, readAsync, readBinary, setWindowTitle; + + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (scriptDirectory.indexOf("blob:") !== 0) { + scriptDirectory = scriptDirectory.substr( + 0, + scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1 + ); + } else { + scriptDirectory = ""; + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array(xhr.response); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + setWindowTitle = (title) => (document.title = title); + } else { + } + + var out = Module["print"] || console.log.bind(console); + + var err = Module["printErr"] || console.warn.bind(console); + + Object.assign(Module, moduleOverrides); + + moduleOverrides = null; + + if (Module["arguments"]) arguments_ = Module["arguments"]; + + if (Module["thisProgram"]) thisProgram = Module["thisProgram"]; + + if (Module["quit"]) quit_ = Module["quit"]; + + var tempRet0 = 0; + + var setTempRet0 = (value) => { + tempRet0 = value; + }; + + var wasmBinary; + + if (Module["wasmBinary"]) wasmBinary = Module["wasmBinary"]; + + var noExitRuntime = Module["noExitRuntime"] || true; + + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + + var wasmMemory; + + var ABORT = false; + + var EXITSTATUS; + + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + + var UTF8Decoder = + typeof TextDecoder != "undefined" ? new TextDecoder("utf8") : undefined; + + function UTF8ArrayToString(heapOrArray, idx, maxBytesToRead) { + var endIdx = idx + maxBytesToRead; + var endPtr = idx; + while (heapOrArray[endPtr] && !(endPtr >= endIdx)) ++endPtr; + if (endPtr - idx > 16 && heapOrArray.buffer && UTF8Decoder) { + return UTF8Decoder.decode(heapOrArray.subarray(idx, endPtr)); + } else { + var str = ""; + while (idx < endPtr) { + var u0 = heapOrArray[idx++]; + if (!(u0 & 128)) { + str += String.fromCharCode(u0); + continue; + } + var u1 = heapOrArray[idx++] & 63; + if ((u0 & 224) == 192) { + str += String.fromCharCode(((u0 & 31) << 6) | u1); + continue; + } + var u2 = heapOrArray[idx++] & 63; + if ((u0 & 240) == 224) { + u0 = ((u0 & 15) << 12) | (u1 << 6) | u2; + } else { + u0 = + ((u0 & 7) << 18) | + (u1 << 12) | + (u2 << 6) | + (heapOrArray[idx++] & 63); + } + if (u0 < 65536) { + str += String.fromCharCode(u0); + } else { + var ch = u0 - 65536; + str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023)); + } + } + } + return str; + } + + function UTF8ToString(ptr, maxBytesToRead) { + return ptr ? UTF8ArrayToString(HEAPU8, ptr, maxBytesToRead) : ""; + } + + function stringToUTF8Array(str, heap, outIdx, maxBytesToWrite) { + if (!(maxBytesToWrite > 0)) return 0; + var startIdx = outIdx; + var endIdx = outIdx + maxBytesToWrite - 1; + for (var i = 0; i < str.length; ++i) { + var u = str.charCodeAt(i); + if (u >= 55296 && u <= 57343) { + var u1 = str.charCodeAt(++i); + u = (65536 + ((u & 1023) << 10)) | (u1 & 1023); + } + if (u <= 127) { + if (outIdx >= endIdx) break; + heap[outIdx++] = u; + } else if (u <= 2047) { + if (outIdx + 1 >= endIdx) break; + heap[outIdx++] = 192 | (u >> 6); + heap[outIdx++] = 128 | (u & 63); + } else if (u <= 65535) { + if (outIdx + 2 >= endIdx) break; + heap[outIdx++] = 224 | (u >> 12); + heap[outIdx++] = 128 | ((u >> 6) & 63); + heap[outIdx++] = 128 | (u & 63); + } else { + if (outIdx + 3 >= endIdx) break; + heap[outIdx++] = 240 | (u >> 18); + heap[outIdx++] = 128 | ((u >> 12) & 63); + heap[outIdx++] = 128 | ((u >> 6) & 63); + heap[outIdx++] = 128 | (u & 63); + } + } + heap[outIdx] = 0; + return outIdx - startIdx; + } + + function stringToUTF8(str, outPtr, maxBytesToWrite) { + return stringToUTF8Array(str, HEAPU8, outPtr, maxBytesToWrite); + } + + function lengthBytesUTF8(str) { + var len = 0; + for (var i = 0; i < str.length; ++i) { + var u = str.charCodeAt(i); + if (u >= 55296 && u <= 57343) + u = (65536 + ((u & 1023) << 10)) | (str.charCodeAt(++i) & 1023); + if (u <= 127) ++len; + else if (u <= 2047) len += 2; + else if (u <= 65535) len += 3; + else len += 4; + } + return len; + } + + var UTF16Decoder = + typeof TextDecoder != "undefined" ? new TextDecoder("utf-16le") : undefined; + + function UTF16ToString(ptr, maxBytesToRead) { + var endPtr = ptr; + var idx = endPtr >> 1; + var maxIdx = idx + maxBytesToRead / 2; + while (!(idx >= maxIdx) && HEAPU16[idx]) ++idx; + endPtr = idx << 1; + if (endPtr - ptr > 32 && UTF16Decoder) { + return UTF16Decoder.decode(HEAPU8.subarray(ptr, endPtr)); + } else { + var str = ""; + for (var i = 0; !(i >= maxBytesToRead / 2); ++i) { + var codeUnit = HEAP16[(ptr + i * 2) >> 1]; + if (codeUnit == 0) break; + str += String.fromCharCode(codeUnit); + } + return str; + } + } + + function stringToUTF16(str, outPtr, maxBytesToWrite) { + if (maxBytesToWrite === undefined) { + maxBytesToWrite = 2147483647; + } + if (maxBytesToWrite < 2) return 0; + maxBytesToWrite -= 2; + var startPtr = outPtr; + var numCharsToWrite = + maxBytesToWrite < str.length * 2 ? maxBytesToWrite / 2 : str.length; + for (var i = 0; i < numCharsToWrite; ++i) { + var codeUnit = str.charCodeAt(i); + HEAP16[outPtr >> 1] = codeUnit; + outPtr += 2; + } + HEAP16[outPtr >> 1] = 0; + return outPtr - startPtr; + } + + function lengthBytesUTF16(str) { + return str.length * 2; + } + + function UTF32ToString(ptr, maxBytesToRead) { + var i = 0; + var str = ""; + while (!(i >= maxBytesToRead / 4)) { + var utf32 = HEAP32[(ptr + i * 4) >> 2]; + if (utf32 == 0) break; + ++i; + if (utf32 >= 65536) { + var ch = utf32 - 65536; + str += String.fromCharCode(55296 | (ch >> 10), 56320 | (ch & 1023)); + } else { + str += String.fromCharCode(utf32); + } + } + return str; + } + + function stringToUTF32(str, outPtr, maxBytesToWrite) { + if (maxBytesToWrite === undefined) { + maxBytesToWrite = 2147483647; + } + if (maxBytesToWrite < 4) return 0; + var startPtr = outPtr; + var endPtr = startPtr + maxBytesToWrite - 4; + for (var i = 0; i < str.length; ++i) { + var codeUnit = str.charCodeAt(i); + if (codeUnit >= 55296 && codeUnit <= 57343) { + var trailSurrogate = str.charCodeAt(++i); + codeUnit = + (65536 + ((codeUnit & 1023) << 10)) | (trailSurrogate & 1023); + } + HEAP32[outPtr >> 2] = codeUnit; + outPtr += 4; + if (outPtr + 4 > endPtr) break; + } + HEAP32[outPtr >> 2] = 0; + return outPtr - startPtr; + } + + function lengthBytesUTF32(str) { + var len = 0; + for (var i = 0; i < str.length; ++i) { + var codeUnit = str.charCodeAt(i); + if (codeUnit >= 55296 && codeUnit <= 57343) ++i; + len += 4; + } + return len; + } + + function allocateUTF8(str) { + var size = lengthBytesUTF8(str) + 1; + var ret = _malloc(size); + if (ret) stringToUTF8Array(str, HEAP8, ret, size); + return ret; + } + + function writeArrayToMemory(array, buffer) { + HEAP8.set(array, buffer); + } + + function writeAsciiToMemory(str, buffer, dontAddNull) { + for (var i = 0; i < str.length; ++i) { + HEAP8[buffer++ >> 0] = str.charCodeAt(i); + } + if (!dontAddNull) HEAP8[buffer >> 0] = 0; + } + + var buffer, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + + function updateGlobalBufferAndViews(buf) { + buffer = buf; + Module["HEAP8"] = HEAP8 = new Int8Array(buf); + Module["HEAP16"] = HEAP16 = new Int16Array(buf); + Module["HEAP32"] = HEAP32 = new Int32Array(buf); + Module["HEAPU8"] = HEAPU8 = new Uint8Array(buf); + Module["HEAPU16"] = HEAPU16 = new Uint16Array(buf); + Module["HEAPU32"] = HEAPU32 = new Uint32Array(buf); + Module["HEAPF32"] = HEAPF32 = new Float32Array(buf); + Module["HEAPF64"] = HEAPF64 = new Float64Array(buf); + } + + var INITIAL_MEMORY = Module["INITIAL_MEMORY"] || 16777216; + + if (Module["wasmMemory"]) { + wasmMemory = Module["wasmMemory"]; + } else { + wasmMemory = new WebAssembly.Memory({ + initial: INITIAL_MEMORY / 65536, + maximum: 2147483648 / 65536, + }); + } + + if (wasmMemory) { + buffer = wasmMemory.buffer; + } + + INITIAL_MEMORY = buffer.byteLength; + + updateGlobalBufferAndViews(buffer); + + var wasmTable; + + var __ATPRERUN__ = []; + + var __ATINIT__ = []; + + var __ATPOSTRUN__ = []; + + var runtimeInitialized = false; + + function keepRuntimeAlive() { + return noExitRuntime; + } + + function preRun() { + if (Module["preRun"]) { + if (typeof Module["preRun"] == "function") + Module["preRun"] = [Module["preRun"]]; + while (Module["preRun"].length) { + addOnPreRun(Module["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + + function postRun() { + if (Module["postRun"]) { + if (typeof Module["postRun"] == "function") + Module["postRun"] = [Module["postRun"]]; + while (Module["postRun"].length) { + addOnPostRun(Module["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + + var runDependencies = 0; + + var runDependencyWatcher = null; + + var dependenciesFulfilled = null; + + function addRunDependency(id) { + runDependencies++; + if (Module["monitorRunDependencies"]) { + Module["monitorRunDependencies"](runDependencies); + } + } + + function removeRunDependency(id) { + runDependencies--; + if (Module["monitorRunDependencies"]) { + Module["monitorRunDependencies"](runDependencies); + } + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + + Module["preloadedImages"] = {}; + + Module["preloadedAudios"] = {}; + + function abort(what) { + { + if (Module["onAbort"]) { + Module["onAbort"](what); + } + } + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -s ASSERTIONS=1 for more info."; + var e = new WebAssembly.RuntimeError(what); + throw e; + } + + var dataURIPrefix = "data:application/octet-stream;base64,"; + + function isDataURI(filename) { + return filename.startsWith(dataURIPrefix); + } + + var wasmBinaryFile; + + wasmBinaryFile = "bergamot-translator-worker.wasm"; + + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + + function getBinary(file) { + try { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + if (readBinary) { + return readBinary(file); + } else { + throw "both async and sync fetching of the wasm failed"; + } + } catch (err) { + abort(err); + } + } + + function getBinaryPromise() { + if (!wasmBinary && (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER)) { + if (typeof fetch == "function") { + return fetch(wasmBinaryFile, { + credentials: "same-origin", + }) + .then(function (response) { + if (!response["ok"]) { + throw ( + "failed to load wasm binary file at '" + wasmBinaryFile + "'" + ); + } + return response["arrayBuffer"](); + }) + .catch(function () { + return getBinary(wasmBinaryFile); + }); + } + } + return Promise.resolve().then(function () { + return getBinary(wasmBinaryFile); + }); + } + + function createWasm() { + var info = { + env: asmLibraryArg, + wasm_gemm: createWasmGemm(), + wasi_snapshot_preview1: asmLibraryArg, + }; + function receiveInstance(instance, module) { + var exports = instance.exports; + Module["asm"] = exports; + wasmTable = Module["asm"]["__indirect_function_table"]; + addOnInit(Module["asm"]["__wasm_call_ctors"]); + exportAsmFunctions(exports); + removeRunDependency("wasm-instantiate"); + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + function instantiateArrayBuffer(receiver) { + return getBinaryPromise() + .then(function (binary) { + return WebAssembly.instantiate(binary, info); + }) + .then(function (instance) { + return instance; + }) + .then(receiver, function (reason) { + err("failed to asynchronously prepare wasm: " + reason); + abort(reason); + }); + } + function instantiateAsync() { + if ( + !wasmBinary && + typeof WebAssembly.instantiateStreaming == "function" && + !isDataURI(wasmBinaryFile) && + typeof fetch == "function" + ) { + return fetch(wasmBinaryFile, { + credentials: "same-origin", + }).then(function (response) { + var result = WebAssembly.instantiateStreaming(response, info); + return result.then(receiveInstantiationResult, function (reason) { + err("wasm streaming compile failed: " + reason); + err("falling back to ArrayBuffer instantiation"); + return instantiateArrayBuffer(receiveInstantiationResult); + }); + }); + } else { + return instantiateArrayBuffer(receiveInstantiationResult); + } + } + if (Module["instantiateWasm"]) { + try { + var exports = Module["instantiateWasm"](info, receiveInstance); + return exports; + } catch (e) { + err("Module.instantiateWasm callback failed with error: " + e); + return false; + } + } + instantiateAsync(); + return {}; + } + + function callRuntimeCallbacks(callbacks) { + while (callbacks.length > 0) { + var callback = callbacks.shift(); + if (typeof callback == "function") { + callback(Module); + continue; + } + var func = callback.func; + if (typeof func == "number") { + if (callback.arg === undefined) { + getWasmTableEntry(func)(); + } else { + getWasmTableEntry(func)(callback.arg); + } + } else { + func(callback.arg === undefined ? null : callback.arg); + } + } + } + + function asmjsMangle(x) { + var unmangledSymbols = ["stackAlloc", "stackSave", "stackRestore"]; + return x.indexOf("dynCall_") == 0 || unmangledSymbols.includes(x) + ? x + : "_" + x; + } + + function exportAsmFunctions(asm) { + var global_object = this; + for (var __exportedFunc in asm) { + var jsname = asmjsMangle(__exportedFunc); + global_object[jsname] = Module[jsname] = asm[__exportedFunc]; + } + } + + var wasmTableMirror = []; + + function getWasmTableEntry(funcPtr) { + var func = wasmTableMirror[funcPtr]; + if (!func) { + if (funcPtr >= wasmTableMirror.length) + wasmTableMirror.length = funcPtr + 1; + wasmTableMirror[funcPtr] = func = wasmTable.get(funcPtr); + } + return func; + } + + function ___assert_fail(condition, filename, line, func) { + abort( + "Assertion failed: " + + UTF8ToString(condition) + + ", at: " + + [ + filename ? UTF8ToString(filename) : "unknown filename", + line, + func ? UTF8ToString(func) : "unknown function", + ] + ); + } + + function ___cxa_allocate_exception(size) { + return _malloc(size + 16) + 16; + } + + var exceptionCaught = []; + + var exceptionLast = 0; + + var uncaughtExceptionCount = 0; + + function ___cxa_rethrow() { + var catchInfo = exceptionCaught.pop(); + if (!catchInfo) { + abort("no exception to throw"); + } + var info = catchInfo.get_exception_info(); + var ptr = catchInfo.get_base_ptr(); + if (!info.get_rethrown()) { + exceptionCaught.push(catchInfo); + info.set_rethrown(true); + info.set_caught(false); + uncaughtExceptionCount++; + } else { + catchInfo.free(); + } + exceptionLast = ptr; + throw ptr; + } + + function ExceptionInfo(excPtr) { + this.excPtr = excPtr; + this.ptr = excPtr - 16; + this.set_type = function (type) { + HEAP32[(this.ptr + 4) >> 2] = type; + }; + this.get_type = function () { + return HEAP32[(this.ptr + 4) >> 2]; + }; + this.set_destructor = function (destructor) { + HEAP32[(this.ptr + 8) >> 2] = destructor; + }; + this.get_destructor = function () { + return HEAP32[(this.ptr + 8) >> 2]; + }; + this.set_refcount = function (refcount) { + HEAP32[this.ptr >> 2] = refcount; + }; + this.set_caught = function (caught) { + caught = caught ? 1 : 0; + HEAP8[(this.ptr + 12) >> 0] = caught; + }; + this.get_caught = function () { + return HEAP8[(this.ptr + 12) >> 0] != 0; + }; + this.set_rethrown = function (rethrown) { + rethrown = rethrown ? 1 : 0; + HEAP8[(this.ptr + 13) >> 0] = rethrown; + }; + this.get_rethrown = function () { + return HEAP8[(this.ptr + 13) >> 0] != 0; + }; + this.init = function (type, destructor) { + this.set_type(type); + this.set_destructor(destructor); + this.set_refcount(0); + this.set_caught(false); + this.set_rethrown(false); + }; + this.add_ref = function () { + var value = HEAP32[this.ptr >> 2]; + HEAP32[this.ptr >> 2] = value + 1; + }; + this.release_ref = function () { + var prev = HEAP32[this.ptr >> 2]; + HEAP32[this.ptr >> 2] = prev - 1; + return prev === 1; + }; + } + + function ___cxa_throw(ptr, type, destructor) { + var info = new ExceptionInfo(ptr); + info.init(type, destructor); + exceptionLast = ptr; + uncaughtExceptionCount++; + throw ptr; + } + + var SYSCALLS = { + buffers: [null, [], []], + printChar: function (stream, curr) { + var buffer = SYSCALLS.buffers[stream]; + if (curr === 0 || curr === 10) { + (stream === 1 ? out : err)(UTF8ArrayToString(buffer, 0)); + buffer.length = 0; + } else { + buffer.push(curr); + } + }, + varargs: undefined, + get: function () { + SYSCALLS.varargs += 4; + var ret = HEAP32[(SYSCALLS.varargs - 4) >> 2]; + return ret; + }, + getStr: function (ptr) { + var ret = UTF8ToString(ptr); + return ret; + }, + get64: function (low, high) { + return low; + }, + }; + + function ___syscall_faccessat(dirfd, path, amode, flags) { + path = SYSCALLS.getStr(path); + path = SYSCALLS.calculateAt(dirfd, path); + return SYSCALLS.doAccess(path, amode); + } + + function ___syscall_fcntl64(fd, cmd, varargs) { + SYSCALLS.varargs = varargs; + return 0; + } + + function ___syscall_fstat64(fd, buf) {} + + function ___syscall_getcwd(buf, size) {} + + function ___syscall_ioctl(fd, op, varargs) { + SYSCALLS.varargs = varargs; + return 0; + } + + function ___syscall_lstat64(path, buf) {} + + function ___syscall_newfstatat(dirfd, path, buf, flags) {} + + function ___syscall_openat(dirfd, path, flags, varargs) { + SYSCALLS.varargs = varargs; + } + + function ___syscall_renameat(olddirfd, oldpath, newdirfd, newpath) {} + + function ___syscall_rmdir(path) {} + + function ___syscall_stat64(path, buf) {} + + function ___syscall_unlinkat(dirfd, path, flags) {} + + var structRegistrations = {}; + + function runDestructors(destructors) { + while (destructors.length) { + var ptr = destructors.pop(); + var del = destructors.pop(); + del(ptr); + } + } + + function simpleReadValueFromPointer(pointer) { + return this["fromWireType"](HEAPU32[pointer >> 2]); + } + + var awaitingDependencies = {}; + + var registeredTypes = {}; + + var typeDependencies = {}; + + var char_0 = 48; + + var char_9 = 57; + + function makeLegalFunctionName(name) { + if (undefined === name) { + return "_unknown"; + } + name = name.replace(/[^a-zA-Z0-9_]/g, "$"); + var f = name.charCodeAt(0); + if (f >= char_0 && f <= char_9) { + return "_" + name; + } + return name; + } + + function createNamedFunction(name, body) { + name = makeLegalFunctionName(name); + return function () { + null; + return body.apply(this, arguments); + }; + } + + function extendError(baseErrorType, errorName) { + var errorClass = createNamedFunction(errorName, function (message) { + this.name = errorName; + this.message = message; + var stack = new Error(message).stack; + if (stack !== undefined) { + this.stack = + this.toString() + "\n" + stack.replace(/^Error(:[^\n]*)?\n/, ""); + } + }); + errorClass.prototype = Object.create(baseErrorType.prototype); + errorClass.prototype.constructor = errorClass; + errorClass.prototype.toString = function () { + if (this.message === undefined) { + return this.name; + } else { + return this.name + ": " + this.message; + } + }; + return errorClass; + } + + var InternalError = undefined; + + function throwInternalError(message) { + throw new InternalError(message); + } + + function whenDependentTypesAreResolved( + myTypes, + dependentTypes, + getTypeConverters + ) { + myTypes.forEach(function (type) { + typeDependencies[type] = dependentTypes; + }); + function onComplete(typeConverters) { + var myTypeConverters = getTypeConverters(typeConverters); + if (myTypeConverters.length !== myTypes.length) { + throwInternalError("Mismatched type converter count"); + } + for (var i = 0; i < myTypes.length; ++i) { + registerType(myTypes[i], myTypeConverters[i]); + } + } + var typeConverters = new Array(dependentTypes.length); + var unregisteredTypes = []; + var registered = 0; + dependentTypes.forEach((dt, i) => { + if (registeredTypes.hasOwnProperty(dt)) { + typeConverters[i] = registeredTypes[dt]; + } else { + unregisteredTypes.push(dt); + if (!awaitingDependencies.hasOwnProperty(dt)) { + awaitingDependencies[dt] = []; + } + awaitingDependencies[dt].push(() => { + typeConverters[i] = registeredTypes[dt]; + ++registered; + if (registered === unregisteredTypes.length) { + onComplete(typeConverters); + } + }); + } + }); + if (0 === unregisteredTypes.length) { + onComplete(typeConverters); + } + } + + function __embind_finalize_value_object(structType) { + var reg = structRegistrations[structType]; + delete structRegistrations[structType]; + var rawConstructor = reg.rawConstructor; + var rawDestructor = reg.rawDestructor; + var fieldRecords = reg.fields; + var fieldTypes = fieldRecords + .map((field) => field.getterReturnType) + .concat(fieldRecords.map((field) => field.setterArgumentType)); + whenDependentTypesAreResolved([structType], fieldTypes, (fieldTypes) => { + var fields = {}; + fieldRecords.forEach((field, i) => { + var fieldName = field.fieldName; + var getterReturnType = fieldTypes[i]; + var getter = field.getter; + var getterContext = field.getterContext; + var setterArgumentType = fieldTypes[i + fieldRecords.length]; + var setter = field.setter; + var setterContext = field.setterContext; + fields[fieldName] = { + read: (ptr) => { + return getterReturnType["fromWireType"](getter(getterContext, ptr)); + }, + write: (ptr, o) => { + var destructors = []; + setter( + setterContext, + ptr, + setterArgumentType["toWireType"](destructors, o) + ); + runDestructors(destructors); + }, + }; + }); + return [ + { + name: reg.name, + fromWireType: function (ptr) { + var rv = {}; + for (var i in fields) { + rv[i] = fields[i].read(ptr); + } + rawDestructor(ptr); + return rv; + }, + toWireType: function (destructors, o) { + for (var fieldName in fields) { + if (!(fieldName in o)) { + throw new TypeError('Missing field: "' + fieldName + '"'); + } + } + var ptr = rawConstructor(); + for (fieldName in fields) { + fields[fieldName].write(ptr, o[fieldName]); + } + if (destructors !== null) { + destructors.push(rawDestructor, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: rawDestructor, + }, + ]; + }); + } + + function __embind_register_bigint( + primitiveType, + name, + size, + minRange, + maxRange + ) {} + + function getShiftFromSize(size) { + switch (size) { + case 1: + return 0; + + case 2: + return 1; + + case 4: + return 2; + + case 8: + return 3; + + default: + throw new TypeError("Unknown type size: " + size); + } + } + + function embind_init_charCodes() { + var codes = new Array(256); + for (var i = 0; i < 256; ++i) { + codes[i] = String.fromCharCode(i); + } + embind_charCodes = codes; + } + + var embind_charCodes = undefined; + + function readLatin1String(ptr) { + var ret = ""; + var c = ptr; + while (HEAPU8[c]) { + ret += embind_charCodes[HEAPU8[c++]]; + } + return ret; + } + + var BindingError = undefined; + + function throwBindingError(message) { + throw new BindingError(message); + } + + function registerType(rawType, registeredInstance, options = {}) { + if (!("argPackAdvance" in registeredInstance)) { + throw new TypeError( + "registerType registeredInstance requires argPackAdvance" + ); + } + var name = registeredInstance.name; + if (!rawType) { + throwBindingError( + 'type "' + name + '" must have a positive integer typeid pointer' + ); + } + if (registeredTypes.hasOwnProperty(rawType)) { + if (options.ignoreDuplicateRegistrations) { + return; + } else { + throwBindingError("Cannot register type '" + name + "' twice"); + } + } + registeredTypes[rawType] = registeredInstance; + delete typeDependencies[rawType]; + if (awaitingDependencies.hasOwnProperty(rawType)) { + var callbacks = awaitingDependencies[rawType]; + delete awaitingDependencies[rawType]; + callbacks.forEach((cb) => cb()); + } + } + + function __embind_register_bool(rawType, name, size, trueValue, falseValue) { + var shift = getShiftFromSize(size); + name = readLatin1String(name); + registerType(rawType, { + name: name, + fromWireType: function (wt) { + return !!wt; + }, + toWireType: function (destructors, o) { + return o ? trueValue : falseValue; + }, + argPackAdvance: 8, + readValueFromPointer: function (pointer) { + var heap; + if (size === 1) { + heap = HEAP8; + } else if (size === 2) { + heap = HEAP16; + } else if (size === 4) { + heap = HEAP32; + } else { + throw new TypeError("Unknown boolean type size: " + name); + } + return this["fromWireType"](heap[pointer >> shift]); + }, + destructorFunction: null, + }); + } + + function ClassHandle_isAliasOf(other) { + if (!(this instanceof ClassHandle)) { + return false; + } + if (!(other instanceof ClassHandle)) { + return false; + } + var leftClass = this.$$.ptrType.registeredClass; + var left = this.$$.ptr; + var rightClass = other.$$.ptrType.registeredClass; + var right = other.$$.ptr; + while (leftClass.baseClass) { + left = leftClass.upcast(left); + leftClass = leftClass.baseClass; + } + while (rightClass.baseClass) { + right = rightClass.upcast(right); + rightClass = rightClass.baseClass; + } + return leftClass === rightClass && left === right; + } + + function shallowCopyInternalPointer(o) { + return { + count: o.count, + deleteScheduled: o.deleteScheduled, + preservePointerOnDelete: o.preservePointerOnDelete, + ptr: o.ptr, + ptrType: o.ptrType, + smartPtr: o.smartPtr, + smartPtrType: o.smartPtrType, + }; + } + + function throwInstanceAlreadyDeleted(obj) { + function getInstanceTypeName(handle) { + return handle.$$.ptrType.registeredClass.name; + } + throwBindingError(getInstanceTypeName(obj) + " instance already deleted"); + } + + var finalizationRegistry = false; + + function detachFinalizer(handle) {} + + function runDestructor($$) { + if ($$.smartPtr) { + $$.smartPtrType.rawDestructor($$.smartPtr); + } else { + $$.ptrType.registeredClass.rawDestructor($$.ptr); + } + } + + function releaseClassHandle($$) { + $$.count.value -= 1; + var toDelete = 0 === $$.count.value; + if (toDelete) { + runDestructor($$); + } + } + + function downcastPointer(ptr, ptrClass, desiredClass) { + if (ptrClass === desiredClass) { + return ptr; + } + if (undefined === desiredClass.baseClass) { + return null; + } + var rv = downcastPointer(ptr, ptrClass, desiredClass.baseClass); + if (rv === null) { + return null; + } + return desiredClass.downcast(rv); + } + + var registeredPointers = {}; + + function getInheritedInstanceCount() { + return Object.keys(registeredInstances).length; + } + + function getLiveInheritedInstances() { + var rv = []; + for (var k in registeredInstances) { + if (registeredInstances.hasOwnProperty(k)) { + rv.push(registeredInstances[k]); + } + } + return rv; + } + + var deletionQueue = []; + + function flushPendingDeletes() { + while (deletionQueue.length) { + var obj = deletionQueue.pop(); + obj.$$.deleteScheduled = false; + obj["delete"](); + } + } + + var delayFunction = undefined; + + function setDelayFunction(fn) { + delayFunction = fn; + if (deletionQueue.length && delayFunction) { + delayFunction(flushPendingDeletes); + } + } + + function init_embind() { + Module["getInheritedInstanceCount"] = getInheritedInstanceCount; + Module["getLiveInheritedInstances"] = getLiveInheritedInstances; + Module["flushPendingDeletes"] = flushPendingDeletes; + Module["setDelayFunction"] = setDelayFunction; + } + + var registeredInstances = {}; + + function getBasestPointer(class_, ptr) { + if (ptr === undefined) { + throwBindingError("ptr should not be undefined"); + } + while (class_.baseClass) { + ptr = class_.upcast(ptr); + class_ = class_.baseClass; + } + return ptr; + } + + function getInheritedInstance(class_, ptr) { + ptr = getBasestPointer(class_, ptr); + return registeredInstances[ptr]; + } + + function makeClassHandle(prototype, record) { + if (!record.ptrType || !record.ptr) { + throwInternalError("makeClassHandle requires ptr and ptrType"); + } + var hasSmartPtrType = !!record.smartPtrType; + var hasSmartPtr = !!record.smartPtr; + if (hasSmartPtrType !== hasSmartPtr) { + throwInternalError("Both smartPtrType and smartPtr must be specified"); + } + record.count = { + value: 1, + }; + return attachFinalizer( + Object.create(prototype, { + $$: { + value: record, + }, + }) + ); + } + + function RegisteredPointer_fromWireType(ptr) { + var rawPointer = this.getPointee(ptr); + if (!rawPointer) { + this.destructor(ptr); + return null; + } + var registeredInstance = getInheritedInstance( + this.registeredClass, + rawPointer + ); + if (undefined !== registeredInstance) { + if (0 === registeredInstance.$$.count.value) { + registeredInstance.$$.ptr = rawPointer; + registeredInstance.$$.smartPtr = ptr; + return registeredInstance["clone"](); + } else { + var rv = registeredInstance["clone"](); + this.destructor(ptr); + return rv; + } + } + function makeDefaultHandle() { + if (this.isSmartPointer) { + return makeClassHandle(this.registeredClass.instancePrototype, { + ptrType: this.pointeeType, + ptr: rawPointer, + smartPtrType: this, + smartPtr: ptr, + }); + } else { + return makeClassHandle(this.registeredClass.instancePrototype, { + ptrType: this, + ptr: ptr, + }); + } + } + var actualType = this.registeredClass.getActualType(rawPointer); + var registeredPointerRecord = registeredPointers[actualType]; + if (!registeredPointerRecord) { + return makeDefaultHandle.call(this); + } + var toType; + if (this.isConst) { + toType = registeredPointerRecord.constPointerType; + } else { + toType = registeredPointerRecord.pointerType; + } + var dp = downcastPointer( + rawPointer, + this.registeredClass, + toType.registeredClass + ); + if (dp === null) { + return makeDefaultHandle.call(this); + } + if (this.isSmartPointer) { + return makeClassHandle(toType.registeredClass.instancePrototype, { + ptrType: toType, + ptr: dp, + smartPtrType: this, + smartPtr: ptr, + }); + } else { + return makeClassHandle(toType.registeredClass.instancePrototype, { + ptrType: toType, + ptr: dp, + }); + } + } + + function attachFinalizer(handle) { + if ("undefined" === typeof FinalizationRegistry) { + attachFinalizer = (handle) => handle; + return handle; + } + finalizationRegistry = new FinalizationRegistry((info) => { + releaseClassHandle(info.$$); + }); + attachFinalizer = (handle) => { + var $$ = handle.$$; + var hasSmartPtr = !!$$.smartPtr; + if (hasSmartPtr) { + var info = { + $$: $$, + }; + finalizationRegistry.register(handle, info, handle); + } + return handle; + }; + detachFinalizer = (handle) => finalizationRegistry.unregister(handle); + return attachFinalizer(handle); + } + + function ClassHandle_clone() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.preservePointerOnDelete) { + this.$$.count.value += 1; + return this; + } else { + var clone = attachFinalizer( + Object.create(Object.getPrototypeOf(this), { + $$: { + value: shallowCopyInternalPointer(this.$$), + }, + }) + ); + clone.$$.count.value += 1; + clone.$$.deleteScheduled = false; + return clone; + } + } + + function ClassHandle_delete() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { + throwBindingError("Object already scheduled for deletion"); + } + detachFinalizer(this); + releaseClassHandle(this.$$); + if (!this.$$.preservePointerOnDelete) { + this.$$.smartPtr = undefined; + this.$$.ptr = undefined; + } + } + + function ClassHandle_isDeleted() { + return !this.$$.ptr; + } + + function ClassHandle_deleteLater() { + if (!this.$$.ptr) { + throwInstanceAlreadyDeleted(this); + } + if (this.$$.deleteScheduled && !this.$$.preservePointerOnDelete) { + throwBindingError("Object already scheduled for deletion"); + } + deletionQueue.push(this); + if (deletionQueue.length === 1 && delayFunction) { + delayFunction(flushPendingDeletes); + } + this.$$.deleteScheduled = true; + return this; + } + + function init_ClassHandle() { + ClassHandle.prototype["isAliasOf"] = ClassHandle_isAliasOf; + ClassHandle.prototype["clone"] = ClassHandle_clone; + ClassHandle.prototype["delete"] = ClassHandle_delete; + ClassHandle.prototype["isDeleted"] = ClassHandle_isDeleted; + ClassHandle.prototype["deleteLater"] = ClassHandle_deleteLater; + } + + function ClassHandle() {} + + function ensureOverloadTable(proto, methodName, humanName) { + if (undefined === proto[methodName].overloadTable) { + var prevFunc = proto[methodName]; + proto[methodName] = function () { + if (!proto[methodName].overloadTable.hasOwnProperty(arguments.length)) { + throwBindingError( + "Function '" + + humanName + + "' called with an invalid number of arguments (" + + arguments.length + + ") - expects one of (" + + proto[methodName].overloadTable + + ")!" + ); + } + return proto[methodName].overloadTable[arguments.length].apply( + this, + arguments + ); + }; + proto[methodName].overloadTable = []; + proto[methodName].overloadTable[prevFunc.argCount] = prevFunc; + } + } + + function exposePublicSymbol(name, value, numArguments) { + if (Module.hasOwnProperty(name)) { + if ( + undefined === numArguments || + (undefined !== Module[name].overloadTable && + undefined !== Module[name].overloadTable[numArguments]) + ) { + throwBindingError("Cannot register public name '" + name + "' twice"); + } + ensureOverloadTable(Module, name, name); + if (Module.hasOwnProperty(numArguments)) { + throwBindingError( + "Cannot register multiple overloads of a function with the same number of arguments (" + + numArguments + + ")!" + ); + } + Module[name].overloadTable[numArguments] = value; + } else { + Module[name] = value; + if (undefined !== numArguments) { + Module[name].numArguments = numArguments; + } + } + } + + function RegisteredClass( + name, + constructor, + instancePrototype, + rawDestructor, + baseClass, + getActualType, + upcast, + downcast + ) { + this.name = name; + this.constructor = constructor; + this.instancePrototype = instancePrototype; + this.rawDestructor = rawDestructor; + this.baseClass = baseClass; + this.getActualType = getActualType; + this.upcast = upcast; + this.downcast = downcast; + this.pureVirtualFunctions = []; + } + + function upcastPointer(ptr, ptrClass, desiredClass) { + while (ptrClass !== desiredClass) { + if (!ptrClass.upcast) { + throwBindingError( + "Expected null or instance of " + + desiredClass.name + + ", got an instance of " + + ptrClass.name + ); + } + ptr = ptrClass.upcast(ptr); + ptrClass = ptrClass.baseClass; + } + return ptr; + } + + function constNoSmartPtrRawPointerToWireType(destructors, handle) { + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + return 0; + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + return ptr; + } + + function genericPointerToWireType(destructors, handle) { + var ptr; + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + if (this.isSmartPointer) { + ptr = this.rawConstructor(); + if (destructors !== null) { + destructors.push(this.rawDestructor, ptr); + } + return ptr; + } else { + return 0; + } + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + if (!this.isConst && handle.$$.ptrType.isConst) { + throwBindingError( + "Cannot convert argument of type " + + (handle.$$.smartPtrType + ? handle.$$.smartPtrType.name + : handle.$$.ptrType.name) + + " to parameter type " + + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + if (this.isSmartPointer) { + if (undefined === handle.$$.smartPtr) { + throwBindingError("Passing raw pointer to smart pointer is illegal"); + } + switch (this.sharingPolicy) { + case 0: + if (handle.$$.smartPtrType === this) { + ptr = handle.$$.smartPtr; + } else { + throwBindingError( + "Cannot convert argument of type " + + (handle.$$.smartPtrType + ? handle.$$.smartPtrType.name + : handle.$$.ptrType.name) + + " to parameter type " + + this.name + ); + } + break; + + case 1: + ptr = handle.$$.smartPtr; + break; + + case 2: + if (handle.$$.smartPtrType === this) { + ptr = handle.$$.smartPtr; + } else { + var clonedHandle = handle["clone"](); + ptr = this.rawShare( + ptr, + Emval.toHandle(function () { + clonedHandle["delete"](); + }) + ); + if (destructors !== null) { + destructors.push(this.rawDestructor, ptr); + } + } + break; + + default: + throwBindingError("Unsupporting sharing policy"); + } + } + return ptr; + } + + function nonConstNoSmartPtrRawPointerToWireType(destructors, handle) { + if (handle === null) { + if (this.isReference) { + throwBindingError("null is not a valid " + this.name); + } + return 0; + } + if (!handle.$$) { + throwBindingError( + 'Cannot pass "' + _embind_repr(handle) + '" as a ' + this.name + ); + } + if (!handle.$$.ptr) { + throwBindingError( + "Cannot pass deleted object as a pointer of type " + this.name + ); + } + if (handle.$$.ptrType.isConst) { + throwBindingError( + "Cannot convert argument of type " + + handle.$$.ptrType.name + + " to parameter type " + + this.name + ); + } + var handleClass = handle.$$.ptrType.registeredClass; + var ptr = upcastPointer(handle.$$.ptr, handleClass, this.registeredClass); + return ptr; + } + + function RegisteredPointer_getPointee(ptr) { + if (this.rawGetPointee) { + ptr = this.rawGetPointee(ptr); + } + return ptr; + } + + function RegisteredPointer_destructor(ptr) { + if (this.rawDestructor) { + this.rawDestructor(ptr); + } + } + + function RegisteredPointer_deleteObject(handle) { + if (handle !== null) { + handle["delete"](); + } + } + + function init_RegisteredPointer() { + RegisteredPointer.prototype.getPointee = RegisteredPointer_getPointee; + RegisteredPointer.prototype.destructor = RegisteredPointer_destructor; + RegisteredPointer.prototype["argPackAdvance"] = 8; + RegisteredPointer.prototype["readValueFromPointer"] = + simpleReadValueFromPointer; + RegisteredPointer.prototype["deleteObject"] = + RegisteredPointer_deleteObject; + RegisteredPointer.prototype["fromWireType"] = + RegisteredPointer_fromWireType; + } + + function RegisteredPointer( + name, + registeredClass, + isReference, + isConst, + isSmartPointer, + pointeeType, + sharingPolicy, + rawGetPointee, + rawConstructor, + rawShare, + rawDestructor + ) { + this.name = name; + this.registeredClass = registeredClass; + this.isReference = isReference; + this.isConst = isConst; + this.isSmartPointer = isSmartPointer; + this.pointeeType = pointeeType; + this.sharingPolicy = sharingPolicy; + this.rawGetPointee = rawGetPointee; + this.rawConstructor = rawConstructor; + this.rawShare = rawShare; + this.rawDestructor = rawDestructor; + if (!isSmartPointer && registeredClass.baseClass === undefined) { + if (isConst) { + this["toWireType"] = constNoSmartPtrRawPointerToWireType; + this.destructorFunction = null; + } else { + this["toWireType"] = nonConstNoSmartPtrRawPointerToWireType; + this.destructorFunction = null; + } + } else { + this["toWireType"] = genericPointerToWireType; + } + } + + function replacePublicSymbol(name, value, numArguments) { + if (!Module.hasOwnProperty(name)) { + throwInternalError("Replacing nonexistant public symbol"); + } + if ( + undefined !== Module[name].overloadTable && + undefined !== numArguments + ) { + Module[name].overloadTable[numArguments] = value; + } else { + Module[name] = value; + Module[name].argCount = numArguments; + } + } + + function dynCallLegacy(sig, ptr, args) { + var f = Module["dynCall_" + sig]; + return args && args.length + ? f.apply(null, [ptr].concat(args)) + : f.call(null, ptr); + } + + function dynCall(sig, ptr, args) { + if (sig.includes("j")) { + return dynCallLegacy(sig, ptr, args); + } + return getWasmTableEntry(ptr).apply(null, args); + } + + function getDynCaller(sig, ptr) { + var argCache = []; + return function () { + argCache.length = 0; + Object.assign(argCache, arguments); + return dynCall(sig, ptr, argCache); + }; + } + + function embind__requireFunction(signature, rawFunction) { + signature = readLatin1String(signature); + function makeDynCaller() { + if (signature.includes("j")) { + return getDynCaller(signature, rawFunction); + } + return getWasmTableEntry(rawFunction); + } + var fp = makeDynCaller(); + if (typeof fp != "function") { + throwBindingError( + "unknown function pointer with signature " + + signature + + ": " + + rawFunction + ); + } + return fp; + } + + var UnboundTypeError = undefined; + + function getTypeName(type) { + var ptr = ___getTypeName(type); + var rv = readLatin1String(ptr); + _free(ptr); + return rv; + } + + function throwUnboundTypeError(message, types) { + var unboundTypes = []; + var seen = {}; + function visit(type) { + if (seen[type]) { + return; + } + if (registeredTypes[type]) { + return; + } + if (typeDependencies[type]) { + typeDependencies[type].forEach(visit); + return; + } + unboundTypes.push(type); + seen[type] = true; + } + types.forEach(visit); + throw new UnboundTypeError( + message + ": " + unboundTypes.map(getTypeName).join([", "]) + ); + } + + function __embind_register_class( + rawType, + rawPointerType, + rawConstPointerType, + baseClassRawType, + getActualTypeSignature, + getActualType, + upcastSignature, + upcast, + downcastSignature, + downcast, + name, + destructorSignature, + rawDestructor + ) { + name = readLatin1String(name); + getActualType = embind__requireFunction( + getActualTypeSignature, + getActualType + ); + if (upcast) { + upcast = embind__requireFunction(upcastSignature, upcast); + } + if (downcast) { + downcast = embind__requireFunction(downcastSignature, downcast); + } + rawDestructor = embind__requireFunction(destructorSignature, rawDestructor); + var legalFunctionName = makeLegalFunctionName(name); + exposePublicSymbol(legalFunctionName, function () { + throwUnboundTypeError( + "Cannot construct " + name + " due to unbound types", + [baseClassRawType] + ); + }); + whenDependentTypesAreResolved( + [rawType, rawPointerType, rawConstPointerType], + baseClassRawType ? [baseClassRawType] : [], + function (base) { + base = base[0]; + var baseClass; + var basePrototype; + if (baseClassRawType) { + baseClass = base.registeredClass; + basePrototype = baseClass.instancePrototype; + } else { + basePrototype = ClassHandle.prototype; + } + var constructor = createNamedFunction(legalFunctionName, function () { + if (Object.getPrototypeOf(this) !== instancePrototype) { + throw new BindingError("Use 'new' to construct " + name); + } + if (undefined === registeredClass.constructor_body) { + throw new BindingError(name + " has no accessible constructor"); + } + var body = registeredClass.constructor_body[arguments.length]; + if (undefined === body) { + throw new BindingError( + "Tried to invoke ctor of " + + name + + " with invalid number of parameters (" + + arguments.length + + ") - expected (" + + Object.keys(registeredClass.constructor_body).toString() + + ") parameters instead!" + ); + } + return body.apply(this, arguments); + }); + var instancePrototype = Object.create(basePrototype, { + constructor: { + value: constructor, + }, + }); + constructor.prototype = instancePrototype; + var registeredClass = new RegisteredClass( + name, + constructor, + instancePrototype, + rawDestructor, + baseClass, + getActualType, + upcast, + downcast + ); + var referenceConverter = new RegisteredPointer( + name, + registeredClass, + true, + false, + false + ); + var pointerConverter = new RegisteredPointer( + name + "*", + registeredClass, + false, + false, + false + ); + var constPointerConverter = new RegisteredPointer( + name + " const*", + registeredClass, + false, + true, + false + ); + registeredPointers[rawType] = { + pointerType: pointerConverter, + constPointerType: constPointerConverter, + }; + replacePublicSymbol(legalFunctionName, constructor); + return [referenceConverter, pointerConverter, constPointerConverter]; + } + ); + } + + function heap32VectorToArray(count, firstElement) { + var array = []; + for (var i = 0; i < count; i++) { + array.push(HEAP32[(firstElement >> 2) + i]); + } + return array; + } + + function __embind_register_class_constructor( + rawClassType, + argCount, + rawArgTypesAddr, + invokerSignature, + invoker, + rawConstructor + ) { + assert(argCount > 0); + var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); + invoker = embind__requireFunction(invokerSignature, invoker); + whenDependentTypesAreResolved([], [rawClassType], function (classType) { + classType = classType[0]; + var humanName = "constructor " + classType.name; + if (undefined === classType.registeredClass.constructor_body) { + classType.registeredClass.constructor_body = []; + } + if ( + undefined !== classType.registeredClass.constructor_body[argCount - 1] + ) { + throw new BindingError( + "Cannot register multiple constructors with identical number of parameters (" + + (argCount - 1) + + ") for class '" + + classType.name + + "'! Overload resolution is currently only performed using the parameter count, not actual type info!" + ); + } + classType.registeredClass.constructor_body[argCount - 1] = () => { + throwUnboundTypeError( + "Cannot construct " + classType.name + " due to unbound types", + rawArgTypes + ); + }; + whenDependentTypesAreResolved([], rawArgTypes, function (argTypes) { + argTypes.splice(1, 0, null); + classType.registeredClass.constructor_body[argCount - 1] = + craftInvokerFunction( + humanName, + argTypes, + null, + invoker, + rawConstructor + ); + return []; + }); + return []; + }); + } + + function craftInvokerFunction( + humanName, + argTypes, + classType, + cppInvokerFunc, + cppTargetFunc + ) { + var argCount = argTypes.length; + if (argCount < 2) { + throwBindingError( + "argTypes array size mismatch! Must at least get return value and 'this' types!" + ); + } + var isClassMethodFunc = argTypes[1] !== null && classType !== null; + var needsDestructorStack = false; + for (var i = 1; i < argTypes.length; ++i) { + if ( + argTypes[i] !== null && + argTypes[i].destructorFunction === undefined + ) { + needsDestructorStack = true; + break; + } + } + var returns = argTypes[0].name !== "void"; + var expectedArgCount = argCount - 2; + var argsWired = new Array(expectedArgCount); + var invokerFuncArgs = []; + var destructors = []; + return function () { + if (arguments.length !== expectedArgCount) { + throwBindingError( + "function " + + humanName + + " called with " + + arguments.length + + " arguments, expected " + + expectedArgCount + + " args!" + ); + } + destructors.length = 0; + var thisWired; + invokerFuncArgs.length = isClassMethodFunc ? 2 : 1; + invokerFuncArgs[0] = cppTargetFunc; + if (isClassMethodFunc) { + thisWired = argTypes[1]["toWireType"](destructors, this); + invokerFuncArgs[1] = thisWired; + } + for (var i = 0; i < expectedArgCount; ++i) { + argsWired[i] = argTypes[i + 2]["toWireType"](destructors, arguments[i]); + invokerFuncArgs.push(argsWired[i]); + } + var rv = cppInvokerFunc.apply(null, invokerFuncArgs); + function onDone(rv) { + if (needsDestructorStack) { + runDestructors(destructors); + } else { + for (var i = isClassMethodFunc ? 1 : 2; i < argTypes.length; i++) { + var param = i === 1 ? thisWired : argsWired[i - 2]; + if (argTypes[i].destructorFunction !== null) { + argTypes[i].destructorFunction(param); + } + } + } + if (returns) { + return argTypes[0]["fromWireType"](rv); + } + } + return onDone(rv); + }; + } + + function __embind_register_class_function( + rawClassType, + methodName, + argCount, + rawArgTypesAddr, + invokerSignature, + rawInvoker, + context, + isPureVirtual + ) { + var rawArgTypes = heap32VectorToArray(argCount, rawArgTypesAddr); + methodName = readLatin1String(methodName); + rawInvoker = embind__requireFunction(invokerSignature, rawInvoker); + whenDependentTypesAreResolved([], [rawClassType], function (classType) { + classType = classType[0]; + var humanName = classType.name + "." + methodName; + if (methodName.startsWith("@@")) { + methodName = Symbol[methodName.substring(2)]; + } + if (isPureVirtual) { + classType.registeredClass.pureVirtualFunctions.push(methodName); + } + function unboundTypesHandler() { + throwUnboundTypeError( + "Cannot call " + humanName + " due to unbound types", + rawArgTypes + ); + } + var proto = classType.registeredClass.instancePrototype; + var method = proto[methodName]; + if ( + undefined === method || + (undefined === method.overloadTable && + method.className !== classType.name && + method.argCount === argCount - 2) + ) { + unboundTypesHandler.argCount = argCount - 2; + unboundTypesHandler.className = classType.name; + proto[methodName] = unboundTypesHandler; + } else { + ensureOverloadTable(proto, methodName, humanName); + proto[methodName].overloadTable[argCount - 2] = unboundTypesHandler; + } + whenDependentTypesAreResolved([], rawArgTypes, function (argTypes) { + var memberFunction = craftInvokerFunction( + humanName, + argTypes, + classType, + rawInvoker, + context + ); + if (undefined === proto[methodName].overloadTable) { + memberFunction.argCount = argCount - 2; + proto[methodName] = memberFunction; + } else { + proto[methodName].overloadTable[argCount - 2] = memberFunction; + } + return []; + }); + return []; + }); + } + + var emval_free_list = []; + + var emval_handle_array = [ + {}, + { + value: undefined, + }, + { + value: null, + }, + { + value: true, + }, + { + value: false, + }, + ]; + + function __emval_decref(handle) { + if (handle > 4 && 0 === --emval_handle_array[handle].refcount) { + emval_handle_array[handle] = undefined; + emval_free_list.push(handle); + } + } + + function count_emval_handles() { + var count = 0; + for (var i = 5; i < emval_handle_array.length; ++i) { + if (emval_handle_array[i] !== undefined) { + ++count; + } + } + return count; + } + + function get_first_emval() { + for (var i = 5; i < emval_handle_array.length; ++i) { + if (emval_handle_array[i] !== undefined) { + return emval_handle_array[i]; + } + } + return null; + } + + function init_emval() { + Module["count_emval_handles"] = count_emval_handles; + Module["get_first_emval"] = get_first_emval; + } + + var Emval = { + toValue: (handle) => { + if (!handle) { + throwBindingError("Cannot use deleted val. handle = " + handle); + } + return emval_handle_array[handle].value; + }, + toHandle: (value) => { + switch (value) { + case undefined: + return 1; + + case null: + return 2; + + case true: + return 3; + + case false: + return 4; + + default: { + var handle = emval_free_list.length + ? emval_free_list.pop() + : emval_handle_array.length; + emval_handle_array[handle] = { + refcount: 1, + value: value, + }; + return handle; + } + } + }, + }; + + function __embind_register_emval(rawType, name) { + name = readLatin1String(name); + registerType(rawType, { + name: name, + fromWireType: function (handle) { + var rv = Emval.toValue(handle); + __emval_decref(handle); + return rv; + }, + toWireType: function (destructors, value) { + return Emval.toHandle(value); + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: null, + }); + } + + function _embind_repr(v) { + if (v === null) { + return "null"; + } + var t = typeof v; + if (t === "object" || t === "array" || t === "function") { + return v.toString(); + } else { + return "" + v; + } + } + + function floatReadValueFromPointer(name, shift) { + switch (shift) { + case 2: + return function (pointer) { + return this["fromWireType"](HEAPF32[pointer >> 2]); + }; + + case 3: + return function (pointer) { + return this["fromWireType"](HEAPF64[pointer >> 3]); + }; + + default: + throw new TypeError("Unknown float type: " + name); + } + } + + function __embind_register_float(rawType, name, size) { + var shift = getShiftFromSize(size); + name = readLatin1String(name); + registerType(rawType, { + name: name, + fromWireType: function (value) { + return value; + }, + toWireType: function (destructors, value) { + return value; + }, + argPackAdvance: 8, + readValueFromPointer: floatReadValueFromPointer(name, shift), + destructorFunction: null, + }); + } + + function integerReadValueFromPointer(name, shift, signed) { + switch (shift) { + case 0: + return signed + ? function readS8FromPointer(pointer) { + return HEAP8[pointer]; + } + : function readU8FromPointer(pointer) { + return HEAPU8[pointer]; + }; + + case 1: + return signed + ? function readS16FromPointer(pointer) { + return HEAP16[pointer >> 1]; + } + : function readU16FromPointer(pointer) { + return HEAPU16[pointer >> 1]; + }; + + case 2: + return signed + ? function readS32FromPointer(pointer) { + return HEAP32[pointer >> 2]; + } + : function readU32FromPointer(pointer) { + return HEAPU32[pointer >> 2]; + }; + + default: + throw new TypeError("Unknown integer type: " + name); + } + } + + function __embind_register_integer( + primitiveType, + name, + size, + minRange, + maxRange + ) { + name = readLatin1String(name); + if (maxRange === -1) { + maxRange = 4294967295; + } + var shift = getShiftFromSize(size); + var fromWireType = (value) => value; + if (minRange === 0) { + var bitshift = 32 - 8 * size; + fromWireType = (value) => (value << bitshift) >>> bitshift; + } + var isUnsignedType = name.includes("unsigned"); + var checkAssertions = (value, toTypeName) => {}; + var toWireType; + if (isUnsignedType) { + toWireType = function (destructors, value) { + checkAssertions(value, this.name); + return value >>> 0; + }; + } else { + toWireType = function (destructors, value) { + checkAssertions(value, this.name); + return value; + }; + } + registerType(primitiveType, { + name: name, + fromWireType: fromWireType, + toWireType: toWireType, + argPackAdvance: 8, + readValueFromPointer: integerReadValueFromPointer( + name, + shift, + minRange !== 0 + ), + destructorFunction: null, + }); + } + + function __embind_register_memory_view(rawType, dataTypeIndex, name) { + var typeMapping = [ + Int8Array, + Uint8Array, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + Float32Array, + Float64Array, + ]; + var TA = typeMapping[dataTypeIndex]; + function decodeMemoryView(handle) { + handle = handle >> 2; + var heap = HEAPU32; + var size = heap[handle]; + var data = heap[handle + 1]; + return new TA(buffer, data, size); + } + name = readLatin1String(name); + registerType( + rawType, + { + name: name, + fromWireType: decodeMemoryView, + argPackAdvance: 8, + readValueFromPointer: decodeMemoryView, + }, + { + ignoreDuplicateRegistrations: true, + } + ); + } + + function __embind_register_smart_ptr( + rawType, + rawPointeeType, + name, + sharingPolicy, + getPointeeSignature, + rawGetPointee, + constructorSignature, + rawConstructor, + shareSignature, + rawShare, + destructorSignature, + rawDestructor + ) { + name = readLatin1String(name); + rawGetPointee = embind__requireFunction(getPointeeSignature, rawGetPointee); + rawConstructor = embind__requireFunction( + constructorSignature, + rawConstructor + ); + rawShare = embind__requireFunction(shareSignature, rawShare); + rawDestructor = embind__requireFunction(destructorSignature, rawDestructor); + whenDependentTypesAreResolved( + [rawType], + [rawPointeeType], + function (pointeeType) { + pointeeType = pointeeType[0]; + var registeredPointer = new RegisteredPointer( + name, + pointeeType.registeredClass, + false, + false, + true, + pointeeType, + sharingPolicy, + rawGetPointee, + rawConstructor, + rawShare, + rawDestructor + ); + return [registeredPointer]; + } + ); + } + + function __embind_register_std_string(rawType, name) { + name = readLatin1String(name); + var stdStringIsUTF8 = name === "std::string"; + registerType(rawType, { + name: name, + fromWireType: function (value) { + var length = HEAPU32[value >> 2]; + var str; + if (stdStringIsUTF8) { + var decodeStartPtr = value + 4; + for (var i = 0; i <= length; ++i) { + var currentBytePtr = value + 4 + i; + if (i == length || HEAPU8[currentBytePtr] == 0) { + var maxRead = currentBytePtr - decodeStartPtr; + var stringSegment = UTF8ToString(decodeStartPtr, maxRead); + if (str === undefined) { + str = stringSegment; + } else { + str += String.fromCharCode(0); + str += stringSegment; + } + decodeStartPtr = currentBytePtr + 1; + } + } + } else { + var a = new Array(length); + for (var i = 0; i < length; ++i) { + a[i] = String.fromCharCode(HEAPU8[value + 4 + i]); + } + str = a.join(""); + } + _free(value); + return str; + }, + toWireType: function (destructors, value) { + if (value instanceof ArrayBuffer) { + value = new Uint8Array(value); + } + var getLength; + var valueIsOfTypeString = typeof value == "string"; + if ( + !( + valueIsOfTypeString || + value instanceof Uint8Array || + value instanceof Uint8ClampedArray || + value instanceof Int8Array + ) + ) { + throwBindingError("Cannot pass non-string to std::string"); + } + if (stdStringIsUTF8 && valueIsOfTypeString) { + getLength = () => lengthBytesUTF8(value); + } else { + getLength = () => value.length; + } + var length = getLength(); + var ptr = _malloc(4 + length + 1); + HEAPU32[ptr >> 2] = length; + if (stdStringIsUTF8 && valueIsOfTypeString) { + stringToUTF8(value, ptr + 4, length + 1); + } else { + if (valueIsOfTypeString) { + for (var i = 0; i < length; ++i) { + var charCode = value.charCodeAt(i); + if (charCode > 255) { + _free(ptr); + throwBindingError( + "String has UTF-16 code units that do not fit in 8 bits" + ); + } + HEAPU8[ptr + 4 + i] = charCode; + } + } else { + for (var i = 0; i < length; ++i) { + HEAPU8[ptr + 4 + i] = value[i]; + } + } + } + if (destructors !== null) { + destructors.push(_free, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: function (ptr) { + _free(ptr); + }, + }); + } + + function __embind_register_std_wstring(rawType, charSize, name) { + name = readLatin1String(name); + var decodeString, encodeString, getHeap, lengthBytesUTF, shift; + if (charSize === 2) { + decodeString = UTF16ToString; + encodeString = stringToUTF16; + lengthBytesUTF = lengthBytesUTF16; + getHeap = () => HEAPU16; + shift = 1; + } else if (charSize === 4) { + decodeString = UTF32ToString; + encodeString = stringToUTF32; + lengthBytesUTF = lengthBytesUTF32; + getHeap = () => HEAPU32; + shift = 2; + } + registerType(rawType, { + name: name, + fromWireType: function (value) { + var length = HEAPU32[value >> 2]; + var HEAP = getHeap(); + var str; + var decodeStartPtr = value + 4; + for (var i = 0; i <= length; ++i) { + var currentBytePtr = value + 4 + i * charSize; + if (i == length || HEAP[currentBytePtr >> shift] == 0) { + var maxReadBytes = currentBytePtr - decodeStartPtr; + var stringSegment = decodeString(decodeStartPtr, maxReadBytes); + if (str === undefined) { + str = stringSegment; + } else { + str += String.fromCharCode(0); + str += stringSegment; + } + decodeStartPtr = currentBytePtr + charSize; + } + } + _free(value); + return str; + }, + toWireType: function (destructors, value) { + if (!(typeof value == "string")) { + throwBindingError( + "Cannot pass non-string to C++ string type " + name + ); + } + var length = lengthBytesUTF(value); + var ptr = _malloc(4 + length + charSize); + HEAPU32[ptr >> 2] = length >> shift; + encodeString(value, ptr + 4, length + charSize); + if (destructors !== null) { + destructors.push(_free, ptr); + } + return ptr; + }, + argPackAdvance: 8, + readValueFromPointer: simpleReadValueFromPointer, + destructorFunction: function (ptr) { + _free(ptr); + }, + }); + } + + function __embind_register_value_object( + rawType, + name, + constructorSignature, + rawConstructor, + destructorSignature, + rawDestructor + ) { + structRegistrations[rawType] = { + name: readLatin1String(name), + rawConstructor: embind__requireFunction( + constructorSignature, + rawConstructor + ), + rawDestructor: embind__requireFunction( + destructorSignature, + rawDestructor + ), + fields: [], + }; + } + + function __embind_register_value_object_field( + structType, + fieldName, + getterReturnType, + getterSignature, + getter, + getterContext, + setterArgumentType, + setterSignature, + setter, + setterContext + ) { + structRegistrations[structType].fields.push({ + fieldName: readLatin1String(fieldName), + getterReturnType: getterReturnType, + getter: embind__requireFunction(getterSignature, getter), + getterContext: getterContext, + setterArgumentType: setterArgumentType, + setter: embind__requireFunction(setterSignature, setter), + setterContext: setterContext, + }); + } + + function __embind_register_void(rawType, name) { + name = readLatin1String(name); + registerType(rawType, { + isVoid: true, + name: name, + argPackAdvance: 0, + fromWireType: function () { + return undefined; + }, + toWireType: function (destructors, o) { + return undefined; + }, + }); + } + + function __emscripten_date_now() { + return Date.now(); + } + + var nowIsMonotonic = true; + + function __emscripten_get_now_is_monotonic() { + return nowIsMonotonic; + } + + function requireRegisteredType(rawType, humanName) { + var impl = registeredTypes[rawType]; + if (undefined === impl) { + throwBindingError( + humanName + " has unknown type " + getTypeName(rawType) + ); + } + return impl; + } + + function __emval_lookupTypes(argCount, argTypes) { + var a = new Array(argCount); + for (var i = 0; i < argCount; ++i) { + a[i] = requireRegisteredType( + HEAP32[(argTypes >> 2) + i], + "parameter " + i + ); + } + return a; + } + + function __emval_call(handle, argCount, argTypes, argv) { + handle = Emval.toValue(handle); + var types = __emval_lookupTypes(argCount, argTypes); + var args = new Array(argCount); + for (var i = 0; i < argCount; ++i) { + var type = types[i]; + args[i] = type["readValueFromPointer"](argv); + argv += type["argPackAdvance"]; + } + var rv = handle.apply(undefined, args); + return Emval.toHandle(rv); + } + + function __emval_incref(handle) { + if (handle > 4) { + emval_handle_array[handle].refcount += 1; + } + } + + function __emval_take_value(type, argv) { + type = requireRegisteredType(type, "_emval_take_value"); + var v = type["readValueFromPointer"](argv); + return Emval.toHandle(v); + } + + function __localtime_js(time, tmPtr) { + var date = new Date(HEAP32[time >> 2] * 1e3); + HEAP32[tmPtr >> 2] = date.getSeconds(); + HEAP32[(tmPtr + 4) >> 2] = date.getMinutes(); + HEAP32[(tmPtr + 8) >> 2] = date.getHours(); + HEAP32[(tmPtr + 12) >> 2] = date.getDate(); + HEAP32[(tmPtr + 16) >> 2] = date.getMonth(); + HEAP32[(tmPtr + 20) >> 2] = date.getFullYear() - 1900; + HEAP32[(tmPtr + 24) >> 2] = date.getDay(); + var start = new Date(date.getFullYear(), 0, 1); + var yday = ((date.getTime() - start.getTime()) / (1e3 * 60 * 60 * 24)) | 0; + HEAP32[(tmPtr + 28) >> 2] = yday; + HEAP32[(tmPtr + 36) >> 2] = -(date.getTimezoneOffset() * 60); + var summerOffset = new Date(date.getFullYear(), 6, 1).getTimezoneOffset(); + var winterOffset = start.getTimezoneOffset(); + var dst = + (summerOffset != winterOffset && + date.getTimezoneOffset() == Math.min(winterOffset, summerOffset)) | 0; + HEAP32[(tmPtr + 32) >> 2] = dst; + } + + function __mmap_js(addr, len, prot, flags, fd, off, allocated, builtin) { + return -52; + } + + function __munmap_js(addr, len, prot, flags, fd, offset) {} + + function _tzset_impl(timezone, daylight, tzname) { + var currentYear = new Date().getFullYear(); + var winter = new Date(currentYear, 0, 1); + var summer = new Date(currentYear, 6, 1); + var winterOffset = winter.getTimezoneOffset(); + var summerOffset = summer.getTimezoneOffset(); + var stdTimezoneOffset = Math.max(winterOffset, summerOffset); + HEAP32[timezone >> 2] = stdTimezoneOffset * 60; + HEAP32[daylight >> 2] = Number(winterOffset != summerOffset); + function extractZone(date) { + var match = date.toTimeString().match(/\(([A-Za-z ]+)\)$/); + return match ? match[1] : "GMT"; + } + var winterName = extractZone(winter); + var summerName = extractZone(summer); + var winterNamePtr = allocateUTF8(winterName); + var summerNamePtr = allocateUTF8(summerName); + if (summerOffset < winterOffset) { + HEAP32[tzname >> 2] = winterNamePtr; + HEAP32[(tzname + 4) >> 2] = summerNamePtr; + } else { + HEAP32[tzname >> 2] = summerNamePtr; + HEAP32[(tzname + 4) >> 2] = winterNamePtr; + } + } + + function __tzset_js(timezone, daylight, tzname) { + if (__tzset_js.called) return; + __tzset_js.called = true; + _tzset_impl(timezone, daylight, tzname); + } + + function _abort() { + abort(""); + } + + function _emscripten_get_heap_max() { + return 2147483648; + } + + var _emscripten_get_now; + + _emscripten_get_now = () => performance.now(); + + function _emscripten_memcpy_big(dest, src, num) { + HEAPU8.copyWithin(dest, src, src + num); + } + + function emscripten_realloc_buffer(size) { + try { + wasmMemory.grow((size - buffer.byteLength + 65535) >>> 16); + updateGlobalBufferAndViews(wasmMemory.buffer); + return 1; + } catch (e) {} + } + + function _emscripten_resize_heap(requestedSize) { + var oldSize = HEAPU8.length; + requestedSize = requestedSize >>> 0; + var maxHeapSize = _emscripten_get_heap_max(); + if (requestedSize > maxHeapSize) { + return false; + } + let alignUp = (x, multiple) => x + ((multiple - (x % multiple)) % multiple); + for (var cutDown = 1; cutDown <= 4; cutDown *= 2) { + var overGrownHeapSize = oldSize * (1 + 0.2 / cutDown); + overGrownHeapSize = Math.min( + overGrownHeapSize, + requestedSize + 100663296 + ); + var newSize = Math.min( + maxHeapSize, + alignUp(Math.max(requestedSize, overGrownHeapSize), 65536) + ); + var replacement = emscripten_realloc_buffer(newSize); + if (replacement) { + return true; + } + } + return false; + } + + var ENV = {}; + + function getExecutableName() { + return thisProgram || "./this.program"; + } + + function getEnvStrings() { + if (!getEnvStrings.strings) { + var lang = + ( + (typeof navigator == "object" && + navigator.languages && + navigator.languages[0]) || + "C" + ).replace("-", "_") + ".UTF-8"; + var env = { + USER: "web_user", + LOGNAME: "web_user", + PATH: "/", + PWD: "/", + HOME: "/home/web_user", + LANG: lang, + _: getExecutableName(), + }; + for (var x in ENV) { + if (ENV[x] === undefined) delete env[x]; + else env[x] = ENV[x]; + } + var strings = []; + for (var x in env) { + strings.push(x + "=" + env[x]); + } + getEnvStrings.strings = strings; + } + return getEnvStrings.strings; + } + + function _environ_get(__environ, environ_buf) { + var bufSize = 0; + getEnvStrings().forEach(function (string, i) { + var ptr = environ_buf + bufSize; + HEAP32[(__environ + i * 4) >> 2] = ptr; + writeAsciiToMemory(string, ptr); + bufSize += string.length + 1; + }); + return 0; + } + + function _environ_sizes_get(penviron_count, penviron_buf_size) { + var strings = getEnvStrings(); + HEAP32[penviron_count >> 2] = strings.length; + var bufSize = 0; + strings.forEach(function (string) { + bufSize += string.length + 1; + }); + HEAP32[penviron_buf_size >> 2] = bufSize; + return 0; + } + + function _exit(status) { + exit(status); + } + + function _fd_close(fd) { + return 0; + } + + function _fd_read(fd, iov, iovcnt, pnum) { + var stream = SYSCALLS.getStreamFromFD(fd); + var num = SYSCALLS.doReadv(stream, iov, iovcnt); + HEAP32[pnum >> 2] = num; + return 0; + } + + function _fd_seek(fd, offset_low, offset_high, whence, newOffset) {} + + function _fd_write(fd, iov, iovcnt, pnum) { + var num = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[iov >> 2]; + var len = HEAP32[(iov + 4) >> 2]; + iov += 8; + for (var j = 0; j < len; j++) { + SYSCALLS.printChar(fd, HEAPU8[ptr + j]); + } + num += len; + } + HEAP32[pnum >> 2] = num; + return 0; + } + + function getRandomDevice() { + if ( + typeof crypto == "object" && + typeof crypto["getRandomValues"] == "function" + ) { + var randomBuffer = new Uint8Array(1); + return function () { + crypto.getRandomValues(randomBuffer); + return randomBuffer[0]; + }; + } else + return function () { + abort("randomDevice"); + }; + } + + function _getentropy(buffer, size) { + if (!_getentropy.randomDevice) { + _getentropy.randomDevice = getRandomDevice(); + } + for (var i = 0; i < size; i++) { + HEAP8[(buffer + i) >> 0] = _getentropy.randomDevice(); + } + return 0; + } + + function _pclose() { + err("missing function: pclose"); + abort(-1); + } + + function _setTempRet0(val) { + setTempRet0(val); + } + + function __isLeapYear(year) { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + } + + function __arraySum(array, index) { + var sum = 0; + for (var i = 0; i <= index; sum += array[i++]) {} + return sum; + } + + var __MONTH_DAYS_LEAP = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + var __MONTH_DAYS_REGULAR = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + function __addDays(date, days) { + var newDate = new Date(date.getTime()); + while (days > 0) { + var leap = __isLeapYear(newDate.getFullYear()); + var currentMonth = newDate.getMonth(); + var daysInCurrentMonth = ( + leap ? __MONTH_DAYS_LEAP : __MONTH_DAYS_REGULAR + )[currentMonth]; + if (days > daysInCurrentMonth - newDate.getDate()) { + days -= daysInCurrentMonth - newDate.getDate() + 1; + newDate.setDate(1); + if (currentMonth < 11) { + newDate.setMonth(currentMonth + 1); + } else { + newDate.setMonth(0); + newDate.setFullYear(newDate.getFullYear() + 1); + } + } else { + newDate.setDate(newDate.getDate() + days); + return newDate; + } + } + return newDate; + } + + function _strftime(s, maxsize, format, tm) { + var tm_zone = HEAP32[(tm + 40) >> 2]; + var date = { + tm_sec: HEAP32[tm >> 2], + tm_min: HEAP32[(tm + 4) >> 2], + tm_hour: HEAP32[(tm + 8) >> 2], + tm_mday: HEAP32[(tm + 12) >> 2], + tm_mon: HEAP32[(tm + 16) >> 2], + tm_year: HEAP32[(tm + 20) >> 2], + tm_wday: HEAP32[(tm + 24) >> 2], + tm_yday: HEAP32[(tm + 28) >> 2], + tm_isdst: HEAP32[(tm + 32) >> 2], + tm_gmtoff: HEAP32[(tm + 36) >> 2], + tm_zone: tm_zone ? UTF8ToString(tm_zone) : "", + }; + var pattern = UTF8ToString(format); + var EXPANSION_RULES_1 = { + "%c": "%a %b %d %H:%M:%S %Y", + "%D": "%m/%d/%y", + "%F": "%Y-%m-%d", + "%h": "%b", + "%r": "%I:%M:%S %p", + "%R": "%H:%M", + "%T": "%H:%M:%S", + "%x": "%m/%d/%y", + "%X": "%H:%M:%S", + "%Ec": "%c", + "%EC": "%C", + "%Ex": "%m/%d/%y", + "%EX": "%H:%M:%S", + "%Ey": "%y", + "%EY": "%Y", + "%Od": "%d", + "%Oe": "%e", + "%OH": "%H", + "%OI": "%I", + "%Om": "%m", + "%OM": "%M", + "%OS": "%S", + "%Ou": "%u", + "%OU": "%U", + "%OV": "%V", + "%Ow": "%w", + "%OW": "%W", + "%Oy": "%y", + }; + for (var rule in EXPANSION_RULES_1) { + pattern = pattern.replace(new RegExp(rule, "g"), EXPANSION_RULES_1[rule]); + } + var WEEKDAYS = [ + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + ]; + var MONTHS = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ]; + function leadingSomething(value, digits, character) { + var str = typeof value == "number" ? value.toString() : value || ""; + while (str.length < digits) { + str = character[0] + str; + } + return str; + } + function leadingNulls(value, digits) { + return leadingSomething(value, digits, "0"); + } + function compareByDay(date1, date2) { + function sgn(value) { + return value < 0 ? -1 : value > 0 ? 1 : 0; + } + var compare; + if ((compare = sgn(date1.getFullYear() - date2.getFullYear())) === 0) { + if ((compare = sgn(date1.getMonth() - date2.getMonth())) === 0) { + compare = sgn(date1.getDate() - date2.getDate()); + } + } + return compare; + } + function getFirstWeekStartDate(janFourth) { + switch (janFourth.getDay()) { + case 0: + return new Date(janFourth.getFullYear() - 1, 11, 29); + + case 1: + return janFourth; + + case 2: + return new Date(janFourth.getFullYear(), 0, 3); + + case 3: + return new Date(janFourth.getFullYear(), 0, 2); + + case 4: + return new Date(janFourth.getFullYear(), 0, 1); + + case 5: + return new Date(janFourth.getFullYear() - 1, 11, 31); + + case 6: + return new Date(janFourth.getFullYear() - 1, 11, 30); + } + } + function getWeekBasedYear(date) { + var thisDate = __addDays( + new Date(date.tm_year + 1900, 0, 1), + date.tm_yday + ); + var janFourthThisYear = new Date(thisDate.getFullYear(), 0, 4); + var janFourthNextYear = new Date(thisDate.getFullYear() + 1, 0, 4); + var firstWeekStartThisYear = getFirstWeekStartDate(janFourthThisYear); + var firstWeekStartNextYear = getFirstWeekStartDate(janFourthNextYear); + if (compareByDay(firstWeekStartThisYear, thisDate) <= 0) { + if (compareByDay(firstWeekStartNextYear, thisDate) <= 0) { + return thisDate.getFullYear() + 1; + } else { + return thisDate.getFullYear(); + } + } else { + return thisDate.getFullYear() - 1; + } + } + var EXPANSION_RULES_2 = { + "%a": function (date) { + return WEEKDAYS[date.tm_wday].substring(0, 3); + }, + "%A": function (date) { + return WEEKDAYS[date.tm_wday]; + }, + "%b": function (date) { + return MONTHS[date.tm_mon].substring(0, 3); + }, + "%B": function (date) { + return MONTHS[date.tm_mon]; + }, + "%C": function (date) { + var year = date.tm_year + 1900; + return leadingNulls((year / 100) | 0, 2); + }, + "%d": function (date) { + return leadingNulls(date.tm_mday, 2); + }, + "%e": function (date) { + return leadingSomething(date.tm_mday, 2, " "); + }, + "%g": function (date) { + return getWeekBasedYear(date).toString().substring(2); + }, + "%G": function (date) { + return getWeekBasedYear(date); + }, + "%H": function (date) { + return leadingNulls(date.tm_hour, 2); + }, + "%I": function (date) { + var twelveHour = date.tm_hour; + if (twelveHour == 0) twelveHour = 12; + else if (twelveHour > 12) twelveHour -= 12; + return leadingNulls(twelveHour, 2); + }, + "%j": function (date) { + return leadingNulls( + date.tm_mday + + __arraySum( + __isLeapYear(date.tm_year + 1900) + ? __MONTH_DAYS_LEAP + : __MONTH_DAYS_REGULAR, + date.tm_mon - 1 + ), + 3 + ); + }, + "%m": function (date) { + return leadingNulls(date.tm_mon + 1, 2); + }, + "%M": function (date) { + return leadingNulls(date.tm_min, 2); + }, + "%n": function () { + return "\n"; + }, + "%p": function (date) { + if (date.tm_hour >= 0 && date.tm_hour < 12) { + return "AM"; + } else { + return "PM"; + } + }, + "%S": function (date) { + return leadingNulls(date.tm_sec, 2); + }, + "%t": function () { + return "\t"; + }, + "%u": function (date) { + return date.tm_wday || 7; + }, + "%U": function (date) { + var days = date.tm_yday + 7 - date.tm_wday; + return leadingNulls(Math.floor(days / 7), 2); + }, + "%V": function (date) { + var val = Math.floor((date.tm_yday + 7 - ((date.tm_wday + 6) % 7)) / 7); + if ((date.tm_wday + 371 - date.tm_yday - 2) % 7 <= 2) { + val++; + } + if (!val) { + val = 52; + var dec31 = (date.tm_wday + 7 - date.tm_yday - 1) % 7; + if ( + dec31 == 4 || + (dec31 == 5 && __isLeapYear((date.tm_year % 400) - 1)) + ) { + val++; + } + } else if (val == 53) { + var jan1 = (date.tm_wday + 371 - date.tm_yday) % 7; + if (jan1 != 4 && (jan1 != 3 || !__isLeapYear(date.tm_year))) val = 1; + } + return leadingNulls(val, 2); + }, + "%w": function (date) { + return date.tm_wday; + }, + "%W": function (date) { + var days = date.tm_yday + 7 - ((date.tm_wday + 6) % 7); + return leadingNulls(Math.floor(days / 7), 2); + }, + "%y": function (date) { + return (date.tm_year + 1900).toString().substring(2); + }, + "%Y": function (date) { + return date.tm_year + 1900; + }, + "%z": function (date) { + var off = date.tm_gmtoff; + var ahead = off >= 0; + off = Math.abs(off) / 60; + off = (off / 60) * 100 + (off % 60); + return (ahead ? "+" : "-") + String("0000" + off).slice(-4); + }, + "%Z": function (date) { + return date.tm_zone; + }, + "%%": function () { + return "%"; + }, + }; + pattern = pattern.replace(/%%/g, "\0\0"); + for (var rule in EXPANSION_RULES_2) { + if (pattern.includes(rule)) { + pattern = pattern.replace( + new RegExp(rule, "g"), + EXPANSION_RULES_2[rule](date) + ); + } + } + pattern = pattern.replace(/\0\0/g, "%"); + var bytes = intArrayFromString(pattern, false); + if (bytes.length > maxsize) { + return 0; + } + writeArrayToMemory(bytes, s); + return bytes.length - 1; + } + + function _strftime_l(s, maxsize, format, tm) { + return _strftime(s, maxsize, format, tm); + } + + InternalError = Module["InternalError"] = extendError(Error, "InternalError"); + + embind_init_charCodes(); + + BindingError = Module["BindingError"] = extendError(Error, "BindingError"); + + init_ClassHandle(); + + init_embind(); + + init_RegisteredPointer(); + + UnboundTypeError = Module["UnboundTypeError"] = extendError( + Error, + "UnboundTypeError" + ); + + init_emval(); + + function intArrayFromString(stringy, dontAddNull, length) { + var len = length > 0 ? length : lengthBytesUTF8(stringy) + 1; + var u8array = new Array(len); + var numBytesWritten = stringToUTF8Array( + stringy, + u8array, + 0, + u8array.length + ); + if (dontAddNull) u8array.length = numBytesWritten; + return u8array; + } + + var asmLibraryArg = { + __assert_fail: ___assert_fail, + __cxa_allocate_exception: ___cxa_allocate_exception, + __cxa_rethrow: ___cxa_rethrow, + __cxa_throw: ___cxa_throw, + __syscall_faccessat: ___syscall_faccessat, + __syscall_fcntl64: ___syscall_fcntl64, + __syscall_fstat64: ___syscall_fstat64, + __syscall_getcwd: ___syscall_getcwd, + __syscall_ioctl: ___syscall_ioctl, + __syscall_lstat64: ___syscall_lstat64, + __syscall_newfstatat: ___syscall_newfstatat, + __syscall_openat: ___syscall_openat, + __syscall_renameat: ___syscall_renameat, + __syscall_rmdir: ___syscall_rmdir, + __syscall_stat64: ___syscall_stat64, + __syscall_unlinkat: ___syscall_unlinkat, + _embind_finalize_value_object: __embind_finalize_value_object, + _embind_register_bigint: __embind_register_bigint, + _embind_register_bool: __embind_register_bool, + _embind_register_class: __embind_register_class, + _embind_register_class_constructor: __embind_register_class_constructor, + _embind_register_class_function: __embind_register_class_function, + _embind_register_emval: __embind_register_emval, + _embind_register_float: __embind_register_float, + _embind_register_integer: __embind_register_integer, + _embind_register_memory_view: __embind_register_memory_view, + _embind_register_smart_ptr: __embind_register_smart_ptr, + _embind_register_std_string: __embind_register_std_string, + _embind_register_std_wstring: __embind_register_std_wstring, + _embind_register_value_object: __embind_register_value_object, + _embind_register_value_object_field: __embind_register_value_object_field, + _embind_register_void: __embind_register_void, + _emscripten_date_now: __emscripten_date_now, + _emscripten_get_now_is_monotonic: __emscripten_get_now_is_monotonic, + _emval_call: __emval_call, + _emval_decref: __emval_decref, + _emval_incref: __emval_incref, + _emval_take_value: __emval_take_value, + _localtime_js: __localtime_js, + _mmap_js: __mmap_js, + _munmap_js: __munmap_js, + _tzset_js: __tzset_js, + abort: _abort, + emscripten_get_heap_max: _emscripten_get_heap_max, + emscripten_get_now: _emscripten_get_now, + emscripten_memcpy_big: _emscripten_memcpy_big, + emscripten_resize_heap: _emscripten_resize_heap, + environ_get: _environ_get, + environ_sizes_get: _environ_sizes_get, + exit: _exit, + fd_close: _fd_close, + fd_read: _fd_read, + fd_seek: _fd_seek, + fd_write: _fd_write, + getentropy: _getentropy, + memory: wasmMemory, + pclose: _pclose, + setTempRet0: _setTempRet0, + strftime_l: _strftime_l, + }; + + var asm = createWasm(); + + var calledRun; + + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = "Program terminated with exit(" + status + ")"; + this.status = status; + } + + dependenciesFulfilled = function runCaller() { + if (!calledRun) run(); + if (!calledRun) dependenciesFulfilled = runCaller; + }; + + function run(args) { + args = args || arguments_; + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) return; + calledRun = true; + Module["calledRun"] = true; + if (ABORT) return; + initRuntime(); + if (Module["onRuntimeInitialized"]) Module["onRuntimeInitialized"](); + postRun(); + } + if (Module["setStatus"]) { + Module["setStatus"]("Running..."); + setTimeout(function () { + setTimeout(function () { + Module["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + + Module["run"] = run; + + function exit(status, implicit) { + EXITSTATUS = status; + procExit(status); + } + + function procExit(code) { + EXITSTATUS = code; + if (!keepRuntimeAlive()) { + if (Module["onExit"]) Module["onExit"](code); + ABORT = true; + } + quit_(code, new ExitStatus(code)); + } + + if (Module["preInit"]) { + if (typeof Module["preInit"] == "function") + Module["preInit"] = [Module["preInit"]]; + while (Module["preInit"].length > 0) { + Module["preInit"].pop()(); + } + } + + run(); + + /* Use an optimized gemm implementation if available, otherwise use the fallback + * implementation. + */ + function createWasmGemm() { + // A map of expected gemm function to the corresponding fallback gemm function names. + const GEMM_TO_FALLBACK_FUNCTIONS_MAP = { + int8_prepare_a: "int8PrepareAFallback", + int8_prepare_b: "int8PrepareBFallback", + int8_prepare_b_from_transposed: "int8PrepareBFromTransposedFallback", + int8_prepare_b_from_quantized_transposed: + "int8PrepareBFromQuantizedTransposedFallback", + int8_prepare_bias: "int8PrepareBiasFallback", + int8_multiply_and_add_bias: "int8MultiplyAndAddBiasFallback", + int8_select_columns_of_b: "int8SelectColumnsOfBFallback", + }; + + // Name of the optimized gemm implementation. + const OPTIMIZED_GEMM = "mozIntGemm"; + + const optimizedGemmModule = WebAssembly[OPTIMIZED_GEMM]; + if (!optimizedGemmModule) { + return fallbackGemm(GEMM_TO_FALLBACK_FUNCTIONS_MAP); + } + + const optimizedGemmModuleExports = new WebAssembly.Instance( + optimizedGemmModule(), + { "": { memory: wasmMemory } } + ).exports; + for (let key in GEMM_TO_FALLBACK_FUNCTIONS_MAP) { + if (!optimizedGemmModuleExports[key]) { + return fallbackGemm(GEMM_TO_FALLBACK_FUNCTIONS_MAP); + } + } + console.log(`Using optimized gemm (${OPTIMIZED_GEMM}) implementation`); + return optimizedGemmModuleExports; + } + + // Return the fallback gemm implementation. + function fallbackGemm(gemmToFallbackFunctionsMap) { + // The fallback gemm implementation + const FALLBACK_GEMM = "asm"; + + let fallbackGemmModuleExports = {}; + for (let key in gemmToFallbackFunctionsMap) { + fallbackGemmModuleExports[key] = (...a) => + Module[FALLBACK_GEMM][gemmToFallbackFunctionsMap[key]](...a); + } + console.log(`Using fallback gemm implementation`); + return fallbackGemmModuleExports; + } + + return Module; +} diff --git a/toolkit/components/translations/bergamot-translator/moz.yaml b/toolkit/components/translations/bergamot-translator/moz.yaml new file mode 100644 index 0000000000..427e7cdc04 --- /dev/null +++ b/toolkit/components/translations/bergamot-translator/moz.yaml @@ -0,0 +1,51 @@ +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Firefox + component: Translation + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: bergamot-translator + + description: The JavaScript emscripten worker to run Project Bergamot. + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://github.com/mozilla/bergamot-translator/ + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: v0.4.4 + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: 5ae1b1ebb3fa9a3eabed8a64ca6798154bd486eb + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: MPL-2.0 + + notes: > + The generated emscripten code contains many global variables. When updating + the code, paste it into the following function to protect the global scope. + + ``` + /* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + function loadBergamot(Module) { + {insertCodeHere} + return Module; + } + ``` + + Then replace `console.log()` calls with `log()` calls so that logging preferences + are respected. diff --git a/toolkit/components/translations/content/language-id-engine-worker.js b/toolkit/components/translations/content/language-id-engine-worker.js new file mode 100644 index 0000000000..1323b505d2 --- /dev/null +++ b/toolkit/components/translations/content/language-id-engine-worker.js @@ -0,0 +1,327 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env mozilla/chrome-worker */ +"use strict"; + +// Throw Promise rejection errors so that they are visible in the console. +self.addEventListener("unhandledrejection", event => { + throw event.reason; +}); + +/* global addOnPostRun FastText loadFastText */ +importScripts( + "chrome://global/content/translations/fasttext.js", + "chrome://global/content/translations/fasttext_wasm.js" +); + +/** + * The number of languages that should be returned when the model analyzes text. + * + * A value of 1 means only the most-likely language will be returned. + * A value of 5 would mean that the top 5 most-likely languages will be returned. + */ +const LANGUAGE_COUNT = 1; + +/** + * The threshold of likelihood in range [0.0, 1.0] that must pass + * for a language to be returned from the model. + * + * A value of 0.0 would mean that a language is always returned with any confidence. + * A value of 0.5 would mean that a language is only returned if the model + * is 50% confident that the analyzed text could be that language. + */ +const CONFIDENCE_THRESHOLD = 0.0; + +// Respect the preference "browser.translations.logLevel". +let _isLoggingEnabled = true; +function log(...args) { + if (_isLoggingEnabled) { + console.log("Translations:", ...args); + } +} + +// Wait for the initialization request. +addEventListener("message", handleInitializationMessage); + +/** + * Initialize the engine, and get it ready to handle language identification requests. + * The "initialize" message must be received before any other message handling + * requests will be processed. + * + * @param {Object} event + * @param {Object} event.data + * @param {string} event.data.type - The message type, expects "initialize". + * @param {ArrayBuffer} event.data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} event.data.modelBuffer - The buffer containing the language-id model binary. + * @param {null | string} event.data.mockedLangTag - The mocked language tag value (only present when mocking). + * @param {null | number} event.data.mockedConfidence - The mocked confidence value (only present when mocking). + * @param {boolean} event.data.isLoggingEnabled + */ +async function handleInitializationMessage({ data }) { + if (data.type !== "initialize") { + throw new Error( + "The LanguageIdEngine worker received a message before it was initialized." + ); + } + + try { + const { isLoggingEnabled } = data; + if (isLoggingEnabled) { + // Respect the "browser.translations.logLevel" preference. + _isLoggingEnabled = true; + } + + /** @type {LanguageIdEngine | MockedLanguageIdEngine} */ + let languageIdEngine; + const { mockedLangTag, mockedConfidence } = data; + if (mockedLangTag !== null && mockedConfidence !== null) { + // Don't actually use the engine as it is mocked. + languageIdEngine = new MockedLanguageIdEngine( + mockedLangTag, + mockedConfidence + ); + } else { + languageIdEngine = await initializeLanguageIdEngine(data); + } + + handleMessages(languageIdEngine); + postMessage({ type: "initialization-success" }); + } catch (error) { + console.error(error); + postMessage({ type: "initialization-error", error: error?.message }); + } + + removeEventListener("message", handleInitializationMessage); +} + +/** + * Initializes the fastText wasm runtime and returns the fastText model. + * + * @param {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @param {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + * @returns {FastTextModel} + */ +function initializeFastTextModel(modelBuffer, wasmBuffer) { + return new Promise((resolve, reject) => { + const initialModule = { + onAbort() { + reject(new Error("Error loading the fastText Wasm Module")); + }, + onRuntimeInitialized() { + addOnPostRun(() => { + const ft = new FastText(initialModule); + const model = ft.loadModelBinary(modelBuffer); + resolve(model); + }); + }, + wasmBinary: wasmBuffer, + }; + loadFastText(initialModule); + }); +} + +/** + * Initialize the LanguageIdEngine from the data payload by loading + * the fastText wasm runtime and model and constructing the engine. + * + * @param {Object} data + * @property {ArrayBuffer} data.wasmBuffer - The buffer containing the wasm binary. + * @property {ArrayBuffer} data.modelBuffer - The buffer containing the language-id model binary. + */ +async function initializeLanguageIdEngine(data) { + const { modelBuffer, wasmBuffer } = data; + if (!modelBuffer) { + throw new Error('LanguageIdEngine initialization missing "modelBuffer"'); + } + if (!wasmBuffer) { + throw new Error('LanguageIdEngine initialization missing "wasmBuffer"'); + } + const model = await initializeFastTextModel(modelBuffer, wasmBuffer); + return new LanguageIdEngine(model); +} + +/** + * Sets up the message handling for the worker. + * + * @param {LanguageIdEngine | MockedLanguageIdEngine} languageIdEngine + */ +function handleMessages(languageIdEngine) { + /** + * Handle any message after the initialization message. + * + * @param {Object} data + * @property {string} data.type - The message type. + * @property {string} data.message - The message text to identify the language of. + * @property {number} data.messageId - The ID of the message. + */ + addEventListener("message", ({ data }) => { + try { + if (data.type === "initialize") { + throw new Error( + "The language-identification engine must not be re-initialized." + ); + } + switch (data.type) { + case "language-id-request": { + const { message, messageId } = data; + try { + const [confidence, langTag] = + languageIdEngine.identifyLanguage(message); + postMessage({ + type: "language-id-response", + langTag, + confidence, + messageId, + }); + } catch (error) { + console.error(error); + postMessage({ + type: "language-id-error", + messageId, + }); + } + break; + } + default: { + console.warn("Unknown message type:", data.type); + } + } + } catch (error) { + // Ensure the unexpected errors are surfaced in the console. + console.error(error); + } + }); +} + +/** + * The LanguageIdEngine wraps around a machine-learning model that can identify text + * as being written in a given human language. The engine is responsible for invoking + * model and returning the language tag in the format that is expected by firefox + * translations code. + */ +class LanguageIdEngine { + /** @type {FastTextModel} */ + #model; + + /** + * @param {FastTextModel} model + */ + constructor(model) { + this.#model = model; + } + + /** + * Formats the language tag returned by the language-identification model to match + * conform to the format used internally by Firefox. + * + * This function is currently configured to handle the fastText language-identification + * model. Updating the language-identification model or moving to something other than + * fastText in the future will likely require updating this function. + * + * @param {string} langTag + * @returns {string} The correctly formatted langTag + */ + #formatLangTag(langTag) { + // The fastText language model returns values of the format "__label__{langTag}". + // As such, this function strips the "__label__" prefix, leaving only the langTag. + let formattedTag = langTag.replace("__label__", ""); + + // fastText is capable of returning any of a predetermined set of 176 langTags: + // https://fasttext.cc/docs/en/language-identification.html + // + // These tags come from ISO639-3: + // https://iso639-3.sil.org/code_tables/deprecated_codes/data + // + // Each of these tags have been cross checked for compatibility with the IANA + // language subtag registry, which is used by BCP 47, and any edge cases are handled below. + // https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry + switch (formattedTag) { + // fastText may return "eml" which is a deprecated ISO639-3 language tag for the language + // Emiliano-Romagnolo. It was split into two separate tags "egl" and "rgn": + // https://iso639-3.sil.org/request/2008-040 + // + // "eml" was once requested to be added to the IANA registry, but it was denied: + // https://www.alvestrand.no/pipermail/ietf-languages/2009-December/009754.html + // + // This case should return either "egl" or "rgn", given that the "eml" tag was split. + // However, given that the fastText model does not distinguish between the two by using + // the deprecated tag, this function will default to "egl" because it is alphabetically first. + // + // At such a time that Firefox Translations may support either of these languages, we should consider + // a way to further distinguish between the two languages at that time. + case "eml": { + formattedTag = "egl"; + break; + } + // The fastText model returns "no" for Norwegian Bokmål. + // + // According to advice from https://r12a.github.io/app-subtags/ + // "no" is a macro language that encompasses the following more specific primary language subtags: "nb" "nn". + // It is recommended to use more specific language subtags as long as it does not break legacy usage of an application. + // As such, this function will return "nb" for Norwegian Bokmål instead of "no" as reported by fastText. + case "no": { + formattedTag = "nb"; + break; + } + } + return formattedTag; + } + + /** + * Identifies the human language in which the message is written and returns + * the BCP 47 language tag of the language it is determined to be along along + * with a rating of how confident the model is that the label is correct. + * + * @param {string} message + * @returns {Array<number | string>} An array containing the confidence and language tag. + * The confidence is a number between 0 and 1, representing a percentage. + * The language tag is a BCP 47 language tag such as "en" for English. + * + * e.g. [0.87, "en"] + */ + identifyLanguage(message) { + const mostLikelyLanguageData = this.#model + .predict(message.trim(), LANGUAGE_COUNT, CONFIDENCE_THRESHOLD) + .get(0); + + // This should never fail as long as + // LANGUAGE_COUNT > 1 && CONFIDENCE_THRESHOLD === 0.0 + if (!mostLikelyLanguageData) { + throw new Error("Unable to identify a language"); + } + + const [confidence, langTag] = mostLikelyLanguageData; + return [confidence, this.#formatLangTag(langTag)]; + } +} + +/** + * For testing purposes, provide a fully mocked engine. This allows for easy integration + * testing of the UI, without having to rely on downloading remote models and remote + * wasm binaries. + */ +class MockedLanguageIdEngine { + /** @type {string} */ + #langTag; + /** @type {number} */ + #confidence; + + /** + * @param {string} langTag + * @param {number} confidence + */ + constructor(langTag, confidence) { + this.#langTag = langTag; + this.#confidence = confidence; + } + + /** + * Mocks identifying a language by returning the mocked engine's pre-determined + * language tag and confidence values. + */ + identifyLanguage(_message) { + return [this.#confidence, this.#langTag]; + } +} diff --git a/toolkit/components/translations/content/simd-detect-worker.js b/toolkit/components/translations/content/simd-detect-worker.js new file mode 100644 index 0000000000..35efce5e25 --- /dev/null +++ b/toolkit/components/translations/content/simd-detect-worker.js @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +let isSimdSupported = false; + +/** + * WebAssembly counts as unsafe eval in privileged contexts, so we have to execute this + * code in a ChromeWorker. The code feature detects SIMD support. The comment above + * the binary code is the .wat version of the .wasm binary. + */ + +try { + new WebAssembly.Module( + new Uint8Array( + // ``` + // ;; Detect SIMD support. + // ;; Compile by running: wat2wasm --enable-all simd-detect.wat + // + // (module + // (func (result v128) + // i32.const 0 + // i8x16.splat + // i8x16.popcnt + // ) + // ) + // ``` + + // prettier-ignore + [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x05, 0x01, 0x60, 0x00, + 0x01, 0x7b, 0x03, 0x02, 0x01, 0x00, 0x0a, 0x0a, 0x01, 0x08, 0x00, 0x41, 0x00, + 0xfd, 0x0f, 0xfd, 0x62, 0x0b + ] + ) + ); + isSimdSupported = true; +} catch (error) { + console.error(`Translations: SIMD not supported`, error); +} + +postMessage({ isSimdSupported }); diff --git a/toolkit/components/translations/content/translations-document.sys.mjs b/toolkit/components/translations/content/translations-document.sys.mjs new file mode 100644 index 0000000000..c1c883dbc8 --- /dev/null +++ b/toolkit/components/translations/content/translations-document.sys.mjs @@ -0,0 +1,1284 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyGetter(lazy, "console", () => { + return console.createInstance({ + maxLogLevelPref: "browser.translations.logLevel", + prefix: "Translations", + }); +}); + +/** + * Map the NodeFilter enums that are used by the TreeWalker into enums that make + * sense for determining the status of the nodes for the TranslationsDocument process. + * This aligns the meanings of the filtering for the translations process. + */ +const NodeStatus = { + // This node is ready to translate as is. + READY_TO_TRANSLATE: NodeFilter.FILTER_ACCEPT, + + // This node contains too many block elements and needs to be subdivided further. + SUBDIVIDE_FURTHER: NodeFilter.FILTER_SKIP, + + // This node should not be considered for translation. + NOT_TRANSLATABLE: NodeFilter.FILTER_REJECT, +}; + +/** + * @typedef {import("../translations").NodeVisibility} NodeVisibility + * @typedef {(message: string) => Promise<string>} TranslationFunction + */ + +/** + * How often the DOM is updated with translations, in milliseconds. + */ +const DOM_UPDATE_INTERVAL_MS = 50; + +/** + * These tags are excluded from translation. + */ +const EXCLUDED_TAGS = new Set([ + // The following are elements that semantically should not be translated. + "CODE", + "KBD", + "SAMP", + "VAR", + "ACRONYM", + + // The following are deprecated tags. + "DIR", + "APPLET", + + // The following are embedded elements, and are not supported (yet). + "SVG", + "MATH", + "EMBED", + "OBJECT", + "IFRAME", + + // These are elements that are treated as opaque by Firefox which causes their + // innerHTML property to be just the raw text node behind it. Any text that is sent as + // HTML must be valid, and there is no guarantee that the innerHTML is valid. + "NOSCRIPT", + "NOEMBED", + "NOFRAMES", + + // The title is handled separately, and a HEAD tag should not be considered. + "HEAD", + + // These are not user-visible tags. + "STYLE", + "SCRIPT", + "TEMPLATE", + + // Textarea elements contain user content, which should not be translated. + "TEXTAREA", +]); + +// Tags that are treated as assumed inline. This list has been created by heuristics +// and excludes some commonly inline tags, due to how they are used practically. +// +// An actual list of inline elements is available here: +// https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elements#list_of_inline_elements +const INLINE_TAGS = new Set([ + "ABBR", + "B", + "CODE", + "DEL", + "EM", + "I", + "INS", + "KBD", + "MARK", + "MATH", + "OUTPUT", + "Q", + "RUBY", + "SMALL", + "STRONG", + "SUB", + "SUP", + "TIME", + "U", + "VAR", + "WBR", + + // These are not really inline, but bergamot-translator treats these as + // sentence-breaking. + "BR", + "TD", + "TH", + "LI", +]); + +/** + * Tags that can't reliably be assumed to be inline or block elements. They default + * to inline, but are often used as block elements. + */ +const GENERIC_TAGS = new Set(["A", "SPAN"]); + +/** + * This class manages the process of translating the DOM from one language to another. + * A translateHTML and a translateText function are injected into the constructor. This + * class is responsible for subdividing a Node into small enough pieces to where it + * contains a reasonable amount of text and inline elements for the translations engine + * to translate. Once a node has been identified as a small enough chunk, its innerHTML + * is read, and sent for translation. The async translation result comes back as an HTML + * string. The DOM node is updated with the new text and potentially changed DOM ordering. + * + * This class also handles mutations of the DOM and will translate nodes as they are added + * to the page, or the when the node's text is changed by content scripts. + */ +export class TranslationsDocument { + /** + * The BCP 47 language tag that is used on the page. + * + * @type {string} */ + documentLanguage; + + /** + * The timeout between the first translation received and the call to update the DOM + * with translations. + */ + #updateTimeout = null; + + /** + * The nodes that need translations. They are queued when the document tree is walked, + * and then they are dispatched for translation based on their visibility. The viewport + * nodes are given the highest priority. + * + * @type {Map<Node, NodeVisibility>} + */ + #queuedNodes = new Map(); + + /** + * The count of how many pending translations have been sent to the translations + * engine. + */ + #pendingTranslationsCount = 0; + + /** + * The list of nodes that need updating with the translated HTML. These are batched + * into an update. + * + * @type {Set<{ node: Node, translatedHTML: string }} + */ + #nodesWithTranslatedHTML = new Set(); + + /** + * The set of nodes that have been subdivided and processed for translation. They + * should not be submitted again unless their contents have been changed. + * + * @type {WeakSet<Node>} + */ + #processedNodes = new WeakSet(); + + /** + * All root elements we're trying to translate. This should be the `document.body` + * and the the `title` element. + * + * @type {Set<Node>} + */ + #rootNodes = new Set(); + + /** + * This promise gets resolved when the initial viewport translations are done. + * This is a key user-visible performance metric. It represents what the user + * actually sees. + * + * @type {Promise<void> | null} + */ + viewportTranslated = null; + + /** + * Construct a new TranslationsDocument. It is tied to a specific Document and cannot + * be re-used. The translation functions are injected since this class shouldn't + * manage the life cycle of the translations engines. + * + * @param {Document} document + * @param {string} documentLanguage - The BCP 47 language tag. + * @param {number} innerWindowId - This is used for better profiler marker reporting. + * @param {TranslationFunction} translateHTML + * @param {TranslationFunction} translateText + */ + constructor( + document, + documentLanguage, + innerWindowId, + translateHTML, + translateText + ) { + /** + * The language of the document. If elements are found that do not match this language, + * then they are skipped. + * + * @type {string} + */ + this.documentLanguage = documentLanguage; + if (documentLanguage.length !== 2) { + throw new Error( + "Expected the language to be a valid 2 letter BCP 47 language tag: " + + documentLanguage + ); + } + + /** @type {TranslationFunction} */ + this.translateHTML = translateHTML; + + /** @type {TranslationFunction} */ + this.translateText = translateText; + + /** @type {number} */ + this.innerWindowId = innerWindowId; + + /** @type {DOMParser} */ + this.domParser = new document.ownerGlobal.DOMParser(); + + /** + * This selector runs to find child nodes that should be excluded. It should be + * basically the same implementation of `isExcludedNode`, but as a selector. + * + * @type {string} + */ + this.excludedNodeSelector = [ + // Use: [lang|=value] to match language codes. + // + // Per: https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors + // + // The elements with an attribute name of attr whose value can be exactly + // value or can begin with value immediately followed by a hyphen, - (U+002D). + // It is often used for language subcode matches. + `[lang]:not([lang|="${this.documentLanguage}"])`, + `[translate=no]`, + `.notranslate`, + `[contenteditable="true"]`, + `[contenteditable=""]`, + [...EXCLUDED_TAGS].join(","), + ].join(","); + + this.observer = new document.ownerGlobal.MutationObserver(mutationsList => { + for (const mutation of mutationsList) { + switch (mutation.type) { + case "childList": + for (const node of mutation.addedNodes) { + this.#processedNodes.delete(node); + this.subdivideNodeForTranslations(node); + } + break; + case "characterData": + this.#processedNodes.delete(mutation); + this.subdivideNodeForTranslations(mutation.target); + break; + default: + break; + } + } + }); + } + + /** + * Add a new element to start translating. This root is tracked for mutations and + * kept up to date with translations. This will be the body element and title tag + * for the document. + * + * @param {Element} [node] + */ + addRootElement(node) { + if (!node) { + return; + } + + if (node.nodeType !== Node.ELEMENT_NODE) { + // This node is not an element, do not add it. + return; + } + + if (this.#rootNodes.has(node)) { + // Exclude nodes that are already targetted. + return; + } + + this.#rootNodes.add(node); + + this.subdivideNodeForTranslations(node); + + this.observer.observe(node, { + characterData: true, + childList: true, + subtree: true, + }); + } + + /** + * Start walking down through a node's subtree and decide which nodes to queue for + * translation. This first node could be the root nodes of the DOM, such as the + * document body, or the title element, or it could be a mutation target. + * + * The nodes go through a process of subdivision until an appropriate sized chunk + * of inline text can be found. + * + * @param {Node} node + */ + subdivideNodeForTranslations(node) { + if (!this.#rootNodes.has(node)) { + // This is a non-root node, which means it came from a mutation observer. + // Ensure that it is a valid node to translate by checking all of its ancestors. + for (let parent of getAncestorsIterator(node)) { + if ( + this.determineTranslationStatus(parent) === + NodeStatus.NOT_TRANSLATABLE + ) { + return; + } + } + } + + switch (this.determineTranslationStatusForUnprocessedNodes(node)) { + case NodeStatus.NOT_TRANSLATABLE: + // This node is rejected as it shouldn't be translated. + return; + + case NodeStatus.READY_TO_TRANSLATE: + // This node is ready for translating, and doesn't need to be subdivided. There + // is no reason to run the TreeWalker, it can be directly submitted for + // translation. + this.queueNodeForTranslation(node); + break; + + case NodeStatus.SUBDIVIDE_FURTHER: + // This node may be translatable, but it needs to be subdivided into smaller + // pieces. Create a TreeWalker to walk the subtree, and find the subtrees/nodes + // that contain enough inline elements to send to be translated. + { + const nodeIterator = node.ownerDocument.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT, + this.determineTranslationStatusForUnprocessedNodes + ); + + // This iterator will contain each node that has been subdivided enough to + // be translated. + let currentNode; + while ((currentNode = nodeIterator.nextNode())) { + this.queueNodeForTranslation(currentNode); + } + } + break; + } + + if (node.nodeName === "BODY") { + this.reportWordsInViewport(); + } + this.dispatchQueuedTranslations(); + } + + /** + * Test whether this is an element we do not want to translate. These are things like + * <code> elements, elements with a different "lang" attribute, and elements that + * have a `translate=no` attribute. + * + * @param {Node} node + */ + isExcludedNode(node) { + // Property access be expensive, so destructure required properties so they are + // not accessed multiple times. + const { nodeType } = node; + + if (nodeType === Node.TEXT_NODE) { + // Text nodes are never excluded. + return false; + } + if (nodeType !== Node.ELEMENT_NODE) { + // Only elements and and text nodes should be considered. + return true; + } + + const { nodeName } = node; + + if (EXCLUDED_TAGS.has(nodeName)) { + // This is an excluded tag. + return true; + } + + if (!this.matchesDocumentLanguage(node)) { + // Exclude nodes that don't match the fromLanguage. + return true; + } + + if (node.getAttribute("translate") === "no") { + // This element has a translate="no" attribute. + // https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/translate + return true; + } + + if (node.classList.contains("notranslate")) { + // Google Translate skips translations if the classList contains "notranslate" + // https://cloud.google.com/translate/troubleshooting + return true; + } + + if (node.isContentEditable) { + // This field is editable, and so exclude it similar to the way that form input + // fields are excluded. + return true; + } + + return false; + } + + /** + * Runs `determineTranslationStatus`, but only on unprocessed nodes. + * + * @param {Node} node + * @return {number} - One of the NodeStatus values. + */ + determineTranslationStatusForUnprocessedNodes = node => { + if (this.#processedNodes.has(node)) { + // Skip nodes that have already been processed. + return NodeStatus.NOT_TRANSLATABLE; + } + + return this.determineTranslationStatus(node); + }; + + /** + * Determines if a node should be submitted for translation, not translatable, or if + * it should be subdivided further. It doesn't check if the node has already been + * processed. + * + * The return result works as a TreeWalker NodeFilter as well. + * + * @param {Node} node + * @returns {number} - One of the `NodeStatus` values. See that object + * for documentation. These values match the filters for the TreeWalker. + * These values also work as a `NodeFilter` value. + */ + determineTranslationStatus(node) { + if (isNodeQueued(node, this.#queuedNodes)) { + // This node or its parent was already queued, reject it. + return NodeStatus.NOT_TRANSLATABLE; + } + + if (this.isExcludedNode(node)) { + // This is an explicitly excluded node. + return NodeStatus.NOT_TRANSLATABLE; + } + + if (node.textContent.trim().length === 0) { + // Do not use subtrees that are empty of text. This textContent call is fairly + // expensive. + return NodeStatus.NOT_TRANSLATABLE; + } + + if (nodeNeedsSubdividing(node)) { + // Skip this node, and dig deeper into its tree to cut off smaller pieces + // to translate. It is presumed to be a wrapper of block elements. + return NodeStatus.SUBDIVIDE_FURTHER; + } + + if ( + containsExcludedNode(node, this.excludedNodeSelector) && + !hasTextNodes(node) + ) { + // Skip this node, and dig deeper into its tree to cut off smaller pieces + // to translate. + return NodeStatus.SUBDIVIDE_FURTHER; + } + + // This node can be treated as entire block to submit for translation. + return NodeStatus.READY_TO_TRANSLATE; + } + + /** + * Queue a node for translation. + * @param {Node} node + */ + queueNodeForTranslation(node) { + /** @type {NodeVisibility} */ + let visibility = "out-of-viewport"; + if (isNodeHidden(node)) { + visibility = "hidden"; + } else if (isNodeInViewport(node)) { + visibility = "in-viewport"; + } + + this.#queuedNodes.set(node, visibility); + } + + /** + * Submit the translations giving priority to nodes in the viewport. + */ + async dispatchQueuedTranslations() { + let inViewportCounts = 0; + let outOfViewportCounts = 0; + let hiddenCounts = 0; + + let inViewportTranslations; + if (!this.viewportTranslated) { + inViewportTranslations = []; + } + + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "in-viewport") { + inViewportCounts++; + const promise = this.submitTranslation(node); + if (inViewportTranslations) { + inViewportTranslations.push(promise); + } + } + } + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "out-of-viewport") { + outOfViewportCounts++; + this.submitTranslation(node); + } + } + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "hidden") { + hiddenCounts++; + this.submitTranslation(node); + } + } + + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + `Translate ${this.#queuedNodes.size} nodes.\n\n` + + `In viewport: ${inViewportCounts}\n` + + `Out of viewport: ${outOfViewportCounts}\n` + + `Hidden: ${hiddenCounts}\n` + ); + + this.#queuedNodes.clear(); + + if (!this.viewportTranslated && inViewportTranslations) { + // Provide a promise that can be used to determine when the initial viewport has + // been translated. This is a key user-visible metric. + this.viewportTranslated = Promise.allSettled(inViewportTranslations); + } + } + + /** + * Record how many words were in the viewport, as this is the most important + * user-visible translation content. + */ + reportWordsInViewport() { + if ( + // This promise gets created for the first dispatchQueuedTranslations + this.viewportTranslated || + this.#queuedNodes.size === 0 + ) { + return; + } + + // TODO(Bug 1814195) - Add telemetry. + // TODO(Bug 1820618) - This whitespace regex will not work in CJK-like languages. + // This requires a segmenter for a proper implementation. + + const whitespace = /\s+/; + let wordCount = 0; + for (const [node, visibility] of this.#queuedNodes) { + if (visibility === "in-viewport") { + wordCount += node.textContent.trim().split(whitespace).length; + } + } + + const message = wordCount + " words are in the viewport."; + lazy.console.log(message); + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + message + ); + } + + /** + * Submit a node for translation to the translations engine. + * + * @param {Node} node + * @returns {Promise<void>} + */ + async submitTranslation(node) { + // Give each element an id that gets passed through the translation so it can be + // reunited later on. + if (node.nodeType === Node.ELEMENT_NODE) { + node.querySelectorAll("*").forEach((el, i) => { + el.dataset.mozTranslationsId = i; + }); + } + + let text, translate; + if (node.nodeType === Node.ELEMENT_NODE) { + text = node.innerHTML; + translate = this.translateHTML; + } else { + text = node.textContent; + translate = this.translateText; + } + + if (text.trim().length === 0) { + return; + } + + // Mark this node as not to be translated again unless the contents are changed + // (which the observer will pick up on) + this.#processedNodes.add(node); + + this.#pendingTranslationsCount++; + try { + const [translatedHTML] = await translate(text); + this.#pendingTranslationsCount--; + this.scheduleNodeUpdateWithTranslation(node, translatedHTML); + } catch (error) { + this.#pendingTranslationsCount--; + lazy.console.error("Translation failed", error); + } + } + + /** + * Start the mutation observer, for instance after applying the translations to the DOM. + */ + startMutationObserver() { + if (Cu.isDeadWrapper(this.observer)) { + // This observer is no longer alive. + return; + } + for (const node of this.#rootNodes) { + if (Cu.isDeadWrapper(node)) { + // This node is no longer alive. + continue; + } + this.observer.observe(node, { + characterData: true, + childList: true, + subtree: true, + }); + } + } + + /** + * Stop the mutation observer, for instance to apply the translations to the DOM. + */ + stopMutationObserver() { + // Was the window already destroyed? + if (!Cu.isDeadWrapper(this.observer)) { + this.observer.disconnect(); + } + } + + /** + * This is called every `DOM_UPDATE_INTERVAL_MS` ms with translations for nodes. + * + * This function is called asynchronously, so nodes may already be dead. Before + * accessing a node make sure and run `Cu.isDeadWrapper` to check that it is alive. + */ + updateNodesWithTranslations() { + // Stop the mutations so that the updates won't trigger observations. + this.stopMutationObserver(); + + for (const { node, translatedHTML } of this.#nodesWithTranslatedHTML) { + if (Cu.isDeadWrapper(node)) { + // The node is no longer alive. + ChromeUtils.addProfilerMarker( + "Translations", + { innerWindowId: this.innerWindowId }, + "Node is no long alive." + ); + continue; + } + switch (node.nodeType) { + case Node.TEXT_NODE: { + if (translatedHTML.trim().length !== 0) { + // Only update the node if there is new text. + node.textContent = translatedHTML; + } + break; + } + case Node.ELEMENT_NODE: { + // TODO (Bug 1820625) - This is slow compared to the original implementation + // in the addon which set the innerHTML directly. We can't set the innerHTML + // here, but perhaps there is another way to get back some of the performance. + const translationsDocument = this.domParser.parseFromString( + `<!DOCTYPE html><div>${translatedHTML}</div>`, + "text/html" + ); + updateElement(translationsDocument, node); + break; + } + } + } + + this.#nodesWithTranslatedHTML.clear(); + this.#updateTimeout = null; + + // Done mutating the DOM. + this.startMutationObserver(); + } + + /** + * Schedule a node to be updated with a translation. + * + * @param {Node} node + * @param {string} translatedHTML + */ + scheduleNodeUpdateWithTranslation(node, translatedHTML) { + // Add the nodes to be populated with the next translation update. + this.#nodesWithTranslatedHTML.add({ node, translatedHTML }); + + if (this.#pendingTranslationsCount === 0) { + // No translations are pending, update the node. + this.updateNodesWithTranslations(); + } else if (!this.#updateTimeout) { + // Schedule an update. + this.#updateTimeout = lazy.setTimeout( + this.updateNodesWithTranslations.bind(this), + DOM_UPDATE_INTERVAL_MS + ); + } else { + // An update has been previously scheduled, do nothing here. + } + } + + /** + * Check to see if a language matches the document language. + * + * @param {Node} node + */ + matchesDocumentLanguage(node) { + if (!node.lang) { + // No `lang` was present, so assume it matches the language. + return true; + } + + // First, cheaply check if language tags match, without canonicalizing. + if (langTagsMatch(this.documentLanguage, node.lang)) { + return true; + } + + try { + // Make sure the local is in the canonical form, and check again. This function + // throws, so don't trust that the language tags are formatting correctly. + const [language] = Intl.getCanonicalLocales(node.lang); + + return langTagsMatch(this.documentLanguage, language); + } catch (_error) { + return false; + } + } +} + +/** + * This function needs to be fairly fast since it's used on many nodes when iterating + * over the DOM to find nodes to translate. + * + * @param {Text | HTMLElement} node + */ +function isNodeHidden(node) { + /** @type {HTMLElement} */ + const element = node.nodeType === Node.TEXT_NODE ? node.parentElement : node; + + // This flushes the style, which is a performance cost. + const style = element.ownerGlobal.getComputedStyle(element); + return style.display === "none" || style.visibility === "hidden"; +} + +/** + * This function cheaply checks that language tags match. + * + * @param {string} knownLanguage + * @param {string} otherLanguage + */ +function langTagsMatch(knownLanguage, otherLanguage) { + if (knownLanguage === otherLanguage) { + // A simple direct match. + return true; + } + if (knownLanguage.length !== 2) { + throw new Error("Expected the knownLanguage to be of length 2."); + } + // Check if the language tags part match, e.g. "en" and "en-US". + return ( + knownLanguage[0] === otherLanguage[0] && + knownLanguage[1] === otherLanguage[1] && + otherLanguage[2] === "-" + ); +} + +/** + * This function runs when walking the DOM, which means it is a hot function. It runs + * fairly fast even though it is computing the bounding box. This is all done in a tight + * loop, and it is done on mutations. Care should be taken with reflows caused by + * getBoundingClientRect, as this is a common performance issue. + * + * The following are the counts of how often this is run on a news site: + * + * Given: + * 1573 DOM nodes + * 504 Text nodes + * 1069 Elements + * + * There were: + * 209 calls to get this funcion. + * + * @param {Node} node + */ +function isNodeInViewport(node) { + const window = node.ownerGlobal; + const document = node.ownerDocument; + + /** @type {HTMLElement} */ + const element = node.nodeType === Node.TEXT_NODE ? node.parentElement : node; + + const rect = element.getBoundingClientRect(); + return ( + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= (window.innerWidth || document.documentElement.clientWidth) + ); +} + +/** + * Actually perform the update of the element with the translated node. This step + * will detach all of the "live" nodes, and match them up in the correct order as provided + * by the translations engine. + * + * @param {Document} translationsDocument + * @param {Element} element + * @returns {void} + */ +function updateElement(translationsDocument, element) { + // This text should have the same layout as the target, but it's not completely + // guaranteed since the content page could change at any time, and the translation process is async. + // + // The document has the following structure: + // + // <html> + // <head> + // <body>{translated content}</body> + // </html> + + const originalHTML = element.innerHTML; + + /** + * The Set of translation IDs for nodes that have been cloned. + * @type {Set<number>} + */ + const clonedNodes = new Set(); + + merge(element, translationsDocument.body.firstChild); + + /** + * Merge the live tree with the translated tree by re-using elements from the live tree. + * + * @param {Node} liveTree + * @param {Node} translatedTree + */ + function merge(liveTree, translatedTree) { + /** @type {Map<number, Element>} */ + const liveElementsById = new Map(); + + /** @type {Array<Text>} */ + const liveTextNodes = []; + + // Remove all the nodes from the liveTree, and categorize them by Text node or + // Element node. + let node; + while ((node = liveTree.firstChild)) { + node.remove(); + + if (node.nodeType === Node.ELEMENT_NODE) { + liveElementsById.set(node.dataset.mozTranslationsId, node); + } else if (node.nodeType === Node.TEXT_NODE) { + liveTextNodes.push(node); + } + } + + // The translated tree dictates the order. + const translatedNodes = translatedTree.childNodes; + for ( + let translatedIndex = 0; + translatedIndex < translatedNodes.length; + translatedIndex++ + ) { + const translatedNode = translatedNodes[translatedIndex]; + + if (translatedNode.nodeType === Node.TEXT_NODE) { + // Copy the translated text to the original Text node and re-append it. + let liveTextNode = liveTextNodes.shift(); + + if (liveTextNode) { + liveTextNode.data = translatedNode.data; + } else { + liveTextNode = translatedNode; + } + + liveTree.appendChild(liveTextNode); + } else if (translatedNode.nodeType === Node.ELEMENT_NODE) { + const translationsId = translatedNode.dataset.mozTranslationsId; + // Element nodes try to use the already existing DOM nodes. + + // Find the element in the live tree that matches the one in the translated tree. + let liveElement = liveElementsById.get(translationsId); + + if (!liveElement) { + lazy.console.warn("Could not find a corresponding live element", { + path: createNodePath(translatedNode, translationsDocument.body), + translationsId, + liveElementsById, + translatedNode, + }); + continue; + } + + // Has this element already been added to the list? Then duplicate it and re-add + // it as a clone. The Translations Engine can sometimes duplicate HTML. + if (liveElement.parentNode) { + liveElement = liveElement.cloneNode(true /* deep clone */); + clonedNodes.add(translationsId); + lazy.console.warn( + "Cloning a node because it was already inserted earlier", + { + path: createNodePath(translatedNode, translationsDocument.body), + translatedNode, + liveElement, + } + ); + } + + if (isNodeTextEmpty(translatedNode)) { + // The original node had text, but the one that came out of translation + // didn't have any text. This scenario might be caused by one of two causes: + // + // 1) The element was duplicated by translation but then not given text + // content. This happens on Wikipedia articles for example. + // + // 2) The translator messed up and could not translate the text. This + // happens on YouTube in the language selector. In that case, having the + // original text is much better than no text at all. + // + // To make sure it is case 1 and not case 2 check whether this is the only occurrence. + for (let i = 0; i < translatedNodes.length; i++) { + if (translatedIndex === i) { + // This is the current node, not a sibling. + continue; + } + const sibling = translatedNodes[i]; + if ( + // Only consider other element nodes. + sibling.nodeType === Node.ELEMENT_NODE && + // If the sibling's translationsId matches, then use the sibling's + // node instead. + translationsId === sibling.dataset.mozTranslationsId + ) { + // This is case 1 from above. Remove this element's original text nodes, + // since a sibling text node now has all of the text nodes. + removeTextNodes(liveElement); + } + } + + // Report this issue to the console. + lazy.console.warn( + "The translated element has no text even though the original did.", + { + path: createNodePath(translatedNode, translationsDocument.body), + translatedNode, + liveElement, + } + ); + } else if (!isNodeTextEmpty(liveElement)) { + // There are still text nodes to find and update, recursively merge. + merge(liveElement, translatedNode); + } + + // Put the live node back in the live branch. But now t has been synced with the + // translated text and order. + liveTree.appendChild(liveElement); + } + } + + const unhandledElements = [...liveElementsById].filter( + ([, element]) => !element.parentNode + ); + + if (unhandledElements.length) { + lazy.console.warn( + `${createNodePath( + translatedTree, + translationsDocument.body + )} Not all nodes unified`, + { + unhandledElements, + clonedNodes, + originalHTML, + translatedHTML: translationsDocument.body.innerHTML, + liveTree: liveTree.outerHTML, + translatedTree: translatedTree.outerHTML, + } + ); + } + } +} + +/** + * For debug purposes, compute a string path to an element. + * + * e.g. "div/div#header/p.bold.string/a" + * + * @param {Node} node + * @param {Node | null} root + */ +function createNodePath(node, root) { + if (root === null) { + root = node.ownerDocument.body; + } + let path = + node.parentNode && node.parentNode !== root + ? createNodePath(node.parentNode) + : ""; + path += `/${node.nodeName}`; + if (node.id) { + path += `#${node.id}`; + } else if (node.className) { + for (const className of node.classList) { + path += "." + className; + } + } + return path; +} + +/** + * @param {Node} node + * @returns {boolean} + */ +function isNodeTextEmpty(node) { + if ("innerText" in node) { + return node.innerText.trim().length === 0; + } + if (node.nodeType === Node.TEXT_NODE && node.nodeValue) { + return node.nodeValue.trim().length === 0; + } + return true; +} + +/** + * @param {Node} node + */ +function removeTextNodes(node) { + for (const child of node.childNodes) { + switch (child.nodeType) { + case Node.TEXT_NODE: + node.removeChild(child); + break; + case Node.ELEMENT_NODE: + removeTextNodes(child); + break; + default: + break; + } + } +} + +/** + * Test whether any of the direct child text nodes of are non-whitespace + * text nodes. + * + * For example: + * - `<p>test</p>`: yes + * - `<p> </p>`: no + * - `<p><b>test</b></p>`: no + * @param {Node} node + * @returns {boolean} + */ +function hasTextNodes(node) { + if (node.nodeType !== Node.ELEMENT_NODE) { + // Only check element nodes. + return false; + } + + for (const child of node.childNodes) { + if (child.nodeType === Node.TEXT_NODE) { + if (child.textContent.trim() === "") { + // This is just whitespace. + continue; + } + // A text node with content was found. + return true; + } + } + + // No text nodes were found. + return false; +} + +/** + * Like `isExcludedNode` but looks at the full subtree. Used to see whether + * we can submit a subtree, or whether we should split it into smaller + * branches first to try to exclude more of the non-translatable content. + * + * @param {Node} node + * @param {string} excludedNodeSelector + * @returns {boolean} + */ +function containsExcludedNode(node, excludedNodeSelector) { + return ( + node.nodeType === Node.ELEMENT_NODE && + node.querySelector(excludedNodeSelector) + ); +} + +/** + * Check if this node has already been queued to be translated. This can be because + * the node is itself is queued, or its parent node is queued. + * + * @param {Node} node + * @param {Map<Node, any>} queuedNodes + * @returns {boolean} + */ +function isNodeQueued(node, queuedNodes) { + if (queuedNodes.has(node)) { + return true; + } + + // If the immediate parent is the body, it is allowed. + if (node.parentNode === node.ownerDocument.body) { + return false; + } + + // Accessing the parentNode is expensive here according to performance profilling. This + // is due to XrayWrappers. Minimize reading attributes by storing a reference to the + // `parentNode` in a named variable, rather than re-accessing it. + let parentNode; + let lastNode = node; + while ((parentNode = lastNode.parentNode)) { + if (queuedNodes.has(parentNode)) { + return parentNode; + } + lastNode = parentNode; + } + + return false; +} + +/** + * Test whether this node should be treated as a wrapper of text, e.g. + * a `<p>`, or as a wrapper for block elements, e.g. `<div>`, based on + * its ratio of assumed inline elements, and assumed "block" elements. If it is a wrapper + * of block elements, then it needs more subdividing. This algorithm is based on + * heuristics and is a best effort attempt at sorting contents without actually computing + * the style of every element. + * + * If it's a Text node, it's inline and doesn't need subdividing. + * + * "Lorem ipsum" + * + * If it is mostly filled with assumed "inline" elements, treat it as inline. + * <p> + * Lorem ipsum dolor sit amet, consectetur adipiscing elit. + * <b>Nullam ut finibus nibh</b>, at tincidunt tellus. + * </p> + * + * Since it has 3 "inline" elements. + * 1. "Lorem ipsum dolor sit amet, consectetur adipiscing elit." + * 2. <b>Nullam ut finibus nibh</b> + * 3. ", at tincidunt tellus." + * + * If it's mostly filled with block elements, do not treat it as inline, as it will + * need more subdividing. + * + * <section> + * Lorem ipsum <strong>dolor sit amet.</strong> + * <div>Nullam ut finibus nibh, at tincidunt tellus.</div> + * <div>Morbi pharetra mauris sed nisl mollis molestie.</div> + * <div>Donec et nibh sit amet velit tincidunt auctor.</div> + * </section> + * + * This node has 2 presumed "inline" elements: + * 1 "Lorem ipsum" + * 2. <strong>dolor sit amet.</strong>. + * + * And the 3 div "block" elements. Since 3 "block" elements > 2 "inline" elements, + * it is presumed to be "inline". + * + * @param {Node} node + * @returns {boolean} + */ +function nodeNeedsSubdividing(node) { + if (node.nodeType === Node.TEXT_NODE) { + // Text nodes are fully subdivided. + return false; + } + + let inlineElements = 0; + let blockElements = 0; + + if (node.nodeName === "TR") { + // TR elements always need subdividing, since the cells are the individual "inline" + // units. For instance the following would be invalid markup: + // + // <tr> + // This is <b>invalid</b> + // </tr> + // + // You will always have the following, which will need more subdividing. + // + // <tr> + // <td>This is <b>valid</b>.</td> + // <td>This is still valid.</td> + // </tr> + return true; + } + + for (let child of node.childNodes) { + switch (child.nodeType) { + case Node.TEXT_NODE: + if (!isNodeTextEmpty(child)) { + inlineElements += 1; + } + break; + case Node.ELEMENT_NODE: { + // Property access can be expensive, so destructure the required properties. + const { nodeName } = child; + if (INLINE_TAGS.has(nodeName)) { + inlineElements += 1; + } else if (GENERIC_TAGS.has(nodeName) && !nodeNeedsSubdividing(child)) { + inlineElements += 1; + } else { + blockElements += 1; + } + break; + } + default: + break; + } + } + + return inlineElements < blockElements; +} + +/** + * Returns an iterator of a node's ancestors. + * + * @param {Node} node + * @returns {Generator<ParentNode>} + */ +function* getAncestorsIterator(node) { + const document = node.ownerDocument; + for ( + let parent = node.parentNode; + parent && parent !== document.documentElement; + parent = parent.parentNode + ) { + yield parent; + } +} diff --git a/toolkit/components/translations/content/translations-engine-worker.js b/toolkit/components/translations/content/translations-engine-worker.js new file mode 100644 index 0000000000..3cf12d1e92 --- /dev/null +++ b/toolkit/components/translations/content/translations-engine-worker.js @@ -0,0 +1,780 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env mozilla/chrome-worker */ +"use strict"; + +/** + * @typedef {import("../translations").Bergamot} Bergamot + * @typedef {import("../translations").LanguageTranslationModelFiles} LanguageTranslationModelFiles + */ + +/* global loadBergamot */ +importScripts("chrome://global/content/translations/bergamot-translator.js"); + +// Respect the preference "browser.translations.logLevel". +let _loggingLevel = "Error"; +function log(...args) { + if (_loggingLevel !== "Error" && _loggingLevel !== "Warn") { + console.log("Translations:", ...args); + } +} +function trace(...args) { + if (_loggingLevel === "Trace" || _loggingLevel === "All") { + console.log("Translations:", ...args); + } +} + +// Throw Promise rejection errors so that they are visible in the console. +self.addEventListener("unhandledrejection", event => { + throw event.reason; +}); + +/** + * The alignment for each file type, file type strings should be same as in the + * model registry. + */ +const MODEL_FILE_ALIGNMENTS = { + model: 256, + lex: 64, + vocab: 64, + qualityModel: 64, + srcvocab: 64, + trgvocab: 64, +}; + +/** + * Initialize the engine, and get it ready to handle translation requests. + * The "initialize" message must be received before any other message handling + * requests will be processed. + */ +addEventListener("message", handleInitializationMessage); + +async function handleInitializationMessage({ data }) { + const startTime = performance.now(); + if (data.type !== "initialize") { + console.error( + "The TranslationEngine worker received a message before it was initialized." + ); + return; + } + + try { + const { fromLanguage, toLanguage, enginePayload, logLevel, innerWindowId } = + data; + + if (!fromLanguage) { + throw new Error('Worker initialization missing "fromLanguage"'); + } + if (!toLanguage) { + throw new Error('Worker initialization missing "toLanguage"'); + } + + if (logLevel) { + // Respect the "browser.translations.logLevel" preference. + _loggingLevel = logLevel; + } + + let engine; + if (enginePayload.isMocked) { + // The engine is testing mode, and no Bergamot wasm is available. + engine = new MockedEngine(fromLanguage, toLanguage); + } else { + const { bergamotWasmArrayBuffer, languageModelFiles } = enginePayload; + const bergamot = await BergamotUtils.initializeWasm( + bergamotWasmArrayBuffer + ); + engine = new Engine( + fromLanguage, + toLanguage, + bergamot, + languageModelFiles + ); + } + + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { startTime, innerWindowId }, + "Translations engine loaded." + ); + + handleMessages(engine); + postMessage({ type: "initialization-success" }); + } catch (error) { + console.error(error); + postMessage({ type: "initialization-error", error: error?.message }); + } + + removeEventListener("message", handleInitializationMessage); +} + +/** + * Sets up the message handling for the worker. + * + * @param {Engine | MockedEngine} engine + */ +function handleMessages(engine) { + let discardPromise; + addEventListener("message", async ({ data }) => { + try { + if (data.type === "initialize") { + throw new Error("The Translations engine must not be re-initialized."); + } + if (data.type === "translation-request") { + // Only show these messages when "All" logging is on, since there are so many + // of them. + trace("Received message", data); + } else { + log("Received message", data); + } + + switch (data.type) { + case "translation-request": { + const { messageBatch, messageId, isHTML, innerWindowId } = data; + if (discardPromise) { + // Wait for messages to be discarded if there are any. + await discardPromise; + } + try { + // Add translations to the work queue, and when they return, post the message + // back. The translation may never return if the translations are discarded + // before they have time to be run. In this case this await is just never + // resolved, and the postMessage is never run. + const translations = await engine.translate( + messageBatch, + isHTML, + innerWindowId + ); + + // This logging level can be very verbose and slow, so only do it under the + // "Trace" level, which is the most verbose. Set the logging level to "Info" to avoid + // these, and get all of the other logs. + trace("Translation complete", { + messageBatch, + translations, + isHTML, + innerWindowId, + }); + + postMessage({ + type: "translation-response", + translations, + messageId, + }); + } catch (error) { + console.error(error); + let message = "An error occurred in the engine worker."; + if (typeof error?.message === "string") { + message = error.message; + } + let stack = "(no stack)"; + if (typeof error?.stack === "string") { + stack = error.stack; + } + postMessage({ + type: "translation-error", + error: { message, stack }, + messageId, + innerWindowId, + }); + } + break; + } + case "discard-translation-queue": { + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { innerWindowId: data.innerWindowId }, + "Translations discard requested" + ); + + discardPromise = engine.discardTranslations(); + await discardPromise; + discardPromise = null; + + // Signal to the "message" listeners in the main thread to stop listening. + postMessage({ + type: "translations-discarded", + }); + break; + } + default: + console.warn("Unknown message type:", data.type); + } + } catch (error) { + // Ensure the unexpected errors are surfaced in the console. + console.error(error); + } + }); +} + +/** + * The Engine is created once for a language pair. The initialization process copies the + * ArrayBuffers for the language buffers from JS-managed ArrayBuffers, to aligned + * internal memory for the wasm heap. + * + * After this the ArrayBuffers are discarded and GC'd. This file should be managed + * from the TranslationsEngine class on the main thread. + * + * This class starts listening for messages only after the Bergamot engine has been + * fully initialized. + */ +class Engine { + /** + * @param {string} fromLanguage + * @param {string} toLanguage + * @param {Bergamot} bergamot + * @param {Array<LanguageTranslationModelFiles>} languageTranslationModelFiles + */ + constructor( + fromLanguage, + toLanguage, + bergamot, + languageTranslationModelFiles + ) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + /** @type {Bergamot} */ + this.bergamot = bergamot; + /** @type {Bergamot["TranslationModel"][]} */ + this.languageTranslationModels = languageTranslationModelFiles.map( + languageTranslationModelFiles => + BergamotUtils.constructSingleTranslationModel( + bergamot, + languageTranslationModelFiles + ) + ); + + /** @type {Bergamot["BlockingService"]} */ + this.translationService = new bergamot.BlockingService({ + // Caching is disabled (see https://github.com/mozilla/firefox-translations/issues/288) + cacheSize: 0, + }); + } + + /** + * Run the translation models to perform a batch of message translations. The + * promise is rejected when the sync version of this function throws an error. + * This function creates an async interface over the synchronous translation + * mechanism. This allows other microtasks such as message handling to still work + * even though the translations are CPU-intensive. + * + * @param {string[]} messageBatch + * @param {boolean} isHTML + * @param {number} innerWindowId - This is required + * + * @param {boolean} withQualityEstimation + * @returns {Promise<string[]>} + */ + translate( + messageBatch, + isHTML, + innerWindowId, + withQualityEstimation = false + ) { + return this.#getWorkQueue(innerWindowId).runTask(() => + this.#syncTranslate( + messageBatch, + isHTML, + innerWindowId, + withQualityEstimation + ) + ); + } + + /** + * Map each innerWindowId to its own WorkQueue. This makes it easy to shut down + * an entire queue of work when the page is unloaded. + * + * @type {Map<number, WorkQueue>} + */ + #workQueues = new Map(); + + /** + * Get or create a `WorkQueue` that is unique to an `innerWindowId`. + * + * @param {number} innerWindowId + * @returns {WorkQueue} + */ + #getWorkQueue(innerWindowId) { + let workQueue = this.#workQueues.get(innerWindowId); + if (workQueue) { + return workQueue; + } + workQueue = new WorkQueue(innerWindowId); + this.#workQueues.set(innerWindowId, workQueue); + return workQueue; + } + + /** + * Cancels any in-progress translations by removing the work queue. + * + * @param {number} innerWindowId + */ + discardTranslations(innerWindowId) { + let workQueue = this.#workQueues.get(innerWindowId); + if (workQueue) { + workQueue.cancelWork(); + this.#workQueues.delete(innerWindowId); + } + } + + /** + * Run the translation models to perform a batch of message translations. This + * blocks the worker thread until it is completed. + * + * @param {string[]} messageBatch + * @param {boolean} isHTML + * @param {number} innerWindowId + * @param {boolean} withQualityEstimation + * @returns {string[]} + */ + #syncTranslate( + messageBatch, + isHTML, + innerWindowId, + withQualityEstimation = false + ) { + const startTime = performance.now(); + let response; + const { messages, options } = BergamotUtils.getTranslationArgs( + this.bergamot, + messageBatch, + isHTML, + withQualityEstimation + ); + try { + if (messages.size() === 0) { + return []; + } + + /** @type {Bergamot["VectorResponse"]} */ + let responses; + + if (this.languageTranslationModels.length === 1) { + responses = this.translationService.translate( + this.languageTranslationModels[0], + messages, + options + ); + } else if (this.languageTranslationModels.length === 2) { + responses = this.translationService.translateViaPivoting( + this.languageTranslationModels[0], + this.languageTranslationModels[1], + messages, + options + ); + } else { + throw new Error( + "Too many models were provided to the translation worker." + ); + } + + // Extract JavaScript values out of the vector. + const translations = BergamotUtils.mapVector(responses, response => + response.getTranslatedText() + ); + + // Report on the time it took to do these translations. + let length = 0; + for (const message of messageBatch) { + length += message.length; + } + ChromeUtils.addProfilerMarker( + "TranslationsWorker", + { startTime, innerWindowId }, + `Translated ${length} code units.` + ); + + return translations; + } finally { + // Free up any memory that was allocated. This will always run. + messages?.delete(); + options?.delete(); + response?.delete(); + } + } +} + +/** + * Static utilities to help work with the Bergamot wasm module. + */ +class BergamotUtils { + /** + * Construct a single translation model. + * + * @param {Bergamot} bergamot + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles + * @returns {Bergamot["TranslationModel"]} + */ + static constructSingleTranslationModel( + bergamot, + languageTranslationModelFiles + ) { + log(`Constructing translation model.`); + + const { model, lex, vocab, qualityModel, srcvocab, trgvocab } = + BergamotUtils.allocateModelMemory( + bergamot, + languageTranslationModelFiles + ); + + // Transform the bytes to mb, like "10.2mb" + const getMemory = memory => `${Math.floor(memory.size() / 100_000) / 10}mb`; + + let memoryLog = `Model memory sizes in wasm heap:`; + memoryLog += `\n Model: ${getMemory(model)}`; + memoryLog += `\n Shortlist: ${getMemory(lex)}`; + + // Set up the vocab list, which could either be a single "vocab" model, or a + // "srcvocab" and "trgvocab" pair. + const vocabList = new bergamot.AlignedMemoryList(); + + if (vocab) { + vocabList.push_back(vocab); + memoryLog += `\n Vocab: ${getMemory(vocab)}`; + } else if (srcvocab && trgvocab) { + vocabList.push_back(srcvocab); + vocabList.push_back(trgvocab); + memoryLog += `\n Src Vocab: ${getMemory(srcvocab)}`; + memoryLog += `\n Trg Vocab: ${getMemory(trgvocab)}`; + } else { + throw new Error("Vocabulary key is not found."); + } + + if (qualityModel) { + memoryLog += `\n QualityModel: ${getMemory(qualityModel)}\n`; + } + + const config = BergamotUtils.generateTextConfig({ + "beam-size": "1", + normalize: "1.0", + "word-penalty": "0", + "max-length-break": "128", + "mini-batch-words": "1024", + workspace: "128", + "max-length-factor": "2.0", + "skip-cost": (!qualityModel).toString(), + "cpu-threads": "0", + quiet: "true", + "quiet-translation": "true", + "gemm-precision": + languageTranslationModelFiles.model.record.name.endsWith("intgemm8.bin") + ? "int8shiftAll" + : "int8shiftAlphaAll", + alignment: "soft", + }); + + log(`Bergamot translation model config: ${config}`); + log(memoryLog); + + return new bergamot.TranslationModel( + config, + model, + lex, + vocabList, + qualityModel ?? null + ); + } + + /** + * The models must be placed in aligned memory that the Bergamot wasm module has access + * to. This function copies over the model blobs into this memory space. + * + * @param {Bergamot} bergamot + * @param {LanguageTranslationModelFiles} languageTranslationModelFiles + * @returns {LanguageTranslationModelFilesAligned} + */ + static allocateModelMemory(bergamot, languageTranslationModelFiles) { + /** @type {LanguageTranslationModelFilesAligned} */ + const results = {}; + + for (const [fileType, file] of Object.entries( + languageTranslationModelFiles + )) { + const alignment = MODEL_FILE_ALIGNMENTS[fileType]; + if (!alignment) { + throw new Error(`Unknown file type: "${fileType}"`); + } + + const alignedMemory = new bergamot.AlignedMemory( + file.buffer.byteLength, + alignment + ); + + alignedMemory.getByteArrayView().set(new Uint8Array(file.buffer)); + + results[fileType] = alignedMemory; + } + + return results; + } + + /** + * Initialize the Bergamot translation engine. It is a wasm compiled version of the + * Marian translation software. The wasm is delivered remotely to cut down on binary size. + * + * https://github.com/mozilla/bergamot-translator/ + * + * @param {ArrayBuffer} wasmBinary + * @returns {Promise<Bergamot>} + */ + static initializeWasm(wasmBinary) { + return new Promise((resolve, reject) => { + /** @type {number} */ + let start = performance.now(); + + /** @type {Bergamot} */ + const bergamot = loadBergamot({ + // This is the amount of memory that a simple run of Bergamot uses, in byte. + INITIAL_MEMORY: 459_276_288, + preRun: [], + onAbort() { + reject(new Error("Error loading Bergamot wasm module.")); + }, + onRuntimeInitialized: async () => { + const duration = performance.now() - start; + log( + `Bergamot wasm runtime initialized in ${duration / 1000} seconds.` + ); + // Await at least one microtask so that the captured `bergamot` variable is + // fully initialized. + await Promise.resolve(); + resolve(bergamot); + }, + wasmBinary, + }); + }); + } + + /** + * Maps the Bergamot Vector to a JS array + * + * @param {Bergamot["Vector"]} vector + * @param {Function} fn + * @returns {Array} + */ + static mapVector(vector, fn) { + const result = []; + for (let index = 0; index < vector.size(); index++) { + result.push(fn(vector.get(index), index)); + } + return result; + } + + /** + * Generate a config for the Marian translation service. It requires specific whitespace. + * + * https://marian-nmt.github.io/docs/cmd/marian-decoder/ + * + * @param {Record<string, string>} config + * @returns {string} + */ + static generateTextConfig(config) { + const indent = " "; + let result = "\n"; + + for (const [key, value] of Object.entries(config)) { + result += `${indent}${key}: ${value}\n`; + } + + return result + indent; + } + + /** + * JS objects need to be translated into wasm objects to configure the translation engine. + * + * @param {Bergamot} bergamot + * @param {string[]} messageBatch + * @param {boolean} withQualityEstimation + * @returns {{ messages: Bergamot["VectorString"], options: Bergamot["VectorResponseOptions"] }} + */ + static getTranslationArgs( + bergamot, + messageBatch, + isHTML, + withQualityEstimation + ) { + const messages = new bergamot.VectorString(); + const options = new bergamot.VectorResponseOptions(); + for (let message of messageBatch) { + message = message.trim(); + // Empty paragraphs break the translation. + if (message === "") { + continue; + } + + if (withQualityEstimation && !isHTML) { + // Bergamot only supports quality estimates with HTML. Purely text content can + // be translated by escaping it as HTML. See: + // https://github.com/mozilla/firefox-translations/blob/431e0d21f22694c1cbc0ff965820d9780cdaeea8/extension/controller/translation/translationWorker.js#L146-L158 + throw new Error( + "Quality estimates on non-hTML is not curently supported." + ); + } + + messages.push_back(message); + options.push_back({ + qualityScores: withQualityEstimation, + alignment: true, + html: isHTML, + }); + } + return { messages, options }; + } +} + +/** + * For testing purposes, provide a fully mocked engine. This allows for easy integration + * testing of the UI, without having to rely on downloading remote models and remote + * wasm binaries. + */ +class MockedEngine { + /** + * @param {string} fromLanguage + * @param {string} toLanguage + */ + constructor(fromLanguage, toLanguage) { + /** @type {string} */ + this.fromLanguage = fromLanguage; + /** @type {string} */ + this.toLanguage = toLanguage; + } + + /** + * Create a fake translation of the text. + * + * @param {string[]} messageBatch + * @param {bool} isHTML + * @returns {string} + */ + translate(messageBatch, isHTML) { + return messageBatch.map(message => { + // Note when an HTML translations is requested. + let html = isHTML ? ", html" : ""; + message = message.toUpperCase(); + + return `${message} [${this.fromLanguage} to ${this.toLanguage}${html}]`; + }); + } + + discardTranslations() {} +} + +/** + * This class takes tasks that may block the thread's event loop, and has them yield + * after a time budget via setTimeout calls to allow other code to execute. + */ +class WorkQueue { + #TIME_BUDGET = 100; // ms + #RUN_IMMEDIATELY_COUNT = 20; + + /** @type {Array<{task: Function, resolve: Function}>} */ + #tasks = []; + #isRunning = false; + #isWorkCancelled = false; + #runImmediately = this.#RUN_IMMEDIATELY_COUNT; + + /** + * @param {number} innerWindowId + */ + constructor(innerWindowId) { + this.innerWindowId = innerWindowId; + } + + /** + * Run the task and return the result. + * + * @template {T} + * @param {() => T} task + * @returns {Promise<T>} + */ + runTask(task) { + if (this.#runImmediately > 0) { + // Run the first N translations immediately, most likely these are the user-visible + // translations on the page, as they are sent in first. The setTimeout of 0 can + // still delay the translations noticeably. + this.#runImmediately--; + return Promise.resolve(task()); + } + return new Promise((resolve, reject) => { + this.#tasks.push({ task, resolve, reject }); + this.#run().catch(error => console.error(error)); + }); + } + + /** + * The internal run function. + */ + async #run() { + if (this.#isRunning) { + // The work queue is already running. + return; + } + + this.#isRunning = true; + + // Measure the timeout + let lastTimeout = null; + + let tasksInBatch = 0; + const addProfilerMarker = () => { + ChromeUtils.addProfilerMarker( + "TranslationsWorker WorkQueue", + { startTime: lastTimeout, innerWindowId: this.innerWindowId }, + `WorkQueue processed ${tasksInBatch} tasks` + ); + }; + + while (this.#tasks.length !== 0) { + if (this.#isWorkCancelled) { + // The work was already cancelled. + break; + } + const now = performance.now(); + + if (lastTimeout === null) { + lastTimeout = now; + // Allow other work to get on the queue. + await new Promise(resolve => setTimeout(resolve, 0)); + } else if (now - lastTimeout > this.#TIME_BUDGET) { + // Perform a timeout with no effective wait. This clears the current + // promise queue from the event loop. + await new Promise(resolve => setTimeout(resolve, 0)); + addProfilerMarker(); + lastTimeout = performance.now(); + } + + // Check this between every `await`. + if (this.#isWorkCancelled) { + break; + } + + tasksInBatch++; + const { task, resolve, reject } = this.#tasks.shift(); + try { + const result = await task(); + + // Check this between every `await`. + if (this.#isWorkCancelled) { + break; + } + // The work is done, resolve the original task. + resolve(result); + } catch (error) { + reject(error); + } + } + addProfilerMarker(); + this.isRunning = false; + } + + async cancelWork() { + this.#isWorkCancelled = true; + this.#tasks = []; + await new Promise(resolve => setTimeout(resolve, 0)); + this.#isWorkCancelled = false; + } +} diff --git a/toolkit/components/translations/content/translations.css b/toolkit/components/translations/content/translations.css new file mode 100644 index 0000000000..094ebde75d --- /dev/null +++ b/toolkit/components/translations/content/translations.css @@ -0,0 +1,174 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +:root { + /* Provide defaults for when this page is viewed in "toolkit". */ + background-color: var(--in-content-page-background, #fff); + color: var(--in-content-page-color, #15141a); + + /* Provide backup values for some of the variables used in "browser" so that the styles + look nice by default in "toolkit". */ + --AT-box-background: var(--in-content-box-background, #fff); + --AT-box-border-color: var(--in-content-box-border-color, #9e9ea0); + --AT-box-info-background: var(--in-content-box-info-background, #f0f0f4); + + /* Variables used in the page layout */ + --AT-page-margin: 20px; + --AT-input-padding: 20px; + /* This is somewhat arbitrary, but works well for the current design. If the computed + header height changes, this will need to be adjusted. */ + --AT-header-height: 156px; + --AT-input-height: calc(min(400px, calc(100vh - var(--AT-header-height)))); + --AT-select-arrow-inset: 5px; +} + +h1 { + /* Provide this style for "toolkit". It is defined in "browser" */ + font-weight: lighter; +} + +body { + display: flex; + justify-content: center; + align-items: center; + inset: 0; + position: absolute; + visibility: hidden; + flex-direction: column; +} + +.about-translations-header { + display: flex; +} + +.about-translations-header > * { + flex: 1; + display: flex; + max-width: 50%; +} + +.about-translations-header-start { + justify-content: start; +} + +.about-translations-header-end { + justify-content: end; +} + +/* Increase the selector specificity to override the base `select` styles. */ +select.about-translations-select { + position: relative; + padding-inline: 10px 20px; + padding-block: 0px; + min-width: 50%; + margin: 5px; + background-position: right var(--AT-select-arrow-inset) center; +} + +select.about-translations-select:dir(rtl) { + background-position-x: left var(--AT-select-arrow-inset); +} + +.about-translations-contents { + display: flex; + flex-direction: column; + box-sizing: border-box; + width: calc(100% - var(--AT-page-margin) * 2); + max-width: 1200px; + background-color: var(--AT-box-background); + border: 1px solid var(--AT-box-border-color); + border-radius: 4px; +} + +.about-translations-input { + display: flex; + width: 100%; + border-top: 1px solid var(--AT-box-border-color); +} + +.about-translations-input-start { + border-inline-end: 1px solid var(--AT-box-border-color); +} + +.about-translations-input > * { + position: relative; + width: 50%; +} + +.about-translations-input-textarea { + /* Override user's dragging of the textarea width. */ + width: 100% !important; + height: var(--AT-input-height); + box-sizing: border-box; + margin: 0; + padding: var(--AT-input-padding); + border: 0; +} + +.about-translations-input-results-blank { + opacity: 0.7; +} + +.about-translations-input-results { + position: absolute; + inset: 0; + padding: var(--AT-input-padding); + box-sizing: border-box; + overflow-y: scroll; +} + +.about-translations-info { + display: none; + padding: 10px; + background-color: var(--AT-box-info-background); + border-radius: 4px; + margin-bottom: var(--AT-input-padding); +} + +.about-translations-info-message { + flex: 1; + align-self: center; +} + +.about-translations-info-icon { + width: 16px; + height: 16px; + margin: 10px; + background-image: url('chrome://global/skin/icons/info.svg'); + -moz-context-properties: fill; + fill: currentColor; +} + +@media (max-width: 700px) { + :root { + --AT-page-margin: 10px; + } + h1 { + margin-top: 15px; + } + body { + padding-bottom: var(--AT-page-margin); + } + .about-translations-input { + flex-direction: column; + flex: 1; + } + .about-translations-input-textarea, + .about-translations-input { + font-size: 16px; + } + .about-translations-input > * { + width: 100%; + flex: 1; + } + .about-translations-input-end { + border-top: 1px solid var(--AT-box-border-color); + } + .about-translations-input-textarea { + height: 100%; + } + .about-translations-contents { + flex: 1; + } +} diff --git a/toolkit/components/translations/content/translations.html b/toolkit/components/translations/content/translations.html new file mode 100644 index 0000000000..bd2c114a0a --- /dev/null +++ b/toolkit/components/translations/content/translations.html @@ -0,0 +1,70 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Security-Policy" content="default-src chrome:; object-src 'none'"> + <meta name="color-scheme" content="light dark"> + <meta name="viewport" content="width=device-width" /> + <title data-l10n-id="about-translations-title"></title> + <link rel="stylesheet" href="chrome://global/skin/global.css"> + <link rel="stylesheet" href="chrome://global/skin/in-content/common.css"> + <link rel="stylesheet" href="chrome://global/content/translations/translations.css"> + <link rel="localization" href="toolkit/branding/brandings.ftl"/> + <link rel="localization" href="locales-preview/aboutTranslations.ftl"/> + <script type="module" src="chrome://global/content/translations/translations.mjs"></script> + </head> + <body> + <h1 data-l10n-id="about-translations-header"></h1> + <main class="about-translations-contents"> + + <header class="about-translations-header"> + <div class="about-translations-header-start"> + <select + class="about-translations-select" + id="language-from" + disabled> + <option data-l10n-id="about-translations-detect" value="detect"></option> + </select> + </div> + <div class="about-translations-header-end"> + <select + class="about-translations-select" + id="language-to" + disabled> + <option data-l10n-id="about-translations-select" value=""></option> + </select> + </div> + </header> + + <main class="about-translations-input"> + <div class="about-translations-input-start"> + <textarea + class="about-translations-input-textarea" + data-l10n-id="about-translations-textarea" + id="translation-from" + ></textarea> + </div> + <div class="about-translations-input-end"> + <div + class="about-translations-input-results about-translations-input-results-blank" + id="translation-to-blank"> + <div class="about-translations-info" id="translation-info"> + <div class="about-translations-info-icon"></div> + <div class="about-translations-info-message" id="translation-info-message"></div> + </div> + <div data-l10n-id="about-translations-results-placeholder"></div> + </div> + <div + class="about-translations-input-results" + id="translation-to"> + </div> + </div> + </main> + + </div> + </body> +</html> diff --git a/toolkit/components/translations/content/translations.mjs b/toolkit/components/translations/content/translations.mjs new file mode 100644 index 0000000000..b279b03b8b --- /dev/null +++ b/toolkit/components/translations/content/translations.mjs @@ -0,0 +1,690 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The following globals are injected via the AboutTranslationsChild actor. +// translations.mjs is running in an unprivileged context, and these injected functions +// allow for the page to get access to additional privileged features. + +/* global AT_getSupportedLanguages, AT_log, AT_getScriptDirection, + AT_logError, AT_destroyTranslationsEngine, AT_createTranslationsEngine, + AT_isTranslationEngineSupported, AT_createLanguageIdEngine, AT_translate, AT_identifyLanguage */ + +// Allow tests to override this value so that they can run faster. +// This is the delay in milliseconds. +window.DEBOUNCE_DELAY = 200; +// Allow tests to test the debounce behavior by counting debounce runs. +window.DEBOUNCE_RUN_COUNT = 0; + +/** + * @typedef {import("../translations").SupportedLanguages} SupportedLanguages + */ + +/** + * The model and controller for initializing about:translations. + */ +class TranslationsState { + /** + * This class is responsible for all UI updated. + * + * @type {TranslationsUI} + */ + ui; + + /** + * The language to translate from, in the form of a BCP 47 language tag, + * e.g. "en" or "fr". + * + * @type {string} + */ + fromLanguage = ""; + + /** + * The language to translate to, in the form of a BCP 47 language tag, + * e.g. "en" or "fr". + * + * @type {string} + */ + toLanguage = ""; + + /** + * The message to translate, cached so that it can be determined if the text + * needs to be re-translated. + * + * @type {string} + */ + messageToTranslate = ""; + + /** + * Only send one translation in at a time to the worker. + * @type {Promise<string[]>} + */ + translationRequest = Promise.resolve([]); + + /** + * The translations engine is only valid for a single language pair, and needs + * to be recreated if the language pair changes. + * + * @type {null | Promise<TranslationsEngine>} + */ + translationsEngine = null; + + /** + * @param {boolean} isSupported + */ + constructor(isSupported) { + /** + * Is the engine supported by the device? + * @type {boolean} + */ + this.isTranslationEngineSupported = isSupported; + + /** + * Allow code to wait for the engine to be created. + * @type {Promise<void>} + */ + this.languageIdEngineCreated = isSupported + ? AT_createLanguageIdEngine() + : Promise.resolve(); + + /** + * @type {SupportedLanguages} + */ + this.supportedLanguages = isSupported + ? AT_getSupportedLanguages() + : Promise.resolve([]); + + this.ui = new TranslationsUI(this); + this.ui.setup(); + + // Set the UI as ready after all of the state promises have settled. + Promise.allSettled([ + this.languageIdEngineCreated, + this.supportedLanguages, + ]).then(() => { + this.ui.setAsReady(); + }); + } + + /** + * Identifies the human language in which the message is written and returns + * the BCP 47 language tag of the language it is determined to be. + * + * e.g. "en" for English. + * + * @param {string} message + */ + async identifyLanguage(message) { + await this.languageIdEngineCreated; + const start = performance.now(); + const { langTag, confidence } = await AT_identifyLanguage(message); + const duration = performance.now() - start; + AT_log( + `[ ${langTag}(${(confidence * 100).toFixed(2)}%) ]`, + `Source language identified in ${duration / 1000} seconds` + ); + return langTag; + } + + /** + * Only request a translation when it's ready. + */ + maybeRequestTranslation = debounce({ + /** + * Debounce the translation requests so that the worker doesn't fire for every + * single keyboard input, but instead the keyboard events are ignored until + * there is a short break, or enough events have happened that it's worth sending + * in a new translation request. + */ + onDebounce: async () => { + // The contents of "this" can change between async steps, store a local variable + // binding of these values. + const { + fromLanguage, + toLanguage, + messageToTranslate, + translationsEngine, + } = this; + + if (!this.isTranslationEngineSupported) { + // Never translate when the engine isn't supported. + return; + } + + if ( + !fromLanguage || + !toLanguage || + !messageToTranslate || + !translationsEngine + ) { + // Not everything is set for translation. + this.ui.updateTranslation(""); + return; + } + + await Promise.all([ + // Ensure the engine is ready to go. + translationsEngine, + // Ensure the previous translation has finished so that only the latest + // translation goes through. + this.translationRequest, + ]); + + if ( + // Check if the current configuration has changed and if this is stale. If so + // then skip this request, as there is already a newer request with more up to + // date information. + this.translationsEngine !== translationsEngine || + this.fromLanguage !== fromLanguage || + this.toLanguage !== toLanguage || + this.messageToTranslate !== messageToTranslate + ) { + return; + } + + const start = performance.now(); + + this.translationRequest = AT_translate([messageToTranslate]); + const [translation] = await this.translationRequest; + + // The measure events will show up in the Firefox Profiler. + performance.measure( + `Translations: Translate "${this.fromLanguage}" to "${this.toLanguage}" with ${messageToTranslate.length} characters.`, + { + start, + end: performance.now(), + } + ); + + this.ui.updateTranslation(translation); + const duration = performance.now() - start; + AT_log(`Translation done in ${duration / 1000} seconds`); + }, + + // Mark the events so that they show up in the Firefox Profiler. This makes it handy + // to visualize the debouncing behavior. + doEveryTime: () => { + performance.mark( + `Translations: input changed to ${this.messageToTranslate.length} characters` + ); + }, + }); + + /** + * Any time a language pair is changed, the TranslationsEngine needs to be rebuilt. + */ + async maybeRebuildWorker() { + // If we may need to re-building the worker, the old translation is no longer valid. + this.ui.updateTranslation(""); + + // These are cases in which it wouldn't make sense or be possible to load any translations models. + if ( + // If fromLanguage or toLanguage are unpopulated we cannot load anything. + !this.fromLanguage || + !this.toLanguage || + // If fromLanguage's value is "detect", rather than a BCP 47 language tag, then no language + // has been detected yet. + this.fromLanguage === "detect" || + // If fromLanguage and toLanguage are the same, this means that the detected language + // is the same as the toLanguage, and we do not want to translate from one language to itself. + this.fromLanguage === this.toLanguage + ) { + if (this.translationsEngine) { + // The engine is no longer needed. + AT_destroyTranslationsEngine(); + this.translationsEngine = null; + } + return; + } + + const start = performance.now(); + AT_log( + `Rebuilding the translations worker for "${this.fromLanguage}" to "${this.toLanguage}"` + ); + + this.translationsEngine = AT_createTranslationsEngine( + this.fromLanguage, + this.toLanguage + ); + this.maybeRequestTranslation(); + + try { + await this.translationsEngine; + const duration = performance.now() - start; + AT_log(`Rebuilt the TranslationsEngine in ${duration / 1000} seconds`); + } catch (error) { + this.ui.showInfo("about-translations-engine-error"); + AT_logError("Failed to get the Translations worker", error); + } + } + + /** + * Updates the fromLanguage to match the detected language only if the + * about-translations-detect option is selected in the language-from dropdown. + * + * If the new fromLanguage is different than the previous fromLanguage this + * may update the UI to display the new language and may rebuild the translations + * worker if there is a valid selected target language. + */ + async maybeUpdateDetectedLanguage() { + if (!this.ui.detectOptionIsSelected() || this.messageToTranslate === "") { + // If we are not detecting languages or if the message has been cleared + // we should ensure that the UI is not displaying a detected language + // and there is no need to run any language detection. + this.ui.setDetectOptionTextContent(""); + return; + } + + const [langTag, supportedLanguages] = await Promise.all([ + this.identifyLanguage(this.messageToTranslate), + this.supportedLanguages, + ]); + + // Only update the language if the detected language matches + // one of our supported languages. + const entry = supportedLanguages.fromLanguages.find( + ({ langTag: existingTag }) => existingTag === langTag + ); + if (entry) { + const { displayName, isBeta } = entry; + await this.setFromLanguage(langTag); + this.ui.setDetectOptionTextContent(displayName, isBeta); + } + } + + /** + * @param {string} lang + */ + async setFromLanguage(lang) { + if (lang !== this.fromLanguage) { + this.fromLanguage = lang; + await this.maybeRebuildWorker(); + } + } + + /** + * @param {string} lang + */ + setToLanguage(lang) { + if (lang !== this.toLanguage) { + this.toLanguage = lang; + this.maybeRebuildWorker(); + } + } + + /** + * @param {string} message + */ + async setMessageToTranslate(message) { + if (message !== this.messageToTranslate) { + this.messageToTranslate = message; + await this.maybeUpdateDetectedLanguage(); + this.maybeRequestTranslation(); + } + } +} + +/** + * + */ +class TranslationsUI { + /** @type {HTMLSelectElement} */ + languageFrom = document.getElementById("language-from"); + /** @type {HTMLSelectElement} */ + languageTo = document.getElementById("language-to"); + /** @type {HTMLTextAreaElement} */ + translationFrom = document.getElementById("translation-from"); + /** @type {HTMLDivElement} */ + translationTo = document.getElementById("translation-to"); + /** @type {HTMLDivElement} */ + translationToBlank = document.getElementById("translation-to-blank"); + /** @type {HTMLDivElement} */ + translationInfo = document.getElementById("translation-info"); + /** @type {HTMLDivElement} */ + translationInfoMessage = document.getElementById("translation-info-message"); + /** @type {TranslationsState} */ + state; + + /** + * The detect-language option element. We want to maintain a handle to this so that + * we can dynamically update its display text to include the detected language. + * + * @type {HTMLOptionElement} + */ + #detectOption; + + /** + * @param {TranslationsState} state + */ + constructor(state) { + this.state = state; + this.translationTo.style.visibility = "visible"; + this.#detectOption = document.querySelector('option[value="detect"]'); + } + + /** + * Do the initial setup. + */ + setup() { + if (!this.state.isTranslationEngineSupported) { + this.showInfo("about-translations-no-support"); + this.disableUI(); + return; + } + this.setupDropdowns(); + this.setupTextarea(); + } + + /** + * Signals that the UI is ready, for tests. + */ + setAsReady() { + document.body.setAttribute("ready", ""); + } + + /** + * Once the models have been synced from remote settings, populate them with the display + * names of the languages. + */ + async setupDropdowns() { + const supportedLanguages = await this.state.supportedLanguages; + + // Update the DOM elements with the display names. + for (const { + langTag, + isBeta, + displayName, + } of supportedLanguages.toLanguages) { + const option = document.createElement("option"); + option.value = langTag; + if (isBeta) { + document.l10n.setAttributes( + option, + "about-translations-displayname-beta", + { language: displayName } + ); + } else { + option.text = displayName; + } + this.languageTo.add(option); + } + + for (const { + langTag, + isBeta, + displayName, + } of supportedLanguages.fromLanguages) { + const option = document.createElement("option"); + option.value = langTag; + if (isBeta) { + document.l10n.setAttributes( + option, + "about-translations-displayname-beta", + { language: displayName } + ); + } else { + option.text = displayName; + } + this.languageFrom.add(option); + } + + // Enable the controls. + this.languageFrom.disabled = false; + this.languageTo.disabled = false; + + // Focus the language dropdowns if they are empty. + if (this.languageFrom.value == "") { + this.languageFrom.focus(); + } else if (this.languageTo.value == "") { + this.languageTo.focus(); + } + + this.state.setFromLanguage(this.languageFrom.value); + this.state.setToLanguage(this.languageTo.value); + this.updateOnLanguageChange(); + + this.languageFrom.addEventListener("input", () => { + this.state.setFromLanguage(this.languageFrom.value); + this.updateOnLanguageChange(); + }); + + this.languageTo.addEventListener("input", () => { + this.state.setToLanguage(this.languageTo.value); + this.updateOnLanguageChange(); + this.translationTo.setAttribute("lang", this.languageTo.value); + }); + } + + /** + * Show an info message to the user. + * + * @param {string} l10nId + */ + showInfo(l10nId) { + this.translationInfoMessage.setAttribute("data-l10n-id", l10nId); + this.translationInfo.style.display = "flex"; + } + + /** + * Hides the info UI. + */ + hideInfo() { + this.translationInfo.style.display = "none"; + } + + /** + * Returns true if about-translations-detect is the currently + * selected option in the language-from dropdown, otherwise false. + * + * @returns {boolean} + */ + detectOptionIsSelected() { + return this.languageFrom.value === "detect"; + } + + /** + * Sets the textContent of the about-translations-detect option in the + * language-from dropdown to include the detected language's display name. + * + * @param {string} displayName + */ + setDetectOptionTextContent(displayName, isBeta = false) { + // Set the text to the fluent value that takes an arg to display the language name. + if (displayName) { + document.l10n.setAttributes( + this.#detectOption, + isBeta + ? "about-translations-detect-lang-beta" + : "about-translations-detect-lang", + { language: displayName } + ); + } else { + // Reset the text to the fluent value that does not display any language name. + document.l10n.setAttributes( + this.#detectOption, + "about-translations-detect" + ); + } + } + + /** + * React to language changes. + */ + updateOnLanguageChange() { + this.#updateDropdownLanguages(); + this.#updateMessageDirections(); + } + + /** + * You cant translate from one language to another language. Hide the options + * if this is the case. + */ + #updateDropdownLanguages() { + for (const option of this.languageFrom.options) { + option.hidden = false; + } + for (const option of this.languageTo.options) { + option.hidden = false; + } + if (this.state.toLanguage) { + const option = this.languageFrom.querySelector( + `[value=${this.state.toLanguage}]` + ); + if (option) { + option.hidden = true; + } + } + if (this.state.fromLanguage) { + const option = this.languageTo.querySelector( + `[value=${this.state.fromLanguage}]` + ); + if (option) { + option.hidden = true; + } + } + this.state.maybeUpdateDetectedLanguage(); + } + + /** + * Define the direction of the language message text, otherwise it might not display + * correctly. For instance English in an RTL UI would display incorrectly like so: + * + * LTR text in LTR UI: + * + * ┌──────────────────────────────────────────────┐ + * │ This is in English. │ + * └──────────────────────────────────────────────┘ + * + * LTR text in RTL UI: + * ┌──────────────────────────────────────────────┐ + * │ .This is in English │ + * └──────────────────────────────────────────────┘ + * + * LTR text in RTL UI, but in an LTR container: + * ┌──────────────────────────────────────────────┐ + * │ This is in English. │ + * └──────────────────────────────────────────────┘ + * + * The effects are similar, but reversed for RTL text in an LTR UI. + */ + #updateMessageDirections() { + if (this.state.toLanguage) { + this.translationTo.setAttribute( + "dir", + AT_getScriptDirection(this.state.toLanguage) + ); + } else { + this.translationTo.removeAttribute("dir"); + } + if (this.state.fromLanguage) { + this.translationFrom.setAttribute( + "dir", + AT_getScriptDirection(this.state.fromLanguage) + ); + } else { + this.translationFrom.removeAttribute("dir"); + } + } + + setupTextarea() { + this.state.setMessageToTranslate(this.translationFrom.value); + this.translationFrom.addEventListener("input", () => { + this.state.setMessageToTranslate(this.translationFrom.value); + }); + } + + disableUI() { + this.translationFrom.disabled = true; + this.languageFrom.disabled = true; + this.languageTo.disabled = true; + } + + /** + * @param {string} message + */ + updateTranslation(message) { + this.translationTo.innerText = message; + if (message) { + this.translationTo.style.visibility = "visible"; + this.translationToBlank.style.visibility = "hidden"; + this.hideInfo(); + } else { + this.translationTo.style.visibility = "hidden"; + this.translationToBlank.style.visibility = "visible"; + } + } +} + +/** + * Listen for events coming from the AboutTranslations actor. + */ +window.addEventListener("AboutTranslationsChromeToContent", ({ detail }) => { + switch (detail.type) { + case "enable": { + // While the feature is in development, hide the feature behind a pref. See the + // "browser.translations.enable" pref in modules/libpref/init/all.js and Bug 971044 + // for the status of enabling this project. + if (window.translationsState) { + throw new Error("about:translations was already initialized."); + } + AT_isTranslationEngineSupported().then(isSupported => { + window.translationsState = new TranslationsState(isSupported); + }); + document.body.style.visibility = "visible"; + break; + } + default: + throw new Error("Unknown AboutTranslationsChromeToContent event."); + } +}); + +/** + * Debounce a function so that it is only called after some wait time with no activity. + * This is good for grouping text entry via keyboard. + * + * @param {Object} settings + * @param {Function} settings.onDebounce + * @param {Function} settings.doEveryTime + * @returns {Function} + */ +function debounce({ onDebounce, doEveryTime }) { + /** @type {number | null} */ + let timeoutId = null; + let lastDispatch = null; + + return (...args) => { + doEveryTime(...args); + + const now = Date.now(); + if (lastDispatch === null) { + // This is the first call to the function. + lastDispatch = now; + } + + const timeLeft = lastDispatch + window.DEBOUNCE_DELAY - now; + + // Always discard the old timeout, either the function will run, or a new + // timer will be scheduled. + clearTimeout(timeoutId); + + if (timeLeft <= 0) { + // It's been long enough to go ahead and call the function. + timeoutId = null; + lastDispatch = null; + window.DEBOUNCE_RUN_COUNT += 1; + onDebounce(...args); + return; + } + + // Re-set the timeout with the current time left. + clearTimeout(timeoutId); + + timeoutId = setTimeout(() => { + // Timeout ended, call the function. + timeoutId = null; + lastDispatch = null; + window.DEBOUNCE_RUN_COUNT += 1; + onDebounce(...args); + }, timeLeft); + }; +} diff --git a/toolkit/components/translations/docs/img/about-translations.png b/toolkit/components/translations/docs/img/about-translations.png Binary files differnew file mode 100644 index 0000000000..51c4d12042 --- /dev/null +++ b/toolkit/components/translations/docs/img/about-translations.png diff --git a/toolkit/components/translations/docs/index.md b/toolkit/components/translations/docs/index.md new file mode 100644 index 0000000000..7a13d5cc1c --- /dev/null +++ b/toolkit/components/translations/docs/index.md @@ -0,0 +1,17 @@ +# Firefox Translations + +Firefox Translations is a project initiative to give Firefox the ability to translate web content from one language to another language as first-class functionality in the browser. + +This project is based initially on the [Firefox Translations WebExtension]. + +## Resources + +```{toctree} +:titlesonly: +:maxdepth: 1 +:glob: + +resources/* +``` + +[Firefox Translations WebExtension]: https://github.com/mozilla/firefox-translations diff --git a/toolkit/components/translations/docs/resources/01_overview.md b/toolkit/components/translations/docs/resources/01_overview.md new file mode 100644 index 0000000000..7f4333265b --- /dev/null +++ b/toolkit/components/translations/docs/resources/01_overview.md @@ -0,0 +1,149 @@ +# Overview + +The following is a high-level overview of the technologies associated with Firefox Translations. + +- [Supported Platforms](#supported-platforms) +- [Language Translations](#language-translation) + - [Technology](#technology) + - [Models](#models) + - [Pivot Translations](#pivot-translations) +- [Language Identification](#language-identification) + - [Technology](#technology-1) + - [Models](#models-1) +- [Remote Settings](#remote-settings) + - [Enabling Firefox Translations](#enabling-firefox-translations) + - [Translating Web Pages](#translating-web-pages) + - [about:translations](#abouttranslations) + +--- +## Supported Platforms + +<input type="checkbox" style="pointer-events: none;" checked><b>Desktop</b><br> +<input type="checkbox" style="pointer-events: none;" checked><b>Android</b><br> +<input type="checkbox" style="pointer-events: none;"><b>iOS</b><br> + +```{note} +- Firefox Translations is available only on devices support [SSE4.1] due to required [SIMD] calculations in [WASM]. +``` + + +--- +## Language Translation + +Firefox Translations utilizes trained machine-learning models that run locally on client +architecture to translate web content from one language to another. + +### Technology + +Firefox Translations utilizes a [WASM] version of the [Bergamot] library to translate from +one language to another. [Bergamot] is powered by [Marian]. + +### Models + +[Bergamot] translation models are single-direction, one-to-one models trained to translate from one language +to one other language (e.g. **`en ⟶ es`**). When Firefox Translations determines a source language and a target language +it utilizes a model specific to this language pair to translate from one to the other. + +### Pivot Translations + +In the event that there is no model to translate directly from a source language and a target language, +Firefox Translations will attempt to satisfy a transitive translation path and will perform a multi-step +translation from the source language to the target language. + + + +```{admonition} Example +> **_No direct translation model exists_**<br> +> <input type="checkbox" style="pointer-events: none;"><b>`es ⟶ fr`</b><br> +> +> **_Transitive dependency satisfied_**<br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`es ⟶ en`</b><br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`en ⟶ fr`</b><br> +> +> **_Pivot translation_**<br> +> <input type="checkbox" style="pointer-events: none;" checked><b>`es ⟶ en ⟶ fr`</b><br> + +In this example, no direct model exists for **`es ⟶ fr`**, but a transitive dependency is satisfied by the two +models for **`es ⟶ en`** and **`en ⟶ fr`**. Firefox Translations will pivot on the **`en`** language by first +translating from **`es`** to **`en`** and then from **`en`** to **`fr`**. +``` +```{note} +- Firefox Translations will not pivot more than once. +- At present, only **`en`** is used as a pivot language. +``` + +--- +## Language Identification + +Firefox Translations utilizes trained machine-learning models that run locally on client +architecture to identify content as being written in a detected language. + +### Technology + +Firefox Translations utilizes a [WASM] version of the [fastText] library to identify in which +language content is written. + +### Models + +Unlike the language translations models in the [section](#language-translations) above, the [fastText] +model is a is a one-to-many model that is capable of detecting all of our supported languages +from the single model. + +--- +## Remote Settings + +Firefox Translations utilizes [Remote Settings] to download [WASM] binaries, [Language Translation](#language-translation) +models and [Language Identification](#language-identification) models to use locally on your system. + +--- +## Using Firefox Translations + +The following documentation describes a high-level overview of using Firefox Translations. + +```{note} +- Firefox Translations is actively under development and is currently available only in [Firefox Nightly]. +``` + +### Enabling Firefox Translations + +Firefox Translations functionality can be enabled by modifying the [translations preferences] in **`about:config`**. + +These configurations are likely to change as the project develops which is why this documentation links to them +in the source code rather than defining them. + +At a time when the preferences are more stable, they can be documented here more clearly. + +### Translating Web Pages + +Once Firefox Translations is enabled, Firefox will analyze each web page to determine if it is translatable +via the available translations models. + +If the web page is translatable, then a translations icon will appear in the URL bar of the browser, allowing +the user to initiate the available translation process. + +### about:translations + +When Firefox Translations is enabled, a page called **`about:translations`** becomes available in the browser. + +This is a test page where there user can select a source language and a target language by typing content into +the source-language text box and seeing the translated text in the target-language text box. + +```{note} +**`about:translations`** is a developer-focused UI that is useful for testing the state, performance, and quality of the language models in an interactive environment. It is fairly unpolished and not intended to be shipped as a product at this time. + +It is, however, useful and fun, so it is documented here. +``` + + + + +<!-- Hyperlinks --> +[Bergamot]: https://browser.mt/ +[fastText]: https://fasttext.cc/ +[Firefox Nightly]: https://www.mozilla.org/en-US/firefox/channel/desktop/ +[Marian]: https://aclanthology.org/P18-4020/ +[Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ +[SIMD]: https://en.wikipedia.org/wiki/Single_instruction,_multiple_data +[SSE4.1]: https://en.wikipedia.org/wiki/SSE4#SSE4.1 +[translations preferences]: https://searchfox.org/mozilla-central/search?q=browser.translations&path=all.js&case=true®exp=false +[WASM]: https://webassembly.org/ diff --git a/toolkit/components/translations/docs/resources/02_contributing.md b/toolkit/components/translations/docs/resources/02_contributing.md new file mode 100644 index 0000000000..227a855d28 --- /dev/null +++ b/toolkit/components/translations/docs/resources/02_contributing.md @@ -0,0 +1,453 @@ +# Contributing + +The following content goes more in-depth than the [Overview](./01_overview.md) section +to provide helpful information regarding contributing to Firefox Translations. + +- [Source Code](#source-code) +- [Architecture](#architecture) + - [JSActors](#jsactors) +- [Remote Settings](#remote-settings) + - [Admin Dashboards](#admin-dashboards) + - [Prod Admin Access](#prod-admin-access) + - [Pulling From Different Sources](#pulling-from-different-sources) + - [Versioning](#versioning) + - [Non-Breaking Changes](#non-breaking-changes) + - [Breaking Changes](#breaking-changes) +- [Building fastText](#building-fasttext) + - [Downloading The Models](#downloading-the-models) + - [Building the WASM Binary](#building-the-wasm-binary) + - [Dependencies](#dependencies) + - [Modifying the EMCXXFLAGS](#modifying-the-emcxxflags) +- [Building Bergamot](#building-bergamot) + +--- +## Source Code + +The primary source code for Firefox Translations lives in the following directory: + +> **[toolkit/components/translations]** + +--- +## Architecture + +### JSActors + +Translations functionality is divided into different classes based on which access privileges are needed. +Generally, this split is between [Parent] and [Child] versions of [JSWindowActors]. + +The [Parent] actors have access to privileged content and is responsible for things like downloading models +from [Remote Settings](#remote-settings), modifying privileged UI components etc. + +The [Child] actors are responsible for interacting with content on the page itself, requesting content from +the [Parent] actors, and creating [Workers] to carry out tasks. + +--- +## Remote Settings + +The machine-learning models and [WASM] binaries are all hosted in Remote Settings and are downloaded/cached when needed. + +### Admin Dashboards + +In order to get access to Firefox Translations content in the Remote Settings admin dashboards, you will need to request +access in the Remote Settings component on [Bugzilla]. + +Once you have access to Firefox Translations content in Remote Settings, you will be able to view it in the admin dashboards: + +**Dev**<br> +> [https://settings.dev.mozaws.net/v0/admin](https://settings.dev.mozaws.net/v1/admin) + +**Stage**<br> +> [https://remote-settings.allizom.org/v0/admin](https://settings-writer.stage.mozaws.net/v1/admin) + +### Prod Admin Access + +In order to access the prod admin dashboard, you must also have access to a VPN that is authorized to view the dashboard. +To gain access to the VPN, follow [Step 3] on this page in the Remote Settings documentation. + +**Prod**<br> +> [https://remote-settings.mozilla.org/v1/admin](https://settings-writer.prod.mozaws.net/v1/admin) + + +### Pulling From Different Sources + +When you are running Firefox, you can choose to pull data from **Dev**, **Stage**, or **Prod** by downloading and installing +the latest [remote-settings-devtools] Firefox extension. + +### Versioning + +Firefox Translations uses semantic versioning for all of its records via the **`version`** property. + +```{note} +**Beta Versions** + +Versions that are below **`1.0`** are considered to be a beta version. For language translation models, +this quality will be communicated to users via the UI. +``` + +#### Non-breaking Changes + +Firefox Translations code will always retrieve the maximum compatible version of each record from Remote Settings. +If two records exist with different versions, (e.g. **`1.0`** and **`1.1`**) then only the version **`1.1`** record +will be considered. + +This allows us to update and ship new versions of models that are compatible with the current source code and wasm runtimes +in both backward-compatible and forward-compatible ways. These can be released through remote settings independent of the +[Firefox Release Schedule]. + +#### Breaking Changes + +Breaking changes for Firefox Translations are a bit more tricky. These are changes that make older-version records +incompatible with the current Firefox source code and/or [WASM] runtimes. + +While a breaking change will result in a change of the semver number (e.g. **`1.1 ⟶ 2.0`**), this alone is not +sufficient. Since Firefox Translations always attempts to use the maximum compatible version, only bumping this number +would result in older versions of Firefox attempting to use a newer-version record that is no longer compatible with the +Firefox source code or [WASM] runtimes. + +To handle these changes, Firefox Translations utilizes Remote Settings [Filter Expressions] to make certain records +available to only particular releases of Firefox. This will allow Firefox Translations to make different sets of Remote Settings records available to different versions +of Firefox. + +```{admonition} Example + +Let's say that Firefox 108 through Firefox 120 is compatible with translations model records in the **`1.*`** major-version range, however Firefox 121 and onward is compatible with only model records in the **`2.*`** major-version range. + +This will allow us to mark the **`1.*`** major-version records with the following filter expression: + +**` +"filter_expression": "env.version|versionCompare('108.a0') >= 0 && env.version|versionCompare('121.a0') < 0" +`** + +This means that these records will only be available in Firefox versions greater than or equal to 108, and less than 121. + +Similarly, we will be able to mark all of the **`2.*`** major-version records with this filter expression: + +**` +"filter_expression": "env.version|versionCompare('121.a0') >= 0" +`** + +This means that these records will only be available in Firefox versions greater than or equal to Firefox 121. + +``` + +Tying breaking changes to releases in this way frees up Firefox Translations to make changes as large as entirely +switching one third-party library for another in the compiled source code, while allowing older versions of Firefox to continue utilizing the old library and allowing newer versions of Firefox to utilize the new library. + +--- +## Building fastText + +### Downloading the Models + +The fastText model that we use can be downloaded directly from the fastText website:<br> +> [https://fasttext.cc/docs/en/language-identification.html](https://fasttext.cc/docs/en/language-identification.html) + +Firefox Translations uses the compressed, **`lid.176.ftz`** model. + +### Building the WASM Binary + +To build the fastText [WASM] binary, we can follow the steps in the [Requirements] section of the fastText website. + +#### Dependencies + +**C++ Compiler**<br> +Any of the C++ compilers from [Getting Set Up To Work On The Firefox Codebase] will be sufficient for this. + +**emskd**<br> +Follow the [Download and Install] instructions for setting up the emscripten sdk. + +#### Modifying the EMCXXFLAGS + +At the time of writing, the a latest commit on the fastText repo ([3697152e0fd772d9185697fdbd4a1d340ca5571d]) +is not compatible by default with the latest version of [emscripten (3.1.35)]. + +A few changes need to be made to the Makefile in order to generate the fastText [WASM] for use in Firefox. + +**1) Disable DYNAMIC_EXECUTION**<br> +In the `Makefile` for the fastText repo, there is a variable called **`EMCXXFLAGS`**.<br> +We need to add the following flag to this variable: + +``` +-s "DYNAMIC_EXECUTION=0" +``` + +If this flag is not set to **`0`**, then emscripten will [generate functions] that use the [eval()] function. +[eval()] is not allowed in the context that fastText runs in FireFox due to security reasons. + +**2) Rename EXTRA_EXPORTED_RUNTIME_METHODS**<br> +In [emscripten (2.0.18)], **`EXTRA_EXPORTED_RUNTIME_METHODS`** was deprecated in favor of **`EXPORTED_RUNTIME_METHODS`**. +The fastText Makefile still has the old flag, so we need to update the name. + +**3) Use the -r Flag When Appropriate**<br> +In [emscripten (2.0.3)] the following change was made: + +> "The default output format is now executable JavaScript. Previously we would default to output objecting files unless, for example, the output name ended in **`.js`**. This is contrary to behavior of clang and gcc. Now emscripten will always produce and executable unless the **`-c`**, **`-r`** or **`-shared`** flags are given. This is true even when the name of the output file ends in **`.o`**. e.g, **`emcc foo.c -o foo.o`** will produce a JavaScript file called **`foo.o`**. This might surprise some users (although it matches the behavior of existing toolchains) so we now produce a warning in this case." + +The Makefile needs to be modified to use the **`-r`** flag when appropriate. These changes are modeled after comments on this [GitHub Issue]. + +**Cumulative Changes**<br> +Here is a diff of the full changes needed for the Makefile at the time of writing: + +```diff +diff --git a/Makefile b/Makefile +index e246f79..396ae0b 100644 +--- a/Makefile ++++ b/Makefile +@@ -73,7 +73,9 @@ clean: + + EMCXX = em++ +-EMCXXFLAGS = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s "EXTRA_EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']" -s "DISABLE_EXCEPTION_CATCHING=0" -s "EXCEPTION_DEBUG=1" -s "FORCE_FILESYSTEM=1" -s "MODULARIZE=1" -s "EXPORT_ES6=1" -s 'EXPORT_NAME="FastTextModule"' -Isrc/ ++EMCXXFLAGS_BASE = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s "EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']" -s "DISABLE_EXCEPTION_CATCHING=0" -s "EXCEPTION_DEBUG=0" -s "DYNAMIC_EXECUTION=0" -s "FORCE_FILESYSTEM=1" -s "MODULARIZE=1" -s "EXPORT_ES6=1" -s 'EXPORT_NAME="FastTextModule"' -Isrc/ ++EMCXXFLAGS = $(EMCXXFLAGS_BASE) -r ++EMCXXFLAGS_JS = $(EMCXXFLAGS_BASE) + EMOBJS = args.bc autotune.bc matrix.bc dictionary.bc loss.bc productquantizer.bc densematrix.bc quantmatrix.bc vector.bc model.bc utils.bc meter.bc fasttext.bc main.bc + + +@@ -120,6 +122,6 @@ fasttext.bc: src/fasttext.cc src/*.h + $(EMCXX) $(EMCXXFLAGS) src/fasttext.cc -o fasttext.bc + + webassembly/fasttext_wasm.js: $(EMOBJS) webassembly/fasttext_wasm.cc Makefile +- $(EMCXX) $(EMCXXFLAGS) $(EMOBJS) -o webassembly/fasttext_wasm.js ++ $(EMCXX) $(EMCXXFLAGS_JS) $(EMOBJS) -o webassembly/fasttext_wasm.js +``` + +After modifying the Makefile in the previous section, running **`make wasm`** in the fastText repo should run without warnings or errors and the following files will be generated in the **`webassembly`** directory: + +``` +webassembly +├── fasttext.js +├── fasttext_wasm.js +└── fasttext_wasm.wasm +``` + +#### Modifying fasttext_wasm.js + +There are a few changes we need to make to the **`fasttext_wasm.js`** file to make it compatible with use in Firefox. + +**1) Define a function, not a module**<br> +The generated code exports a module, but this needs to be modified into a function for use in [importScripts()] in a worker. + +At the top of the file we need to make the following changes: + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..4802343da2a03 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -1,9 +1,6 @@ + +-var FastTextModule = (() => { +- var _scriptDir = import.meta.url; +- +- return ( +-async function(FastTextModule = {}) { ++async function loadFastTextModule(FastTextModule = {}) { ++ const _scriptDir = null; + + // include: shell.js + // The Module object: Our interface to the outside world. We import +``` + +Here we are defining a function rather than a variable, and we are setting **`_scriptDir`** to null +because **`import.meta.url`** is only available for use within modules. + +Next we need to modify the bottom of the file to match these changes: + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..0a6fca3f524e4 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -8287,7 +8287,3 @@ run(); + + return FastTextModule.ready + } +- +-); +-})(); +-export default FastTextModule; +``` + +**2) Remove unneeded environment checks**<br> +Next we need to remove unneeded checks for different environments: + +```JavaScript +if (ENVIRONMENT_IS_NODE) { + // ... +} else +if (ENVIRONMENT_IS_SHELL) { + // ... +} else +if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + // ... +} else +{ + throw new Error('environment detection error'); +} +``` + +Since this code will only be run inside of a worker, we want to delete the blocks that deal with **`ENVIRONMENT_IS_NODE`** and **`ENVIRONMENT_IS_SHELL`**. In fact, this code will fail to be imported by [importScripts()] if we don't do this. + +**3) Remove the use of `import.meta.url`**<br> +Finally, there is a use of **`import.meta.url`** that we need to remove. + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js +index 64c6184a85851..746cbae2ec952 100644 +--- a/toolkit/components/translations/fasttext/fasttext_wasm.js ++++ b/toolkit/components/translations/fasttext/fasttext_wasm.js +@@ -746,7 +746,7 @@ if (Module['locateFile']) { + } + } else { + // Use bundler-friendly `new URL(..., import.meta.url)` pattern; works in browsers too. +- wasmBinaryFile = new URL('fasttext_wasm.wasm', import.meta.url).href; ++ wasmBinaryFile = null; + } + + function getBinary(file) { +``` + +As mentioned before, **`import.meta.url`** is not allowed outside of modules and cannot be used with [importScripts()] +in the worker code that we are creating. + +It is okay to set this to null here, because we will be providing the **`wasmBinaryFile`** via [Remote Settings]. + +**4) Minifying the file**<br> +The generated **`fasttext_wasm.js`** file is very large. To minimize the impact on the size of the code in the Firefox source tree, we want to minify the file using the [minify] tool. + +``` +Size Name +291k ├── fasttext_wasm.js (original) +109k └── fasttext_wasm.js (minified) +``` + +**5) Adding the license**<br> +Finally, we should add a copy of the current fastText MIT license to the top of the minified **`fasttext_wasm.js`** file. +You should be able to paste this from the generated **`fasttext.js`** file. + +#### Modifying fasttext.js + +```{note} +It is likely that the source file in tree already has these changes and is already sufficient, +even if **`fasttext_wasm.js`** has been recently updated. Try running it first as-is before replacing +and re-modifying. +``` + +Next we need to modify **`fasttext.js`** to utilize the changes that we made to **`fasttext_wasm.js`** and also to +not be a module so that we can import it using [importScripts()]. + +These changes do the following: + +1) Define a variable called **`fastTextModule`** for use in the worker scripts. +2) Utilize the **`loadFastTextModule()`** function that we defined in **`fasttext_wasm.js`** +3) Add a function **`loadModelBinary()`** that takes the wasm binary directly, which we will provide through [Remote Settings]. +4) Remove any module exports. + +```diff +diff --git a/toolkit/components/translations/fasttext/fasttext.js b/toolkit/components/translations/fasttext/fasttext.js +index 86600b9ac9e28..2c49b3faaeedc 100644 +--- a/toolkit/components/translations/fasttext/fasttext.js ++++ b/toolkit/components/translations/fasttext/fasttext.js +@@ -6,20 +6,30 @@ + * LICENSE file in the root directory of this source tree. + */ + +-import fastTextModularized from './fasttext_wasm.js'; +-const fastTextModule = fastTextModularized(); ++let fastTextModule; ++ ++const _initFastTextModule = async function (wasmModule) { ++ try { ++ fastTextModule = await loadFastTextModule(wasmModule); ++ } catch(e) { ++ console.error(e); ++ } ++ return true ++} + + let postRunFunc = null; + const addOnPostRun = function(func) { + postRunFunc = func; + }; + +-fastTextModule.addOnPostRun(() => { +- if (postRunFunc) { +- postRunFunc(); +- } +-}); + ++const loadFastText = (wasmModule) => { ++ _initFastTextModule(wasmModule).then((res) => { ++ if (postRunFunc) { ++ postRunFunc(); ++ } ++ }) ++} + const thisModule = this; + const trainFileInWasmFs = 'train.txt'; + const testFileInWasmFs = 'test.txt'; +@@ -41,7 +51,7 @@ const getFloat32ArrayFromHeap = (len) => { + const heapToFloat32 = (r) => new Float32Array(r.buffer, r.ptr, r.size); + + class FastText { +- constructor() { ++ constructor(fastTextModule) { + this.f = new fastTextModule.FastText(); + } + +@@ -77,6 +87,15 @@ class FastText { + }); + } + ++ loadModelBinary(buffer) { ++ const fastTextNative = this.f; ++ const byteArray = new Uint8Array(buffer); ++ const FS = fastTextModule.FS; ++ FS.writeFile(modelFileInWasmFs, byteArray); ++ fastTextNative.loadModel(modelFileInWasmFs); ++ return new FastTextModel(fastTextNative); ++ } ++ + _train(url, modelName, kwargs = {}, callback = null) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; +@@ -515,6 +534,3 @@ class FastTextModel { + }); + } + } +- +- +-export {FastText, addOnPostRun}; +``` + +--- +## Building Bergamot + +TODO + + +<!-- Hyperlinks --> +[3697152e0fd772d9185697fdbd4a1d340ca5571d]: https://github.com/facebookresearch/fastText/tree/3697152e0fd772d9185697fdbd4a1d340ca5571d +[Bugzilla]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Cloud%20Services&component=Server%3A%20Remote%20Settings +[Child]: https://searchfox.org/mozilla-central/search?q=TranslationsChild +[Download and Install]: https://emscripten.org/docs/getting_started/downloads.html#download-and-install +[emscripten (2.0.3)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#203-09102020 +[emscripten (2.0.18)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#2018-04232021 +[emscripten (3.1.35)]: https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3135---040323 +[Environments]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#environments +[eval()]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval +[Filter Expressions]: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions +[Firefox Release Schedule]: https://wiki.mozilla.org/Release_Management/Calendar +[generate functions]: https://emscripten.org/docs/api_reference/emscripten.h.html?highlight=dynamic_execution#functions +[Getting Set Up To Work On The Firefox Codebase]: https://firefox-source-docs.mozilla.org/setup/index.html +[GitHub Issue]: https://github.com/facebookresearch/fastText/pull/1227#issuecomment-1353830003 +[importScripts()]: https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/importScripts +[JSWindowActors]: https://firefox-source-docs.mozilla.org/dom/ipc/jsactors.html#jswindowactor +[minify]: https://github.com/tdewolff/minify +[Parent]: https://searchfox.org/mozilla-central/search?q=TranslationsParent +[Step 3]: https://remote-settings.readthedocs.io/en/latest/getting-started.html#create-a-new-official-type-of-remote-settings +[remote-settings-devtools]: https://github.com/mozilla-extensions/remote-settings-devtools/releases +[Remote Settings]: https://remote-settings.readthedocs.io/en/latest/ +[Requirements]: https://fasttext.cc/docs/en/webassembly-module.html#requirements +[toolkit/components/translations]: https://searchfox.org/mozilla-central/search?q=toolkit%2Fcomponents%2Ftranslations +[WASM]: https://webassembly.org/ +[Workers]: https://searchfox.org/mozilla-central/search?q=%2Ftranslations.*worker&path=&case=false®exp=true diff --git a/toolkit/components/translations/fasttext/LICENSE b/toolkit/components/translations/fasttext/LICENSE new file mode 100644 index 0000000000..5a14f2864a --- /dev/null +++ b/toolkit/components/translations/fasttext/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016-present, Facebook, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkit/components/translations/fasttext/fasttext.js b/toolkit/components/translations/fasttext/fasttext.js new file mode 100644 index 0000000000..a79dfeffa0 --- /dev/null +++ b/toolkit/components/translations/fasttext/fasttext.js @@ -0,0 +1,536 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +let fastTextModule; + +const _initFastTextModule = async function (wasmModule) { + try { + fastTextModule = await loadFastTextModule(wasmModule); + } catch(e) { + console.error(e); + } + return true +} + +let postRunFunc = null; +const addOnPostRun = function (func) { + postRunFunc = func; +}; + + +const loadFastText = (wasmModule) => { + _initFastTextModule(wasmModule).then((res) => { + if (postRunFunc) { + postRunFunc(); + } + }) +} +const thisModule = this; +const trainFileInWasmFs = 'train.txt'; +const testFileInWasmFs = 'test.txt'; +const modelFileInWasmFs = 'model.bin'; + +const getFloat32ArrayFromHeap = (len) => { + const dataBytes = len * Float32Array.BYTES_PER_ELEMENT; + const dataPtr = fastTextModule._malloc(dataBytes); + const dataHeap = new Uint8Array(fastTextModule.HEAPU8.buffer, + dataPtr, + dataBytes); + return { + 'ptr':dataHeap.byteOffset, + 'size':len, + 'buffer':dataHeap.buffer + }; +}; + +const heapToFloat32 = (r) => new Float32Array(r.buffer, r.ptr, r.size); + +class FastText { + constructor(fastTextModule) { + this.f = new fastTextModule.FastText(); + } + + /** + * loadModel + * + * Loads the model file from the specified url, and returns the + * corresponding `FastTextModel` object. + * + * @param {string} url + * the url of the model file. + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + loadModel(url) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + + const fastTextNative = this.f; + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(modelFileInWasmFs, byteArray); + }).then(() => { + fastTextNative.loadModel(modelFileInWasmFs); + resolve(new FastTextModel(fastTextNative)); + }).catch(error => { + reject(error); + }); + }); + } + + loadModelBinary(buffer) { + const fastTextNative = this.f; + const byteArray = new Uint8Array(buffer); + const FS = fastTextModule.FS; + FS.writeFile(modelFileInWasmFs, byteArray); + fastTextNative.loadModel(modelFileInWasmFs); + return new FastTextModel(fastTextNative); + } + + _train(url, modelName, kwargs = {}, callback = null) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; + + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(trainFileInWasmFs, byteArray); + }).then(() => { + const argsList = ['lr', 'lrUpdateRate', 'dim', 'ws', 'epoch', + 'minCount', 'minCountLabel', 'neg', 'wordNgrams', 'loss', + 'model', 'bucket', 'minn', 'maxn', 't', 'label', 'verbose', + 'pretrainedVectors', 'saveOutput', 'seed', 'qout', 'retrain', + 'qnorm', 'cutoff', 'dsub', 'qnorm', 'autotuneValidationFile', + 'autotuneMetric', 'autotunePredictions', 'autotuneDuration', + 'autotuneModelSize']; + const args = new fastTextModule.Args(); + argsList.forEach(k => { + if (k in kwargs) { + args[k] = kwargs[k]; + } + }); + args.model = fastTextModule.ModelName[modelName]; + args.loss = ('loss' in kwargs) ? + fastTextModule.LossName[kwargs['loss']] : 'hs'; + args.thread = 1; + args.input = trainFileInWasmFs; + + fastTextNative.train(args, callback); + + resolve(new FastTextModel(fastTextNative)); + }).catch(error => { + reject(error); + }); + }); + } + + /** + * trainSupervised + * + * Downloads the input file from the specified url, trains a supervised + * model and returns a `FastTextModel` object. + * + * @param {string} url + * the url of the input file. + * The input file must must contain at least one label per line. For an + * example consult the example datasets which are part of the fastText + * repository such as the dataset pulled by classification-example.sh. + * + * @param {dict} kwargs + * train parameters. + * For example {'lr': 0.5, 'epoch': 5} + * + * @param {function} callback + * train callback function + * `callback` function is called regularly from the train loop: + * `callback(progress, loss, wordsPerSec, learningRate, eta)` + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + trainSupervised(url, kwargs = {}, callback) { + const self = this; + return new Promise(function(resolve, reject) { + self._train(url, 'supervised', kwargs, callback).then(model => { + resolve(model); + }).catch(error => { + reject(error); + }); + }); + } + + /** + * trainUnsupervised + * + * Downloads the input file from the specified url, trains an unsupervised + * model and returns a `FastTextModel` object. + * + * @param {string} url + * the url of the input file. + * The input file must not contain any labels or use the specified label + * prefixunless it is ok for those words to be ignored. For an example + * consult the dataset pulled by the example script word-vector-example.sh + * which is part of the fastText repository. + * + * @param {string} modelName + * Model to be used for unsupervised learning. `cbow` or `skipgram`. + * + * @param {dict} kwargs + * train parameters. + * For example {'lr': 0.5, 'epoch': 5} + * + * @param {function} callback + * train callback function + * `callback` function is called regularly from the train loop: + * `callback(progress, loss, wordsPerSec, learningRate, eta)` + * + * @return {Promise} promise object that resolves to a `FastTextModel` + * + */ + trainUnsupervised(url, modelName, kwargs = {}, callback) { + const self = this; + return new Promise(function(resolve, reject) { + self._train(url, modelName, kwargs, callback).then(model => { + resolve(model); + }).catch(error => { + reject(error); + }); + }); + } + +} + + +class FastTextModel { + /** + * `FastTextModel` represents a trained model. + * + * @constructor + * + * @param {object} fastTextNative + * webassembly object that makes the bridge between js and C++ + */ + constructor(fastTextNative) { + this.f = fastTextNative; + } + + /** + * isQuant + * + * @return {bool} true if the model is quantized + * + */ + isQuant() { + return this.f.isQuant; + } + + /** + * getDimension + * + * @return {int} the dimension (size) of a lookup vector (hidden layer) + * + */ + getDimension() { + return this.f.args.dim; + } + + /** + * getWordVector + * + * @param {string} word + * + * @return {Float32Array} the vector representation of `word`. + * + */ + getWordVector(word) { + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getWordVector(b, word); + + return heapToFloat32(b); + } + + /** + * getSentenceVector + * + * @param {string} text + * + * @return {Float32Array} the vector representation of `text`. + * + */ + getSentenceVector(text) { + if (text.indexOf('\n') != -1) { + "sentence vector processes one line at a time (remove '\\n')"; + } + text += '\n'; + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getSentenceVector(b, text); + + return heapToFloat32(b); + } + + /** + * getNearestNeighbors + * + * returns the nearest `k` neighbors of `word`. + * + * @param {string} word + * @param {int} k + * + * @return {Array.<Pair.<number, string>>} + * words and their corresponding cosine similarities. + * + */ + getNearestNeighbors(word, k = 10) { + return this.f.getNN(word, k); + } + + /** + * getAnalogies + * + * returns the nearest `k` neighbors of the operation + * `wordA - wordB + wordC`. + * + * @param {string} wordA + * @param {string} wordB + * @param {string} wordC + * @param {int} k + * + * @return {Array.<Pair.<number, string>>} + * words and their corresponding cosine similarities + * + */ + getAnalogies(wordA, wordB, wordC, k) { + return this.f.getAnalogies(k, wordA, wordB, wordC); + } + + /** + * getWordId + * + * Given a word, get the word id within the dictionary. + * Returns -1 if word is not in the dictionary. + * + * @return {int} word id + * + */ + getWordId(word) { + return this.f.getWordId(word); + } + + /** + * getSubwordId + * + * Given a subword, return the index (within input matrix) it hashes to. + * + * @return {int} subword id + * + */ + getSubwordId(subword) { + return this.f.getSubwordId(subword); + } + + /** + * getSubwords + * + * returns the subwords and their indicies. + * + * @param {string} word + * + * @return {Pair.<Array.<string>, Array.<int>>} + * words and their corresponding indicies + * + */ + getSubwords(word) { + return this.f.getSubwords(word); + } + + /** + * getInputVector + * + * Given an index, get the corresponding vector of the Input Matrix. + * + * @param {int} ind + * + * @return {Float32Array} the vector of the `ind`'th index + * + */ + getInputVector(ind) { + const b = getFloat32ArrayFromHeap(this.getDimension()); + this.f.getInputVector(b, ind); + + return heapToFloat32(b); + } + + /** + * predict + * + * Given a string, get a list of labels and a list of corresponding + * probabilities. k controls the number of returned labels. + * + * @param {string} text + * @param {int} k, the number of predictions to be returned + * @param {number} probability threshold + * + * @return {Array.<Pair.<number, string>>} + * labels and their probabilities + * + */ + predict(text, k = 1, threshold = 0.0) { + return this.f.predict(text, k, threshold); + } + + /** + * getInputMatrix + * + * Get a reference to the full input matrix of a Model. This only + * works if the model is not quantized. + * + * @return {DenseMatrix} + * densematrix with functions: `rows`, `cols`, `at(i,j)` + * + * example: + * let inputMatrix = model.getInputMatrix(); + * let value = inputMatrix.at(1, 2); + */ + getInputMatrix() { + if (this.isQuant()) { + throw new Error("Can't get quantized Matrix"); + } + return this.f.getInputMatrix(); + } + + /** + * getOutputMatrix + * + * Get a reference to the full input matrix of a Model. This only + * works if the model is not quantized. + * + * @return {DenseMatrix} + * densematrix with functions: `rows`, `cols`, `at(i,j)` + * + * example: + * let outputMatrix = model.getOutputMatrix(); + * let value = outputMatrix.at(1, 2); + */ + getOutputMatrix() { + if (this.isQuant()) { + throw new Error("Can't get quantized Matrix"); + } + return this.f.getOutputMatrix(); + } + + /** + * getWords + * + * Get the entire list of words of the dictionary including the frequency + * of the individual words. This does not include any subwords. For that + * please consult the function get_subwords. + * + * @return {Pair.<Array.<string>, Array.<int>>} + * words and their corresponding frequencies + * + */ + getWords() { + return this.f.getWords(); + } + + /** + * getLabels + * + * Get the entire list of labels of the dictionary including the frequency + * of the individual labels. + * + * @return {Pair.<Array.<string>, Array.<int>>} + * labels and their corresponding frequencies + * + */ + getLabels() { + return this.f.getLabels(); + } + + /** + * getLine + * + * Split a line of text into words and labels. Labels must start with + * the prefix used to create the model (__label__ by default). + * + * @param {string} text + * + * @return {Pair.<Array.<string>, Array.<string>>} + * words and labels + * + */ + getLine(text) { + return this.f.getLine(text); + } + + /** + * saveModel + * + * Saves the model file in web assembly in-memory FS and returns a blob + * + * @return {Blob} blob data of the file saved in web assembly FS + * + */ + saveModel() { + this.f.saveModel(modelFileInWasmFs); + const content = fastTextModule.FS.readFile(modelFileInWasmFs, + { encoding: 'binary' }); + return new Blob( + [new Uint8Array(content, content.byteOffset, content.length)], + { type: ' application/octet-stream' } + ); + } + + /** + * test + * + * Downloads the test file from the specified url, evaluates the supervised + * model with it. + * + * @param {string} url + * @param {int} k, the number of predictions to be returned + * @param {number} probability threshold + * + * @return {Promise} promise object that resolves to a `Meter` object + * + * example: + * model.test("/absolute/url/to/test.txt", 1, 0.0).then((meter) => { + * console.log(meter.precision); + * console.log(meter.recall); + * console.log(meter.f1Score); + * console.log(meter.nexamples()); + * }); + * + */ + test(url, k, threshold) { + const fetchFunc = (thisModule && thisModule.fetch) || fetch; + const fastTextNative = this.f; + + return new Promise(function(resolve, reject) { + fetchFunc(url).then(response => { + return response.arrayBuffer(); + }).then(bytes => { + const byteArray = new Uint8Array(bytes); + const FS = fastTextModule.FS; + FS.writeFile(testFileInWasmFs, byteArray); + }).then(() => { + const meter = fastTextNative.test(testFileInWasmFs, k, threshold); + resolve(meter); + }).catch(error => { + reject(error); + }); + }); + } +} diff --git a/toolkit/components/translations/fasttext/fasttext_wasm.js b/toolkit/components/translations/fasttext/fasttext_wasm.js new file mode 100644 index 0000000000..84d05459a3 --- /dev/null +++ b/toolkit/components/translations/fasttext/fasttext_wasm.js @@ -0,0 +1,8 @@ +/** + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +async function loadFastTextModule(aJ={}){var b_=null,b=typeof aJ!='undefined'?aJ:{},bV,aA,bT,e$,bR,e_,aO,R,N,eZ,A,aQ,an,aR,eV,Z,r,gz,gy,gx,gw,ae,eS,af,aD,eR,gv,q,v,C,ah,h,l,au,aF,aE,bz,by,gu,bv,aZ,eC,Q,O,ad,W,bi,K,k,s,aw,bG,J,am,y,cG,cF,o,H,G,j,cp,aS,a,t,ap,Y,P,ax,cg,ce,bZ,aI,bU,T,aU,bt,ak,ac,aa,bw,x,S,bq,dU,bb,a$,bm,bo,bp,av,az,eE,bH,gg,gf,ge,eM,bJ,eO,gd,eQ,bK,B,e,al,bM,eW,eX,aP,f,g,b$,fa,fb,fc,fd,fe,ff,fg,fh,fi,fj,fk,fl,fm,fn,fo,gc,fq,fr,ft,gb,gA,eN,eT,ay;if(b.ready=new Promise(function(a,b){bV=a,aA=b}),["_main","getExceptionMessage","___get_exception_message","_free","___getTypeName","__embind_initialize_bindings","_fflush","onRuntimeInitialized"].forEach(a=>{Object.getOwnPropertyDescriptor(b.ready,a)||Object.defineProperty(b.ready,a,{get:()=>p('You are getting '+a+' on the Promise object, instead of the instance. Use .then() to get called back with the instance, see the MODULARIZE docs in src/settings.js'),set:()=>p('You are setting '+a+' on the Promise object, instead of the instance. Use .then() to get called back with the instance, see the MODULARIZE docs in src/settings.js')})}),bT=Object.assign({},b),e$=[],bR='./this.program',e_=(b,a)=>{throw a},aO=typeof window=='object',R=typeof importScripts=='function',N=typeof process=='object'&&typeof process.versions=='object'&&typeof process.versions.node=='string',eZ=!aO&&!N&&!R,b.ENVIRONMENT)throw new Error('Module.ENVIRONMENT has been deprecated. To force the environment, use the ENVIRONMENT compile-time option (for example, -sENVIRONMENT=web or -sENVIRONMENT=node)');A='';function eY(a){return b.locateFile?b.locateFile(a,A):A+a}if(aO||R){if(R?A=self.location.href:typeof document!='undefined'&&document.currentScript&&(A=document.currentScript.src),b_&&(A=b_),A.indexOf('blob:')!==0?A=A.substr(0,A.replace(/[?#].*/,"").lastIndexOf('/')+1):A='',!(typeof window=='object'||typeof importScripts=='function'))throw new Error('not compiled for this environment (did you build to HTML and try to run it not on the web, or set ENVIRONMENT to something - like node - and run it someplace else - like on the web?)');aQ=b=>{var a=new XMLHttpRequest;return a.open('GET',b,!1),a.send(null),a.responseText},R&&(aR=b=>{var a=new XMLHttpRequest;return a.open('GET',b,!1),a.responseType='arraybuffer',a.send(null),new Uint8Array(a.response)}),an=(c,d,b)=>{var a=new XMLHttpRequest;a.open('GET',c,!0),a.responseType='arraybuffer',a.onload=()=>{if(a.status==200||a.status==0&&a.response){d(a.response);return}b()},a.onerror=b,a.send(null)},eV=a=>document.title=a}else throw new Error('environment detection error');Z=b.print||console.log.bind(console),r=b.printErr||console.warn.bind(console),Object.assign(b,bT),bT=null,ga(),b.arguments&&(e$=b.arguments),F('arguments','arguments_'),b.thisProgram&&(bR=b.thisProgram),F('thisProgram','thisProgram'),b.quit&&(e_=b.quit),F('quit','quit_'),c(typeof b.memoryInitializerPrefixURL=='undefined','Module.memoryInitializerPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.pthreadMainPrefixURL=='undefined','Module.pthreadMainPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.cdInitializerPrefixURL=='undefined','Module.cdInitializerPrefixURL option was removed, use Module.locateFile instead'),c(typeof b.filePackagePrefixURL=='undefined','Module.filePackagePrefixURL option was removed, use Module.locateFile instead'),c(typeof b.read=='undefined','Module.read option was removed (modify read_ in JS)'),c(typeof b.readAsync=='undefined','Module.readAsync option was removed (modify readAsync in JS)'),c(typeof b.readBinary=='undefined','Module.readBinary option was removed (modify readBinary in JS)'),c(typeof b.setWindowTitle=='undefined','Module.setWindowTitle option was removed (modify setWindowTitle in JS)'),c(typeof b.TOTAL_MEMORY=='undefined','Module.TOTAL_MEMORY has been renamed Module.INITIAL_MEMORY'),F('read','read_'),F('readAsync','readAsync'),F('readBinary','readBinary'),F('setWindowTitle','setWindowTitle'),gz='IDBFS is no longer included by default; build with -lidbfs.js',gy='PROXYFS is no longer included by default; build with -lproxyfs.js',gx='WORKERFS is no longer included by default; build with -lworkerfs.js',gw='NODEFS is no longer included by default; build with -lnodefs.js',c(!eZ,"shell environment detected but not enabled at build time. Add 'shell' to `-sENVIRONMENT` to enable."),b.wasmBinary&&(ae=b.wasmBinary),F('wasmBinary','wasmBinary'),eS=b.noExitRuntime||!0,F('noExitRuntime','noExitRuntime'),typeof WebAssembly!='object'&&p('no native wasm support detected'),aD=!1;function c(b,a){b||p('Assertion failed'+(a?': '+a:''))}function bI(){var a=af.buffer;b.HEAP8=q=new Int8Array(a),b.HEAP16=C=new Int16Array(a),b.HEAP32=h=new Int32Array(a),b.HEAPU8=v=new Uint8Array(a),b.HEAPU16=ah=new Uint16Array(a),b.HEAPU32=l=new Uint32Array(a),b.HEAPF32=au=new Float32Array(a),b.HEAPF64=aF=new Float64Array(a)}c(!b.STACK_SIZE,'STACK_SIZE can no longer be set at runtime. Use -sSTACK_SIZE at link time'),c(typeof Int32Array!='undefined'&&typeof Float64Array!='undefined'&&Int32Array.prototype.subarray!=void 0&&Int32Array.prototype.set!=void 0,'JS engine does not provide full typed array support'),c(!b.wasmMemory,'Use of `wasmMemory` detected. Use -sIMPORTED_MEMORY to define wasmMemory externally'),c(!b.INITIAL_MEMORY,'Detected runtime INITIAL_MEMORY setting. Use -sIMPORTED_MEMORY to define wasmMemory dynamically');function eI(){var a=aP();c((a&3)==0),a==0&&(a+=4),l[a>>2]=34821223,l[a+4>>2]=2310721022,l[0]=1668509029}function aV(){var a,b,c;if(aD)return;a=aP(),a==0&&(a+=4),b=l[a>>2],c=l[a+4>>2],(b!=34821223||c!=2310721022)&&p('Stack overflow! Stack cookie has been overwritten at '+$(a)+', expected hex dwords 0x89BACDFE and 0x2135467, but received '+$(c)+' '+$(b)),l[0]!==1668509029&&p('Runtime error: The application has corrupted its heap memory area (address zero)!')}(function(){var a=new Int16Array(1),b=new Int8Array(a.buffer);if(a[0]=25459,b[0]!==115||b[1]!==99)throw'Runtime error: expected the system to be little-endian! (Run with -sSUPPORT_BIG_ENDIAN to bypass)'})(),bz=[],by=[],gu=[],bv=[],aZ=!1,eC=0;function go(){return eS||eC>0}function eB(){if(b.preRun)for(typeof b.preRun=='function'&&(b.preRun=[b.preRun]);b.preRun.length;)et(b.preRun.shift());aK(bz)}function ew(){c(!aZ),aZ=!0,aV(),!b.noFSInit&&!a.init.initialized&&a.init(),a.ignorePermissions=!1,G.init(),aK(by)}function ev(){if(aV(),b.postRun)for(typeof b.postRun=='function'&&(b.postRun=[b.postRun]);b.postRun.length;)br(b.postRun.shift());aK(bv)}function et(a){bz.unshift(a)}function eh(a){by.unshift(a)}function gt(a){}function br(a){bv.unshift(a)}c(Math.imul,'This browser does not support Math.imul(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.fround,'This browser does not support Math.fround(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.clz32,'This browser does not support Math.clz32(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),c(Math.trunc,'This browser does not support Math.trunc(), build with LEGACY_VM_SUPPORT or POLYFILL_OLD_MATH_FUNCTIONS to add in a polyfill'),Q=0,O=null,ad=null,W={};function bh(a){for(var b=a;1;){if(!W[a])return a;a=b+Math.random()}}function aH(a){Q++,b.monitorRunDependencies&&b.monitorRunDependencies(Q),a?(c(!W[a]),W[a]=1,O===null&&typeof setInterval!='undefined'&&(O=setInterval(function(){var a,b;if(aD){clearInterval(O),O=null;return}a=!1;for(b in W)a||(a=!0,r('still waiting on run dependencies:')),r('dependency: '+b);a&&r('(end of list)')},1e4))):r('warning: run dependency added without ID')}function ai(a){if(Q--,b.monitorRunDependencies&&b.monitorRunDependencies(Q),a?(c(W[a]),delete W[a]):r('warning: run dependency removed without ID'),Q==0)if(O!==null&&(clearInterval(O),O=null),ad){var d=ad;ad=null,d()}}function p(a){b.onAbort&&b.onAbort(a),a='Aborted('+a+')',r(a),aD=!0,eR=1;var c=new WebAssembly.RuntimeError(a);throw aA(c),c}bi='data:application/octet-stream;base64,';function bd(a){return a.startsWith(bi)}function bc(a){return a.startsWith('file://')}function n(a,d){return function(){var f=a,e=d;return d||(e=b.asm),c(aZ,'native function `'+f+'` called before runtime initialization'),e[a]||c(e[a],'exported native function `'+f+'` not found'),e[a].apply(null,arguments)}}class d extends Error{}class gs extends d{}class ba extends d{constructor(a){super(a),this.excPtr=a;const b=bx(a);this.name=b[0],this.message=b[1]}}b.locateFile?(K='fasttext_wasm.wasm',bd(K)||(K=eY(K))):K=null;function bj(a){try{if(a==K&&ae)return new Uint8Array(ae);if(aR)return aR(a);throw"both async and sync fetching of the wasm failed"}catch(a){p(a)}}function dH(a){if(!ae&&(aO||R)){if(typeof fetch=='function'&&!bc(a))return fetch(a,{credentials:'same-origin'}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+a+"'";return b.arrayBuffer()}).catch(function(){return bj(a)});if(an)return new Promise(function(b,c){an(a,function(a){b(new Uint8Array(a))},c)})}return Promise.resolve().then(function(){return bj(a)})}function bF(a,b,c){return dH(a).then(function(a){return WebAssembly.instantiate(a,b)}).then(function(a){return a}).then(c,function(a){r('failed to asynchronously prepare wasm: '+a),bc(K)&&r('warning: Loading from a file URI ('+K+') is not supported in most browsers. See https://emscripten.org/docs/getting_started/FAQ.html#how-do-i-run-a-local-webserver-for-testing-why-does-my-program-stall-in-downloading-or-preparing'),p(a)})}function dz(d,a,b,c){return!d&&typeof WebAssembly.instantiateStreaming=='function'&&!bd(a)&&!bc(a)&&!N&&typeof fetch=='function'?fetch(a,{credentials:'same-origin'}).then(function(d){var e=WebAssembly.instantiateStreaming(d,b);return e.then(c,function(d){return r('wasm streaming compile failed: '+d),r('falling back to ArrayBuffer instantiation'),bF(a,b,c)})}):bF(a,b,c)}function dx(){var a={env:bH,wasi_snapshot_preview1:bH},e;function d(d,e){var a=d.exports;return b.asm=a,af=b.asm.memory,c(af,"memory not found in wasm exports"),bI(),aE=b.asm.__indirect_function_table,c(aE,"table not found in wasm exports"),eh(b.asm.__wasm_call_ctors),ai('wasm-instantiate'),a}aH('wasm-instantiate'),e=b;function f(a){c(b===e,'the Module object should not be replaced during async compilation - perhaps the order of HTML elements is wrong?'),e=null,d(a.instance)}if(b.instantiateWasm)try{return b.instantiateWasm(a,d)}catch(a){r('Module.instantiateWasm callback failed with error: '+a),aA(a)}return dz(ae,K,a,f).catch(aA),{}}function F(a,c){Object.getOwnPropertyDescriptor(b,a)||Object.defineProperty(b,a,{configurable:!0,get:function(){p('Module.'+a+' has been replaced with plain '+c+' (the initial value can be provided on Module, but after startup the value is only looked for on a local variable of that name)')}})}function dt(a){Object.getOwnPropertyDescriptor(b,a)&&p('`Module.'+a+'` was supplied but `'+a+'` not included in INCOMING_MODULE_JS_API')}function bN(a){return a==='FS_createPath'||a==='FS_createDataFile'||a==='FS_createPreloadedFile'||a==='FS_unlink'||a==='addRunDependency'||a==='FS_createLazyFile'||a==='FS_createDevice'||a==='removeRunDependency'}function dq(a,b){typeof globalThis!='undefined'&&Object.defineProperty(globalThis,a,{configurable:!0,get:function(){D('`'+a+'` is not longer defined by emscripten. '+b)}})}dq('buffer','Please use HEAP8.buffer or wasmMemory.buffer');function dp(a){typeof globalThis!='undefined'&&!Object.getOwnPropertyDescriptor(globalThis,a)&&Object.defineProperty(globalThis,a,{configurable:!0,get:function(){var b='`'+a+'` is a library symbol and not included by default; add it to your library.js __deps or to DEFAULT_LIBRARY_FUNCS_TO_INCLUDE on the command line',c=a;c.startsWith('_')||(c='$'+a),b+=" (e.g. -sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE="+c+")",bN(a)&&(b+='. Alternatively, forcing filesystem support (-sFORCE_FILESYSTEM) can export this for you'),D(b)}}),bX(a)}function bX(a){Object.getOwnPropertyDescriptor(b,a)||Object.defineProperty(b,a,{configurable:!0,get:function(){var b="'"+a+"' was not exported. add it to EXPORTED_RUNTIME_METHODS (see the FAQ)";bN(a)&&(b+='. Alternatively, forcing filesystem support (-sFORCE_FILESYSTEM) can export this for you'),p(b)}})}function gr(a){console.error(a)}function gq(a){this.name='ExitStatus',this.message='Program terminated with exit('+a+')',this.status=a}function aK(a){while(a.length>0)a.shift()(b)}aw=[];function i(a){var b=aw[a];return b||(a>=aw.length&&(aw.length=a+1),aw[a]=b=aE.get(a)),c(aE.get(a)==b,"JavaScript-side Wasm function table mirror is out of date!"),b}function bS(a){if(a.release_ref()&&!a.get_rethrown()){var b=a.get_destructor();b&&i(b)(a.excPtr),eM(a.excPtr)}}function X(a){this.excPtr=a,this.ptr=a-24,this.set_type=function(a){l[this.ptr+4>>2]=a},this.get_type=function(){return l[this.ptr+4>>2]},this.set_destructor=function(a){l[this.ptr+8>>2]=a},this.get_destructor=function(){return l[this.ptr+8>>2]},this.set_refcount=function(a){h[this.ptr>>2]=a},this.set_caught=function(a){a=a?1:0,q[this.ptr+12>>0]=a},this.get_caught=function(){return q[this.ptr+12>>0]!=0},this.set_rethrown=function(a){a=a?1:0,q[this.ptr+13>>0]=a},this.get_rethrown=function(){return q[this.ptr+13>>0]!=0},this.init=function(a,b){this.set_adjusted_ptr(0),this.set_type(a),this.set_destructor(b),this.set_refcount(0),this.set_caught(!1),this.set_rethrown(!1)},this.add_ref=function(){var a=h[this.ptr>>2];h[this.ptr>>2]=a+1},this.release_ref=function(){var a=h[this.ptr>>2];return h[this.ptr>>2]=a-1,c(a>0),a===1},this.set_adjusted_ptr=function(a){l[this.ptr+16>>2]=a},this.get_adjusted_ptr=function(){return l[this.ptr+16>>2]},this.get_exception_ptr=function(){var b=fd(this.get_type()),a;return b?l[this.excPtr>>2]:(a=this.get_adjusted_ptr(),a!==0)?a:this.excPtr}}function bP(a){if(!a)return;bS(new X(a))}function gp(a){bP(a)}function df(a){var b=f(),c=a();return g(b),c}bG=typeof TextDecoder!='undefined'?new TextDecoder('utf8'):void 0;function U(c,b,i){for(var j=b+i,d=b,e,a,f,g,h;c[d]&&!(d>=j);)++d;if(d-b>16&&c.buffer&&bG)return bG.decode(c.subarray(b,d));for(e='';b<d;){if(a=c[b++],!(a&128)){e+=String.fromCharCode(a);continue}if(f=c[b++]&63,(a&224)==192){e+=String.fromCharCode((a&31)<<6|f);continue}g=c[b++]&63,(a&240)==224?a=(a&15)<<12|f<<6|g:((a&248)!=240&&D('Invalid UTF-8 leading byte '+$(a)+' encountered when deserializing a UTF-8 string in wasm memory to a JS string!'),a=(a&7)<<18|f<<12|g<<6|c[b++]&63),a<65536?e+=String.fromCharCode(a):(h=a-65536,e+=String.fromCharCode(55296|h>>10,56320|h&1023))}return e}function L(a,b){return c(typeof a=='number'),a?U(v,a,b):''}function ca(a){return df(function(){var c=b$(4),d=b$(4),e,b,g,f;return fb(a,c,d),e=l[c>>2],b=l[d>>2],g=L(e),B(e),b&&(f=L(b),B(b)),[g,f]})}function bx(a){return ca(a)}b.getExceptionMessage=bx;function gn(a,b='i8'){switch(b.endsWith('*')&&(b='*'),b){case'i1':return q[a>>0];case'i8':return q[a>>0];case'i16':return C[a>>1];case'i32':return h[a>>2];case'i64':return h[a>>2];case'float':return au[a>>2];case'double':return aF[a>>3];case'*':return l[a>>2];default:p('invalid type for getValue: '+b)}}function aY(a){a.add_ref()}function bs(a){if(!a)return;aY(new X(a))}function gm(a){bs(a)}function $(a){return c(typeof a=='number'),'0x'+a.toString(16).padStart(8,'0')}function gl(a,b,c='i8'){switch(c.endsWith('*')&&(c='*'),c){case'i1':q[a>>0]=b;break;case'i8':q[a>>0]=b;break;case'i16':C[a>>1]=b;break;case'i32':h[a>>2]=b;break;case'i64':s=[b>>>0,(k=b,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[a>>2]=s[0],h[a+4>>2]=s[1];break;case'float':au[a>>2]=b;break;case'double':aF[a>>3]=b;break;case'*':l[a>>2]=b;break;default:p('invalid type for setValue: '+c)}}function D(a){D.shown||(D.shown={}),D.shown[a]||(D.shown[a]=1,N&&(a='warning: '+a),r(a))}function cR(c,a,d,b){p('Assertion failed: '+L(c)+', at: '+[a?L(a):'unknown filename',d,b?L(b):'unknown function'])}J=[],am=0;function cP(b){var a=new X(b);return a.get_caught()||(a.set_caught(!0),am--),a.set_rethrown(!1),J.push(a),aY(a),a.get_exception_ptr()}function cO(){if(!J.length)return 0;var a=J[J.length-1];return aY(a),a.excPtr}y=0;function cN(){e(0),c(J.length>0);var a=J.pop();bS(a),y=0}function cH(a){throw y||(y=new ba(a)),y}function bk(){var a=y&&y.excPtr,d,b,e,c,f;if(!a)return al(0),0;if(d=new X(a),d.set_adjusted_ptr(a),b=d.get_type(),!b)return al(0),a;for(e=0;e<arguments.length;e++){if(c=arguments[e],c===0||c===b)break;if(f=d.ptr+16,fc(c,b,f))return al(c),a}return al(b),a}cG=bk,cF=bk;function bn(){var a=J.pop(),b;throw a||p('no exception to throw'),b=a.excPtr,a.get_rethrown()||(J.push(a),a.set_rethrown(!0),a.set_caught(!1),am++),y=new ba(b),y}function cC(a){if(!a)return;var b=new X(a);J.push(b),b.set_rethrown(!0),bn()}function cz(a,b,c){var d=new X(a);throw d.init(b,c),y=new ba(a),am++,y}function cy(){return am}function cx(a){return h[eQ()>>2]=a,a}o={isAbs:a=>a.charAt(0)==='/',splitPath:a=>{var b=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;return b.exec(a).slice(1)},normalizeArray:(a,e)=>{for(var c=0,b=a.length-1,d;b>=0;b--)d=a[b],d==='.'?a.splice(b,1):d==='..'?(a.splice(b,1),c++):c&&(a.splice(b,1),c--);if(e)for(;c;c--)a.unshift('..');return a},normalize:a=>{var b=o.isAbs(a),c=a.substr(-1)==='/';return a=o.normalizeArray(a.split('/').filter(a=>!!a),!b).join('/'),!a&&!b&&(a='.'),a&&c&&(a+='/'),(b?'/':'')+a},dirname:d=>{var b=o.splitPath(d),c=b[0],a=b[1];return!c&&!a?'.':(a&&(a=a.substr(0,a.length-1)),c+a)},basename:a=>{if(a==='/')return'/';a=o.normalize(a),a=a.replace(/\/$/,"");var b=a.lastIndexOf('/');return b===-1?a:a.substr(b+1)},join:function(){var a=Array.prototype.slice.call(arguments);return o.normalize(a.join('/'))},join2:(a,b)=>o.normalize(a+'/'+b)};function cu(){var a,b,c;if(typeof crypto=='object'&&typeof crypto.getRandomValues=='function')return a=>crypto.getRandomValues(a);if(N)try{return a=require('crypto'),b=a.randomFillSync,b?b=>a.randomFillSync(b):(c=a.randomBytes,a=>(a.set(c(a.byteLength)),a))}catch(a){}p("no cryptographic support found for randomDevice. consider polyfilling it if you want to use something insecure like Math.random(), e.g. put this in a --pre-js: var crypto = { getRandomValues: function(array) { for (var i = 0; i < array.length; i++) array[i] = (Math.random()*256)|0 } };")}function bu(a){return(bu=cu())(a)}H={resolve:function(){for(var b='',c=!1,d=arguments.length-1,e;d>=-1&&!c;d--){if(e=d>=0?arguments[d]:a.cwd(),typeof e!='string')throw new TypeError('Arguments to path.resolve must be strings');if(!e)return'';b=e+'/'+b,c=o.isAbs(e)}return b=o.normalizeArray(b.split('/').filter(a=>!!a),!c).join('/'),(c?'/':'')+b||'.'},relative:(g,h)=>{var d,e,f,c,a,b;g=H.resolve(g).substr(1),h=H.resolve(h).substr(1);function i(b){for(var a=0,c;a<b.length;a++)if(b[a]!=='')break;for(c=b.length-1;c>=0;c--)if(b[c]!=='')break;return a>c?[]:b.slice(a,c-a+1)}d=i(g.split('/')),e=i(h.split('/')),f=Math.min(d.length,e.length),c=f;for(a=0;a<f;a++)if(d[a]!==e[a]){c=a;break}b=[];for(a=c;a<d.length;a++)b.push('..');return b=b.concat(e.slice(c)),b.join('/')}};function aX(d){for(var a=0,b=0,c;b<d.length;++b)c=d.charCodeAt(b),c<=127?a++:c<=2047?a+=2:c>=55296&&c<=57343?(a+=4,++b):a+=3;return a}function aW(f,c,b,g){var h,e,d,a,i;if(!(g>0))return 0;h=b,e=b+g-1;for(d=0;d<f.length;++d)if(a=f.charCodeAt(d),a>=55296&&a<=57343&&(i=f.charCodeAt(++d),a=65536+((a&1023)<<10)|i&1023),a<=127){if(b>=e)break;c[b++]=a}else if(a<=2047){if(b+1>=e)break;c[b++]=192|a>>6,c[b++]=128|a&63}else if(a<=65535){if(b+2>=e)break;c[b++]=224|a>>12,c[b++]=128|a>>6&63,c[b++]=128|a&63}else{if(b+3>=e)break;a>1114111&&D('Invalid Unicode code point '+$(a)+' encountered when serializing a JS string to a UTF-8 string in wasm memory! (Valid unicode code points should be in range 0-0x10FFFF).'),c[b++]=240|a>>18,c[b++]=128|a>>12&63,c[b++]=128|a>>6&63,c[b++]=128|a&63}return c[b]=0,b-h}function aB(b,d,c){var e=c>0?c:aX(b)+1,a=new Array(e),f=aW(b,a,0,a.length);return d&&(a.length=f),a}G={ttys:[],init:function(){},shutdown:function(){},register:function(b,c){G.ttys[b]={input:[],output:[],ops:c},a.registerDevice(b,G.stream_ops)},stream_ops:{open:function(b){var c=G.ttys[b.node.rdev];if(!c)throw new a.ErrnoError(43);b.tty=c,b.seekable=!1},close:function(a){a.tty.ops.fsync(a.tty)},fsync:function(a){a.tty.ops.fsync(a.tty)},read:function(c,h,f,g,i){var d,e,b;if(!c.tty||!c.tty.ops.get_char)throw new a.ErrnoError(60);d=0;for(e=0;e<g;e++){try{b=c.tty.ops.get_char(c.tty)}catch(b){throw new a.ErrnoError(29)}if(b===void 0&&d===0)throw new a.ErrnoError(6);if(b===null||b===void 0)break;d++,h[f+e]=b}return d&&(c.node.timestamp=Date.now()),d},write:function(b,e,f,d,g){if(!b.tty||!b.tty.ops.put_char)throw new a.ErrnoError(60);try{for(var c=0;c<d;c++)b.tty.ops.put_char(b.tty,e[f+c])}catch(b){throw new a.ErrnoError(29)}return d&&(b.node.timestamp=Date.now()),c}},default_tty_ops:{get_char:function(c){var a,d,e,b;if(!c.input.length){if(a=null,N){d=256,e=Buffer.alloc(d),b=0;try{b=fs.readSync(process.stdin.fd,e,0,d,-1)}catch(a){if(a.toString().includes('EOF'))b=0;else throw a}b>0?a=e.slice(0,b).toString('utf-8'):a=null}else typeof window!='undefined'&&typeof window.prompt=='function'?(a=window.prompt('Input: '),a!==null&&(a+='\n')):typeof readline=='function'&&(a=readline(),a!==null&&(a+='\n'));if(!a)return null;c.input=aB(a,!0)}return c.input.shift()},put_char:function(b,a){a===null||a===10?(Z(U(b.output,0)),b.output=[]):a!=0&&b.output.push(a)},fsync:function(a){a.output&&a.output.length>0&&(Z(U(a.output,0)),a.output=[])}},default_tty1_ops:{put_char:function(b,a){a===null||a===10?(r(U(b.output,0)),b.output=[]):a!=0&&b.output.push(a)},fsync:function(a){a.output&&a.output.length>0&&(r(U(a.output,0)),a.output=[])}}};function gk(a,b){return v.fill(0,a,a+b),a}function gj(b,a){return c(a,"alignment argument is required"),Math.ceil(b/a)*a}function bC(a){p('internal error: mmapAlloc called but `emscripten_builtin_memalign` native symbol not exported')}j={ops_table:null,mount:function(a){return j.createNode(null,'/',16384|511,0)},createNode:function(c,e,d,f){if(a.isBlkdev(d)||a.isFIFO(d))throw new a.ErrnoError(63);j.ops_table||(j.ops_table={dir:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr,lookup:j.node_ops.lookup,mknod:j.node_ops.mknod,rename:j.node_ops.rename,unlink:j.node_ops.unlink,rmdir:j.node_ops.rmdir,readdir:j.node_ops.readdir,symlink:j.node_ops.symlink},stream:{llseek:j.stream_ops.llseek}},file:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr},stream:{llseek:j.stream_ops.llseek,read:j.stream_ops.read,write:j.stream_ops.write,allocate:j.stream_ops.allocate,mmap:j.stream_ops.mmap,msync:j.stream_ops.msync}},link:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr,readlink:j.node_ops.readlink},stream:{}},chrdev:{node:{getattr:j.node_ops.getattr,setattr:j.node_ops.setattr},stream:a.chrdev_stream_ops}});var b=a.createNode(c,e,d,f);return a.isDir(b.mode)?(b.node_ops=j.ops_table.dir.node,b.stream_ops=j.ops_table.dir.stream,b.contents={}):a.isFile(b.mode)?(b.node_ops=j.ops_table.file.node,b.stream_ops=j.ops_table.file.stream,b.usedBytes=0,b.contents=null):a.isLink(b.mode)?(b.node_ops=j.ops_table.link.node,b.stream_ops=j.ops_table.link.stream):a.isChrdev(b.mode)&&(b.node_ops=j.ops_table.chrdev.node,b.stream_ops=j.ops_table.chrdev.stream),b.timestamp=Date.now(),c&&(c.contents[e]=b,c.timestamp=b.timestamp),b},getFileDataAsTypedArray:function(a){return a.contents?a.contents.subarray?a.contents.subarray(0,a.usedBytes):new Uint8Array(a.contents):new Uint8Array(0)},expandFileStorage:function(a,b){var c=a.contents?a.contents.length:0,d,e;if(c>=b)return;d=1024*1024,b=Math.max(b,c*(c<d?2:1.125)>>>0),c!=0&&(b=Math.max(b,256)),e=a.contents,a.contents=new Uint8Array(b),a.usedBytes>0&&a.contents.set(e.subarray(0,a.usedBytes),0)},resizeFileStorage:function(a,b){if(a.usedBytes==b)return;if(b==0)a.contents=null,a.usedBytes=0;else{var c=a.contents;a.contents=new Uint8Array(b),c&&a.contents.set(c.subarray(0,Math.min(b,a.usedBytes))),a.usedBytes=b}},node_ops:{getattr:function(c){var b={};return b.dev=a.isChrdev(c.mode)?c.id:1,b.ino=c.id,b.mode=c.mode,b.nlink=1,b.uid=0,b.gid=0,b.rdev=c.rdev,a.isDir(c.mode)?b.size=4096:a.isFile(c.mode)?b.size=c.usedBytes:a.isLink(c.mode)?b.size=c.link.length:b.size=0,b.atime=new Date(c.timestamp),b.mtime=new Date(c.timestamp),b.ctime=new Date(c.timestamp),b.blksize=4096,b.blocks=Math.ceil(b.size/b.blksize),b},setattr:function(b,a){a.mode!==void 0&&(b.mode=a.mode),a.timestamp!==void 0&&(b.timestamp=a.timestamp),a.size!==void 0&&j.resizeFileStorage(b,a.size)},lookup:function(b,c){throw a.genericErrors[44]},mknod:function(a,b,c,d){return j.createNode(a,b,c,d)},rename:function(b,c,e){var d,f;if(a.isDir(b.mode)){try{d=a.lookupNode(c,e)}catch(a){}if(d)for(f in d.contents)throw new a.ErrnoError(55)}delete b.parent.contents[b.name],b.parent.timestamp=Date.now(),b.name=e,c.contents[e]=b,c.timestamp=b.parent.timestamp,b.parent=c},unlink:function(a,b){delete a.contents[b],a.timestamp=Date.now()},rmdir:function(b,c){var d=a.lookupNode(b,c),e;for(e in d.contents)throw new a.ErrnoError(55);delete b.contents[c],b.timestamp=Date.now()},readdir:function(a){var b=['.','..'],c;for(c in a.contents){if(!a.contents.hasOwnProperty(c))continue;b.push(c)}return b},symlink:function(b,c,d){var a=j.createNode(b,c,511|40960,0);return a.link=d,a},readlink:function(b){if(!a.isLink(b.mode))throw new a.ErrnoError(28);return b.link}},stream_ops:{read:function(f,h,g,i,b){var e=f.node.contents,a,d;if(b>=f.node.usedBytes)return 0;if(a=Math.min(f.node.usedBytes-b,i),c(a>=0),a>8&&e.subarray)h.set(e.subarray(b,b+a),g);else for(d=0;d<a;d++)h[g+d]=e[b+d];return a},write:function(i,d,e,b,f,h){var a,g;if(c(!(d instanceof ArrayBuffer)),d.buffer===q.buffer&&(h=!1),!b)return 0;if(a=i.node,a.timestamp=Date.now(),d.subarray&&(!a.contents||a.contents.subarray)){if(h)return c(f===0,'canOwn must imply no weird position inside the file'),a.contents=d.subarray(e,e+b),a.usedBytes=b,b;if(a.usedBytes===0&&f===0)return a.contents=d.slice(e,e+b),a.usedBytes=b,b;if(f+b<=a.usedBytes)return a.contents.set(d.subarray(e,e+b),f),b}if(j.expandFileStorage(a,f+b),a.contents.subarray&&d.subarray)a.contents.set(d.subarray(e,e+b),f);else for(g=0;g<b;g++)a.contents[f+g]=d[e+g];return a.usedBytes=Math.max(a.usedBytes,f+b),b},llseek:function(c,e,d){var b=e;if(d===1?b+=c.position:d===2&&a.isFile(c.node.mode)&&(b+=c.node.usedBytes),b<0)throw new a.ErrnoError(28);return b},allocate:function(a,b,c){j.expandFileStorage(a.node,b+c),a.node.usedBytes=Math.max(a.node.usedBytes,b+c)},mmap:function(g,f,c,i,h){var d,e,b;if(!a.isFile(g.node.mode))throw new a.ErrnoError(43);if(b=g.node.contents,!(h&2)&&b.buffer===q.buffer)e=!1,d=b.byteOffset;else{if((c>0||c+f<b.length)&&(b.subarray?b=b.subarray(c,c+f):b=Array.prototype.slice.call(b,c,c+f)),e=!0,d=bC(f),!d)throw new a.ErrnoError(48);q.set(b,d)}return{ptr:d,allocated:e}},msync:function(a,b,c,d,e){return j.stream_ops.write(a,b,0,d,c,!1),0}}};function cq(a,e,d,f){var b=f?'':bh('al '+a);an(a,d=>{c(d,'Loading data file "'+a+'" failed (no arrayBuffer).'),e(new Uint8Array(d)),b&&ai(b)},b=>{if(d)d();else throw'Loading data file "'+a+'" failed.'}),b&&aH(b)}cp={0:"Success",1:"Arg list too long",2:"Permission denied",3:"Address already in use",4:"Address not available",5:"Address family not supported by protocol family",6:"No more processes",7:"Socket already connected",8:"Bad file number",9:"Trying to read unreadable message",10:"Mount device busy",11:"Operation canceled",12:"No children",13:"Connection aborted",14:"Connection refused",15:"Connection reset by peer",16:"File locking deadlock error",17:"Destination address required",18:"Math arg out of domain of func",19:"Quota exceeded",20:"File exists",21:"Bad address",22:"File too large",23:"Host is unreachable",24:"Identifier removed",25:"Illegal byte sequence",26:"Connection already in progress",27:"Interrupted system call",28:"Invalid argument",29:"I/O error",30:"Socket is already connected",31:"Is a directory",32:"Too many symbolic links",33:"Too many open files",34:"Too many links",35:"Message too long",36:"Multihop attempted",37:"File or path name too long",38:"Network interface is not configured",39:"Connection reset by network",40:"Network is unreachable",41:"Too many open files in system",42:"No buffer space available",43:"No such device",44:"No such file or directory",45:"Exec format error",46:"No record locks available",47:"The link has been severed",48:"Not enough core",49:"No message of desired type",50:"Protocol not available",51:"No space left on device",52:"Function not implemented",53:"Socket is not connected",54:"Not a directory",55:"Directory not empty",56:"State not recoverable",57:"Socket operation on non-socket",59:"Not a typewriter",60:"No such device or address",61:"Value too large for defined data type",62:"Previous owner died",63:"Not super-user",64:"Broken pipe",65:"Protocol error",66:"Unknown protocol",67:"Protocol wrong type for socket",68:"Math result not representable",69:"Read only file system",70:"Illegal seek",71:"No such process",72:"Stale file handle",73:"Connection timed out",74:"Text file busy",75:"Cross-device link",100:"Device not a stream",101:"Bad font file fmt",102:"Invalid slot",103:"Invalid request code",104:"No anode",105:"Block device required",106:"Channel number out of range",107:"Level 3 halted",108:"Level 3 reset",109:"Link number out of range",110:"Protocol driver not attached",111:"No CSI structure available",112:"Level 2 halted",113:"Invalid exchange",114:"Invalid request descriptor",115:"Exchange full",116:"No data (for no delay io)",117:"Timer expired",118:"Out of streams resources",119:"Machine is not on the network",120:"Package not installed",121:"The object is remote",122:"Advertise error",123:"Srmount error",124:"Communication error on send",125:"Cross mount point (not really error)",126:"Given log. name not unique",127:"f.d. invalid for this operation",128:"Remote address changed",129:"Can access a needed shared lib",130:"Accessing a corrupted shared lib",131:".lib section in a.out corrupted",132:"Attempting to link in too many libs",133:"Attempting to exec a shared library",135:"Streams pipe error",136:"Too many users",137:"Socket type not supported",138:"Not supported",139:"Protocol family not supported",140:"Can't send after socket shutdown",141:"Too many references",142:"Host is down",148:"No medium (in tape drive)",156:"Level 2 not synchronized"},aS={};function co(a){return D('warning: build with -sDEMANGLE_SUPPORT to link in libcxxabi demangling'),a}function ck(a){var b=/\b_Z[\w\d_]+/g;return a.replace(b,function(a){var b=co(a);return a===b?a:b+' ['+a+']'})}a={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:"/",initialized:!1,ignorePermissions:!0,ErrnoError:null,genericErrors:{},filesystems:null,syncFSRequests:0,lookupPath:(h,d={})=>{var i,f,b,c,e,g,j,k,l;if(h=H.resolve(h),!h)return{path:'',node:null};if(i={follow_mount:!0,recurse_count:0},d=Object.assign(i,d),d.recurse_count>8)throw new a.ErrnoError(32);f=h.split('/').filter(a=>!!a),b=a.root,c='/';for(e=0;e<f.length;e++){if(g=e===f.length-1,g&&d.parent)break;if(b=a.lookupNode(b,f[e]),c=o.join2(c,f[e]),a.isMountpoint(b)&&(!g||g&&d.follow_mount)&&(b=b.mounted.root),!g||d.follow)for(j=0;a.isLink(b.mode);)if(k=a.readlink(c),c=H.resolve(o.dirname(c),k),l=a.lookupPath(c,{recurse_count:d.recurse_count+1}),b=l.node,j++>40)throw new a.ErrnoError(32)}return{path:c,node:b}},getPath:b=>{for(var c,d;!0;){if(a.isRoot(b))return d=b.mount.mountpoint,!c?d:d[d.length-1]!=='/'?d+'/'+c:d+c;c=c?b.name+'/'+c:b.name,b=b.parent}},hashName:(e,d)=>{for(var b=0,c=0;c<d.length;c++)b=(b<<5)-b+d.charCodeAt(c)|0;return(e+b>>>0)%a.nameTable.length},hashAddNode:b=>{var c=a.hashName(b.parent.id,b.name);b.name_next=a.nameTable[c],a.nameTable[c]=b},hashRemoveNode:b=>{var d=a.hashName(b.parent.id,b.name),c;if(a.nameTable[d]===b)a.nameTable[d]=b.name_next;else for(c=a.nameTable[d];c;){if(c.name_next===b){c.name_next=b.name_next;break}c=c.name_next}},lookupNode:(c,d)=>{var e=a.mayLookup(c),f,b,g;if(e)throw new a.ErrnoError(e,c);f=a.hashName(c.id,d);for(b=a.nameTable[f];b;b=b.name_next)if(g=b.name,b.parent.id===c.id&&g===d)return b;return a.lookup(c,d)},createNode:(b,e,f,g)=>{c(typeof b=='object');var d=new a.FSNode(b,e,f,g);return a.hashAddNode(d),d},destroyNode:b=>{a.hashRemoveNode(b)},isRoot:a=>a===a.parent,isMountpoint:a=>!!a.mounted,isFile:a=>(a&61440)===32768,isDir:a=>(a&61440)===16384,isLink:a=>(a&61440)===40960,isChrdev:a=>(a&61440)===8192,isBlkdev:a=>(a&61440)===24576,isFIFO:a=>(a&61440)===4096,isSocket:a=>(a&49152)===49152,flagModes:{r:0,"r+":2,w:577,"w+":578,a:1089,"a+":1090},modeStringToFlags:b=>{var c=a.flagModes[b];if(typeof c=='undefined')throw new Error('Unknown file open mode: '+b);return c},flagsToPermissionString:a=>{var b=['r','w','rw'][a&3];return a&512&&(b+='w'),b},nodePermissions:(b,c)=>a.ignorePermissions?0:c.includes('r')&&!(b.mode&292)?2:c.includes('w')&&!(b.mode&146)?2:c.includes('x')&&!(b.mode&73)?2:0,mayLookup:b=>{var c=a.nodePermissions(b,'x');return c?c:b.node_ops.lookup?0:2},mayCreate:(b,c)=>{try{var d=a.lookupNode(b,c);return 20}catch(a){}return a.nodePermissions(b,'wx')},mayDelete:(c,e,f)=>{var b,d;try{b=a.lookupNode(c,e)}catch(a){return a.errno}if(d=a.nodePermissions(c,'wx'),d)return d;if(f){{if(!a.isDir(b.mode))return 54;if(a.isRoot(b)||a.getPath(b)===a.cwd())return 10}}else if(a.isDir(b.mode))return 31;return 0},mayOpen:(b,c)=>{if(!b)return 44;if(a.isLink(b.mode))return 32;if(a.isDir(b.mode))if(a.flagsToPermissionString(c)!=='r'||c&512)return 31;return a.nodePermissions(b,a.flagsToPermissionString(c))},MAX_OPEN_FDS:4096,nextfd:(c=0,d=a.MAX_OPEN_FDS)=>{for(var b=c;b<=d;b++)if(!a.streams[b])return b;throw new a.ErrnoError(33)},getStream:b=>a.streams[b],createStream:(b,d,e)=>{a.FSStream||(a.FSStream=function(){this.shared={}},a.FSStream.prototype={},Object.defineProperties(a.FSStream.prototype,{object:{get:function(){return this.node},set:function(a){this.node=a}},isRead:{get:function(){return(this.flags&2097155)!==1}},isWrite:{get:function(){return(this.flags&2097155)!==0}},isAppend:{get:function(){return this.flags&1024}},flags:{get:function(){return this.shared.flags},set:function(a){this.shared.flags=a}},position:{get:function(){return this.shared.position},set:function(a){this.shared.position=a}}})),b=Object.assign(new a.FSStream,b);var c=a.nextfd(d,e);return b.fd=c,a.streams[c]=b,b},closeStream:b=>{a.streams[b]=null},chrdev_stream_ops:{open:b=>{var c=a.getDevice(b.node.rdev);b.stream_ops=c.stream_ops,b.stream_ops.open&&b.stream_ops.open(b)},llseek:()=>{throw new a.ErrnoError(70)}},major:a=>a>>8,minor:a=>a&255,makedev:(a,b)=>a<<8|b,registerDevice:(b,c)=>{a.devices[b]={stream_ops:c}},getDevice:b=>a.devices[b],getMounts:d=>{for(var b=[],a=[d],c;a.length;)c=a.pop(),b.push(c),a.push.apply(a,c.mounts);return b},syncfs:(b,f)=>{var e,g;typeof b=='function'&&(f=b,b=!1),a.syncFSRequests++,a.syncFSRequests>1&&r('warning: '+a.syncFSRequests+' FS.syncfs operations in flight at once, probably just doing extra work'),e=a.getMounts(a.root.mount),g=0;function h(b){return c(a.syncFSRequests>0),a.syncFSRequests--,f(b)}function d(a){if(a){if(!d.errored)return d.errored=!0,h(a);return}++g>=e.length&&h(null)}e.forEach(a=>{if(!a.type.syncfs)return d(null);a.type.syncfs(a,b,d)})},mount:(f,j,d)=>{var g,i,b,h,c,e;if(typeof f=='string')throw f;if(g=d==='/',i=!d,g&&a.root)throw new a.ErrnoError(10);if(!g&&!i){if(h=a.lookupPath(d,{follow_mount:!1}),d=h.path,b=h.node,a.isMountpoint(b))throw new a.ErrnoError(10);if(!a.isDir(b.mode))throw new a.ErrnoError(54)}return c={type:f,opts:j,mountpoint:d,mounts:[]},e=f.mount(c),e.mount=c,c.root=e,g?a.root=e:b&&(b.mounted=c,b.mount&&b.mount.mounts.push(c)),e},unmount:g=>{var d=a.lookupPath(g,{follow_mount:!1}),b,e,h,f;if(!a.isMountpoint(d.node))throw new a.ErrnoError(28);b=d.node,e=b.mounted,h=a.getMounts(e),Object.keys(a.nameTable).forEach(c=>{for(var b=a.nameTable[c],d;b;)d=b.name_next,h.includes(b.mount)&&a.destroyNode(b),b=d}),b.mounted=null,f=b.mount.mounts.indexOf(e),c(f!==-1),b.mount.mounts.splice(f,1)},lookup:(a,b)=>a.node_ops.lookup(a,b),mknod:(d,f,g)=>{var h=a.lookupPath(d,{parent:!0}),c=h.node,b=o.basename(d),e;if(!b||b==='.'||b==='..')throw new a.ErrnoError(28);if(e=a.mayCreate(c,b),e)throw new a.ErrnoError(e);if(!c.node_ops.mknod)throw new a.ErrnoError(63);return c.node_ops.mknod(c,b,f,g)},create:(c,b)=>(b=b!==void 0?b:438,b&=4095,b|=32768,a.mknod(c,b,0)),mkdir:(c,b)=>(b=b!==void 0?b:511,b&=511|512,b|=16384,a.mknod(c,b,0)),mkdirTree:(e,f)=>{for(var c=e.split('/'),d='',b=0;b<c.length;++b){if(!c[b])continue;d+='/'+c[b];try{a.mkdir(d,f)}catch(a){if(a.errno!=20)throw a}}},mkdev:(d,b,c)=>(typeof c=='undefined'&&(c=b,b=438),b|=8192,a.mknod(d,b,c)),symlink:(e,f)=>{var g,b,c,d;if(!H.resolve(e))throw new a.ErrnoError(44);if(g=a.lookupPath(f,{parent:!0}),b=g.node,!b)throw new a.ErrnoError(44);if(c=o.basename(f),d=a.mayCreate(b,c),d)throw new a.ErrnoError(d);if(!b.node_ops.symlink)throw new a.ErrnoError(63);return b.node_ops.symlink(b,c,e)},rename:(h,i)=>{var p=o.dirname(h),n=o.dirname(i),l=o.basename(h),j=o.basename(i),g=a.lookupPath(h,{parent:!0}),b=g.node,c,e,k,f,m,d;if(g=a.lookupPath(i,{parent:!0}),c=g.node,!b||!c)throw new a.ErrnoError(44);if(b.mount!==c.mount)throw new a.ErrnoError(75);if(e=a.lookupNode(b,l),k=H.relative(h,n),k.charAt(0)!=='.')throw new a.ErrnoError(28);if(k=H.relative(i,p),k.charAt(0)!=='.')throw new a.ErrnoError(55);try{f=a.lookupNode(c,j)}catch(a){}if(e===f)return;if(m=a.isDir(e.mode),d=a.mayDelete(b,l,m),d)throw new a.ErrnoError(d);if(d=f?a.mayDelete(c,j,m):a.mayCreate(c,j),d)throw new a.ErrnoError(d);if(!b.node_ops.rename)throw new a.ErrnoError(63);if(a.isMountpoint(e)||f&&a.isMountpoint(f))throw new a.ErrnoError(10);if(c!==b)if(d=a.nodePermissions(b,'w'),d)throw new a.ErrnoError(d);a.hashRemoveNode(e);try{b.node_ops.rename(e,c,j)}catch(a){throw a}finally{a.hashAddNode(e)}},rmdir:d=>{var g=a.lookupPath(d,{parent:!0}),b=g.node,c=o.basename(d),e=a.lookupNode(b,c),f=a.mayDelete(b,c,!0);if(f)throw new a.ErrnoError(f);if(!b.node_ops.rmdir)throw new a.ErrnoError(63);if(a.isMountpoint(e))throw new a.ErrnoError(10);b.node_ops.rmdir(b,c),a.destroyNode(e)},readdir:c=>{var d=a.lookupPath(c,{follow:!0}),b=d.node;if(!b.node_ops.readdir)throw new a.ErrnoError(54);return b.node_ops.readdir(b)},unlink:d=>{var g=a.lookupPath(d,{parent:!0}),b=g.node,c,e,f;if(!b)throw new a.ErrnoError(44);if(c=o.basename(d),e=a.lookupNode(b,c),f=a.mayDelete(b,c,!1),f)throw new a.ErrnoError(f);if(!b.node_ops.unlink)throw new a.ErrnoError(63);if(a.isMountpoint(e))throw new a.ErrnoError(10);b.node_ops.unlink(b,c),a.destroyNode(e)},readlink:c=>{var d=a.lookupPath(c),b=d.node;if(!b)throw new a.ErrnoError(44);if(!b.node_ops.readlink)throw new a.ErrnoError(28);return H.resolve(a.getPath(b.parent),b.node_ops.readlink(b))},stat:(c,d)=>{var e=a.lookupPath(c,{follow:!d}),b=e.node;if(!b)throw new a.ErrnoError(44);if(!b.node_ops.getattr)throw new a.ErrnoError(63);return b.node_ops.getattr(b)},lstat:b=>a.stat(b,!0),chmod:(c,d,e)=>{var b,f;if(typeof c=='string'?(f=a.lookupPath(c,{follow:!e}),b=f.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);b.node_ops.setattr(b,{mode:d&4095|b.mode&~4095,timestamp:Date.now()})},lchmod:(b,c)=>{a.chmod(b,c,!0)},fchmod:(c,d)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);a.chmod(b.node,d)},chown:(c,f,g,d)=>{var b,e;if(typeof c=='string'?(e=a.lookupPath(c,{follow:!d}),b=e.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);b.node_ops.setattr(b,{timestamp:Date.now()})},lchown:(b,c,d)=>{a.chown(b,c,d,!0)},fchown:(c,d,e)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);a.chown(b.node,d,e)},truncate:(c,e)=>{var b,f,d;if(e<0)throw new a.ErrnoError(28);if(typeof c=='string'?(f=a.lookupPath(c,{follow:!0}),b=f.node):b=c,!b.node_ops.setattr)throw new a.ErrnoError(63);if(a.isDir(b.mode))throw new a.ErrnoError(31);if(!a.isFile(b.mode))throw new a.ErrnoError(28);if(d=a.nodePermissions(b,'w'),d)throw new a.ErrnoError(d);b.node_ops.setattr(b,{size:e,timestamp:Date.now()})},ftruncate:(c,d)=>{var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);if((b.flags&2097155)===0)throw new a.ErrnoError(28);a.truncate(b.node,d)},utime:(c,d,e)=>{var f=a.lookupPath(c,{follow:!0}),b=f.node;b.node_ops.setattr(b,{timestamp:Math.max(d,e)})},open:(e,c,f)=>{var d,j,h,i,g;if(e==="")throw new a.ErrnoError(44);if(c=typeof c=='string'?a.modeStringToFlags(c):c,f=typeof f=='undefined'?438:f,c&64?f=f&4095|32768:f=0,typeof e=='object')d=e;else{e=o.normalize(e);try{j=a.lookupPath(e,{follow:!(c&131072)}),d=j.node}catch(a){}}if(h=!1,c&64)if(d){if(c&128)throw new a.ErrnoError(20)}else d=a.mknod(e,f,0),h=!0;if(!d)throw new a.ErrnoError(44);if(a.isChrdev(d.mode)&&(c&=~512),c&65536&&!a.isDir(d.mode))throw new a.ErrnoError(54);if(!h)if(i=a.mayOpen(d,c),i)throw new a.ErrnoError(i);return c&512&&!h&&a.truncate(d,0),c&=~(128|512|131072),g=a.createStream({node:d,path:a.getPath(d),flags:c,seekable:!0,position:0,stream_ops:d.stream_ops,ungotten:[],error:!1}),g.stream_ops.open&&g.stream_ops.open(g),b.logReadFiles&&!(c&1)&&(a.readFiles||(a.readFiles={}),e in a.readFiles||(a.readFiles[e]=1)),g},close:b=>{if(a.isClosed(b))throw new a.ErrnoError(8);b.getdents&&(b.getdents=null);try{b.stream_ops.close&&b.stream_ops.close(b)}catch(a){throw a}finally{a.closeStream(b.fd)}b.fd=null},isClosed:a=>a.fd===null,llseek:(b,d,c)=>{if(a.isClosed(b))throw new a.ErrnoError(8);if(!b.seekable||!b.stream_ops.llseek)throw new a.ErrnoError(70);if(c!=0&&c!=1&&c!=2)throw new a.ErrnoError(28);return b.position=b.stream_ops.llseek(b,d,c),b.ungotten=[],b.position},read:(b,g,h,f,c)=>{var d,e;if(f<0||c<0)throw new a.ErrnoError(28);if(a.isClosed(b))throw new a.ErrnoError(8);if((b.flags&2097155)===1)throw new a.ErrnoError(8);if(a.isDir(b.node.mode))throw new a.ErrnoError(31);if(!b.stream_ops.read)throw new a.ErrnoError(28);if(d=typeof c!='undefined',d){if(!b.seekable)throw new a.ErrnoError(70)}else c=b.position;return e=b.stream_ops.read(b,g,h,f,c),d||(b.position+=e),e},write:(b,i,g,f,c,h)=>{var e,d;if(f<0||c<0)throw new a.ErrnoError(28);if(a.isClosed(b))throw new a.ErrnoError(8);if((b.flags&2097155)===0)throw new a.ErrnoError(8);if(a.isDir(b.node.mode))throw new a.ErrnoError(31);if(!b.stream_ops.write)throw new a.ErrnoError(28);if(b.seekable&&b.flags&1024&&a.llseek(b,0,2),e=typeof c!='undefined',e){if(!b.seekable)throw new a.ErrnoError(70)}else c=b.position;return d=b.stream_ops.write(b,i,g,f,c,h),e||(b.position+=d),d},allocate:(b,c,d)=>{if(a.isClosed(b))throw new a.ErrnoError(8);if(c<0||d<=0)throw new a.ErrnoError(28);if((b.flags&2097155)===0)throw new a.ErrnoError(8);if(!a.isFile(b.node.mode)&&!a.isDir(b.node.mode))throw new a.ErrnoError(43);if(!b.stream_ops.allocate)throw new a.ErrnoError(138);b.stream_ops.allocate(b,c,d)},mmap:(b,e,f,c,d)=>{if((c&2)!==0&&(d&2)===0&&(b.flags&2097155)!==2)throw new a.ErrnoError(2);if((b.flags&2097155)===1)throw new a.ErrnoError(2);if(!b.stream_ops.mmap)throw new a.ErrnoError(43);return b.stream_ops.mmap(b,e,f,c,d)},msync:(a,b,c,d,e)=>a.stream_ops.msync?a.stream_ops.msync(a,b,c,d,e):0,munmap:a=>0,ioctl:(b,c,d)=>{if(!b.stream_ops.ioctl)throw new a.ErrnoError(59);return b.stream_ops.ioctl(b,c,d)},readFile:(h,b={})=>{var d,e,g,f,c;if(b.flags=b.flags||0,b.encoding=b.encoding||'binary',b.encoding!=='utf8'&&b.encoding!=='binary')throw new Error('Invalid encoding type "'+b.encoding+'"');return e=a.open(h,b.flags),g=a.stat(h),f=g.size,c=new Uint8Array(f),a.read(e,c,0,f,0),b.encoding==='utf8'?d=U(c,0):b.encoding==='binary'&&(d=c),a.close(e),d},writeFile:(g,b,c={})=>{var d,e,f;if(c.flags=c.flags||577,d=a.open(g,c.flags,c.mode),typeof b=='string')e=new Uint8Array(aX(b)+1),f=aW(b,e,0,e.length),a.write(d,e,0,f,void 0,c.canOwn);else if(ArrayBuffer.isView(b))a.write(d,b,0,b.byteLength,void 0,c.canOwn);else throw new Error('Unsupported data type');a.close(d)},cwd:()=>a.currentPath,chdir:d=>{var b=a.lookupPath(d,{follow:!0}),c;if(b.node===null)throw new a.ErrnoError(44);if(!a.isDir(b.node.mode))throw new a.ErrnoError(54);if(c=a.nodePermissions(b.node,'x'),c)throw new a.ErrnoError(c);a.currentPath=b.path},createDefaultDirectories:()=>{a.mkdir('/tmp'),a.mkdir('/home'),a.mkdir('/home/web_user')},createDefaultDevices:()=>{var c,b,d;a.mkdir('/dev'),a.registerDevice(a.makedev(1,3),{read:()=>0,write:(b,c,d,a,e)=>a}),a.mkdev('/dev/null',a.makedev(1,3)),G.register(a.makedev(5,0),G.default_tty_ops),G.register(a.makedev(6,0),G.default_tty1_ops),a.mkdev('/dev/tty',a.makedev(5,0)),a.mkdev('/dev/tty1',a.makedev(6,0)),c=new Uint8Array(1024),b=0,d=()=>(b===0&&(b=bu(c).byteLength),c[--b]),a.createDevice('/dev','random',d),a.createDevice('/dev','urandom',d),a.mkdir('/dev/shm'),a.mkdir('/dev/shm/tmp')},createSpecialDirectories:()=>{a.mkdir('/proc');var b=a.mkdir('/proc/self');a.mkdir('/proc/self/fd'),a.mount({mount:()=>{var c=a.createNode(b,'fd',16384|511,73);return c.node_ops={lookup:(f,d)=>{var e=+d,c=a.getStream(e),b;if(!c)throw new a.ErrnoError(8);return b={parent:null,mount:{mountpoint:'fake'},node_ops:{readlink:()=>c.path}},b.parent=b,b}},c}},{},'/proc/self/fd')},createStandardStreams:()=>{var d,e,f;b.stdin?a.createDevice('/dev','stdin',b.stdin):a.symlink('/dev/tty','/dev/stdin'),b.stdout?a.createDevice('/dev','stdout',null,b.stdout):a.symlink('/dev/tty','/dev/stdout'),b.stderr?a.createDevice('/dev','stderr',null,b.stderr):a.symlink('/dev/tty1','/dev/stderr'),d=a.open('/dev/stdin',0),e=a.open('/dev/stdout',1),f=a.open('/dev/stderr',1),c(d.fd===0,'invalid handle for stdin ('+d.fd+')'),c(e.fd===1,'invalid handle for stdout ('+e.fd+')'),c(f.fd===2,'invalid handle for stderr ('+f.fd+')')},ensureErrnoError:()=>{if(a.ErrnoError)return;a.ErrnoError=function(a,b){this.name='ErrnoError',this.node=b,this.setErrno=function(a){this.errno=a;for(var b in aS)if(aS[b]===a){this.code=b;break}},this.setErrno(a),this.message=cp[a],this.stack&&(Object.defineProperty(this,"stack",{value:(new Error).stack,writable:!0}),this.stack=ck(this.stack))},a.ErrnoError.prototype=new Error,a.ErrnoError.prototype.constructor=a.ErrnoError,[44].forEach(b=>{a.genericErrors[b]=new a.ErrnoError(b),a.genericErrors[b].stack='<generic error, no stack>'})},staticInit:()=>{a.ensureErrnoError(),a.nameTable=new Array(4096),a.mount(j,{},'/'),a.createDefaultDirectories(),a.createDefaultDevices(),a.createSpecialDirectories(),a.filesystems={MEMFS:j}},init:(d,e,f)=>{c(!a.init.initialized,'FS.init was previously called. If you want to initialize later with custom parameters, remove any earlier calls (note that one is automatically added to the generated code)'),a.init.initialized=!0,a.ensureErrnoError(),b.stdin=d||b.stdin,b.stdout=e||b.stdout,b.stderr=f||b.stderr,a.createStandardStreams()},quit:()=>{var b,c;a.init.initialized=!1,bK(0);for(b=0;b<a.streams.length;b++){if(c=a.streams[b],!c)continue;a.close(c)}},getMode:(b,c)=>{var a=0;return b&&(a|=292|73),c&&(a|=146),a},findObject:(c,d)=>{var b=a.analyzePath(c,d);return b.exists?b.object:null},analyzePath:(d,e)=>{var c,b;try{c=a.lookupPath(d,{follow:!e}),d=c.path}catch(a){}b={isRoot:!1,exists:!1,error:0,name:null,path:null,object:null,parentExists:!1,parentPath:null,parentObject:null};try{c=a.lookupPath(d,{parent:!0}),b.parentExists=!0,b.parentPath=c.path,b.parentObject=c.node,b.name=o.basename(d),c=a.lookupPath(d,{follow:!e}),b.exists=!0,b.path=c.path,b.object=c.node,b.name=c.node.name,b.isRoot=c.path==='/'}catch(a){b.error=a.errno}return b},createPath:(b,f,g,h)=>{var d,e,c;for(b=typeof b=='string'?b:a.getPath(b),d=f.split('/').reverse();d.length;){if(e=d.pop(),!e)continue;c=o.join2(b,e);try{a.mkdir(c)}catch(a){}b=c}return c},createFile:(b,c,h,d,e)=>{var f=o.join2(typeof b=='string'?b:a.getPath(b),c),g=a.getMode(d,e);return a.create(f,g)},createDataFile:(c,f,b,m,l,k)=>{var i=f,g,e,h,d,n,j;if(c&&(c=typeof c=='string'?c:a.getPath(c),i=f?o.join2(c,f):c),g=a.getMode(m,l),e=a.create(i,g),b){if(typeof b=='string'){h=new Array(b.length);for(d=0,n=b.length;d<n;++d)h[d]=b.charCodeAt(d);b=h}a.chmod(e,g|146),j=a.open(e,577),a.write(j,b,0,b.length,0,k),a.close(j),a.chmod(e,g)}return e},createDevice:(c,f,d,b)=>{var g=o.join2(typeof c=='string'?c:a.getPath(c),f),h=a.getMode(!!d,!!b),e;return a.createDevice.major||(a.createDevice.major=64),e=a.makedev(a.createDevice.major++,0),a.registerDevice(e,{open:a=>{a.seekable=!1},close:a=>{b&&b.buffer&&b.buffer.length&&b(10)},read:(h,i,f,g,j)=>{for(var c=0,e=0,b;e<g;e++){try{b=d()}catch(b){throw new a.ErrnoError(29)}if(b===void 0&&c===0)throw new a.ErrnoError(6);if(b===null||b===void 0)break;c++,i[f+e]=b}return c&&(h.node.timestamp=Date.now()),c},write:(e,f,g,d,h)=>{for(var c=0;c<d;c++)try{b(f[g+c])}catch(b){throw new a.ErrnoError(29)}return d&&(e.node.timestamp=Date.now()),c}}),a.mkdev(g,h,e)},forceLoadFile:b=>{if(b.isDevice||b.isFolder||b.link||b.contents)return!0;if(typeof XMLHttpRequest!='undefined')throw new Error("Lazy loading should have been performed (contents set) in createLazyFile, but it was not. Lazy loading only works in web workers. Use --embed-file or --preload-file in emcc on the main thread.");if(aQ)try{b.contents=aB(aQ(b.url),!0),b.usedBytes=b.contents.length}catch(b){throw new a.ErrnoError(29)}else throw new Error('Cannot load without read() or XMLHttpRequest.')},createLazyFile:(k,l,f,m,n)=>{var h,d,b,e,i;function g(){this.lengthKnown=!1,this.chunks=[]}if(g.prototype.get=function(a){var b,c;return a>this.length-1||a<0?void 0:(b=a%this.chunkSize,c=a/this.chunkSize|0,this.getter(c)[b])},g.prototype.setDataGetter=function(a){this.getter=a},g.prototype.cacheLength=function(){var b=new XMLHttpRequest,a,e,g,h,c,i,d;if(b.open('HEAD',f,!1),b.send(null),!(b.status>=200&&b.status<300||b.status===304))throw new Error("Couldn't load "+f+". Status: "+b.status);a=Number(b.getResponseHeader("Content-length")),g=(e=b.getResponseHeader("Accept-Ranges"))&&e==="bytes",h=(e=b.getResponseHeader("Content-Encoding"))&&e==="gzip",c=1024*1024,g||(c=a),i=(e,d)=>{if(e>d)throw new Error("invalid range ("+e+", "+d+") or no bytes requested!");if(d>a-1)throw new Error("only "+a+" bytes available! programmer error!");var b=new XMLHttpRequest;if(b.open('GET',f,!1),a!==c&&b.setRequestHeader("Range","bytes="+e+"-"+d),b.responseType='arraybuffer',b.overrideMimeType&&b.overrideMimeType('text/plain; charset=x-user-defined'),b.send(null),!(b.status>=200&&b.status<300||b.status===304))throw new Error("Couldn't load "+f+". Status: "+b.status);return b.response!==void 0?new Uint8Array(b.response||[]):aB(b.responseText||'',!0)},d=this,d.setDataGetter(b=>{var f=b*c,e=(b+1)*c-1;if(e=Math.min(e,a-1),typeof d.chunks[b]=='undefined'&&(d.chunks[b]=i(f,e)),typeof d.chunks[b]=='undefined')throw new Error('doXHR failed!');return d.chunks[b]}),(h||!a)&&(c=a=1,a=this.getter(0).length,c=a,Z("LazyFiles on gzip forces download of the whole file when length is accessed")),this._length=a,this._chunkSize=c,this.lengthKnown=!0},typeof XMLHttpRequest!='undefined'){if(!R)throw'Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc';h=new g,Object.defineProperties(h,{length:{get:function(){return this.lengthKnown||this.cacheLength(),this._length}},chunkSize:{get:function(){return this.lengthKnown||this.cacheLength(),this._chunkSize}}}),d={isDevice:!1,contents:h}}else d={isDevice:!1,url:f};b=a.createFile(k,l,d,m,n),d.contents?b.contents=d.contents:d.url&&(b.contents=null,b.url=d.url),Object.defineProperties(b,{usedBytes:{get:function(){return this.contents.length}}}),e={},i=Object.keys(b.stream_ops),i.forEach(c=>{var d=b.stream_ops[c];e[c]=function(){return a.forceLoadFile(b),d.apply(null,arguments)}});function j(i,g,f,h,e){var b=i.node.contents,d,a;if(e>=b.length)return 0;if(d=Math.min(b.length-e,h),c(d>=0),b.slice)for(a=0;a<d;a++)g[f+a]=b[e+a];else for(a=0;a<d;a++)g[f+a]=b.get(e+a);return d}return e.read=(c,d,e,f,g)=>(a.forceLoadFile(b),j(c,d,e,f,g)),e.mmap=(e,d,f,g,h)=>{a.forceLoadFile(b);var c=bC(d);if(!c)throw new a.ErrnoError(48);return j(e,q,c,d,f),{ptr:c,allocated:!0}},b.stream_ops=e,b},createPreloadedFile:(f,c,d,m,n,g,b,k,l,i)=>{var j=c?H.resolve(o.join2(f,c)):f,e=bh('cp '+j);function h(d){function h(b){i&&i(),k||a.createDataFile(f,c,b,m,n,l),g&&g(),ai(e)}if(Browser.handledByPreloadPlugin(d,j,h,()=>{b&&b(),ai(e)}))return;h(d)}aH(e),typeof d=='string'?cq(d,a=>h(a),b):h(d)},absolutePath:()=>{p('FS.absolutePath has been removed; use PATH_FS.resolve instead')},createFolder:()=>{p('FS.createFolder has been removed; use FS.mkdir instead')},createLink:()=>{p('FS.createLink has been removed; use FS.symlink instead')},joinPath:()=>{p('FS.joinPath has been removed; use PATH.join instead')},mmapAlloc:()=>{p('FS.mmapAlloc has been replaced by the top level function mmapAlloc')},standardizePath:()=>{p('FS.standardizePath has been removed; use PATH.normalize instead')}},t={DEFAULT_POLLMASK:5,calculateAt:function(d,c,f){var b,e;if(o.isAbs(c))return c;if(d===-100?b=a.cwd():(e=t.getStreamFromFD(d),b=e.path),c.length==0){if(!f)throw new a.ErrnoError(44);return b}return o.join2(b,c)},doStat:function(i,g,b){var c,e,f,d;try{c=i(g)}catch(b){if(b&&b.node&&o.normalize(g)!==o.normalize(a.getPath(b.node)))return-54;throw b}return h[b>>2]=c.dev,h[b+8>>2]=c.ino,h[b+12>>2]=c.mode,l[b+16>>2]=c.nlink,h[b+20>>2]=c.uid,h[b+24>>2]=c.gid,h[b+28>>2]=c.rdev,(s=[c.size>>>0,(k=c.size,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+40>>2]=s[0],h[b+44>>2]=s[1]),h[b+48>>2]=4096,h[b+52>>2]=c.blocks,e=c.atime.getTime(),f=c.mtime.getTime(),d=c.ctime.getTime(),(s=[Math.floor(e/1e3)>>>0,(k=Math.floor(e/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+56>>2]=s[0],h[b+60>>2]=s[1]),l[b+64>>2]=e%1e3*1e3,(s=[Math.floor(f/1e3)>>>0,(k=Math.floor(f/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+72>>2]=s[0],h[b+76>>2]=s[1]),l[b+80>>2]=f%1e3*1e3,(s=[Math.floor(d/1e3)>>>0,(k=Math.floor(d/1e3),+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+88>>2]=s[0],h[b+92>>2]=s[1]),l[b+96>>2]=d%1e3*1e3,(s=[c.ino>>>0,(k=c.ino,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[b+104>>2]=s[0],h[b+108>>2]=s[1]),0},doMsync:function(b,c,d,e,f){if(!a.isFile(c.node.mode))throw new a.ErrnoError(43);if(e&2)return 0;var g=v.slice(b,b+d);a.msync(c,g,f,d,e)},varargs:void 0,get:function(){c(t.varargs!=void 0),t.varargs+=4;var a=h[t.varargs-4>>2];return a},getStr:function(a){var b=L(a);return b},getStreamFromFD:function(c){var b=a.getStream(c);if(!b)throw new a.ErrnoError(8);return b}};function cj(h,f,g){var c,b,d,e;t.varargs=g;try{switch(c=t.getStreamFromFD(h),f){case 0:return b=t.get(),b<0?-28:(d=a.createStream(c,b),d.fd);case 1:case 2:return 0;case 3:return c.flags;case 4:return b=t.get(),c.flags|=b,0;case 5:return b=t.get(),e=0,C[b+e>>1]=2,0;case 6:case 7:return 0;case 16:case 8:return-28;case 9:return cx(28),-1;default:return-28}}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}function ci(e,d,f){var b,c;t.varargs=f;try{switch(b=t.getStreamFromFD(e),d){case 21509:case 21505:return b.tty?0:-59;case 21510:case 21511:case 21512:case 21506:case 21507:case 21508:return b.tty?0:-59;case 21519:return b.tty?(c=t.get(),h[c>>2]=0,0):-59;case 21520:return b.tty?-28:-59;case 21531:return c=t.get(),a.ioctl(b,d,c);case 21523:return b.tty?0:-59;case 21524:return b.tty?0:-59;default:return-28}}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}function ch(d,b,e,c){t.varargs=c;try{b=t.getStr(b),b=t.calculateAt(d,b);var f=c?t.get():0;return a.open(b,e,f).fd}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return-b.errno}}ap={};function aq(a){for(var b,c;a.length;)b=a.pop(),c=a.pop(),c(b)}function _(a){return this.fromWireType(h[a>>2])}Y={},P={},ax={},cg=48,ce=57;function bW(a){if(void 0===a)return'_unknown';a=a.replace(/[^a-zA-Z0-9_]/g,'$');var b=a.charCodeAt(0);return b>=cg&&b<=ce?'_'+a:a}function aM(a,b){return a=bW(a),{[a]:function(){return b.apply(this,arguments)}}[a]}function aL(c,b){var a=aM(b,function(a){this.name=b,this.message=a;var c=new Error(a).stack;c!==void 0&&(this.stack=this.toString()+'\n'+c.replace(/^Error(:[^\n]*)?\n/,''))});return a.prototype=Object.create(c.prototype),a.prototype.constructor=a,a.prototype.toString=function(){return this.message===void 0?this.name:this.name+': '+this.message},a}bZ=void 0;function aG(a){throw new bZ(a)}function E(b,d,g){var a,c,e;b.forEach(function(a){ax[a]=d});function f(d){var c=g(d),a;c.length!==b.length&&aG('Mismatched type converter count');for(a=0;a<b.length;++a)z(b[a],c[a])}a=new Array(d.length),c=[],e=0,d.forEach((b,d)=>{P.hasOwnProperty(b)?a[d]=P[b]:(c.push(b),Y.hasOwnProperty(b)||(Y[b]=[]),Y[b].push(()=>{a[d]=P[b],++e,e===c.length&&f(a)}))}),0===c.length&&f(a)}function cb(d){var c=ap[d],a,b,f,g,e;delete ap[d],a=c.elements,b=a.length,f=a.map(function(a){return a.getterReturnType}).concat(a.map(function(a){return a.setterArgumentType})),g=c.rawConstructor,e=c.rawDestructor,E([d],f,function(d){return a.forEach((a,c)=>{var e=d[c],f=a.getter,g=a.getterContext,h=d[c+b],i=a.setter,j=a.setterContext;a.read=a=>e.fromWireType(f(g,a)),a.write=(b,c)=>{var a=[];i(j,b,h.toWireType(a,c)),aq(a)}}),[{name:c.name,fromWireType:function(d){for(var f=new Array(b),c=0;c<b;++c)f[c]=a[c].read(d);return e(d),f},toWireType:function(i,h){var f,d;if(b!==h.length)throw new TypeError("Incorrect number of tuple elements for "+c.name+": expected="+b+", actual="+h.length);f=g();for(d=0;d<b;++d)a[d].write(f,h[d]);return i!==null&&i.push(e,f),f},argPackAdvance:8,readValueFromPointer:_,destructorFunction:e}]})}aI={};function cc(c){var a=aI[c],e,d,b,f;delete aI[c],e=a.rawConstructor,d=a.rawDestructor,b=a.fields,f=b.map(a=>a.getterReturnType).concat(b.map(a=>a.setterArgumentType)),E([c],f,f=>{var c={};return b.forEach((a,d)=>{var e=a.fieldName,g=f[d],h=a.getter,i=a.getterContext,j=f[d+b.length],k=a.setter,l=a.setterContext;c[e]={read:a=>g.fromWireType(h(i,a)),write:(b,c)=>{var a=[];k(l,b,j.toWireType(a,c)),aq(a)}}}),[{name:a.name,fromWireType:function(a){var b={},e;for(e in c)b[e]=c[e].read(a);return d(a),b},toWireType:function(f,g){var a,b;for(a in c)if(!(a in g))throw new TypeError('Missing field: "'+a+'"');b=e();for(a in c)c[a].write(b,g[a]);return f!==null&&f.push(d,b),b},argPackAdvance:8,readValueFromPointer:_,destructorFunction:d}]})}function cd(a,b,c,d,e){}function aC(a){switch(a){case 1:return 0;case 2:return 1;case 4:return 2;case 8:return 3;default:throw new TypeError('Unknown type size: '+a)}}function cf(){for(var b=new Array(256),a=0;a<256;++a)b[a]=String.fromCharCode(a);bU=b}bU=void 0;function u(c){for(var a="",b=c;v[b];)a+=bU[v[b++]];return a}T=void 0;function m(a){throw new T(a)}function z(a,b,e={}){var c,d;if(!('argPackAdvance'in b))throw new TypeError('registerType registeredInstance requires argPackAdvance');if(c=b.name,a||m('type "'+c+'" must have a positive integer typeid pointer'),P.hasOwnProperty(a)){if(e.ignoreDuplicateRegistrations)return;m("Cannot register type '"+c+"' twice")}P[a]=b,delete ax[a],Y.hasOwnProperty(a)&&(d=Y[a],delete Y[a],d.forEach(a=>a()))}function cl(c,a,b,d,e){var f=aC(b);a=u(a),z(c,{name:a,fromWireType:function(a){return!!a},toWireType:function(b,a){return a?d:e},argPackAdvance:8,readValueFromPointer:function(d){var c;if(b===1)c=q;else if(b===2)c=C;else if(b===4)c=h;else throw new TypeError("Unknown boolean type size: "+a);return this.fromWireType(c[d>>f])},destructorFunction:null})}function cm(e){var a,c,b,d;if(!(this instanceof M))return!1;if(!(e instanceof M))return!1;for(a=this.$$.ptrType.registeredClass,c=this.$$.ptr,b=e.$$.ptrType.registeredClass,d=e.$$.ptr;a.baseClass;)c=a.upcast(c),a=a.baseClass;while(b.baseClass)d=b.upcast(d),b=b.baseClass;return a===b&&c===d}function cn(a){return{count:a.count,deleteScheduled:a.deleteScheduled,preservePointerOnDelete:a.preservePointerOnDelete,ptr:a.ptr,ptrType:a.ptrType,smartPtr:a.smartPtr,smartPtrType:a.smartPtrType}}function aN(a){function b(a){return a.$$.ptrType.registeredClass.name}m(b(a)+' instance already deleted')}aU=!1;function bE(a){}function cr(a){a.smartPtr?a.smartPtrType.rawDestructor(a.smartPtr):a.ptrType.registeredClass.rawDestructor(a.ptr)}function bB(a){a.count.value-=1;var b=0===a.count.value;b&&cr(a)}function bA(b,c,a){if(c===a)return b;if(void 0===a.baseClass)return null;var d=bA(b,c,a.baseClass);return d===null?null:a.downcast(d)}bt={};function cv(){return Object.keys(aa).length}function cw(){var a=[],b;for(b in aa)aa.hasOwnProperty(b)&&a.push(aa[b]);return a}ak=[];function a_(){while(ak.length){var a=ak.pop();a.$$.deleteScheduled=!1,a.delete()}}ac=void 0;function cA(a){ac=a,ak.length&&ac&&ac(a_)}function cB(){b.getInheritedInstanceCount=cv,b.getLiveInheritedInstances=cw,b.flushPendingDeletes=a_,b.setDelayFunction=cA}aa={};function cD(a,b){for(b===void 0&&m('ptr should not be undefined');a.baseClass;)b=a.upcast(b),a=a.baseClass;return b}function cE(b,a){return a=cD(b,a),aa[a]}function at(d,a){var b,c;return(!a.ptrType||!a.ptr)&&aG('makeClassHandle requires ptr and ptrType'),b=!!a.smartPtrType,c=!!a.smartPtr,b!==c&&aG('Both smartPtrType and smartPtr must be specified'),a.count={value:1},aj(Object.create(d,{$$:{value:a}}))}function bl(a){var d=this.getPointee(a),c,h,i,e,b,f;if(!d)return this.destructor(a),null;if(c=cE(this.registeredClass,d),void 0!==c)return 0===c.$$.count.value?(c.$$.ptr=d,c.$$.smartPtr=a,c.clone()):(h=c.clone(),this.destructor(a),h);function g(){return this.isSmartPointer?at(this.registeredClass.instancePrototype,{ptrType:this.pointeeType,ptr:d,smartPtrType:this,smartPtr:a}):at(this.registeredClass.instancePrototype,{ptrType:this,ptr:a})}return i=this.registeredClass.getActualType(d),e=bt[i],!e?g.call(this):(this.isConst?b=e.constPointerType:b=e.pointerType,f=bA(d,this.registeredClass,b.registeredClass),f===null)?g.call(this):this.isSmartPointer?at(b.registeredClass.instancePrototype,{ptrType:b,ptr:f,smartPtrType:this,smartPtr:a}):at(b.registeredClass.instancePrototype,{ptrType:b,ptr:f})}function aj(a){return'undefined'==typeof FinalizationRegistry?(aj=a=>a,a):(aU=new FinalizationRegistry(a=>{console.warn(a.leakWarning.stack.replace(/^Error: /,'')),bB(a.$$)}),aj=a=>{var b=a.$$,d=!!b.smartPtr,c,e;return d&&(c={$$:b},e=b.ptrType.registeredClass,c.leakWarning=new Error("Embind found a leaked C++ instance "+e.name+" <"+$(b.ptr)+">.\n"+"We'll free it automatically in this case, but this functionality is not reliable across various environments.\n"+"Make sure to invoke .delete() manually once you're done with the instance instead.\n"+"Originally allocated"),'captureStackTrace'in Error&&Error.captureStackTrace(c.leakWarning,bl),aU.register(a,c,a)),a},bE=a=>aU.unregister(a),aj(a))}function cI(){if(this.$$.ptr||aN(this),this.$$.preservePointerOnDelete)return this.$$.count.value+=1,this;var a=aj(Object.create(Object.getPrototypeOf(this),{$$:{value:cn(this.$$)}}));return a.$$.count.value+=1,a.$$.deleteScheduled=!1,a}function cJ(){this.$$.ptr||aN(this),this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete&&m('Object already scheduled for deletion'),bE(this),bB(this.$$),this.$$.preservePointerOnDelete||(this.$$.smartPtr=void 0,this.$$.ptr=void 0)}function cK(){return!this.$$.ptr}function cL(){return this.$$.ptr||aN(this),this.$$.deleteScheduled&&!this.$$.preservePointerOnDelete&&m('Object already scheduled for deletion'),ak.push(this),ak.length===1&&ac&&ac(a_),this.$$.deleteScheduled=!0,this}function cM(){M.prototype.isAliasOf=cm,M.prototype.clone=cI,M.prototype.delete=cJ,M.prototype.isDeleted=cK,M.prototype.deleteLater=cL}function M(){}function bg(a,b,d){if(void 0===a[b].overloadTable){var c=a[b];a[b]=function(){return a[b].overloadTable.hasOwnProperty(arguments.length)||m("Function '"+d+"' called with an invalid number of arguments ("+arguments.length+") - expects one of ("+a[b].overloadTable+")!"),a[b].overloadTable[arguments.length].apply(this,arguments)},a[b].overloadTable=[],a[b].overloadTable[c.argCount]=c}}function bf(a,d,c){b.hasOwnProperty(a)?((void 0===c||void 0!==b[a].overloadTable&&void 0!==b[a].overloadTable[c])&&m("Cannot register public name '"+a+"' twice"),bg(b,a,a),b.hasOwnProperty(c)&&m("Cannot register multiple overloads of a function with the same number of arguments ("+c+")!"),b[a].overloadTable[c]=d):(b[a]=d,void 0!==c&&(b[a].numArguments=c))}function cQ(a,b,c,d,e,f,g,h){this.name=a,this.constructor=b,this.instancePrototype=c,this.rawDestructor=d,this.baseClass=e,this.getActualType=f,this.upcast=g,this.downcast=h,this.pureVirtualFunctions=[]}function ao(b,a,c){while(a!==c)a.upcast||m("Expected null or instance of "+c.name+", got an instance of "+a.name),b=a.upcast(b),a=a.baseClass;return b}function cS(d,a){var b,c;return a===null?(this.isReference&&m('null is not a valid '+this.name),0):(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),b=a.$$.ptrType.registeredClass,c=ao(a.$$.ptr,b,this.registeredClass),c)}function cT(c,a){var b,d,e;if(a===null)return this.isReference&&m('null is not a valid '+this.name),this.isSmartPointer?(b=this.rawConstructor(),c!==null&&c.push(this.rawDestructor,b),b):0;if(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),!this.isConst&&a.$$.ptrType.isConst&&m('Cannot convert argument of type '+(a.$$.smartPtrType?a.$$.smartPtrType.name:a.$$.ptrType.name)+' to parameter type '+this.name),d=a.$$.ptrType.registeredClass,b=ao(a.$$.ptr,d,this.registeredClass),this.isSmartPointer)switch(void 0===a.$$.smartPtr&&m('Passing raw pointer to smart pointer is illegal'),this.sharingPolicy){case 0:a.$$.smartPtrType===this?b=a.$$.smartPtr:m('Cannot convert argument of type '+(a.$$.smartPtrType?a.$$.smartPtrType.name:a.$$.ptrType.name)+' to parameter type '+this.name);break;case 1:b=a.$$.smartPtr;break;case 2:a.$$.smartPtrType===this?b=a.$$.smartPtr:(e=a.clone(),b=this.rawShare(b,S.toHandle(function(){e.delete()})),c!==null&&c.push(this.rawDestructor,b));break;default:m('Unsupporting sharing policy')}return b}function cU(d,a){var b,c;return a===null?(this.isReference&&m('null is not a valid '+this.name),0):(a.$$||m('Cannot pass "'+V(a)+'" as a '+this.name),a.$$.ptr||m('Cannot pass deleted object as a pointer of type '+this.name),a.$$.ptrType.isConst&&m('Cannot convert argument of type '+a.$$.ptrType.name+' to parameter type '+this.name),b=a.$$.ptrType.registeredClass,c=ao(a.$$.ptr,b,this.registeredClass),c)}function cV(a){return this.rawGetPointee&&(a=this.rawGetPointee(a)),a}function cW(a){this.rawDestructor&&this.rawDestructor(a)}function cX(a){a!==null&&a.delete()}function cY(){I.prototype.getPointee=cV,I.prototype.destructor=cW,I.prototype.argPackAdvance=8,I.prototype.readValueFromPointer=_,I.prototype.deleteObject=cX,I.prototype.fromWireType=bl}function I(d,a,e,b,c,f,g,h,i,j,k){this.name=d,this.registeredClass=a,this.isReference=e,this.isConst=b,this.isSmartPointer=c,this.pointeeType=f,this.sharingPolicy=g,this.rawGetPointee=h,this.rawConstructor=i,this.rawShare=j,this.rawDestructor=k,!c&&a.baseClass===void 0?b?(this.toWireType=cS,this.destructorFunction=null):(this.toWireType=cU,this.destructorFunction=null):this.toWireType=cT}function c_(a,d,c){b.hasOwnProperty(a)||aG('Replacing nonexistant public symbol'),void 0!==b[a].overloadTable&&void 0!==c?b[a].overloadTable[c]=d:(b[a]=d,b[a].argCount=c)}function c$(d,e,a){c('dynCall_'+d in b,"bad function pointer type - dynCall function not found for sig '"+d+"'"),a&&a.length?c(a.length===d.substring(1).replace(/j/g,'--').length):c(d.length==1);var f=b['dynCall_'+d];return a&&a.length?f.apply(null,[e].concat(a)):f.call(null,e)}function da(b,a,d){if(b.includes('j'))return c$(b,a,d);c(i(a),'missing table entry in dynCall: '+a);var e=i(a).apply(null,d);return e}function db(a,d){c(a.includes('j')||a.includes('p'),'getDynCaller should only be called with i64 sigs');var b=[];return function(){return b.length=0,Object.assign(b,arguments),da(a,d,b)}}function w(a,b){a=u(a);function d(){return a.includes('j')?db(a,b):i(b)}var c=d();return typeof c!="function"&&m("unknown function pointer with signature "+a+": "+b),c}bw=void 0;function bD(b){var a=eO(b),c=u(a);return B(a),c}function ag(d,e){var a=[],b={};function c(d){if(b[d])return;if(P[d])return;if(ax[d]){ax[d].forEach(c);return}a.push(d),b[d]=!0}throw e.forEach(c),new bw(d+': '+a.map(bD).join([', ']))}function dg(h,m,k,c,j,f,n,b,i,d,a,l,e){a=u(a),f=w(j,f),b&&(b=w(n,b)),d&&(d=w(i,d)),e=w(l,e);var g=bW(a);bf(g,function(){ag('Cannot construct '+a+' due to unbound types',[c])}),E([h,m,k],c?[c]:[],function(n){var k,l,j,m,i,q,o,p;return n=n[0],c?(k=n.registeredClass,l=k.instancePrototype):l=M.prototype,j=aM(g,function(){if(Object.getPrototypeOf(this)!==m)throw new T("Use 'new' to construct "+a);if(void 0===i.constructor_body)throw new T(a+" has no accessible constructor");var b=i.constructor_body[arguments.length];if(void 0===b)throw new T("Tried to invoke ctor of "+a+" with invalid number of parameters ("+arguments.length+") - expected ("+Object.keys(i.constructor_body).toString()+") parameters instead!");return b.apply(this,arguments)}),m=Object.create(l,{constructor:{value:j}}),j.prototype=m,i=new cQ(a,j,m,e,k,f,b,d),q=new I(a,i,!0,!1,!1),o=new I(a+'*',i,!1,!1,!1),p=new I(a+' const*',i,!1,!0,!1),bt[h]={pointerType:o,constPointerType:p},c_(g,j),[q,o,p]})}function bO(c,d){for(var b=[],a=0;a<c;a++)b.push(l[d+a*4>>2]);return b}function be(q,a,p,o,n,k){var i=a.length,h,j,f,l,e,g,b,d;i<2&&m("argTypes array size mismatch! Must at least get return value and 'this' types!"),c(!k,'Async bindings are only supported with JSPI.'),h=a[1]!==null&&p!==null,j=!1;for(f=1;f<a.length;++f)if(a[f]!==null&&a[f].destructorFunction===void 0){j=!0;break}return l=a[0].name!=="void",e=i-2,g=new Array(e),b=[],d=[],function(){var f,c,i;arguments.length!==e&&m('function '+q+' called with '+arguments.length+' arguments, expected '+e+' args!'),d.length=0,b.length=h?2:1,b[0]=n,h&&(f=a[1].toWireType(d,this),b[1]=f);for(c=0;c<e;++c)g[c]=a[c+2].toWireType(d,arguments[c]),b.push(g[c]);i=o.apply(null,b);function k(e){var b,c;if(j)aq(d);else for(b=h?1:2;b<a.length;b++)c=b===1?f:g[b-2],a[b].destructorFunction!==null&&a[b].destructorFunction(c);if(l)return a[0].fromWireType(e)}return k(i)}}function dj(h,a,f,g,d,e){var b,i,j;c(a>0),b=bO(a,f),d=w(g,d),i=[e],j=[],E([],[h],function(c){c=c[0];var f='constructor '+c.name;if(void 0===c.registeredClass.constructor_body&&(c.registeredClass.constructor_body=[]),void 0!==c.registeredClass.constructor_body[a-1])throw new T("Cannot register multiple constructors with identical number of parameters ("+(a-1)+") for class '"+c.name+"'! Overload resolution is currently only performed using the parameter count, not actual type info!");return c.registeredClass.constructor_body[a-1]=()=>{ag('Cannot construct '+c.name+' due to unbound types',b)},E([],b,function(b){return b.splice(1,0,null),c.registeredClass.constructor_body[a-1]=be(f,b,null,d,e),[]}),[]})}function dk(e,a,b,j,f,c,g,h,i){var d=bO(b,j);a=u(a),c=w(f,c),E([],[e],function(e){var k,f,j;e=e[0],k=e.name+'.'+a,a.startsWith("@@")&&(a=Symbol[a.substring(2)]),h&&e.registeredClass.pureVirtualFunctions.push(a);function l(){ag('Cannot call '+k+' due to unbound types',d)}return f=e.registeredClass.instancePrototype,j=f[a],void 0===j||void 0===j.overloadTable&&j.className!==e.name&&j.argCount===b-2?(l.argCount=b-2,l.className=e.name,f[a]=l):(bg(f,a,k),f[a].overloadTable[b-2]=l),E([],d,function(h){var d=be(k,h,e,c,g,i);return void 0===f[a].overloadTable?(d.argCount=b-2,f[a]=d):f[a].overloadTable[b-2]=d,[]}),[]})}function bY(a,c,b){return a instanceof Object||m(b+' with invalid "this": '+a),a instanceof c.registeredClass.constructor||m(b+' incompatible with "this" of type '+a.constructor.name),a.$$.ptr||m('cannot call emscripten binding method '+b+' on deleted object'),ao(a.$$.ptr,a.$$.ptrType.registeredClass,c.registeredClass)}function dm(h,b,c,f,e,g,d,i,a,j){b=u(b),e=w(f,e),E([],[h],function(f){var h,k;return f=f[0],h=f.name+'.'+b,k={get:function(){ag('Cannot access '+h+' due to unbound types',[c,d])},enumerable:!0,configurable:!0},a?k.set=()=>{ag('Cannot access '+h+' due to unbound types',[c,d])}:k.set=a=>{m(h+' is a read-only property')},Object.defineProperty(f.registeredClass.instancePrototype,b,k),E([],a?[c,d]:[c],function(c){var k=c[0],d={get:function(){var a=bY(this,f,h+' getter');return k.fromWireType(e(g,a))},enumerable:!0},l;return a&&(a=w(i,a),l=c[1],d.set=function(c){var d=bY(this,f,h+' setter'),b=[];a(j,d,l.toWireType(b,c)),aq(b)}),Object.defineProperty(f.registeredClass.instancePrototype,b,d),[]}),[]})}function dn(){this.allocated=[void 0],this.freelist=[],this.get=function(a){return c(this.allocated[a]!==void 0,'invalid handle: '+a),this.allocated[a]},this.allocate=function(b){let a=this.freelist.pop()||this.allocated.length;return this.allocated[a]=b,a},this.free=function(a){c(this.allocated[a]!==void 0),this.allocated[a]=void 0,this.freelist.push(a)}}x=new dn;function bQ(a){a>=x.reserved&&0===--x.get(a).refcount&&x.free(a)}function dr(){for(var b=0,a=x.reserved;a<x.allocated.length;++a)x.allocated[a]!==void 0&&++b;return b}function ds(){x.allocated.push({value:void 0},{value:null},{value:!0},{value:!1}),x.reserved=x.allocated.length,b.count_emval_handles=dr}S={toValue:a=>(a||m('Cannot use deleted val. handle = '+a),x.get(a).value),toHandle:a=>{switch(a){case void 0:return 1;case null:return 2;case!0:return 3;case!1:return 4;default:return x.allocate({refcount:1,value:a})}}};function du(b,a){a=u(a),z(b,{name:a,fromWireType:function(a){var b=S.toValue(a);return bQ(a),b},toWireType:function(b,a){return S.toHandle(a)},argPackAdvance:8,readValueFromPointer:_,destructorFunction:null})}function dv(b,c,a){switch(c){case 0:return function(b){var c=a?q:v;return this.fromWireType(c[b])};case 1:return function(b){var c=a?C:ah;return this.fromWireType(c[b>>1])};case 2:return function(b){var c=a?h:l;return this.fromWireType(c[b>>2])};default:throw new TypeError("Unknown integer type: "+b)}}function dw(c,a,d,e){var f=aC(d);a=u(a);function b(){}b.values={},z(c,{name:a,constructor:b,fromWireType:function(a){return this.constructor.values[a]},toWireType:function(b,a){return a.value},argPackAdvance:8,readValueFromPointer:dv(a,f,e),destructorFunction:null}),bf(a,b)}function aT(a,c){var b=P[a];return void 0===b&&m(c+" has unknown type "+bD(a)),b}function dy(f,a,c){var b=aT(f,'enum'),d,e;a=u(a),d=b.constructor,e=Object.create(b.constructor.prototype,{value:{value:c},constructor:{value:aM(b.name+'_'+a,function(){})}}),d.values[c]=e,d[a]=e}function V(a){if(a===null)return'null';var b=typeof a;return b==='object'||b==='array'||b==='function'?a.toString():''+a}function dA(a,b){switch(b){case 2:return function(a){return this.fromWireType(au[a>>2])};case 3:return function(a){return this.fromWireType(aF[a>>3])};default:throw new TypeError("Unknown float type: "+a)}}function dB(b,a,c){var d=aC(c);a=u(a),z(b,{name:a,fromWireType:function(a){return a},toWireType:function(b,a){if(typeof a!="number"&&typeof a!="boolean")throw new TypeError('Cannot convert "'+V(a)+'" to '+this.name);return a},argPackAdvance:8,readValueFromPointer:dA(a,d),destructorFunction:null})}function dC(b,c,a){switch(c){case 0:return a?function(a){return q[a]}:function(a){return v[a]};case 1:return a?function(a){return C[a>>1]}:function(a){return ah[a>>1]};case 2:return a?function(a){return h[a>>2]}:function(a){return l[a>>2]};default:throw new TypeError("Unknown integer type: "+b)}}function dD(k,a,h,b,c){var i,e,f,j,g,d;a=u(a),c===-1&&(c=4294967295),i=aC(h),e=a=>a,b===0&&(f=32-8*h,e=a=>a<<f>>>f),j=a.includes('unsigned'),g=(d,e)=>{if(typeof d!="number"&&typeof d!="boolean")throw new TypeError('Cannot convert "'+V(d)+'" to '+e);if(d<b||d>c)throw new TypeError('Passing a number "'+V(d)+'" from JS side to C/C++ side to an argument of type "'+a+'", which is outside the valid range ['+b+', '+c+']!')},j?d=function(b,a){return g(a,this.name),a>>>0}:d=function(b,a){return g(a,this.name),a},z(k,{name:a,fromWireType:e,toWireType:d,argPackAdvance:8,readValueFromPointer:dC(a,i,b!==0),destructorFunction:null})}function dE(c,d,a){var e=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array],f=e[d];function b(a){var b,c,d;return a=a>>2,b=l,c=b[a],d=b[a+1],new f(b.buffer,d,c)}a=u(a),z(c,{name:a,fromWireType:b,argPackAdvance:8,readValueFromPointer:b},{ignoreDuplicateRegistrations:!0})}function dF(b,d,a){return c(typeof a=='number','stringToUTF8(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),aW(b,v,d,a)}function dG(c,a){a=u(a);var b=a==="std::string";z(c,{name:a,fromWireType:function(e){var d=l[e>>2],f=e+4,c,g,a,h,k,i,j;if(b){{g=f;for(a=0;a<=d;++a)h=f+a,(a==d||v[h]==0)&&(k=h-g,i=L(g,k),c===void 0?c=i:(c+=String.fromCharCode(0),c+=i),g=h+1)}}else{j=new Array(d);for(a=0;a<d;++a)j[a]=String.fromCharCode(v[f+a]);c=j.join('')}return B(e),c},toWireType:function(i,a){var d,e,f,g,c,h;if(a instanceof ArrayBuffer&&(a=new Uint8Array(a)),e=typeof a=='string',e||a instanceof Uint8Array||a instanceof Uint8ClampedArray||a instanceof Int8Array||m('Cannot pass non-string to std::string'),b&&e?d=aX(a):d=a.length,f=bJ(4+d+1),g=f+4,l[f>>2]=d,b&&e)dF(a,g,d+1);else if(e)for(c=0;c<d;++c)h=a.charCodeAt(c),h>255&&(B(g),m('String has UTF-16 code units that do not fit in 8 bits')),v[g+c]=h;else for(c=0;c<d;++c)v[g+c]=a[c];return i!==null&&i.push(B,f),f},argPackAdvance:8,readValueFromPointer:_,destructorFunction:function(a){B(a)}})}bq=typeof TextDecoder!='undefined'?new TextDecoder('utf-16le'):void 0;function dI(b,i){var d,a,h,g,e,f;for(c(b%2==0,'Pointer passed to UTF16ToString must be aligned to two bytes!'),d=b,a=d>>1,h=a+i/2;!(a>=h)&&ah[a];)++a;if(d=a<<1,d-b>32&&bq)return bq.decode(v.subarray(b,d));g='';for(e=0;!(e>=i/2);++e){if(f=C[b+e*2>>1],f==0)break;g+=String.fromCharCode(f)}return g}function dJ(e,b,a){var f,g,d,h;if(c(b%2==0,'Pointer passed to stringToUTF16 must be aligned to two bytes!'),c(typeof a=='number','stringToUTF16(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),a===void 0&&(a=2147483647),a<2)return 0;a-=2,f=b,g=a<e.length*2?a/2:e.length;for(d=0;d<g;++d)h=e.charCodeAt(d),C[b>>1]=h,b+=2;return C[b>>1]=0,b-f}function dK(a){return a.length*2}function dL(f,g){var b,d,a,e;for(c(f%4==0,'Pointer passed to UTF32ToString must be aligned to four bytes!'),b=0,d='';!(b>=g/4);){if(a=h[f+b*4>>2],a==0)break;++b,a>=65536?(e=a-65536,d+=String.fromCharCode(55296|e>>10,56320|e&1023)):d+=String.fromCharCode(a)}return d}function dM(g,a,d){var f,i,e,b,j;if(c(a%4==0,'Pointer passed to stringToUTF32 must be aligned to four bytes!'),c(typeof d=='number','stringToUTF32(str, outPtr, maxBytesToWrite) is missing the third parameter that specifies the length of the output buffer!'),d===void 0&&(d=2147483647),d<4)return 0;f=a,i=f+d-4;for(e=0;e<g.length;++e)if(b=g.charCodeAt(e),b>=55296&&b<=57343&&(j=g.charCodeAt(++e),b=65536+((b&1023)<<10)|j&1023),h[a>>2]=b,a+=4,a+4>i)break;return h[a>>2]=0,a-f}function dN(b){for(var c=0,a=0,d;a<b.length;++a)d=b.charCodeAt(a),d>=55296&&d<=57343&&++a,c+=4;return c}function dO(h,a,b){b=u(b);var e,f,g,d,c;a===2?(e=dI,f=dJ,d=dK,g=()=>ah,c=1):a===4&&(e=dL,f=dM,d=dN,g=()=>l,c=2),z(h,{name:b,fromWireType:function(d){for(var j=l[d>>2],n=g(),h=d+4,f=0,b,i,m,k;f<=j;++f)i=d+4+f*a,(f==j||n[i>>c]==0)&&(m=i-h,k=e(h,m),b===void 0?b=k:(b+=String.fromCharCode(0),b+=k),h=i+a);return B(d),b},toWireType:function(i,h){var g,e;return typeof h=='string'||m('Cannot pass non-string to C++ string type '+b),g=d(h),e=bJ(4+g+a),l[e>>2]=g>>c,f(h,e+4,g+a),i!==null&&i.push(B,e),e},argPackAdvance:8,readValueFromPointer:_,destructorFunction:function(a){B(a)}})}function dP(a,b,c,d,e,f){ap[a]={name:u(b),rawConstructor:w(c,d),rawDestructor:w(e,f),elements:[]}}function dQ(a,b,c,d,e,f,g,h,i){ap[a].elements.push({getterReturnType:b,getter:w(c,d),getterContext:e,setterArgumentType:f,setter:w(g,h),setterContext:i})}function dR(a,b,c,d,e,f){aI[a]={name:u(b),rawConstructor:w(c,d),rawDestructor:w(e,f),fields:[]}}function dS(a,b,c,d,e,f,g,h,i,j){aI[a].fields.push({fieldName:u(b),getterReturnType:c,getter:w(d,e),getterContext:f,setterArgumentType:g,setter:w(h,i),setterContext:j})}function dT(b,a){a=u(a),z(b,{isVoid:!0,name:a,argPackAdvance:0,fromWireType:function(){},toWireType:function(a,b){}})}dU=!0;function dV(){return dU}function dW(b,d){for(var c=new Array(b),a=0;a<b;++a)c[a]=aT(l[d+a*4>>2],"parameter "+a);return c}function dX(d,b,i,g){var h,c,a,e,f;d=S.toValue(d),h=dW(b,i),c=new Array(b);for(a=0;a<b;++a)e=h[a],c[a]=e.readValueFromPointer(g),g+=e.argPackAdvance;return f=d.apply(void 0,c),S.toHandle(f)}function dY(a){a>4&&(x.get(a).refcount+=1)}function dZ(a,b){a=aT(a,'_emval_take_value');var c=a.readValueFromPointer(b);return S.toHandle(c)}function d_(){p('native code called abort()')}function d$(){return Date.now()}N?bb=()=>{var a=process.hrtime();return a[0]*1e3+a[1]/1e6}:bb=()=>performance.now();function eb(b,a,c){v.copyWithin(b,a,a+c)}function ec(){return 2147483648}function ed(a){var b=af.buffer;try{return af.grow(a-b.byteLength+65535>>>16),bI(),1}catch(c){r('emscripten_realloc_buffer: Attempted to grow heap from '+b.byteLength+' bytes to '+a+' bytes, but got error: '+c)}}function ee(a){var b=v.length,d,e,f,g,i;if(a=a>>>0,c(a>b),d=ec(),a>d)return r('Cannot enlarge memory, asked to go up to '+a+' bytes, but the limit is '+d+' bytes!'),!1;let h=(b,a)=>b+(a-b%a)%a;for(e=1;e<=4;e*=2)if(f=b*(1+.2/e),f=Math.min(f,a+100663296),g=Math.min(d,h(Math.max(a,f),65536)),i=ed(g),i)return!0;return r('Failed to grow the heap from '+b+' bytes to '+g+' bytes, not enough memory!'),!1}a$={};function eg(){return bR||'./this.program'}function ab(){var d,b,a,c;if(!ab.strings){d=(typeof navigator=='object'&&navigator.languages&&navigator.languages[0]||'C').replace('-','_')+'.UTF-8',b={USER:'web_user',LOGNAME:'web_user',PATH:'/',PWD:'/',HOME:'/home/web_user',LANG:d,_:eg()};for(a in a$)a$[a]===void 0?delete b[a]:b[a]=a$[a];c=[];for(a in b)c.push(a+'='+b[a]);ab.strings=c}return ab.strings}function ei(b,d){for(var a=0;a<b.length;++a)c(b.charCodeAt(a)===(b.charCodeAt(a)&255)),q[d++>>0]=b.charCodeAt(a);q[d>>0]=0}function ej(b,c){var a=0;return ab().forEach(function(d,f){var e=c+a;l[b+f*4>>2]=e,ei(d,e),a+=d.length+1}),0}function ek(c,d){var a=ab(),b;return l[c>>2]=a.length,b=0,a.forEach(function(a){b+=a.length+1}),l[d>>2]=b,0}function el(b){try{var c=t.getStreamFromFD(b);return a.close(c),0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function em(h,c,j,d){for(var e=0,f=0,i,g,b;f<j;f++){if(i=l[c>>2],g=l[c+4>>2],c+=8,b=a.read(h,q,i,g,d),b<0)return-1;if(e+=b,b<g)break;typeof d!='undefined'&&(d+=b)}return e}function en(d,e,f,g){var b,c;try{return b=t.getStreamFromFD(d),c=em(b,e,f),l[g>>2]=c,0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function eo(a,b){return c(a==a>>>0||a==(a|0)),c(b===(b|0)),b+2097152>>>0<4194305-!!a?(a>>>0)+b*4294967296:NaN}function ep(i,f,g,d,e){var c,b;try{return c=eo(f,g),isNaN(c)?61:(b=t.getStreamFromFD(i),a.llseek(b,c,d),(s=[b.position>>>0,(k=b.position,+Math.abs(k)>=1?k>0?(Math.min(+Math.floor(k/4294967296),4294967295)|0)>>>0:~~+Math.ceil((k-+(~~k>>>0))/4294967296)>>>0:0)],h[e>>2]=s[0],h[e+4>>2]=s[1]),b.getdents&&c===0&&d===0&&(b.getdents=null),0)}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function eq(g,b,j,d){for(var e=0,f=0,h,i,c;f<j;f++){if(h=l[b>>2],i=l[b+4>>2],b+=8,c=a.write(g,q,h,i,d),c<0)return-1;e+=c,typeof d!='undefined'&&(d+=c)}return e}function er(d,e,f,g){var b,c;try{return b=t.getStreamFromFD(d),c=eq(b,e,f),l[g>>2]=c,0}catch(b){if(typeof a=='undefined'||!(b.name==='ErrnoError'))throw b;return b.errno}}function es(a){return a}function ar(a){return a%4===0&&(a%100!==0||a%400===0)}function eu(c,d){for(var a=0,b=0;b<=d;a+=c[b++]);return a}bm=[31,29,31,30,31,30,31,31,30,31,30,31],bo=[31,28,31,30,31,30,31,31,30,31,30,31];function ex(e,b){for(var a=new Date(e.getTime()),f,c,d;b>0;)if(f=ar(a.getFullYear()),c=a.getMonth(),d=(f?bm:bo)[c],b>d-a.getDate())b-=d-a.getDate()+1,a.setDate(1),c<11?a.setMonth(c+1):(a.setMonth(0),a.setFullYear(a.getFullYear()+1));else return a.setDate(a.getDate()+b),a;return a}function ey(a,b){c(a.length>=0,'writeArrayToMemory array must have a length (should be an array or typed array)'),q.set(a,b)}function ez(s,r,q,b){var j=h[b+40>>2],p={tm_sec:h[b>>2],tm_min:h[b+4>>2],tm_hour:h[b+8>>2],tm_mday:h[b+12>>2],tm_mon:h[b+16>>2],tm_year:h[b+20>>2],tm_wday:h[b+24>>2],tm_yday:h[b+28>>2],tm_isdst:h[b+32>>2],tm_gmtoff:h[b+36>>2],tm_zone:j?L(j):''},c=L(q),i={'%c':'%a %b %d %H:%M:%S %Y','%D':'%m/%d/%y','%F':'%Y-%m-%d','%h':'%b','%r':'%I:%M:%S %p','%R':'%H:%M','%T':'%H:%M:%S','%x':'%m/%d/%y','%X':'%H:%M:%S','%Ec':'%c','%EC':'%C','%Ex':'%m/%d/%y','%EX':'%H:%M:%S','%Ey':'%y','%EY':'%Y','%Od':'%d','%Oe':'%e','%OH':'%H','%OI':'%I','%Om':'%m','%OM':'%M','%OS':'%S','%Ou':'%u','%OU':'%U','%OV':'%V','%Ow':'%w','%OW':'%W','%Oy':'%y'},d,f,l,n,e;for(d in i)c=c.replace(new RegExp(d,'g'),i[d]);f=['Sunday','Monday','Tuesday','Wednesday','Thursday','Friday','Saturday'],l=['January','February','March','April','May','June','July','August','September','October','November','December'];function m(b,c,d){for(var a=typeof b=='number'?b.toString():b||'';a.length<c;)a=d[0]+a;return a}function a(a,b){return m(a,b,'0')}function o(b,c){function d(a){return a<0?-1:a>0?1:0}var a;return(a=d(b.getFullYear()-c.getFullYear()))===0&&(a=d(b.getMonth()-c.getMonth()))===0&&(a=d(b.getDate()-c.getDate())),a}function g(a){switch(a.getDay()){case 0:return new Date(a.getFullYear()-1,11,29);case 1:return a;case 2:return new Date(a.getFullYear(),0,3);case 3:return new Date(a.getFullYear(),0,2);case 4:return new Date(a.getFullYear(),0,1);case 5:return new Date(a.getFullYear()-1,11,31);case 6:return new Date(a.getFullYear()-1,11,30)}}function k(b){var a=ex(new Date(b.tm_year+1900,0,1),b.tm_yday),c=new Date(a.getFullYear(),0,4),d=new Date(a.getFullYear()+1,0,4),e=g(c),f=g(d);return o(e,a)<=0?o(f,a)<=0?a.getFullYear()+1:a.getFullYear():a.getFullYear()-1}n={'%a':function(a){return f[a.tm_wday].substring(0,3)},'%A':function(a){return f[a.tm_wday]},'%b':function(a){return l[a.tm_mon].substring(0,3)},'%B':function(a){return l[a.tm_mon]},'%C':function(b){var c=b.tm_year+1900;return a(c/100|0,2)},'%d':function(b){return a(b.tm_mday,2)},'%e':function(a){return m(a.tm_mday,2,' ')},'%g':function(a){return k(a).toString().substring(2)},'%G':function(a){return k(a)},'%H':function(b){return a(b.tm_hour,2)},'%I':function(c){var b=c.tm_hour;return b==0?b=12:b>12&&(b-=12),a(b,2)},'%j':function(b){return a(b.tm_mday+eu(ar(b.tm_year+1900)?bm:bo,b.tm_mon-1),3)},'%m':function(b){return a(b.tm_mon+1,2)},'%M':function(b){return a(b.tm_min,2)},'%n':function(){return'\n'},'%p':function(a){return a.tm_hour>=0&&a.tm_hour<12?'AM':'PM'},'%S':function(b){return a(b.tm_sec,2)},'%t':function(){return' '},'%u':function(a){return a.tm_wday||7},'%U':function(b){var c=b.tm_yday+7-b.tm_wday;return a(Math.floor(c/7),2)},'%V':function(b){var c=Math.floor((b.tm_yday+7-(b.tm_wday+6)%7)/7),d,e;return(b.tm_wday+371-b.tm_yday-2)%7<=2&&c++,c?c==53&&(e=(b.tm_wday+371-b.tm_yday)%7,e!=4&&(e!=3||!ar(b.tm_year))&&(c=1)):(c=52,d=(b.tm_wday+7-b.tm_yday-1)%7,(d==4||d==5&&ar(b.tm_year%400-1))&&c++),a(c,2)},'%w':function(a){return a.tm_wday},'%W':function(b){var c=b.tm_yday+7-(b.tm_wday+6)%7;return a(Math.floor(c/7),2)},'%y':function(a){return(a.tm_year+1900).toString().substring(2)},'%Y':function(a){return a.tm_year+1900},'%z':function(b){var a=b.tm_gmtoff,c=a>=0;return a=Math.abs(a)/60,a=a/60*100+a%60,(c?'+':'-')+String("0000"+a).slice(-4)},'%Z':function(a){return a.tm_zone},'%%':function(){return'%'}},c=c.replace(/%%/g,'\0\0');for(d in n)c.includes(d)&&(c=c.replace(new RegExp(d,'g'),n[d](p)));return c=c.replace(/\0\0/g,'%'),e=aB(c,!1),e.length>r?0:(ey(e,s),e.length-1)}function eA(a,b,c,d,e){return ez(a,b,c,d)}bp=function(b,c,d,e){b||(b=this),this.parent=b,this.mount=b.mount,this.mounted=null,this.id=a.nextInode++,this.name=c,this.mode=d,this.node_ops={},this.stream_ops={},this.rdev=e},av=292|73,az=146,Object.defineProperties(bp.prototype,{read:{get:function(){return(this.mode&av)===av},set:function(a){a?this.mode|=av:this.mode&=~av}},write:{get:function(){return(this.mode&az)===az},set:function(a){a?this.mode|=az:this.mode&=~az}},isFolder:{get:function(){return a.isDir(this.mode)}},isDevice:{get:function(){return a.isChrdev(this.mode)}}}),a.FSNode=bp,a.staticInit(),b.FS_createPath=a.createPath,b.FS_createDataFile=a.createDataFile,b.FS_createPreloadedFile=a.createPreloadedFile,b.FS_unlink=a.unlink,b.FS_createLazyFile=a.createLazyFile,b.FS_createDevice=a.createDevice,aS={EPERM:63,ENOENT:44,ESRCH:71,EINTR:27,EIO:29,ENXIO:60,E2BIG:1,ENOEXEC:45,EBADF:8,ECHILD:12,EAGAIN:6,EWOULDBLOCK:6,ENOMEM:48,EACCES:2,EFAULT:21,ENOTBLK:105,EBUSY:10,EEXIST:20,EXDEV:75,ENODEV:43,ENOTDIR:54,EISDIR:31,EINVAL:28,ENFILE:41,EMFILE:33,ENOTTY:59,ETXTBSY:74,EFBIG:22,ENOSPC:51,ESPIPE:70,EROFS:69,EMLINK:34,EPIPE:64,EDOM:18,ERANGE:68,ENOMSG:49,EIDRM:24,ECHRNG:106,EL2NSYNC:156,EL3HLT:107,EL3RST:108,ELNRNG:109,EUNATCH:110,ENOCSI:111,EL2HLT:112,EDEADLK:16,ENOLCK:46,EBADE:113,EBADR:114,EXFULL:115,ENOANO:104,EBADRQC:103,EBADSLT:102,EDEADLOCK:16,EBFONT:101,ENOSTR:100,ENODATA:116,ETIME:117,ENOSR:118,ENONET:119,ENOPKG:120,EREMOTE:121,ENOLINK:47,EADV:122,ESRMNT:123,ECOMM:124,EPROTO:65,EMULTIHOP:36,EDOTDOT:125,EBADMSG:9,ENOTUNIQ:126,EBADFD:127,EREMCHG:128,ELIBACC:129,ELIBBAD:130,ELIBSCN:131,ELIBMAX:132,ELIBEXEC:133,ENOSYS:52,ENOTEMPTY:55,ENAMETOOLONG:37,ELOOP:32,EOPNOTSUPP:138,EPFNOSUPPORT:139,ECONNRESET:15,ENOBUFS:42,EAFNOSUPPORT:5,EPROTOTYPE:67,ENOTSOCK:57,ENOPROTOOPT:50,ESHUTDOWN:140,ECONNREFUSED:14,EADDRINUSE:3,ECONNABORTED:13,ENETUNREACH:40,ENETDOWN:38,ETIMEDOUT:73,EHOSTDOWN:142,EHOSTUNREACH:23,EINPROGRESS:26,EALREADY:7,EDESTADDRREQ:17,EMSGSIZE:35,EPROTONOSUPPORT:66,ESOCKTNOSUPPORT:137,EADDRNOTAVAIL:4,ENETRESET:39,EISCONN:30,ENOTCONN:53,ETOOMANYREFS:141,EUSERS:136,EDQUOT:19,ESTALE:72,ENOTSUP:138,ENOMEDIUM:148,EILSEQ:25,EOVERFLOW:61,ECANCELED:11,ENOTRECOVERABLE:56,EOWNERDEAD:62,ESTRPIPE:135},bZ=b.InternalError=aL(Error,'InternalError'),cf(),T=b.BindingError=aL(Error,'BindingError'),cM(),cB(),cY(),bw=b.UnboundTypeError=aL(Error,'UnboundTypeError'),ds(),eE=typeof atob=='function'?atob:function(a){var d='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=',b='',c=0,k,h,i,g,e,f,j;a=a.replace(/[^A-Za-z0-9\+\/\=]/g,'');do i=d.indexOf(a.charAt(c++)),g=d.indexOf(a.charAt(c++)),e=d.indexOf(a.charAt(c++)),f=d.indexOf(a.charAt(c++)),j=i<<2|g>>4,k=(g&15)<<4|e>>2,h=(e&3)<<6|f,b=b+String.fromCharCode(j),e!==64&&(b=b+String.fromCharCode(k)),f!==64&&(b=b+String.fromCharCode(h));while(c<a.length)return b};function eF(e){var b,c,d,a;if(typeof N=='boolean'&&N)return b=Buffer.from(e,'base64'),new Uint8Array(b.buffer,b.byteOffset,b.byteLength);try{c=eE(e),d=new Uint8Array(c.length);for(a=0;a<c.length;++a)d[a]=c.charCodeAt(a);return d}catch(a){throw new Error('Converting base64 string to bytes failed.')}}function gi(a){if(!bd(a))return;return eF(a.slice(bi.length))}function ga(){dt('fetchSettings')}bH={__assert_fail:cR,__cxa_begin_catch:cP,__cxa_current_primary_exception:cO,__cxa_decrement_exception_refcount:bP,__cxa_end_catch:cN,__cxa_find_matching_catch_2:cG,__cxa_find_matching_catch_3:cF,__cxa_increment_exception_refcount:bs,__cxa_rethrow:bn,__cxa_rethrow_primary_exception:cC,__cxa_throw:cz,__cxa_uncaught_exceptions:cy,__resumeException:cH,__syscall_fcntl64:cj,__syscall_ioctl:ci,__syscall_openat:ch,_embind_finalize_value_array:cb,_embind_finalize_value_object:cc,_embind_register_bigint:cd,_embind_register_bool:cl,_embind_register_class:dg,_embind_register_class_constructor:dj,_embind_register_class_function:dk,_embind_register_class_property:dm,_embind_register_emval:du,_embind_register_enum:dw,_embind_register_enum_value:dy,_embind_register_float:dB,_embind_register_integer:dD,_embind_register_memory_view:dE,_embind_register_std_string:dG,_embind_register_std_wstring:dO,_embind_register_value_array:dP,_embind_register_value_array_element:dQ,_embind_register_value_object:dR,_embind_register_value_object_field:dS,_embind_register_void:dT,_emscripten_get_now_is_monotonic:dV,_emval_call:dX,_emval_decref:bQ,_emval_incref:dY,_emval_take_value:dZ,abort:d_,emscripten_date_now:d$,emscripten_get_now:bb,emscripten_memcpy_big:eb,emscripten_resize_heap:ee,environ_get:ej,environ_sizes_get:ek,fd_close:el,fd_read:en,fd_seek:ep,fd_write:er,invoke_dii:fK,invoke_diii:fp,invoke_fi:fS,invoke_fiii:fu,invoke_fiij:de,invoke_i:fJ,invoke_ii:fB,invoke_iii:fw,invoke_iiif:fL,invoke_iiii:fx,invoke_iiiii:fI,invoke_iiiiid:fZ,invoke_iiiiif:fX,invoke_iiiiii:fD,invoke_iiiiiii:fF,invoke_iiiiiiii:f_,invoke_iiiiiiiiiii:f$,invoke_iiiiiiiiiiii:eL,invoke_iiiiiiiiiiiii:fv,invoke_iiiiij:ef,invoke_iij:eG,invoke_iiji:dc,invoke_iijj:ct,invoke_j:dl,invoke_ji:cZ,invoke_jiiii:eD,invoke_jij:eH,invoke_v:fC,invoke_vi:fy,invoke_vif:fN,invoke_viffddj:di,invoke_viffi:fO,invoke_vifii:fW,invoke_vii:fA,invoke_viif:fT,invoke_viifi:fV,invoke_viifii:fU,invoke_viii:fE,invoke_viiifi:fH,invoke_viiifii:fR,invoke_viiii:fz,invoke_viiiid:fY,invoke_viiiif:fQ,invoke_viiiifi:fP,invoke_viiiii:fG,invoke_viiiiii:fM,invoke_viiiiiii:eP,invoke_viiiiiiiiii:eK,invoke_viiiiiiiiiiiiiii:eJ,invoke_viijf:dd,invoke_viijii:ea,invoke_vij:dh,invoke_vijj:cs,llvm_eh_typeid_for:es,strftime_l:eA},gg=dx(),gf=n("__wasm_call_ctors"),ge=n("getTempRet0"),eM=n("__cxa_free_exception"),bJ=n("malloc"),eO=b.___getTypeName=n("__getTypeName"),gd=b.__embind_initialize_bindings=n("_embind_initialize_bindings"),eQ=n("__errno_location"),bK=b._fflush=n("fflush"),B=b._free=n("free"),e=n("setThrew"),al=n("setTempRet0"),bM=function(){return(bM=b.asm.emscripten_stack_init).apply(null,arguments)},eW=function(){return(eW=b.asm.emscripten_stack_get_free).apply(null,arguments)},eX=function(){return(eX=b.asm.emscripten_stack_get_base).apply(null,arguments)},aP=function(){return(aP=b.asm.emscripten_stack_get_end).apply(null,arguments)},f=n("stackSave"),g=n("stackRestore"),b$=n("stackAlloc"),fa=function(){return(fa=b.asm.emscripten_stack_get_current).apply(null,arguments)},fb=b.___get_exception_message=n("__get_exception_message"),fc=n("__cxa_can_catch"),fd=n("__cxa_is_pointer_type"),fe=b.dynCall_iij=n("dynCall_iij"),ff=b.dynCall_vijj=n("dynCall_vijj"),fg=b.dynCall_fiij=n("dynCall_fiij"),fh=b.dynCall_viijf=n("dynCall_viijf"),fi=b.dynCall_ji=n("dynCall_ji"),fj=b.dynCall_vij=n("dynCall_vij"),fk=b.dynCall_jij=n("dynCall_jij"),fl=b.dynCall_viffddj=n("dynCall_viffddj"),fm=b.dynCall_iiji=n("dynCall_iiji"),fn=b.dynCall_iijj=n("dynCall_iijj"),fo=b.dynCall_j=n("dynCall_j"),gc=b.dynCall_jiji=n("dynCall_jiji"),fq=b.dynCall_viijii=n("dynCall_viijii"),fr=b.dynCall_iiiiij=n("dynCall_iiiiij"),ft=b.dynCall_jiiii=n("dynCall_jiiii"),gb=b.dynCall_iiiiijj=n("dynCall_iiiiijj"),gA=b.dynCall_iiiiiijj=n("dynCall_iiiiiijj");function fw(a,b,c){var h=f();try{return i(a)(b,c)}catch(a){if(g(h),!(a instanceof d))throw a;e(1,0)}}function fx(a,b,c,h){var j=f();try{return i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function fy(a,b){var c=f();try{i(a)(b)}catch(a){if(g(c),!(a instanceof d))throw a;e(1,0)}}function fz(a,b,c,h,j){var k=f();try{i(a)(b,c,h,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function fA(a,b,c){var h=f();try{i(a)(b,c)}catch(a){if(g(h),!(a instanceof d))throw a;e(1,0)}}function fB(a,b){var c=f();try{return i(a)(b)}catch(a){if(g(c),!(a instanceof d))throw a;e(1,0)}}function fC(a){var b=f();try{i(a)()}catch(a){if(g(b),!(a instanceof d))throw a;e(1,0)}}function fD(a,b,c,h,j,k){var l=f();try{return i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fE(a,b,c,h){var j=f();try{i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function fF(a,b,c,h,j,k,l){var m=f();try{return i(a)(b,c,h,j,k,l)}catch(a){if(g(m),!(a instanceof d))throw a;e(1,0)}}function fG(a,b,c,h,j,k){var l=f();try{i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fH(a,b,c,h,j,k){var l=f();try{i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fI(a,b,c,h,j){var k=f();try{return i(a)(b,c,h,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function fJ(a){var b=f();try{return i(a)()}catch(a){if(g(b),!(a instanceof d))throw a;e(1,0)}}function fK(a,b,c){var h=f();try{return i(a)(b,c)}catch(a){if(g(h),!(a instanceof d))throw a;e(1,0)}}function fL(a,b,c,h){var j=f();try{return i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function fM(a,b,c,h,j,k,l){var m=f();try{i(a)(b,c,h,j,k,l)}catch(a){if(g(m),!(a instanceof d))throw a;e(1,0)}}function fN(a,b,c){var h=f();try{i(a)(b,c)}catch(a){if(g(h),!(a instanceof d))throw a;e(1,0)}}function fO(a,b,c,h,j){var k=f();try{i(a)(b,c,h,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function fP(a,b,c,h,j,k,l){var m=f();try{i(a)(b,c,h,j,k,l)}catch(a){if(g(m),!(a instanceof d))throw a;e(1,0)}}function fQ(a,b,c,h,j,k){var l=f();try{i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fR(a,b,c,h,j,k,l){var m=f();try{i(a)(b,c,h,j,k,l)}catch(a){if(g(m),!(a instanceof d))throw a;e(1,0)}}function fS(a,b){var c=f();try{return i(a)(b)}catch(a){if(g(c),!(a instanceof d))throw a;e(1,0)}}function fT(a,b,c,h){var j=f();try{i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function fU(a,b,c,h,j,k){var l=f();try{i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fV(a,b,c,h,j){var k=f();try{i(a)(b,c,h,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function fW(a,b,c,h,j){var k=f();try{i(a)(b,c,h,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function fX(a,b,c,h,j,k){var l=f();try{return i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fY(a,b,c,h,j,k){var l=f();try{i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function fZ(a,b,c,h,j,k){var l=f();try{return i(a)(b,c,h,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function f_(a,b,c,h,j,k,l,m){var n=f();try{return i(a)(b,c,h,j,k,l,m)}catch(a){if(g(n),!(a instanceof d))throw a;e(1,0)}}function f$(a,b,c,h,j,k,l,m,n,o,p){var q=f();try{return i(a)(b,c,h,j,k,l,m,n,o,p)}catch(a){if(g(q),!(a instanceof d))throw a;e(1,0)}}function fv(a,b,c,h,j,k,l,m,n,o,p,q,r){var s=f();try{return i(a)(b,c,h,j,k,l,m,n,o,p,q,r)}catch(a){if(g(s),!(a instanceof d))throw a;e(1,0)}}function fu(a,b,c,h){var j=f();try{return i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function fp(a,b,c,h){var j=f();try{return i(a)(b,c,h)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function eP(a,b,c,h,j,k,l,m){var n=f();try{i(a)(b,c,h,j,k,l,m)}catch(a){if(g(n),!(a instanceof d))throw a;e(1,0)}}function eL(a,b,c,h,j,k,l,m,n,o,p,q){var r=f();try{return i(a)(b,c,h,j,k,l,m,n,o,p,q)}catch(a){if(g(r),!(a instanceof d))throw a;e(1,0)}}function eK(a,b,c,h,j,k,l,m,n,o,p){var q=f();try{i(a)(b,c,h,j,k,l,m,n,o,p)}catch(a){if(g(q),!(a instanceof d))throw a;e(1,0)}}function eJ(a,b,c,h,j,k,l,m,n,o,p,q,r,s,t,u){var v=f();try{i(a)(b,c,h,j,k,l,m,n,o,p,q,r,s,t,u)}catch(a){if(g(v),!(a instanceof d))throw a;e(1,0)}}function eH(a,b,c,h){var i=f();try{return fk(a,b,c,h)}catch(a){if(g(i),!(a instanceof d))throw a;e(1,0)}}function eG(a,b,c,h){var i=f();try{return fe(a,b,c,h)}catch(a){if(g(i),!(a instanceof d))throw a;e(1,0)}}function cs(a,b,c,h,i,j){var k=f();try{ff(a,b,c,h,i,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function ct(a,b,c,h,i,j){var k=f();try{return fn(a,b,c,h,i,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function cZ(a,b){var c=f();try{return fi(a,b)}catch(a){if(g(c),!(a instanceof d))throw a;e(1,0)}}function dc(a,b,c,h,i){var j=f();try{return fm(a,b,c,h,i)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function dd(a,b,c,h,i,j){var k=f();try{fh(a,b,c,h,i,j)}catch(a){if(g(k),!(a instanceof d))throw a;e(1,0)}}function de(a,b,c,h,i){var j=f();try{return fg(a,b,c,h,i)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}function dh(a,b,c,h){var i=f();try{fj(a,b,c,h)}catch(a){if(g(i),!(a instanceof d))throw a;e(1,0)}}function di(a,b,c,h,i,j,k,l){var m=f();try{fl(a,b,c,h,i,j,k,l)}catch(a){if(g(m),!(a instanceof d))throw a;e(1,0)}}function dl(a){var b=f();try{return fo(a)}catch(a){if(g(b),!(a instanceof d))throw a;e(1,0)}}function ea(a,b,c,h,i,j,k){var l=f();try{fq(a,b,c,h,i,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function ef(a,b,c,h,i,j,k){var l=f();try{return fr(a,b,c,h,i,j,k)}catch(a){if(g(l),!(a instanceof d))throw a;e(1,0)}}function eD(a,b,c,h,i){var j=f();try{return ft(a,b,c,h,i)}catch(a){if(g(j),!(a instanceof d))throw a;e(1,0)}}b.addOnPostRun=br,b.addRunDependency=aH,b.removeRunDependency=ai,b.FS_createPath=a.createPath,b.FS_createDataFile=a.createDataFile,b.FS_createPreloadedFile=a.createPreloadedFile,b.FS_createLazyFile=a.createLazyFile,b.FS_createDevice=a.createDevice,b.FS_unlink=a.unlink,b.FS=a,eN=['exitJS','ydayFromDate','inetPton4','inetNtop4','inetPton6','inetNtop6','readSockaddr','writeSockaddr','getHostByName','traverseStack','getCallstack','emscriptenLog','convertPCtoSourceLocation','readEmAsmArgs','jstoi_q','jstoi_s','listenOnce','autoResumeAudioContext','handleException','runtimeKeepalivePush','runtimeKeepalivePop','callUserCallback','maybeExit','safeSetTimeout','asmjsMangle','getNativeTypeSize','STACK_SIZE','STACK_ALIGN','POINTER_SIZE','ASSERTIONS','writeI53ToI64','writeI53ToI64Clamped','writeI53ToI64Signaling','writeI53ToU64Clamped','writeI53ToU64Signaling','readI53FromI64','readI53FromU64','convertI32PairToI53','convertU32PairToI53','getCFunc','ccall','cwrap','uleb128Encode','sigToWasmTypes','generateFuncType','convertJsFunctionToWasm','getEmptyTableSlot','updateTableMap','getFunctionAddress','addFunction','removeFunction','reallyNegative','unSign','strLen','reSign','formatString','intArrayToString','AsciiToString','stringToNewUTF8','stringToUTF8OnStack','getSocketFromFD','getSocketAddress','registerKeyEventCallback','maybeCStringToJsString','findEventTarget','findCanvasEventTarget','getBoundingClientRect','fillMouseEventData','registerMouseEventCallback','registerWheelEventCallback','registerUiEventCallback','registerFocusEventCallback','fillDeviceOrientationEventData','registerDeviceOrientationEventCallback','fillDeviceMotionEventData','registerDeviceMotionEventCallback','screenOrientation','fillOrientationChangeEventData','registerOrientationChangeEventCallback','fillFullscreenChangeEventData','registerFullscreenChangeEventCallback','JSEvents_requestFullscreen','JSEvents_resizeCanvasForFullscreen','registerRestoreOldStyle','hideEverythingExceptGivenElement','restoreHiddenElements','setLetterbox','softFullscreenResizeWebGLRenderTarget','doRequestFullscreen','fillPointerlockChangeEventData','registerPointerlockChangeEventCallback','registerPointerlockErrorEventCallback','requestPointerLock','fillVisibilityChangeEventData','registerVisibilityChangeEventCallback','registerTouchEventCallback','fillGamepadEventData','registerGamepadEventCallback','registerBeforeUnloadEventCallback','fillBatteryEventData','battery','registerBatteryEventCallback','setCanvasElementSize','getCanvasElementSize','jsStackTrace','stackTrace','checkWasiClock','wasiRightsToMuslOFlags','wasiOFlagsToMuslOFlags','createDyncallWrapper','setImmediateWrapped','clearImmediateWrapped','polyfillSetImmediate','getPromise','makePromise','makePromiseCallback','setMainLoop','_setNetworkCallback','heapObjectForWebGLType','heapAccessShiftForWebGLHeap','webgl_enable_ANGLE_instanced_arrays','webgl_enable_OES_vertex_array_object','webgl_enable_WEBGL_draw_buffers','webgl_enable_WEBGL_multi_draw','emscriptenWebGLGet','computeUnpackAlignedImageSize','colorChannelsInGlTextureFormat','emscriptenWebGLGetTexPixelData','__glGenObject','emscriptenWebGLGetUniform','webglGetUniformLocation','webglPrepareUniformLocationsBeforeFirstUse','webglGetLeftBracePos','emscriptenWebGLGetVertexAttrib','__glGetActiveAttribOrUniform','writeGLArray','registerWebGlEventCallback','runAndAbortIfError','SDL_unicode','SDL_ttfContext','SDL_audio','GLFW_Window','ALLOC_NORMAL','ALLOC_STACK','allocate','writeStringToMemory','writeAsciiToMemory','registerInheritedInstance','unregisterInheritedInstance','getStringOrSymbol','craftEmvalAllocator','emval_get_global','emval_allocateDestructors','emval_addMethodCaller'],eN.forEach(dp),eT=['run','addOnPreRun','addOnInit','addOnPreMain','addOnExit','FS_createFolder','FS_createLink','out','err','callMain','abort','keepRuntimeAlive','wasmMemory','stackAlloc','stackSave','stackRestore','getTempRet0','setTempRet0','writeStackCookie','checkStackCookie','ptrToString','zeroMemory','getHeapMax','emscripten_realloc_buffer','ENV','MONTH_DAYS_REGULAR','MONTH_DAYS_LEAP','MONTH_DAYS_REGULAR_CUMULATIVE','MONTH_DAYS_LEAP_CUMULATIVE','isLeapYear','arraySum','addDays','ERRNO_CODES','ERRNO_MESSAGES','setErrNo','DNS','Protocols','Sockets','initRandomFill','randomFill','timers','warnOnce','UNWIND_CACHE','readEmAsmArgsArray','getExecutableName','dynCallLegacy','getDynCaller','dynCall','asyncLoad','alignMemory','mmapAlloc','HandleAllocator','convertI32PairToI53Checked','freeTableIndexes','functionsInTableMap','setValue','getValue','PATH','PATH_FS','UTF8Decoder','UTF8ArrayToString','UTF8ToString','stringToUTF8Array','stringToUTF8','lengthBytesUTF8','intArrayFromString','stringToAscii','UTF16Decoder','UTF16ToString','stringToUTF16','lengthBytesUTF16','UTF32ToString','stringToUTF32','lengthBytesUTF32','writeArrayToMemory','SYSCALLS','JSEvents','specialHTMLTargets','currentFullscreenStrategy','restoreOldWindowedStyle','demangle','demangleAll','ExitStatus','getEnvStrings','doReadv','doWritev','dlopenMissingError','promiseMap','uncaughtExceptionCount','exceptionLast','exceptionCaught','ExceptionInfo','exception_addRef','exception_decRef','getExceptionMessageCommon','incrementExceptionRefcount','decrementExceptionRefcount','getExceptionMessage','Browser','wget','MEMFS','TTY','PIPEFS','SOCKFS','tempFixedLengthArray','miniTempWebGLFloatBuffers','miniTempWebGLIntBuffers','GL','emscripten_webgl_power_preferences','AL','GLUT','EGL','GLEW','IDBStore','SDL','SDL_gfx','GLFW','allocateUTF8','allocateUTF8OnStack','InternalError','BindingError','UnboundTypeError','PureVirtualError','init_embind','throwInternalError','throwBindingError','throwUnboundTypeError','ensureOverloadTable','exposePublicSymbol','replacePublicSymbol','extendError','createNamedFunction','embindRepr','registeredInstances','getBasestPointer','getInheritedInstance','getInheritedInstanceCount','getLiveInheritedInstances','registeredTypes','awaitingDependencies','typeDependencies','registeredPointers','registerType','whenDependentTypesAreResolved','embind_charCodes','embind_init_charCodes','readLatin1String','getTypeName','heap32VectorToArray','requireRegisteredType','getShiftFromSize','integerReadValueFromPointer','enumReadValueFromPointer','floatReadValueFromPointer','simpleReadValueFromPointer','runDestructors','craftInvokerFunction','embind__requireFunction','tupleRegistrations','structRegistrations','genericPointerToWireType','constNoSmartPtrRawPointerToWireType','nonConstNoSmartPtrRawPointerToWireType','init_RegisteredPointer','RegisteredPointer','RegisteredPointer_getPointee','RegisteredPointer_destructor','RegisteredPointer_deleteObject','RegisteredPointer_fromWireType','runDestructor','releaseClassHandle','finalizationRegistry','detachFinalizer_deps','detachFinalizer','attachFinalizer','makeClassHandle','init_ClassHandle','ClassHandle','ClassHandle_isAliasOf','throwInstanceAlreadyDeleted','ClassHandle_clone','ClassHandle_delete','deletionQueue','ClassHandle_isDeleted','ClassHandle_deleteLater','flushPendingDeletes','delayFunction','setDelayFunction','RegisteredClass','shallowCopyInternalPointer','downcastPointer','upcastPointer','validateThis','char_0','char_9','makeLegalFunctionName','emval_handles','emval_symbols','init_emval','count_emval_handles','Emval','emval_newers','emval_lookupTypes','emval_methodCallers','emval_registeredMethods'],eT.forEach(bX),ad=function a(){ay||bL(),ay||(ad=a)};function eU(){bM(),eI()}function bL(){if(Q>0)return;if(eU(),eB(),Q>0)return;function a(){if(ay)return;if(ay=!0,b.calledRun=!0,aD)return;ew(),bV(b),b.onRuntimeInitialized&&b.onRuntimeInitialized(),c(!b._main,'compiled without a main, but one is present. if you added it from JS, use Module["onRuntimeInitialized"]'),ev()}b.setStatus?(b.setStatus('Running...'),setTimeout(function(){setTimeout(function(){b.setStatus('')},1),a()},1)):a(),aV()}function gh(){var c=Z,d=r,b=!1;Z=r=a=>{b=!0};try{bK(0),['stdout','stderr'].forEach(function(e){var d=a.analyzePath('/dev/'+e),f,g,c;if(!d)return;f=d.object,g=f.rdev,c=G.ttys[g],c&&c.output&&c.output.length&&(b=!0)})}catch(a){}Z=c,r=d,b&&D('stdio streams had content in them that was not flushed. you should set EXIT_RUNTIME to 1 (see the FAQ), or make sure to emit a newline when you printf etc.')}if(b.preInit)for(typeof b.preInit=='function'&&(b.preInit=[b.preInit]);b.preInit.length>0;)b.preInit.pop()();return bL(),aJ.ready} diff --git a/toolkit/components/translations/fasttext/moz.yaml b/toolkit/components/translations/fasttext/moz.yaml new file mode 100644 index 0000000000..69fa4df06a --- /dev/null +++ b/toolkit/components/translations/fasttext/moz.yaml @@ -0,0 +1,44 @@ +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Firefox + component: Translation + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: fasttext + + description: The JavaScript emscripten worker to run fastText + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://github.com/facebookresearch/fastText + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: v0.9.2 + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: 3697152e0fd772d9185697fdbd4a1d340ca5571d + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: MIT + + notes: > + This code was generated from the fastText repository on the following revision: + 3697152e0fd772d9185697fdbd4a1d340ca5571d + + https://github.com/facebookresearch/fastText + + There are detailed instructions in the Firefox Source Docs on how to build these + dependencies locally. + + https://firefox-source-docs.mozilla.org/toolkit/components/translations/resources/02_contributing.html#building-fasttext diff --git a/toolkit/components/translations/jar.mn b/toolkit/components/translations/jar.mn new file mode 100644 index 0000000000..09acdcab81 --- /dev/null +++ b/toolkit/components/translations/jar.mn @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +toolkit.jar: + content/global/translations/bergamot-translator.js (bergamot-translator/bergamot-translator.js) + content/global/translations/fasttext.js (fasttext/fasttext.js) + content/global/translations/fasttext_wasm.js (fasttext/fasttext_wasm.js) + content/global/translations/language-id-engine-worker.js (content/language-id-engine-worker.js) + content/global/translations/simd-detect-worker.js (content/simd-detect-worker.js) + content/global/translations/translations-document.sys.mjs (content/translations-document.sys.mjs) + content/global/translations/translations-engine-worker.js (content/translations-engine-worker.js) + content/global/translations/translations.html (content/translations.html) + content/global/translations/translations.css (content/translations.css) + content/global/translations/translations.mjs (content/translations.mjs) + content/global/translations/TranslationsTelemetry.sys.mjs (TranslationsTelemetry.sys.mjs) diff --git a/toolkit/components/translations/metrics.yaml b/toolkit/components/translations/metrics.yaml new file mode 100644 index 0000000000..d20c9ea62f --- /dev/null +++ b/toolkit/components/translations/metrics.yaml @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public↩ +# License, v. 2.0. If a copy of the MPL was not distributed with this↩ +# file, You can obtain one at http://mozilla.org/MPL/2.0/.↩ + +# Adding a new metric? We have docs for that!↩ +# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html↩ + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 +$tags: + - 'Firefox :: Translation' + +translations: + requests_count: + type: counter + description: > + The count of translation requests. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: 122 + + error_rate: + type: rate + description: > + The rate of failed translations requests. + denominator_metric: translations.requests_count + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: 122 + + error: + type: event + description: > + The specific translations error that caused a full-page translation failure. + extra_keys: + reason: + type: string + description: The reason for the error. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836436 + data_sensitivity: + - technical + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: 122 + + translation_request: + type: event + description: > + Triggers when a full-page translation request is sent. + extra_keys: + from_language: + type: string + description: The language being translated from. + to_language: + type: string + description: The language being translated into. + auto_translate: + type: boolean + description: > + True if this translation was triggered automatically + due to configured always-translate-language settings. + False if this translation was triggered manually. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836381 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1836381 + data_sensitivity: + - interaction + notification_emails: + - translations-telemetry-alerts@mozilla.com + expires: 122 diff --git a/toolkit/components/translations/moz.build b/toolkit/components/translations/moz.build new file mode 100644 index 0000000000..0da031a0fb --- /dev/null +++ b/toolkit/components/translations/moz.build @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SPHINX_TREES["/toolkit/components/translations"] = "docs" + +JAR_MANIFESTS += ["jar.mn"] + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Translation") + +DIRS += ["actors"] + +BROWSER_CHROME_MANIFESTS += ["tests/browser/browser.ini"] + +with Files("docs/**"): + SCHEDULES.exclusive = ["docs"] diff --git a/toolkit/components/translations/tests/browser/browser.ini b/toolkit/components/translations/tests/browser/browser.ini new file mode 100644 index 0000000000..3cbd829f4b --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser.ini @@ -0,0 +1,22 @@ +[DEFAULT] +support-files = + head.js + shared-head.js + translations-test.mjs + translations-tester-en.html + translations-tester-es.html + translations-tester-es-2.html + translations-tester-no-tag.html +[browser_about_translations_debounce.js] +skip-if = + os == 'linux' # Bug 1821461 +[browser_about_translations_directions.js] +[browser_about_translations_dropdowns.js] +[browser_about_translations_enabling.js] +[browser_about_translations_translations.js] +[browser_full_page.js] +[browser_remote_settings.js] +[browser_translation_document.js] +[browser_translations_actor.js] +[browser_translations_actor_preferred_language.js] +[browser_translations_lang_tags.js] diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js b/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js new file mode 100644 index 0000000000..f31a240ee8 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_debounce.js @@ -0,0 +1,84 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Test the debounce behavior. + */ +add_task(async function test_about_translations_debounce() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr", isBeta: false }, + { fromLang: "fr", toLang: "en", isBeta: false }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + // Do not allow the debounce to come to completion. + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + function setInput(element, value) { + element.value = value; + element.dispatchEvent(new Event("input")); + } + setInput(fromSelect, "en"); + setInput(toSelect, "fr"); + setInput(translationTextarea, "T"); + + info("Get the translations into a stable translationed state"); + await assertTranslationResult("T [en to fr]"); + + info("Reset and pause the debounce state."); + Cu.waiveXrays(window).DEBOUNCE_DELAY = 1_000_000_000; + Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT = 0; + + info("Input text which will be debounced."); + setInput(translationTextarea, "T"); + setInput(translationTextarea, "Te"); + setInput(translationTextarea, "Tex"); + is(Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT, 0, "Debounce has not run."); + + info("Allow the debounce to actually come to completion."); + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; + setInput(translationTextarea, "Text"); + + await assertTranslationResult("TEXT [en to fr]"); + is(Cu.waiveXrays(window).DEBOUNCE_RUN_COUNT, 1, "Debounce ran once."); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_directions.js b/toolkit/components/translations/tests/browser/browser_about_translations_directions.js new file mode 100644 index 0000000000..153ab1f1d9 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_directions.js @@ -0,0 +1,73 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_about_translations_language_directions() { + await openAboutTranslations({ + languagePairs: [ + // English (en) is LTR and Arabic (ar) is RTL. + { fromLang: "en", toLang: "ar", isBeta: true }, + { fromLang: "ar", toLang: "en", isBeta: true }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "ar"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This text starts as LTR."; + translationTextarea.dispatchEvent(new Event("input")); + + is( + window.getComputedStyle(translationTextarea).direction, + "ltr", + "The English input is LTR" + ); + is( + window.getComputedStyle(translationResult).direction, + "rtl", + "The Arabic results are RTL" + ); + + toSelect.value = ""; + toSelect.dispatchEvent(new Event("input")); + fromSelect.value = "ar"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "en"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This text starts as RTL."; + translationTextarea.dispatchEvent(new Event("input")); + + is( + window.getComputedStyle(translationTextarea).direction, + "rtl", + "The Arabic input is RTL" + ); + is( + window.getComputedStyle(translationResult).direction, + "ltr", + "The English results are LTR" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js b/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js new file mode 100644 index 0000000000..67c5da1af9 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_dropdowns.js @@ -0,0 +1,131 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_task(async function test_about_translations_dropdowns() { + let languagePairs = [ + { fromLang: "en", toLang: "es", isBeta: false }, + { fromLang: "es", toLang: "en", isBeta: false }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en", isBeta: true }, + ]; + await openAboutTranslations({ + languagePairs, + dataForContent: languagePairs, + runInPage: async ({ dataForContent: languagePairs, selectors }) => { + const { document } = content; + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** + * Some languages can be marked as hidden in the dropbdown. This function + * asserts the configuration of the options. + * + * @param {object} args + * @param {string} args.message + * @param {HTMLSelectElement} args.select + * @param {string[]} args.availableOptions + * @param {string} args.selectedValue + */ + function assertOptions({ + message, + select, + availableOptions, + selectedValue, + }) { + const options = [...select.options]; + const betaL10nId = "about-translations-displayname-beta"; + for (const option of options) { + for (const languagePair of languagePairs) { + if ( + languagePair.fromLang === option.value || + languagePair.toLang === option.value + ) { + if (option.getAttribute("data-l10n-id") === betaL10nId) { + is( + languagePair.isBeta, + true, + `Since data-l10n-id was ${betaL10nId} for ${option.value}, then it must be part of a beta language pair, but it was not.` + ); + } + if (!languagePair.isBeta) { + is( + option.getAttribute("data-l10n-id") === betaL10nId, + false, + `Since the languagePair is non-beta, the language option ${option.value} should not have a data-l10-id of ${betaL10nId}, but it does.` + ); + } + } + } + } + info(message); + Assert.deepEqual( + options.filter(option => !option.hidden).map(option => option.value), + availableOptions, + "The available options match." + ); + + is(selectedValue, select.value, "The selected value matches."); + } + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + + assertOptions({ + message: 'From languages have "detect" already selected.', + select: fromSelect, + availableOptions: ["detect", "en", "is", "es"], + selectedValue: "detect", + }); + + assertOptions({ + message: + 'The "to" options do not have "detect" in the list, and nothing is selected.', + select: toSelect, + availableOptions: ["", "en", "es"], + selectedValue: "", + }); + + info('Switch the "to" language to "es".'); + toSelect.value = "es"; + toSelect.dispatchEvent(new Event("input")); + + assertOptions({ + message: 'The "from" languages no longer suggest "es".', + select: fromSelect, + availableOptions: ["detect", "en", "is"], + selectedValue: "detect", + }); + + assertOptions({ + message: 'The "to" options remain the same, but "es" is selected.', + select: toSelect, + availableOptions: ["", "en", "es"], + selectedValue: "es", + }); + + info('Switch the "from" language to English.'); + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + + assertOptions({ + message: 'The "to" languages no longer suggest "en".', + select: toSelect, + availableOptions: ["", "es"], + selectedValue: "es", + }); + + assertOptions({ + message: 'The "from" options remain the same, but "en" is selected.', + select: fromSelect, + availableOptions: ["detect", "en", "is"], + selectedValue: "en", + }); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js b/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js new file mode 100644 index 0000000000..662abc3995 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_enabling.js @@ -0,0 +1,124 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Checks that the page renders without issue, and that the expected elements + * are there. + */ +add_task(async function test_about_translations_enabled() { + await openAboutTranslations({ + runInPage: async ({ selectors }) => { + const { document, window } = content; + + await ContentTaskUtils.waitForCondition(() => { + const trElement = document.querySelector(selectors.translationResult); + const trBlankElement = document.querySelector( + selectors.translationResultBlank + ); + const { visibility: trVisibility } = window.getComputedStyle(trElement); + const { visibility: trBlankVisibility } = + window.getComputedStyle(trBlankElement); + return trVisibility === "hidden" && trBlankVisibility === "visible"; + }, `Waiting for placeholder text to be visible."`); + + function checkElementIsVisible(expectVisible, name) { + const expected = expectVisible ? "visible" : "hidden"; + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is( + visibility, + expected, + `Element ${name} was not ${expected} but should be.` + ); + } + + checkElementIsVisible(true, "pageHeader"); + checkElementIsVisible(true, "fromLanguageSelect"); + checkElementIsVisible(true, "toLanguageSelect"); + checkElementIsVisible(true, "translationTextarea"); + checkElementIsVisible(true, "translationResultBlank"); + + checkElementIsVisible(false, "translationResult"); + }, + }); +}); + +/** + * Checks that the page does not show the content when disabled. + */ +add_task(async function test_about_translations_disabled() { + await openAboutTranslations({ + disabled: true, + runInPage: async ({ selectors }) => { + const { document, window } = content; + + await ContentTaskUtils.waitForCondition(() => { + const element = document.querySelector(selectors.translationResult); + const { visibility } = window.getComputedStyle(element); + return visibility === "hidden"; + }, `Waiting for translated text to be hidden."`); + + function checkElementIsInvisible(name) { + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is(visibility, "hidden", `Element ${name} was invisible.`); + } + + checkElementIsInvisible("pageHeader"); + checkElementIsInvisible("fromLanguageSelect"); + checkElementIsInvisible("toLanguageSelect"); + checkElementIsInvisible("translationTextarea"); + checkElementIsInvisible("translationResult"); + checkElementIsInvisible("translationResultBlank"); + }, + }); +}); + +/** + * Test that the page is properly disabled when the engine isn't supported. + */ +add_task(async function test_about_translations_disabling() { + await openAboutTranslations({ + prefs: [["browser.translations.simulateUnsupportedEngine", true]], + runInPage: async ({ selectors }) => { + const { document, window } = content; + + info('Checking for the "no support" message.'); + await ContentTaskUtils.waitForCondition( + () => document.querySelector(selectors.noSupportMessage), + 'Waiting for the "no support" message.' + ); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + + ok(fromSelect.disabled, "The from select is disabled"); + ok(toSelect.disabled, "The to select is disabled"); + ok(translationTextarea.disabled, "The textarea is disabled"); + + function checkElementIsVisible(expectVisible, name) { + const expected = expectVisible ? "visible" : "hidden"; + const element = document.querySelector(selectors[name]); + ok(Boolean(element), `Element ${name} was found.`); + const { visibility } = window.getComputedStyle(element); + is( + visibility, + expected, + `Element ${name} was not ${expected} but should be.` + ); + } + + checkElementIsVisible(true, "translationInfo"); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_about_translations_translations.js b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js new file mode 100644 index 0000000000..e519f2d0d2 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_about_translations_translations.js @@ -0,0 +1,248 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Test that the UI actually translates text, but use a mocked translations engine. + * The results of the "translation" will be modifying the text to be full width latin + * characters, so that the results visually appear modified. + */ +add_task(async function test_about_translations_translations() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr", isBeta: false }, + { fromLang: "fr", toLang: "en", isBeta: false }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en", isBeta: true }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr]"); + is( + translationResult.getAttribute("lang"), + "fr", + "The result is listed as in French." + ); + + // Blank out the "to" select so it doesn't try to translate between is to fr. + toSelect.value = ""; + toSelect.dispatchEvent(new Event("input")); + + fromSelect.value = "is"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "en"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "This is the second translation."; + translationTextarea.dispatchEvent(new Event("input")); + + await assertTranslationResult( + "THIS IS THE SECOND TRANSLATION. [is to en]" + ); + is( + translationResult.getAttribute("lang"), + "en", + "The result is listed as in English." + ); + }, + }); +}); + +/** + * Test the useHTML pref. + */ +add_task(async function test_about_translations_html() { + await openAboutTranslations({ + languagePairs: [ + { fromLang: "en", toLang: "fr", isBeta: false }, + { fromLang: "fr", toLang: "en", isBeta: false }, + ], + prefs: [["browser.translations.useHTML", true]], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is( + translation, + translationResult.innerText, + "The text runs through the mocked translations engine." + ); + } + + fromSelect.value = "en"; + fromSelect.dispatchEvent(new Event("input")); + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr, html]"); + }, + }); +}); + +add_task(async function test_about_translations_language_identification() { + await openAboutTranslations({ + detectedLangTag: "en", + detectedLanguageConfidence: "0.98", + languagePairs: [ + { fromLang: "en", toLang: "fr", isBeta: false }, + { fromLang: "fr", toLang: "en", isBeta: false }, + ], + runInPage: async ({ selectors }) => { + const { document, window } = content; + Cu.waiveXrays(window).DEBOUNCE_DELAY = 5; // Make the timer run faster for tests. + + await ContentTaskUtils.waitForCondition(() => { + return document.body.hasAttribute("ready"); + }, "Waiting for the document to be ready."); + + /** @type {HTMLSelectElement} */ + const fromSelect = document.querySelector(selectors.fromLanguageSelect); + /** @type {HTMLSelectElement} */ + const toSelect = document.querySelector(selectors.toLanguageSelect); + /** @type {HTMLTextAreaElement} */ + const translationTextarea = document.querySelector( + selectors.translationTextarea + ); + /** @type {HTMLDivElement} */ + const translationResult = document.querySelector( + selectors.translationResult + ); + + async function assertTranslationResult(translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === translationResult.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + is( + translation, + translationResult.innerText, + "The language identification engine correctly informs the translation." + ); + } + + const fromSelectStartValue = fromSelect.value; + const detectStartText = fromSelect.options[0].textContent; + + is( + fromSelectStartValue, + "detect", + 'The fromSelect starting value is "detect"' + ); + + translationTextarea.value = "Text to translate."; + translationTextarea.dispatchEvent(new Event("input")); + + toSelect.value = "fr"; + toSelect.dispatchEvent(new Event("input")); + + await ContentTaskUtils.waitForCondition(() => { + const element = document.querySelector( + selectors.translationResultBlank + ); + const { visibility } = window.getComputedStyle(element); + return visibility === "hidden"; + }, `Waiting for placeholder text to be visible."`); + + const fromSelectFinalValue = fromSelect.value; + is( + fromSelectFinalValue, + fromSelectStartValue, + "The fromSelect value has not changed" + ); + + // The mocked translations make the text uppercase and reports the models used. + await assertTranslationResult("TEXT TO TRANSLATE. [en to fr]"); + + const detectFinalText = fromSelect.options[0].textContent; + is( + true, + detectFinalText.startsWith(detectStartText) && + detectFinalText.length > detectStartText.length, + `fromSelect starting display text (${detectStartText}) should be a substring of the final text (${detectFinalText})` + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_full_page.js b/toolkit/components/translations/tests/browser/browser_full_page.js new file mode 100644 index 0000000000..035b484d4f --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_full_page.js @@ -0,0 +1,138 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Check that the full page translation feature works. + */ +add_task(async function test_full_page_translation() { + await autoTranslatePage({ + page: TRANSLATIONS_TESTER_ES, + languagePairs: [ + { fromLang: "es", toLang: "en", isBeta: false }, + { fromLang: "en", toLang: "es", isBeta: false }, + ], + runInPage: async TranslationsTest => { + const selectors = TranslationsTest.getSelectors(); + + await TranslationsTest.assertTranslationResult( + "The main title gets translated.", + selectors.getH1, + "DON QUIJOTE DE LA MANCHA [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "The last paragraph gets translated. It is out of the viewport.", + selectors.getLastParagraph, + "— PUES, AUNQUE MOVÁIS MÁS BRAZOS QUE LOS DEL GIGANTE BRIAREO, ME LO HABÉIS DE PAGAR. [es to en, html]" + ); + + selectors.getH1().innerText = "Este es un titulo"; + + await TranslationsTest.assertTranslationResult( + "Mutations get tracked", + selectors.getH1, + "ESTE ES UN TITULO [es to en]" + ); + + await TranslationsTest.assertTranslationResult( + "Other languages do not get translated.", + selectors.getHeader, + "The following is an excerpt from Don Quijote de la Mancha, which is in the public domain" + ); + }, + }); +}); + +/** + * Check that the full page translation feature doesn't translate pages in the app's + * locale. + */ +add_task(async function test_about_translations_enabled() { + const { appLocaleAsBCP47 } = Services.locale; + if (!appLocaleAsBCP47.startsWith("en")) { + console.warn( + "This test assumes to be running in an 'en' app locale, however the app locale " + + `is set to ${appLocaleAsBCP47}. Skipping the test.` + ); + ok(true, "Skipping test."); + return; + } + + await autoTranslatePage({ + page: TRANSLATIONS_TESTER_EN, + languagePairs: [ + { fromLang: "es", toLang: "en", isBeta: false }, + { fromLang: "en", toLang: "es", isBeta: false }, + ], + runInPage: async () => { + const { document } = content; + + for (let i = 0; i < 5; i++) { + // There is no way to directly check the non-existence of a translation, as + // the translations engine works async, and you can't dispatch a CustomEvent + // to listen for translations, as this script runs after the initial translations + // check. So resort to a setTimeout and check a few times. This relies on timing, + // but _cannot fail_ if it's working correctly. It _will most likely fail_ if + // this page accidentally gets translated. + + const timeout = 10; + + info("Waiting for the timeout."); + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + await new Promise(resolve => setTimeout(resolve, timeout)); + is( + document.querySelector("h1").innerText, + `"The Wonderful Wizard of Oz" by L. Frank Baum`, + `The page remains untranslated after ${(i + 1) * timeout}ms.` + ); + } + }, + }); +}); + +/** + * Check that the full page translation feature works. + */ +add_task(async function test_language_identification_for_page_translation() { + await autoTranslatePage({ + page: TRANSLATIONS_TESTER_NO_TAG, + detectedLangTag: "es", + detectedLanguageConfidence: 0.95, + resolveLanguageIdDownloads: true, + languagePairs: [ + { fromLang: "es", toLang: "en", isBeta: false }, + { fromLang: "en", toLang: "es", isBeta: false }, + ], + runInPage: async TranslationsTest => { + const selectors = TranslationsTest.getSelectors(); + + await TranslationsTest.assertTranslationResult( + "The main title gets translated.", + selectors.getH1, + "DON QUIJOTE DE LA MANCHA [es to en, html]" + ); + + await TranslationsTest.assertTranslationResult( + "The last paragraph gets translated. It is out of the viewport.", + selectors.getLastParagraph, + "— PUES, AUNQUE MOVÁIS MÁS BRAZOS QUE LOS DEL GIGANTE BRIAREO, ME LO HABÉIS DE PAGAR. [es to en, html]" + ); + + selectors.getH1().innerText = "Este es un titulo"; + + await TranslationsTest.assertTranslationResult( + "Mutations get tracked", + selectors.getH1, + "ESTE ES UN TITULO [es to en]" + ); + + await TranslationsTest.assertTranslationResult( + "Other languages do not get translated.", + selectors.getHeader, + "The following is an excerpt from Don Quijote de la Mancha, which is in the public domain" + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_remote_settings.js b/toolkit/components/translations/tests/browser/browser_remote_settings.js new file mode 100644 index 0000000000..c77201e45d --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_remote_settings.js @@ -0,0 +1,303 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * @typedef {import("../translations").RemoteSettingsClient} RemoteSettingsClient + * @typedef {import("../../translations").TranslationModelRecord} TranslationModelRecord + */ + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +// The full Firefox version string. +const firefoxFullVersion = AppConstants.MOZ_APP_VERSION_DISPLAY; + +// The Firefox major version string (i.e. the first set of digits). +const firefoxMajorVersion = firefoxFullVersion.match(/\d+/); + +// The Firefox "AlphaZero" version string. +// This is a version that is less than even the latest Nightly +// which is of the form `${firefoxMajorVersion}.a1`. +const firefoxAlphaZeroVersion = `${firefoxMajorVersion}.a0`; + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {string} mockedKey + * @returns {RemoteSettingsClient} + */ +async function createRemoteSettingsClient(mockedKey) { + const client = RemoteSettings(mockedKey); + await client.db.clear(); + await client.db.importChanges({}, Date.now()); + return client; +} + +// The following test ensures the capabilities of `filter_expression` in remote settings +// to successfully discriminate against the Firefox version when retrieving records. +// +// This is used when making major breaking changes that would require particular records +// to only show up in certain versions of Firefox, such as actual code changes that no +// longer allow compatibility with given records. +// +// Some examples might be: +// +// - Modifying a wasm.js file that is no longer compatible with a previous wasm binary record. +// In such a case, the old binary record would need to be shipped in versions with the old +// wasm.js file, and the new binary record would need to be shipped in version with the new +// wasm.js file. +// +// - Switching to a different library for translation or language detection. +// Using a different library would not only mean major changes to the code, but it would +// certainly mean that the records for the old libraries are no longer compatible. +// We will need to ship those records only in older versions of Firefox that utilize the old +// libraries, and ship new records in the new versions of Firefox. +add_task(async function test_filter_current_firefox_version() { + // Create a new RemoteSettingsClient just for this test. + let client = await createRemoteSettingsClient( + "test_filter_current_firefox_version" + ); + + // Create a list of records that are expected to pass the filter_expression and be + // successfully retrieved from the remote settings client. + const expectedPresentRecords = [ + { + name: `undefined filter expression`, + filter_expression: undefined, + }, + { + name: `null filter expression`, + filter_expression: null, + }, + { + name: `empty filter expression`, + filter_expression: ``, + }, + { + name: `env.version == ${firefoxFullVersion}`, + filter_expression: `env.version|versionCompare('${firefoxFullVersion}') == 0`, + }, + { + name: `env.version > ${firefoxAlphaZeroVersion}`, + filter_expression: `env.version|versionCompare('${firefoxAlphaZeroVersion}') > 0`, + }, + ]; + for (let record of expectedPresentRecords) { + client.db.create(record); + } + + // Create a list of records that are expected fo fail the filter_expression and be + // absent when we retrieve the records from the remote settings client. + const expectedAbsentRecords = [ + { + name: `env.version < 1`, + filter_expression: `env.version|versionCompare('1') < 0`, + }, + ]; + for (let record of expectedAbsentRecords) { + client.db.create(record); + } + + const retrievedRecords = await client.get(); + + // Ensure that each record that is expected to be present exists in the retrieved records. + for (let expectedPresentRecord of expectedPresentRecords) { + is( + retrievedRecords.some( + record => record.name == expectedPresentRecord.name + ), + true, + `The following record was expected to be present but was not found: ${expectedPresentRecord.name}\n` + ); + } + + // Ensure that each record that is expected to be absent does not exist in the retrieved records. + for (let expectedAbsentRecord of expectedAbsentRecords) { + is( + retrievedRecords.some(record => record.name == expectedAbsentRecord.name), + false, + `The following record was expected to be absent but was found: ${expectedAbsentRecord.name}\n` + ); + } + + // Ensure that the length of the retrieved records is exactly the length of the records expected to be present. + is( + retrievedRecords.length, + expectedPresentRecords.length, + `Expected ${expectedPresentRecords.length} items but got ${retrievedRecords.length} items\n` + ); +}); + +// The following test ensures that we are able to always retrieve the maximum +// compatible version of records. These are for version changes that do not +// require shipping different records based on a particular Firefox version, +// but rather for changes to model content or wasm runtimes that are fully +// compatible with the existing source code. +add_task(async function test_get_records_with_multiple_versions() { + // Create a new RemoteSettingsClient just for this test. + let client = await createRemoteSettingsClient( + "test_get_translation_model_records" + ); + + const lookupKey = record => + `${record.name}${TranslationsParent.languagePairKey( + record.fromLang, + record.toLang + )}`; + + // A mapping of each record name to its max version. + const maxVersionMap = {}; + + // Create a list of records that are all version 1.0 + /** @type {TranslationModelRecord[]} */ + const versionOneRecords = [ + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "en", + toLang: "es", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "es", + toLang: "en", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "lex.50.50.enes.s2t.bin", + fromLang: "en", + toLang: "es", + fileType: "lex", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "model.enes.intgemm.alphas.bin", + fromLang: "en", + toLang: "es", + fileType: "model", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "vocab.deen.spm", + fromLang: "en", + toLang: "de", + fileType: "vocab", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "lex.50.50.ende.s2t.bin", + fromLang: "en", + toLang: "de", + fileType: "lex", + version: "1.0", + }, + { + id: crypto.randomUUID(), + name: "model.ende.intgemm.alphas.bin", + fromLang: "en", + toLang: "de", + fileType: "model", + version: "1.0", + }, + ]; + versionOneRecords.reduce((map, record) => { + map[lookupKey(record)] = record.version; + return map; + }, maxVersionMap); + for (const record of versionOneRecords) { + client.db.create(record); + } + + // Create a list of records that are identical to some of the above, but with higher version numbers. + const higherVersionRecords = [ + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.1", + }, + { + id: crypto.randomUUID(), + name: "qualityModel.enes.bin", + fromLang: "en", + toLang: "es", + fileType: "qualityModel", + version: "1.2", + }, + { + id: crypto.randomUUID(), + name: "vocab.esen.spm", + fromLang: "en", + toLang: "es", + fileType: "vocab", + version: "1.1", + }, + ]; + higherVersionRecords.reduce((map, record) => { + const key = lookupKey(record); + if (record.version > map[key]) { + map[key] = record.version; + } + return map; + }, maxVersionMap); + for (const record of higherVersionRecords) { + client.db.create(record); + } + + TranslationsParent.mockTranslationsEngine( + client, + await createTranslationsWasmRemoteClient() + ); + + const retrievedRecords = await TranslationsParent.getMaxVersionRecords( + client, + { lookupKey } + ); + + for (const record of retrievedRecords) { + is( + lookupKey(record) in maxVersionMap, + true, + `Expected record ${record.name} to be contained in the nameToVersionMap, but found none\n` + ); + is( + record.version, + maxVersionMap[lookupKey(record)], + `Expected record ${record.name} to be version ${ + maxVersionMap[lookupKey(record)] + }, but found version ${record.version}\n` + ); + } + + const expectedSize = Object.keys(maxVersionMap).length; + is( + retrievedRecords.length, + expectedSize, + `Expected retrieved records to be the same size as the name-to-version map ( + ${expectedSize} + ), but found ${retrievedRecords.length}\n` + ); + + TranslationsParent.unmockTranslationsEngine(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translation_document.js b/toolkit/components/translations/tests/browser/browser_translation_document.js new file mode 100644 index 0000000000..ca309e6cc3 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translation_document.js @@ -0,0 +1,938 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * @type {typeof import("../../content/translations-document.sys.mjs")} + */ +const { TranslationsDocument } = ChromeUtils.importESModule( + "chrome://global/content/translations/translations-document.sys.mjs" +); + +/** + * @param {string} html + * @param {{ + * fakeTranslator?: (message: string) => Promise<string> + * }} [options] + */ +async function createDoc(html, options) { + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ], + }); + + const parser = new DOMParser(); + const document = parser.parseFromString(html, "text/html"); + + /** + * Fake translations by converting them to uppercase. + * @param {string} message + */ + async function fakeTranslator(message) { + /** + * @param {Node} node + */ + function upperCaseNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.toUpperCase(); + } + for (const childNode of node.childNodes) { + upperCaseNode(childNode); + } + } + const translatedDoc = parser.parseFromString(message, "text/html"); + upperCaseNode(translatedDoc.body); + return [translatedDoc.body.innerHTML]; + } + + const translationsDocument = new TranslationsDocument( + document, + "en", + 0, // This is a fake innerWindowID + options?.fakeTranslator ?? fakeTranslator, + options?.fakeTranslator ?? fakeTranslator + ); + + /** + * Test utility to check that the document matches the expected markup + * + * @param {string} html + */ + async function htmlMatches(message, html) { + const expected = naivelyPrettify(html); + try { + await BrowserTestUtils.waitForCondition( + () => naivelyPrettify(document.body.innerHTML) === expected, + "Waiting for HTML to match." + ); + ok(true, message); + } catch (error) { + console.error(error); + + // Provide a nice error message. + const actual = naivelyPrettify(document.body.innerHTML); + ok( + false, + `${message}\n\nExpected HTML:\n\n${expected}\n\nActual HTML:\n\n${actual}\n\n` + ); + } + } + + function translate() { + info("Running translation."); + translationsDocument.addRootElement(document.body); + } + + function cleanup() { + SpecialPowers.popPrefEnv(); + } + + return { translate, htmlMatches, cleanup, translationsDocument, document }; +} + +add_task(async function test_translated_div_element() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + await htmlMatches( + "The document starts out as expected.", + /* html */ ` + <div> + This is a simple translation. + </div> + ` + ); + + translate(); + + await htmlMatches( + "A single element with a single text node is translated into uppercase.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_translated_textnode() { + const { translate, htmlMatches, cleanup } = await createDoc( + "This is a simple text translation." + ); + + translate(); + + await htmlMatches( + "A Text node at the root is translated into all caps", + "THIS IS A SIMPLE TEXT TRANSLATION." + ); + + cleanup(); +}); + +add_task(async function test_no_text_trees() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <div></div> + <span></span> + </div> + `); + + translate(); + + await htmlMatches( + "Trees with no text are not affected", + /* html */ ` + <div> + <div></div> + <span></span> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_no_text_trees() { + const { translate, htmlMatches, cleanup } = await createDoc(""); + translate(); + await htmlMatches("No text is still no text", ""); + cleanup(); +}); + +add_task(async function test_translated_title() { + const { cleanup, document, translationsDocument } = + await createDoc(/* html */ ` + <!DOCTYPE html> + <html> + <head> + <meta charset="utf-8" /> + <title>This is an actual full page.</title> + </head> + <body> + + </body> + </html> + `); + + info("The title element is the only <head> element that is used as a root."); + translationsDocument.addRootElement( + document.getElementsByTagName("title")[0] + ); + + const translatedTitle = "THIS IS AN ACTUAL FULL PAGE."; + try { + await BrowserTestUtils.waitForCondition( + () => document.title === translatedTitle + ); + } catch (error) {} + is(document.title, translatedTitle, "The title was changed."); + + cleanup(); +}); + +add_task(async function test_translated_nested_elements() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div class="menu-main-menu-container"> + <ul class="menu-list"> + <li class="menu-item menu-item-top-level"> + <a href="/">Latest Work</a> + </li> + <li class="menu-item menu-item-top-level"> + <a href="/category/interactive/">Creative Coding</a> + </li> + <li id="menu-id-categories" class="menu-item menu-item-top-level"> + <a href="#"><span class='category-arrow'>Categories</span></a> + </li> + </ul> + </div> + `); + + translate(); + + await htmlMatches( + "The nested elements are translated into all caps.", + /* html */ ` + <div class="menu-main-menu-container"> + <ul class="menu-list"> + <li class="menu-item menu-item-top-level" data-moz-translations-id="0"> + <a href="/" data-moz-translations-id="1">LATEST WORK</a> + </li> + <li class="menu-item menu-item-top-level" data-moz-translations-id="2"> + <a href="/category/interactive/" data-moz-translations-id="3">CREATIVE CODING</a> + </li> + <li id="menu-id-categories" class="menu-item menu-item-top-level" data-moz-translations-id="4"> + <a href="#" data-moz-translations-id="5"> + <span class="category-arrow" data-moz-translations-id="6">CATEGORIES</span> + </a> + </li> + </ul> + </div> + ` + ); + + cleanup(); +}); + +/** + * Only translate elements with a matching "from" language. + */ +add_task(async function test_translated_language() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + <div> + No lang property + </div> + <div lang="en"> + Language matches + </div> + <div lang="fr"> + Language mismatch is ignored. + </div> + <div lang="en-US"> + Language match with region. + </div> + <div lang="fr"> + <div> + Language mismatch with + </div> + <div> + nested elements. + </div> + </div> + </div> + `); + + translate(); + + await htmlMatches( + "Language matching of elements behaves as expected.", + /* html */ ` + <div> + <div> + NO LANG PROPERTY + </div> + <div lang="en"> + LANGUAGE MATCHES + </div> + <div lang="fr"> + Language mismatch is ignored. + </div> + <div lang="en-US"> + LANGUAGE MATCH WITH REGION. + </div> + <div lang="fr"> + <div> + Language mismatch with + </div> + <div> + nested elements. + </div> + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test elements that have been marked as ignored. + */ +add_task(async function test_ignored_translations() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div translate="yes"> + This is translated. + </div> + <div translate="no"> + This is not translated. + </div> + <div class="notranslate"> + This is not translated. + </div> + <div class="class-before notranslate class-after"> + This is not translated. + </div> + <div contenteditable> + This is not translated. + </div> + <div contenteditable="true"> + This is not translated. + </div> + <div contenteditable="false"> + This is translated. + </div> + `); + + translate(); + + await htmlMatches( + "Language matching of elements behaves as expected.", + /* html */ ` + <div translate="yes"> + THIS IS TRANSLATED. + </div> + <div translate="no"> + This is not translated. + </div> + <div class="notranslate"> + This is not translated. + </div> + <div class="class-before notranslate class-after"> + This is not translated. + </div> + <div contenteditable=""> + This is not translated. + </div> + <div contenteditable="true"> + This is not translated. + </div> + <div contenteditable="false"> + THIS IS TRANSLATED. + </div> + ` + ); + + cleanup(); +}); + +/** + * Test excluded tags. + */ +add_task(async function test_excluded_tags() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <div> + This is translated. + </div> + <code> + This is ignored + </code> + <script> + This is ignored + </script> + <textarea> + This is ignored + </textarea> + `); + + translate(); + + await htmlMatches( + "EXCLUDED_TAGS are not translated", + /* html */ ` + <div> + THIS IS TRANSLATED. + </div> + <code> + This is ignored + </code> + <script> + This is ignored + </script> + <textarea> + This is ignored + </textarea> + ` + ); + + cleanup(); +}); + +add_task(async function test_comments() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <!-- Comments don't make it to the DOM --> + <div> + <!-- These will be ignored in the translation. --> + This is translated. + </div> + `); + + translate(); + + await htmlMatches( + "Comments do not affect things.", + /* html */ ` + <div> + THIS IS TRANSLATED. + </div> + ` + ); + + cleanup(); +}); + +/** + * Test the batching behavior on what is sent in for a translation. + */ +add_task(async function test_translation_batching() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + This is a simple section. + </div> + <div> + <span>This entire</span> section continues in a <b>batch</b>. + </div> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + aaaa aa a aaaaaa aaaaaaa. + </div> + <div> + <span data-moz-translations-id="0"> + bbbb bbbbbb + </span> + bbbbbbb bbbbbbbbb bb b + <b data-moz-translations-id="1"> + bbbbb + </b> + . + </div> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_many_inlines() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + <span> + This is a + </span> + <span> + much longer + </span> + <span> + section that includes + </span> + <span> + many span elements + </span> + <span> + to test what happens + </span> + <span> + in cases like this. + </span> + </div> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + <span data-moz-translations-id="0"> + aaaa aa a + </span> + <span data-moz-translations-id="1"> + aaaa aaaaaa + </span> + <span data-moz-translations-id="2"> + aaaaaaa aaaa aaaaaaaa + </span> + <span data-moz-translations-id="3"> + aaaa aaaa aaaaaaaa + </span> + <span data-moz-translations-id="4"> + aa aaaa aaaa aaaaaaa + </span> + <span data-moz-translations-id="5"> + aa aaaaa aaaa aaaa. + </span> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_many_inlines() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + <div> + This is a + </div> + <div> + much longer + </div> + <div> + section that includes + </div> + <div> + many div elements + </div> + <div> + to test what happens + </div> + <div> + in cases like this. + </div> + </div> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Batching", + /* html */ ` + <div> + <div> + aaaa aa a + </div> + <div> + bbbb bbbbbb + </div> + <div> + ccccccc cccc cccccccc + </div> + <div> + dddd ddd dddddddd + </div> + <div> + ee eeee eeee eeeeeee + </div> + <div> + ff fffff ffff ffff. + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test the "presumed" inline content behavior. + */ +add_task(async function test_presumed_inlines1() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <div>Block element</div> + </div> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Mixing a text node with otherwise block elements will send it all in as one batch.", + /* html */ ` + <div> + aaaa aaaa + <div data-moz-translations-id="0"> + aaaaa aaaaaaa + </div> + </div> + ` + ); + + cleanup(); +}); + +/** + * Test what happens when there are many inline elements. + */ +add_task(async function test_presumed_inlines2() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <span>Inline</span> + <div>Block Element</div> + </div> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Conflicting inline and block elements will be sent in together if there are more inlines", + /* html */ ` + <div> + aaaa aaaa + <span data-moz-translations-id="0"> + aaaaaa + </span> + <div data-moz-translations-id="1"> + aaaaa aaaaaaa + </div> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_presumed_inlines3() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <div> + Text node + <span>Inline</span> + <div>Block Element</div> + <div>Block Element</div> + <div>Block Element</div> + </span> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + // TODO - There is a bug in the implementation, see the untranslated "Text node" below. + // this was in the original implementation. + await htmlMatches( + "Conflicting inlines will be sent in as separate blocks if there are more block elements", + /* html */ ` + <div> + Text node + <span> + aaaaaa + </span> + <div> + bbbbb bbbbbbb + </div> + <div> + ccccc ccccccc + </div> + <div> + ddddd ddddddd + </div> + </div> + ` + ); + + cleanup(); +}); + +add_task(async function test_chunking_large_text() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <pre> + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque fermentum est ante, ut porttitor enim molestie et. Nam mattis ullamcorper justo a ultrices. Ut ac sodales lorem. Sed feugiat ultricies lacus. Proin dapibus sit amet nunc a ullamcorper. Donec leo purus, convallis quis urna non, semper pulvinar augue. Nulla placerat turpis arcu, sit amet imperdiet sapien tincidunt ut. Donec sit amet luctus lorem, sed consectetur lectus. Pellentesque est nisi, feugiat et ipsum quis, vestibulum blandit nulla. + + Proin accumsan sapien ut nibh mattis tincidunt. Donec facilisis nibh sodales, mattis risus et, malesuada lorem. Nam suscipit lacinia venenatis. Praesent ac consectetur ante. Vestibulum pulvinar ut massa in viverra. Nunc tincidunt tortor nunc. Vivamus sit amet hendrerit mi. Aliquam posuere velit non ante facilisis euismod. In ullamcorper, lacus vel hendrerit tincidunt, dui justo iaculis nulla, sit amet tincidunt nisl magna et urna. Sed varius tincidunt ligula. Interdum et malesuada fames ac ante ipsum primis in faucibus. Nam sed gravida ligula. Donec tincidunt arcu eros, ac maximus magna auctor eu. Vivamus suscipit neque velit, in ullamcorper elit pulvinar et. Morbi auctor tempor risus, imperdiet placerat velit gravida vel. Duis ultricies accumsan libero quis molestie. + + Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam nec arcu dapibus enim vulputate vulputate aliquet a libero. Nam hendrerit pulvinar libero, eget posuere quam porta eu. Pellentesque dignissim justo eu leo accumsan, sit amet suscipit ante gravida. Vivamus eu faucibus orci. Quisque sagittis tortor eget orci venenatis porttitor. Quisque mollis ipsum a dignissim dignissim. + + Aenean sagittis nisi lectus, non lacinia orci dapibus viverra. Donec diam lorem, tincidunt sed massa vel, vulputate tincidunt metus. In quam felis, egestas et faucibus faucibus, vestibulum quis tortor. Morbi odio mi, suscipit vitae leo in, consequat interdum augue. Quisque purus velit, dictum ac ante eget, volutpat dapibus ante. Suspendisse quis augue vitae velit elementum dictum nec aliquet nisl. Maecenas vestibulum quam augue, eu maximus urna blandit eu. Donec nunc risus, elementum id ligula nec, ultrices venenatis libero. Suspendisse ullamcorper ex ante, malesuada pulvinar sem placerat vel. + + In hac habitasse platea dictumst. Duis vulputate tellus arcu, at posuere ligula viverra luctus. Fusce ultrices malesuada neque vitae vehicula. Aliquam blandit nisi sed nibh facilisis, non varius turpis venenatis. Vestibulum ut velit laoreet, sagittis leo ac, pharetra ex. Aenean mollis risus sed nibh auctor, et feugiat neque iaculis. Fusce fermentum libero metus, at consectetur massa euismod sed. Mauris ut metus sit amet leo porttitor mollis. Vivamus tincidunt lorem non purus suscipit sollicitudin. Maecenas ut tristique elit. Ut eu volutpat turpis. Suspendisse nec tristique augue. Nullam faucibus egestas volutpat. Sed tempor eros et mi ultrices, nec feugiat eros egestas. + </pre> + `, + { fakeTranslator: createBatchFakeTranslator() } + ); + + translate(); + + await htmlMatches( + "Large chunks of text can be still sent in for translation in one big pass, " + + "this could be slow bad behavior for the user.", + /* html */ ` + <pre> + aaaaa aaaaa aaaaa aaa aaaa, aaaaaaaaaaa aaaaaaaaaa aaaa. aaaaaaa aaaaaaaaa aaa aaaa, aa aaaaaaaaa aaaa aaaaaaaa aa. aaa aaaaaa aaaaaaaaaaa aaaaa a aaaaaaaa. aa aa aaaaaaa aaaaa. aaa aaaaaaa aaaaaaaaa aaaaa. aaaaa aaaaaaa aaa aaaa aaaa a aaaaaaaaaaa. aaaaa aaa aaaaa, aaaaaaaaa aaaa aaaa aaa, aaaaaa aaaaaaaa aaaaa. aaaaa aaaaaaaa aaaaaa aaaa, aaa aaaa aaaaaaaaa aaaaaa aaaaaaaaa aa. aaaaa aaa aaaa aaaaaa aaaaa, aaa aaaaaaaaaaa aaaaaa. aaaaaaaaaaaa aaa aaaa, aaaaaaa aa aaaaa aaaa, aaaaaaaaaa aaaaaaa aaaaa. + + aaaaa aaaaaaaa aaaaaa aa aaaa aaaaaa aaaaaaaaa. aaaaa aaaaaaaaa aaaa aaaaaaa, aaaaaa aaaaa aa, aaaaaaaaa aaaaa. aaa aaaaaaaa aaaaaaa aaaaaaaaa. aaaaaaaa aa aaaaaaaaaaa aaaa. aaaaaaaaaa aaaaaaaa aa aaaaa aa aaaaaaa. aaaa aaaaaaaaa aaaaaa aaaa. aaaaaaa aaa aaaa aaaaaaaaa aa. aaaaaaa aaaaaaa aaaaa aaa aaaa aaaaaaaaa aaaaaaa. aa aaaaaaaaaaa, aaaaa aaa aaaaaaaaa aaaaaaaaa, aaa aaaaa aaaaaaa aaaaa, aaa aaaa aaaaaaaaa aaaa aaaaa aa aaaa. aaa aaaaaa aaaaaaaaa aaaaaa. aaaaaaaa aa aaaaaaaaa aaaaa aa aaaa aaaaa aaaaaa aa aaaaaaaa. aaa aaa aaaaaaa aaaaaa. aaaaa aaaaaaaaa aaaa aaaa, aa aaaaaaa aaaaa aaaaaa aa. aaaaaaa aaaaaaaa aaaaa aaaaa, aa aaaaaaaaaaa aaaa aaaaaaaa aa. aaaaa aaaaaa aaaaaa aaaaa, aaaaaaaaa aaaaaaaa aaaaa aaaaaaa aaa. aaaa aaaaaaaaa aaaaaaaa aaaaaa aaaa aaaaaaaa. + + aaaaaaaaaa aaaa aaaaa aaaaaa aa aaaaaaaa aaaa aaaaaa aa aaaaaaaa aaaaaaa aaaaaaa aaaaa; aaaaa aaa aaaa aaaaaaa aaaa aaaaaaaaa aaaaaaaaa aaaaaaa a aaaaaa. aaa aaaaaaaaa aaaaaaaa aaaaaa, aaaa aaaaaaa aaaa aaaaa aa. aaaaaaaaaaaa aaaaaaaaa aaaaa aa aaa aaaaaaaa, aaa aaaa aaaaaaaa aaaa aaaaaaa. aaaaaaa aa aaaaaaaa aaaa. aaaaaaa aaaaaaaa aaaaaa aaaa aaaa aaaaaaaaa aaaaaaaaa. aaaaaaa aaaaaa aaaaa a aaaaaaaaa aaaaaaaaa. + + aaaaaa aaaaaaaa aaaa aaaaaa, aaa aaaaaaa aaaa aaaaaaa aaaaaaa. aaaaa aaaa aaaaa, aaaaaaaaa aaa aaaaa aaa, aaaaaaaaa aaaaaaaaa aaaaa. aa aaaa aaaaa, aaaaaaa aa aaaaaaaa aaaaaaaa, aaaaaaaaaa aaaa aaaaaa. aaaaa aaaa aa, aaaaaaaa aaaaa aaa aa, aaaaaaaaa aaaaaaaa aaaaa. aaaaaaa aaaaa aaaaa, aaaaaa aa aaaa aaaa, aaaaaaaa aaaaaaa aaaa. aaaaaaaaaaa aaaa aaaaa aaaaa aaaaa aaaaaaaaa aaaaaa aaa aaaaaaa aaaa. aaaaaaaa aaaaaaaaaa aaaa aaaaa, aa aaaaaaa aaaa aaaaaaa aa. aaaaa aaaa aaaaa, aaaaaaaaa aa aaaaaa aaa, aaaaaaaa aaaaaaaaa aaaaaa. aaaaaaaaaaa aaaaaaaaaaa aa aaaa, aaaaaaaaa aaaaaaaa aaa aaaaaaaa aaa. + + aa aaa aaaaaaaaa aaaaaa aaaaaaaa. aaaa aaaaaaaaa aaaaaa aaaa, aa aaaaaaa aaaaaa aaaaaaa aaaaaa. aaaaa aaaaaaaa aaaaaaaaa aaaaa aaaaa aaaaaaaa. aaaaaaa aaaaaaa aaaa aaa aaaa aaaaaaaaa, aaa aaaaaa aaaaaa aaaaaaaaa. aaaaaaaaaa aa aaaaa aaaaaaa, aaaaaaaa aaa aa, aaaaaaaa aa. aaaaaa aaaaaa aaaaa aaa aaaa aaaaaa, aa aaaaaaa aaaaa aaaaaaa. aaaaa aaaaaaaaa aaaaaa aaaaa, aa aaaaaaaaaaa aaaaa aaaaaaa aaa. aaaaaa aa aaaaa aaa aaaa aaa aaaaaaaaa aaaaaa. aaaaaaa aaaaaaaaa aaaaa aaa aaaaa aaaaaaaa aaaaaaaaaaaa. aaaaaaaa aa aaaaaaaaa aaaa. aa aa aaaaaaaa aaaaaa. aaaaaaaaaaa aaa aaaaaaaaa aaaaa. aaaaaa aaaaaaaa aaaaaaa aaaaaaaa. aaa aaaaaa aaaa aa aa aaaaaaaa, aaa aaaaaaa aaaa aaaaaaa. + </pre> + ` + ); + + cleanup(); +}); + +add_task(async function test_reordering() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + <span> + B - This was first. + </span> + <span> + A - This was second. + </span> + <span> + C - This was third. + </span> + `, + { fakeTranslator: reorderingTranslator } + ); + + translate(); + + await htmlMatches( + "Nodes can be re-ordered by the translator", + /* html */ ` + <span data-moz-translations-id="1"> + A - THIS WAS SECOND. + </span> + <span data-moz-translations-id="0"> + B - THIS WAS FIRST. + </span> + <span data-moz-translations-id="2"> + C - THIS WAS THIRD. + </span> + ` + ); + + cleanup(); +}); + +add_task(async function test_reordering2() { + const { translate, htmlMatches, cleanup } = await createDoc( + /* html */ ` + B - This was first. + <span> + A - This was second. + </span> + C - This was third. + `, + { fakeTranslator: reorderingTranslator } + ); + + translate(); + + // Note: ${" "} is used below to ensure that the whitespace is not stripped from + // the test. + await htmlMatches( + "Text nodes can be re-ordered.", + /* html */ ` + <span data-moz-translations-id="0"> + A - THIS WAS SECOND. + </span> + B - THIS WAS FIRST. +${" "} + C - THIS WAS THIRD. + ` + ); + + cleanup(); +}); + +add_task(async function test_mutations() { + const { translate, htmlMatches, cleanup, document } = + await createDoc(/* html */ ` + <div> + This is a simple translation. + </div> + `); + + translate(); + + await htmlMatches( + "It translates.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + ` + ); + + info('Trigger the "childList" mutation.'); + const div = document.createElement("div"); + div.innerText = "This is an added node."; + document.body.appendChild(div); + + await htmlMatches( + "The added node gets translated.", + /* html */ ` + <div> + THIS IS A SIMPLE TRANSLATION. + </div> + <div> + THIS IS AN ADDED NODE. + </div> + ` + ); + + info('Trigger the "characterData" mutation.'); + document.querySelector("div").firstChild.nodeValue = + "This is a changed node."; + + await htmlMatches( + "The changed node gets translated", + /* html */ ` + <div> + THIS IS A CHANGED NODE. + </div> + <div> + THIS IS AN ADDED NODE. + </div> + ` + ); + cleanup(); +}); + +add_task(async function test_tables() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <table> + <tr> + <th>Table header 1</th> + <th>Table header 2</th> + </tr> + <tr> + <td>Table data 1</td> + <td>Table data 2</td> + </tr> + </table> + `); + + translate(); + + await htmlMatches( + "Tables are correctly translated.", + /* html */ ` + <table> + <tbody> + <tr> + <th> + TABLE HEADER 1 + </th> + <th> + TABLE HEADER 2 + </th> + </tr> + <tr> + <td> + TABLE DATA 1 + </td> + <td> + TABLE DATA 2 + </td> + </tr> + </tbody> + </table> + ` + ); + + cleanup(); +}); + +// TODO(Bug 1819205) - Attribute support needs to be added.ç +add_task(async function test_attributes() { + const { translate, htmlMatches, cleanup } = await createDoc(/* html */ ` + <label title="Titles are user visible">Enter information:</label> + <input type="text" placeholder="This is a placeholder"> + `); + + translate(); + + // This is what this test should assert: + // eslint-disable-next-line no-unused-vars + const actualExpected = /* html */ ` + <label title="TITLES ARE USER VISIBLE"> + ENTER INFORMATION: + </label> + <input type="text" placeholder="THIS IS A PLACEHOLDER" > + `; + + await htmlMatches( + "Placeholders support needs to be added", + /* html */ ` + <label title="Titles are user visible"> + ENTER INFORMATION: + </label> + <input type="text" placeholder="This is a placeholder"> + ` + ); + + cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor.js b/toolkit/components/translations/tests/browser/browser_translations_actor.js new file mode 100644 index 0000000000..03ee1e2948 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor.js @@ -0,0 +1,217 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This file contains unit tests for the translations actor. Generally it's preferable + * to test behavior in a full integration test, but occasionally it's useful to test + * specific implementation behavior. + */ + +/** + * Enforce the pivot language behavior in ensureLanguagePairsHavePivots. + */ +add_task(async function test_pivot_language_behavior() { + info( + "Expect 4 console.error messages notifying of the lack of a pivot language." + ); + + const { actor, cleanup } = await setupActorTest({ + languagePairs: [ + { fromLang: "en", toLang: "es", isBeta: false }, + { fromLang: "es", toLang: "en", isBeta: false }, + { fromLang: "en", toLang: "yue", isBeta: true }, + { fromLang: "yue", toLang: "en", isBeta: true }, + // This is not a bi-directional translation. + { fromLang: "is", toLang: "en", isBeta: false }, + // These are non-pivot languages. + { fromLang: "zh", toLang: "ja", isBeta: true }, + { fromLang: "ja", toLang: "zh", isBeta: true }, + ], + }); + + const { languagePairs } = await actor.getSupportedLanguages(); + + // The pairs aren't guaranteed to be sorted. + languagePairs.sort((a, b) => + TranslationsParent.languagePairKey(a.fromLang, a.toLang).localeCompare( + TranslationsParent.languagePairKey(b.fromLang, b.toLang) + ) + ); + + Assert.deepEqual( + languagePairs, + [ + { fromLang: "en", toLang: "es", isBeta: false }, + { fromLang: "en", toLang: "yue", isBeta: true }, + { fromLang: "es", toLang: "en", isBeta: false }, + { fromLang: "is", toLang: "en", isBeta: false }, + { fromLang: "yue", toLang: "en", isBeta: true }, + ], + "Non-pivot languages were removed." + ); + + return cleanup(); +}); + +async function usingAppLocale(locale, callback) { + info(`Mocking the locale "${locale}", expect missing resource errors.`); + const { availableLocales, requestedLocales } = Services.locale; + Services.locale.availableLocales = [locale]; + Services.locale.requestedLocales = [locale]; + + if (Services.locale.appLocaleAsBCP47 !== locale) { + throw new Error("Unable to change the app locale."); + } + await callback(); + + // Reset back to the originals. + Services.locale.availableLocales = availableLocales; + Services.locale.requestedLocales = requestedLocales; +} + +add_task(async function test_translating_to_and_from_app_language() { + const PIVOT_LANGUAGE = "en"; + + const { actor, cleanup } = await setupActorTest({ + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + /** + * Each language pair has multiple models. De-duplicate the language pairs and + * return a sorted list. + */ + function getUniqueLanguagePairs(records) { + const langPairs = new Set(); + for (const { fromLang, toLang } of records) { + langPairs.add(TranslationsParent.languagePairKey(fromLang, toLang)); + } + return Array.from(langPairs) + .sort() + .map(langPair => { + const [fromLang, toLang] = langPair.split(","); + return { + fromLang, + toLang, + }; + }); + } + + function assertLanguagePairs({ + app, + requested, + message, + languagePairs, + isForDeletion, + }) { + return usingAppLocale(app, async () => { + Assert.deepEqual( + getUniqueLanguagePairs( + await actor.getRecordsForTranslatingToAndFromAppLanguage( + requested, + isForDeletion + ) + ), + languagePairs, + message + ); + }); + } + + await assertLanguagePairs({ + message: + "When the app locale is the pivot language, download another language.", + app: PIVOT_LANGUAGE, + requested: "fr", + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "When a pivot language is required, they are both downloaded.", + app: "fr", + requested: "pl", + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "When downloading the pivot language, only download the one for the app's locale.", + app: "es", + requested: PIVOT_LANGUAGE, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "Delete just the requested language when the app locale is the pivot language", + app: PIVOT_LANGUAGE, + requested: "fr", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the requested language, and not the pivot.", + app: "fr", + requested: "pl", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the requested language, and not the pivot.", + app: "fr", + requested: "pl", + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: "Delete just the pivot → app and app → pivot.", + app: "es", + requested: PIVOT_LANGUAGE, + isForDeletion: true, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + ], + }); + + await assertLanguagePairs({ + message: + "If the app and request language are the same, nothing is returned.", + app: "fr", + requested: "fr", + languagePairs: [], + }); + + return cleanup(); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js b/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js new file mode 100644 index 0000000000..50b548b6f1 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_actor_preferred_language.js @@ -0,0 +1,81 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +function waitForAppLocaleChanged() { + new Promise(resolve => { + function onChange() { + Services.obs.removeObserver(onChange, "intl:app-locales-changed"); + resolve(); + } + Services.obs.addObserver(onChange, "intl:app-locales-changed"); + }); +} + +async function testWithLocales({ + systemLocales, + appLocales, + webLanguages, + test, +}) { + const cleanup = await mockLocales({ + systemLocales, + appLocales, + webLanguages, + }); + test(); + return cleanup(); +} + +add_task(async function test_preferred_language() { + await testWithLocales({ + systemLocales: ["en-US"], + appLocales: ["en-US"], + webLanguages: ["en-US"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + ["en"], + "When all locales are English, only English is preferred." + ); + }, + }); + + await testWithLocales({ + systemLocales: ["es-ES"], + appLocales: ["en-US"], + webLanguages: ["en-US"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + ["en", "es"], + "When the operating system differs, it is added tot he end of the preferred languages." + ); + }, + }); + + await testWithLocales({ + systemLocales: ["zh-TW", "zh-CN", "de"], + appLocales: ["pt-BR", "pl"], + webLanguages: ["cs", "hu"], + test() { + Assert.deepEqual( + TranslationsParent.getPreferredLanguages(), + [ + // appLocales, notice that "en" is the last fallback. + "pt", + "pl", + "en", + // webLanguages + "cs", + "hu", + // systemLocales + "zh", + "de", + ], + "Demonstrate an unrealistic but complicated locale situation." + ); + }, + }); +}); diff --git a/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js b/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js new file mode 100644 index 0000000000..8320d1ac29 --- /dev/null +++ b/toolkit/components/translations/tests/browser/browser_translations_lang_tags.js @@ -0,0 +1,125 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const PIVOT_LANGUAGE = "en"; + +const LANGUAGE_PAIRS = [ + { fromLang: PIVOT_LANGUAGE, toLang: "es" }, + { fromLang: "es", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + { fromLang: PIVOT_LANGUAGE, toLang: "pl" }, + { fromLang: "pl", toLang: PIVOT_LANGUAGE }, +]; + +async function runLangTagsTest( + { + systemLocales, + appLocales, + webLanguages, + page, + languagePairs = LANGUAGE_PAIRS, + }, + langTags +) { + const cleanupLocales = await mockLocales({ + systemLocales, + appLocales, + webLanguages, + }); + + const { cleanup: cleanupTestPage } = await loadTestPage({ + page, + languagePairs, + }); + const actor = getTranslationsParent(); + + await TestUtils.waitForCondition( + async () => (await actor.getLangTagsForTranslation())?.docLangTag, + "Waiting for a document language tag to be found." + ); + + Assert.deepEqual(await actor.getLangTagsForTranslation(), langTags); + + await cleanupLocales(); + await cleanupTestPage(); +} + +add_task(async function test_lang_tags_direct_translations() { + info( + "Test the detected languages for translations when a translation pair is available" + ); + await runLangTagsTest( + { + systemLocales: ["en"], + appLocales: ["en"], + webLanguages: ["en"], + page: TRANSLATIONS_TESTER_ES, + }, + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_pivots() { + info("Test the detected languages for translations when a pivot is needed."); + await runLangTagsTest( + { + systemLocales: ["fr"], + appLocales: ["fr", "en"], + webLanguages: ["fr", "en"], + page: TRANSLATIONS_TESTER_ES, + }, + { + docLangTag: "es", + userLangTag: "fr", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_pivots_second_preferred() { + info( + "Test using a pivot language when the first preferred lang tag doesn't match" + ); + await runLangTagsTest( + { + systemLocales: ["it"], + appLocales: ["it", "en"], + webLanguages: ["it", "en"], + page: TRANSLATIONS_TESTER_ES, + }, + { + docLangTag: "es", + userLangTag: "en", + isDocLangTagSupported: true, + } + ); +}); + +add_task(async function test_lang_tags_with_non_supported_doc_language() { + info("Test using a pivot language when the doc language isn't supported"); + await runLangTagsTest( + { + systemLocales: ["fr"], + appLocales: ["fr", "en"], + webLanguages: ["fr", "en"], + page: TRANSLATIONS_TESTER_ES, + languagePairs: [ + { fromLang: PIVOT_LANGUAGE, toLang: "fr" }, + { fromLang: "fr", toLang: PIVOT_LANGUAGE }, + // No Spanish support. + ], + }, + { + docLangTag: "es", + userLangTag: "fr", + isDocLangTagSupported: false, + } + ); +}); diff --git a/toolkit/components/translations/tests/browser/head.js b/toolkit/components/translations/tests/browser/head.js new file mode 100644 index 0000000000..08c5793a46 --- /dev/null +++ b/toolkit/components/translations/tests/browser/head.js @@ -0,0 +1,9 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/toolkit/components/translations/tests/browser/shared-head.js", + this +); diff --git a/toolkit/components/translations/tests/browser/shared-head.js b/toolkit/components/translations/tests/browser/shared-head.js new file mode 100644 index 0000000000..7ccc461b83 --- /dev/null +++ b/toolkit/components/translations/tests/browser/shared-head.js @@ -0,0 +1,1011 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Avoid about:blank's non-standard behavior. +const BLANK_PAGE = + "data:text/html;charset=utf-8,<!DOCTYPE html><title>Blank</title>Blank page"; + +const URL_COM_PREFIX = "https://example.com/browser/"; +const URL_ORG_PREFIX = "https://example.org/browser/"; +const CHROME_URL_PREFIX = "chrome://mochitests/content/browser/"; +const DIR_PATH = "toolkit/components/translations/tests/browser/"; +const TRANSLATIONS_TESTER_EN = + URL_COM_PREFIX + DIR_PATH + "translations-tester-en.html"; +const TRANSLATIONS_TESTER_ES = + URL_COM_PREFIX + DIR_PATH + "translations-tester-es.html"; +const TRANSLATIONS_TESTER_ES_2 = + URL_COM_PREFIX + DIR_PATH + "translations-tester-es-2.html"; +const TRANSLATIONS_TESTER_ES_DOT_ORG = + URL_ORG_PREFIX + DIR_PATH + "translations-tester-es.html"; +const TRANSLATIONS_TESTER_NO_TAG = + URL_COM_PREFIX + DIR_PATH + "translations-tester-no-tag.html"; + +/** + * The mochitest runs in the parent process. This function opens up a new tab, + * opens up about:translations, and passes the test requirements into the content process. + * + * @template T + * + * @param {object} options + * + * @param {T} options.dataForContent + * The data must support structural cloning and will be passed into the + * content process. + * + * @param {(args: { dataForContent: T, selectors: Record<string, string> }) => Promise<void>} options.runInPage + * This function must not capture any values, as it will be cloned in the content process. + * Any required data should be passed in using the "dataForContent" parameter. The + * "selectors" property contains any useful selectors for the content. + * + * @param {boolean} [options.disabled] + * Disable the panel through a pref. + * + * @param {number} detectedLanguageConfidence + * This is the value for the MockedLanguageIdEngine to give as a confidence score for + * the mocked detected language. + * + * @param {string} detectedLangTag + * This is the BCP 47 language tag for the MockedLanguageIdEngine to return as + * the mocked detected language. + * + * @param {Array<{ fromLang: string, toLang: string, isBeta: boolean }>} options.languagePairs + * The translation languages pairs to mock for the test. + * + * @param {Array<[string, string]>} options.prefs + * Prefs to push on for the test. + */ +async function openAboutTranslations({ + dataForContent, + disabled, + runInPage, + detectedLanguageConfidence, + detectedLangTag, + languagePairs = DEFAULT_LANGUAGE_PAIRS, + prefs, +}) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", !disabled], + ["browser.translations.logLevel", "All"], + ...(prefs ?? []), + ], + }); + + /** + * Collect any relevant selectors for the page here. + */ + const selectors = { + pageHeader: '[data-l10n-id="about-translations-header"]', + fromLanguageSelect: "select#language-from", + toLanguageSelect: "select#language-to", + translationTextarea: "textarea#translation-from", + translationResult: "#translation-to", + translationResultBlank: "#translation-to-blank", + translationInfo: "#translation-info", + noSupportMessage: "[data-l10n-id='about-translations-no-support']", + }; + + // Start the tab at a blank page. + let tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { removeMocks, remoteClients } = await createAndMockRemoteSettings({ + languagePairs, + // TODO(Bug 1814168) - Do not test download behavior as this is not robustly + // handled for about:translations yet. + autoDownloadFromRemoteSettings: true, + detectedLangTag, + detectedLanguageConfidence, + }); + + // Now load the about:translations page, since the actor could be mocked. + BrowserTestUtils.loadURIString(tab.linkedBrowser, "about:translations"); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + // Resolve the files. + await remoteClients.languageIdModels.resolvePendingDownloads(1); + // The language id and translation engine each have a wasm file, so expect 2 downloads. + await remoteClients.translationsWasm.resolvePendingDownloads(2); + await remoteClients.translationModels.resolvePendingDownloads( + languagePairs.length * FILES_PER_LANGUAGE_PAIR + ); + + await ContentTask.spawn( + tab.linkedBrowser, + { dataForContent, selectors }, + runInPage + ); + + removeMocks(); + + BrowserTestUtils.removeTab(tab); + await SpecialPowers.popPrefEnv(); +} + +/** + * Naively prettify's html based on the opening and closing tags. This is not robust + * for general usage, but should be adequate for these tests. + * @param {string} html + * @returns {string} + */ +function naivelyPrettify(html) { + let result = ""; + let indent = 0; + + function addText(actualEndIndex) { + const text = html.slice(startIndex, actualEndIndex).trim(); + if (text) { + for (let i = 0; i < indent; i++) { + result += " "; + } + result += text + "\n"; + } + startIndex = actualEndIndex; + } + + let startIndex = 0; + let endIndex = 0; + for (; endIndex < html.length; endIndex++) { + if ( + html[endIndex] === " " || + html[endIndex] === "\t" || + html[endIndex] === "n" + ) { + // Skip whitespace. + // " <div>foobar</div>" + // ^^^ + startIndex = endIndex; + continue; + } + + // Find all of the text. + // "<div>foobar</div>" + // ^^^^^^ + while (endIndex < html.length && html[endIndex] !== "<") { + endIndex++; + } + + addText(endIndex); + + if (html[endIndex] === "<") { + if (html[endIndex + 1] === "/") { + // "<div>foobar</div>" + // ^ + while (endIndex < html.length && html[endIndex] !== ">") { + endIndex++; + } + indent--; + addText(endIndex + 1); + } else { + // "<div>foobar</div>" + // ^ + while (endIndex < html.length && html[endIndex] !== ">") { + endIndex++; + } + // "<div>foobar</div>" + // ^ + addText(endIndex + 1); + indent++; + } + } + } + + return result.trim(); +} + +/** + * This fake translator reports on the batching of calls by replacing the text + * with a letter. Each call of the function moves the letter forward alphabetically. + * + * So consecutive calls would transform things like: + * "First translation" -> "aaaa aaaaaaaaa" + * "Second translation" -> "bbbbb bbbbbbbbb" + * "Third translation" -> "cccc ccccccccc" + * + * This can visually show what the translation batching behavior looks like. + */ +function createBatchFakeTranslator() { + let letter = "a"; + /** + * @param {string} message + */ + return async function fakeTranslator(message) { + /** + * @param {Node} node + */ + function transformNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.replace(/\w/g, letter); + } + for (const childNode of node.childNodes) { + transformNode(childNode); + } + } + + const parser = new DOMParser(); + const translatedDoc = parser.parseFromString(message, "text/html"); + transformNode(translatedDoc.body); + + // "Increment" the letter. + letter = String.fromCodePoint(letter.codePointAt(0) + 1); + + return [translatedDoc.body.innerHTML]; + }; +} + +/** + * This fake translator reorders Nodes to be in alphabetical order, and then + * uppercases the text. This allows for testing the reordering behavior of the + * translation engine. + * + * @param {string} message + */ +async function reorderingTranslator(message) { + /** + * @param {Node} node + */ + function transformNode(node) { + if (typeof node.nodeValue === "string") { + node.nodeValue = node.nodeValue.toUpperCase(); + } + const nodes = [...node.childNodes]; + nodes.sort((a, b) => + (a.textContent?.trim() ?? "").localeCompare(b.textContent?.trim() ?? "") + ); + for (const childNode of nodes) { + childNode.remove(); + } + for (const childNode of nodes) { + // Re-append in sorted order. + node.appendChild(childNode); + transformNode(childNode); + } + } + + const parser = new DOMParser(); + const translatedDoc = parser.parseFromString(message, "text/html"); + transformNode(translatedDoc.body); + + return [translatedDoc.body.innerHTML]; +} + +/** + * @returns {import("../../actors/TranslationsParent.sys.mjs").TranslationsParent} + */ +function getTranslationsParent() { + return gBrowser.selectedBrowser.browsingContext.currentWindowGlobal.getActor( + "Translations" + ); +} + +/** + * This is for tests that don't need a browser page to run. + */ +async function setupActorTest({ + languagePairs, + prefs, + detectedLanguageConfidence, + detectedLangTag, +}) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ...(prefs ?? []), + ], + }); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + detectedLangTag, + detectedLanguageConfidence, + }); + + // Create a new tab so each test gets a new actor, and doesn't re-use the old one. + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + return { + actor: getTranslationsParent(), + remoteClients, + cleanup() { + BrowserTestUtils.removeTab(tab); + removeMocks(); + return SpecialPowers.popPrefEnv(); + }, + }; +} + +/** + * Provide some default language pairs when none are provided. + */ +const DEFAULT_LANGUAGE_PAIRS = [ + { fromLang: "en", toLang: "es", isBeta: false }, + { fromLang: "es", toLang: "en", isBeta: false }, +]; + +async function createAndMockRemoteSettings({ + languagePairs = DEFAULT_LANGUAGE_PAIRS, + detectedLanguageConfidence = 0.5, + detectedLangTag = "en", + autoDownloadFromRemoteSettings = false, +}) { + const remoteClients = { + translationModels: await createTranslationModelsRemoteClient( + autoDownloadFromRemoteSettings, + languagePairs + ), + translationsWasm: await createTranslationsWasmRemoteClient( + autoDownloadFromRemoteSettings + ), + languageIdModels: await createLanguageIdModelsRemoteClient( + autoDownloadFromRemoteSettings + ), + }; + + TranslationsParent.mockTranslationsEngine( + remoteClients.translationModels.client, + remoteClients.translationsWasm.client + ); + + TranslationsParent.mockLanguageIdentification( + detectedLangTag, + detectedLanguageConfidence, + remoteClients.languageIdModels.client + ); + return { + removeMocks() { + TranslationsParent.unmockTranslationsEngine(); + TranslationsParent.unmockLanguageIdentification(); + }, + remoteClients, + }; +} + +async function loadTestPage({ + languagePairs, + autoDownloadFromRemoteSettings = false, + detectedLanguageConfidence, + detectedLangTag, + page, + prefs, + permissionsUrls = [], +}) { + Services.fog.testResetFOG(); + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ...(prefs ?? []), + ], + }); + await SpecialPowers.pushPermissions( + permissionsUrls.map(url => ({ + type: "translations", + allow: true, + context: url, + })) + ); + + // Start the tab at a blank page. + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + detectedLanguageConfidence, + detectedLangTag, + autoDownloadFromRemoteSettings, + }); + + BrowserTestUtils.loadURIString(tab.linkedBrowser, page); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + return { + tab, + remoteClients, + + /** + * @param {number} count - Count of the language pairs expected. + */ + async resolveDownloads(count) { + await remoteClients.translationsWasm.resolvePendingDownloads(1); + await remoteClients.translationModels.resolvePendingDownloads( + FILES_PER_LANGUAGE_PAIR * count + ); + }, + + /** + * @param {number} count - Count of the language pairs expected. + */ + async rejectDownloads(count) { + await remoteClients.translationsWasm.rejectPendingDownloads(1); + await remoteClients.translationModels.rejectPendingDownloads( + FILES_PER_LANGUAGE_PAIR * count + ); + }, + + async resolveLanguageIdDownloads() { + await remoteClients.translationsWasm.resolvePendingDownloads(1); + await remoteClients.languageIdModels.resolvePendingDownloads(1); + }, + + /** + * @returns {Promise<void>} + */ + cleanup() { + removeMocks(); + Services.fog.testResetFOG(); + BrowserTestUtils.removeTab(tab); + return Promise.all([ + SpecialPowers.popPrefEnv(), + SpecialPowers.popPermissions(), + ]); + }, + + /** + * Runs a callback in the content page. The function's contents are serialized as + * a string, and run in the page. The `translations-test.mjs` module is made + * available to the page. + * + * @param {(TranslationsTest: import("./translations-test.mjs")) => any} callback + * @returns {Promise<void>} + */ + runInPage(callback) { + // ContentTask.spawn runs the `Function.prototype.toString` on this function in + // order to send it into the content process. The following function is doing its + // own string manipulation in order to load in the TranslationsTest module. + const fn = new Function(/* js */ ` + const TranslationsTest = ChromeUtils.importESModule( + "chrome://mochitests/content/browser/toolkit/components/translations/tests/browser/translations-test.mjs" + ); + + // Pass in the values that get injected by the task runner. + TranslationsTest.setup({Assert, ContentTaskUtils, content}); + + return (${callback.toString()})(TranslationsTest); + `); + + return ContentTask.spawn( + tab.linkedBrowser, + {}, // Data to inject. + fn + ); + }, + }; +} + +/** + * Captures any reported errors in the TranslationsParent. + * + * @param {Function} callback + * @returns {Array<{ error: Error, args: any[] }>} + */ +async function captureTranslationsError(callback) { + const { reportError } = TranslationsParent; + + let errors = []; + TranslationsParent.reportError = (error, ...args) => { + errors.push({ error, args }); + }; + + await callback(); + + // Restore the original function. + TranslationsParent.reportError = reportError; + return errors; +} + +/** + * Load a test page and run + * @param {Object} options - The options for `loadTestPage` plus a `runInPage` function. + */ +async function autoTranslatePage(options) { + const { cleanup, runInPage } = await loadTestPage({ + autoDownloadFromRemoteSettings: true, + prefs: [ + ["browser.translations.autoTranslate", true], + ...(options.prefs ?? []), + ], + ...options, + }); + await runInPage(options.runInPage); + await cleanup(); +} + +/** + * @param {RemoteSettingsClient} client + * @param {boolean} autoDownloadFromRemoteSettings - Skip the manual download process, + * and automatically download the files. Normally it's preferrable to manually trigger + * the downloads to trigger the download behavior, but this flag lets you bypass this + * and automatically download the files. + */ +function createAttachmentMock(client, autoDownloadFromRemoteSettings) { + const pendingDownloads = []; + client.attachments.download = record => + new Promise((resolve, reject) => { + console.log("Download requested:", client.collectionName, record.name); + if (autoDownloadFromRemoteSettings) { + resolve({ buffer: new ArrayBuffer() }); + } else { + pendingDownloads.push({ record, resolve, reject }); + } + }); + + function resolvePendingDownloads(expectedDownloadCount) { + info( + `Resolving ${expectedDownloadCount} mocked downloads for "${client.collectionName}"` + ); + return downloadHandler(expectedDownloadCount, download => + download.resolve({ buffer: new ArrayBuffer() }) + ); + } + + async function rejectPendingDownloads(expectedDownloadCount) { + info( + `Intentionally rejecting ${expectedDownloadCount} mocked downloads for "${client.collectionName}"` + ); + + // Add 1 to account for the original attempt. + const attempts = TranslationsParent.MAX_DOWNLOAD_RETRIES + 1; + return downloadHandler(expectedDownloadCount * attempts, download => + download.reject(new Error("Intentionally rejecting downloads.")) + ); + } + + async function downloadHandler(expectedDownloadCount, action) { + const names = []; + let maxTries = 100; + while (names.length < expectedDownloadCount && maxTries-- > 0) { + await new Promise(resolve => setTimeout(resolve, 0)); + let download = pendingDownloads.shift(); + if (!download) { + // Uncomment the following to debug download issues: + // console.log(`No pending download:`, client.collectionName, names.length); + continue; + } + console.log(`Handling download:`, client.collectionName); + action(download); + names.push(download.record.name); + } + + // This next check is not guaranteed to catch an unexpected download, but wait + // at least one event loop tick to see if any more downloads were added. + await new Promise(resolve => setTimeout(resolve, 0)); + + if (pendingDownloads.length) { + throw new Error( + `An unexpected download was found, only expected ${expectedDownloadCount} downloads` + ); + } + + return names.sort((a, b) => a.localeCompare(b)); + } + + async function assertNoNewDownloads() { + await new Promise(resolve => setTimeout(resolve, 0)); + is( + pendingDownloads.length, + 0, + `No downloads happened for "${client.collectionName}"` + ); + } + + return { + client, + pendingDownloads, + resolvePendingDownloads, + rejectPendingDownloads, + assertNoNewDownloads, + }; +} + +/** + * The amount of files that are generated per mocked language pair. + */ +const FILES_PER_LANGUAGE_PAIR = 3; + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @param {Object[]} langPairs + * @returns {RemoteSettingsClient} + */ +async function createTranslationModelsRemoteClient( + autoDownloadFromRemoteSettings, + langPairs +) { + const records = []; + for (const { fromLang, toLang, isBeta } of langPairs) { + const lang = fromLang + toLang; + const models = [ + { fileType: "model", name: `model.${lang}.intgemm.alphas.bin` }, + { fileType: "lex", name: `lex.50.50.${lang}.s2t.bin` }, + { fileType: "vocab", name: `vocab.${lang}.spm` }, + ]; + + if (models.length !== FILES_PER_LANGUAGE_PAIR) { + throw new Error("Files per language pair was wrong."); + } + + for (const { fileType, name } of models) { + records.push({ + id: crypto.randomUUID(), + name, + fromLang, + toLang, + fileType, + version: isBeta ? "0.1" : "1.0", + last_modified: Date.now(), + schema: Date.now(), + }); + } + } + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const client = RemoteSettings("test-translation-models"); + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock(client, autoDownloadFromRemoteSettings); +} + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @returns {RemoteSettingsClient} + */ +async function createTranslationsWasmRemoteClient( + autoDownloadFromRemoteSettings +) { + const records = ["bergamot-translator", "fasttext-wasm"].map(name => ({ + id: crypto.randomUUID(), + name, + version: "1.0", + last_modified: Date.now(), + schema: Date.now(), + })); + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const client = RemoteSettings("test-translations-wasm"); + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock(client, autoDownloadFromRemoteSettings); +} + +/** + * Creates a local RemoteSettingsClient for use within tests. + * + * @param {boolean} autoDownloadFromRemoteSettings + * @returns {RemoteSettingsClient} + */ +async function createLanguageIdModelsRemoteClient( + autoDownloadFromRemoteSettings +) { + const records = [ + { + id: crypto.randomUUID(), + name: "lid.176.ftz", + version: "1.0", + last_modified: Date.now(), + schema: Date.now(), + }, + ]; + + const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" + ); + const client = RemoteSettings("test-language-id-models"); + const metadata = {}; + await client.db.clear(); + await client.db.importChanges(metadata, Date.now(), records); + + return createAttachmentMock(client, autoDownloadFromRemoteSettings); +} + +async function selectAboutPreferencesElements() { + const document = gBrowser.selectedBrowser.contentDocument; + + const rows = await TestUtils.waitForCondition(() => { + const elements = document.querySelectorAll(".translations-manage-language"); + if (elements.length !== 3) { + return false; + } + return elements; + }, "Waiting for manage language rows."); + + const [downloadAllRow, frenchRow, spanishRow] = rows; + + const downloadAllLabel = downloadAllRow.querySelector("label"); + const downloadAll = downloadAllRow.querySelector( + "#translations-manage-install-all" + ); + const deleteAll = downloadAllRow.querySelector( + "#translations-manage-delete-all" + ); + const frenchLabel = frenchRow.querySelector("label"); + const frenchDownload = frenchRow.querySelector( + `[data-l10n-id="translations-manage-download-button"]` + ); + const frenchDelete = frenchRow.querySelector( + `[data-l10n-id="translations-manage-delete-button"]` + ); + const spanishLabel = spanishRow.querySelector("label"); + const spanishDownload = spanishRow.querySelector( + `[data-l10n-id="translations-manage-download-button"]` + ); + const spanishDelete = spanishRow.querySelector( + `[data-l10n-id="translations-manage-delete-button"]` + ); + + return { + document, + downloadAllLabel, + downloadAll, + deleteAll, + frenchLabel, + frenchDownload, + frenchDelete, + spanishLabel, + spanishDownload, + spanishDelete, + }; +} + +function click(button, message) { + info(message); + if (button.hidden) { + throw new Error("The button was hidden when trying to click it."); + } + button.click(); +} + +/** + * @param {Object} options + * @param {string} options.message + * @param {Record<string, Element[]>} options.visible + * @param {Record<string, Element[]>} options.hidden + */ +async function assertVisibility({ message, visible, hidden }) { + info(message); + try { + // First wait for the condition to be met. + await TestUtils.waitForCondition(() => { + for (const element of Object.values(visible)) { + if (element.hidden) { + return false; + } + } + for (const element of Object.values(hidden)) { + if (!element.hidden) { + return false; + } + } + return true; + }); + } catch (error) { + // Ignore, this will get caught below. + } + // Now report the conditions. + for (const [name, element] of Object.entries(visible)) { + ok(!element.hidden, `${name} is visible.`); + } + for (const [name, element] of Object.entries(hidden)) { + ok(element.hidden, `${name} is hidden.`); + } +} + +async function setupAboutPreferences(languagePairs) { + await SpecialPowers.pushPrefEnv({ + set: [ + // Enabled by default. + ["browser.translations.enable", true], + ["browser.translations.logLevel", "All"], + ], + }); + const tab = await BrowserTestUtils.openNewForegroundTab( + gBrowser, + BLANK_PAGE, + true // waitForLoad + ); + + const { remoteClients, removeMocks } = await createAndMockRemoteSettings({ + languagePairs, + }); + + BrowserTestUtils.loadURIString(tab.linkedBrowser, "about:preferences"); + await BrowserTestUtils.browserLoaded(tab.linkedBrowser); + + const elements = await selectAboutPreferencesElements(); + + async function cleanup() { + gBrowser.removeCurrentTab(); + removeMocks(); + await SpecialPowers.popPrefEnv(); + } + + return { + cleanup, + remoteClients, + elements, + }; +} + +function waitForAppLocaleChanged() { + new Promise(resolve => { + function onChange() { + Services.obs.removeObserver(onChange, "intl:app-locales-changed"); + resolve(); + } + Services.obs.addObserver(onChange, "intl:app-locales-changed"); + }); +} + +async function mockLocales({ systemLocales, appLocales, webLanguages }) { + const appLocaleChanged1 = waitForAppLocaleChanged(); + + TranslationsParent.mockedSystemLocales = systemLocales; + const { availableLocales, requestedLocales } = Services.locale; + + info("Mocking locales, so expect potential .ftl resource errors."); + Services.locale.availableLocales = appLocales; + Services.locale.requestedLocales = appLocales; + + await appLocaleChanged1; + + await SpecialPowers.pushPrefEnv({ + set: [["intl.accept_languages", webLanguages.join(",")]], + }); + + return async () => { + const appLocaleChanged2 = waitForAppLocaleChanged(); + + // Reset back to the originals. + TranslationsParent.mockedSystemLocales = null; + Services.locale.availableLocales = availableLocales; + Services.locale.requestedLocales = requestedLocales; + + await appLocaleChanged2; + + await SpecialPowers.popPrefEnv(); + }; +} + +/** + * Helpful test functions for translations telemetry + */ +class TestTranslationsTelemetry { + /** + * Asserts qualities about a counter telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} counter - The Glean counter object. + * @param {Object} expectedCount - The expected value of the counter. + */ + static async assertCounter(name, counter, expectedCount) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const count = counter.testGetValue() ?? 0; + is( + count, + expectedCount, + `Telemetry counter ${name} should have expected count` + ); + } + + /** + * Asserts qualities about an event telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} event - The Glean event object. + * @param {Object} expectations - The test expectations. + * @param {number} expectations.expectedLength - The expected length of the event. + * @param {Array<function>} [expectations.allValuePredicates=[]] + * - An array of function predicates to assert for all event values. + * @param {Array<function>} [expectations.finalValuePredicates=[]] + * - An array of function predicates to assert for only the final event value. + */ + static async assertEvent( + name, + event, + { expectedLength, allValuePredicates = [], finalValuePredicates = [] } + ) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const values = event.testGetValue() ?? []; + const length = values.length; + + is( + length, + expectedLength, + `Telemetry event ${name} should have length ${expectedLength}` + ); + + if (allValuePredicates.length !== 0) { + is( + length > 0, + true, + `Telemetry event ${name} should contain values if allPredicates are specified` + ); + for (const value of values) { + for (const predicate of allValuePredicates) { + is( + predicate(value), + true, + `Telemetry event ${name} allPredicate { ${predicate.toString()} } should pass for each value` + ); + } + } + } + + if (finalValuePredicates.length !== 0) { + is( + length > 0, + true, + `Telemetry event ${name} should contain values if finalPredicates are specified` + ); + for (const predicate of finalValuePredicates) { + is( + predicate(values[length - 1]), + true, + `Telemetry event ${name} finalPredicate { ${predicate.toString()} } should pass for final value` + ); + } + } + } + + /** + * Asserts qualities about a rate telemetry metric. + * + * @param {string} name - The name of the metric. + * @param {Object} rate - The Glean rate object. + * @param {Object} expectations - The test expectations. + * @param {number} expectations.expectedNumerator - The expected value of the numerator. + * @param {number} expectations.expectedDenominator - The expected value of the denominator. + */ + static async assertRate( + name, + rate, + { expectedNumerator, expectedDenominator } + ) { + // Ensures that glean metrics are collected from all child processes + // so that calls to testGetValue() are up to date. + await Services.fog.testFlushAllChildren(); + const { numerator = 0, denominator = 0 } = rate.testGetValue() ?? {}; + is( + numerator, + expectedNumerator, + `Telemetry rate ${name} should have expected numerator` + ); + is( + denominator, + expectedDenominator, + `Telemetry rate ${name} should have expected denominator` + ); + } +} diff --git a/toolkit/components/translations/tests/browser/translations-test.mjs b/toolkit/components/translations/tests/browser/translations-test.mjs new file mode 100644 index 0000000000..d15a141837 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-test.mjs @@ -0,0 +1,66 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// eslint-disable-next-line no-unused-vars +let ok; +let is; +// eslint-disable-next-line no-unused-vars +let isnot; +let ContentTaskUtils; + +/** @type {{ document: Document, window: Window }} */ +let content; + +/** + * Inject the global variables from the test scope into the ES module scope. + */ +export function setup(config) { + // When a function is provided to `ContentTask.spawn`, that function is provided the + // Assert library through variable capture. In this case, this code is an ESM module, + // and does not have access to that scope. To work around this issue, pass any any + // relevant variables to that can be bound to the module scope. + // + // See: https://searchfox.org/mozilla-central/rev/cdddec7fd690700efa4d6b48532cf70155e0386b/testing/mochitest/BrowserTestUtils/content/content-task.js#78 + const { Assert } = config; + ok = Assert.ok.bind(Assert); + is = Assert.equal.bind(Assert); + isnot = Assert.notEqual.bind(Assert); + + ContentTaskUtils = config.ContentTaskUtils; + content = config.content; +} + +export function getSelectors() { + return { + getH1() { + return content.document.querySelector("h1"); + }, + getLastParagraph() { + return content.document.querySelector("p:last-child"); + }, + getHeader() { + return content.document.querySelector("header"); + }, + }; +} + +/** + * Asserts that a page was translated with a specific result. + * + * @param {string} message The assertion message. + * @param {Function} getNode A function to get the node. + * @param {string} translation The translated message. + */ +export async function assertTranslationResult(message, getNode, translation) { + try { + await ContentTaskUtils.waitForCondition( + () => translation === getNode()?.innerText, + `Waiting for: "${translation}"` + ); + } catch (error) { + // The result wasn't found, but the assertion below will report the error. + console.error(error); + } + + is(translation, getNode()?.innerText, message); +} diff --git a/toolkit/components/translations/tests/browser/translations-tester-en.html b/toolkit/components/translations/tests/browser/translations-tester-en.html new file mode 100644 index 0000000000..f32a7486ed --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-en.html @@ -0,0 +1,34 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <!-- The following is an excerpt from The Wondeful Wizard of Oz, which is in the public domain --> + <h1>"The Wonderful Wizard of Oz" by L. Frank Baum</h1> + <p>The little girl, seeing she had lost one of her pretty shoes, grew angry, and said to the Witch, “Give me back my shoe!”</p> + <p>“I will not,” retorted the Witch, “for it is now my shoe, and not yours.”</p> + <p>“You are a wicked creature!” cried Dorothy. “You have no right to take my shoe from me.”</p> + <p>“I shall keep it, just the same,” said the Witch, laughing at her, “and someday I shall get the other one from you, too.”</p> + <p>This made Dorothy so very angry that she picked up the bucket of water that stood near and dashed it over the Witch, wetting her from head to foot.</p> + <p>Instantly the wicked woman gave a loud cry of fear, and then, as Dorothy looked at her in wonder, the Witch began to shrink and fall away.</p> + <p>“See what you have done!” she screamed. “In a minute I shall melt away.”</p> + <p>“I’m very sorry, indeed,” said Dorothy, who was truly frightened to see the Witch actually melting away like brown sugar before her very eyes.</p> + <p>“Didn’t you know water would be the end of me?” asked the Witch, in a wailing, despairing voice.</p> + <p>“Of course not,” answered Dorothy. “How should I?”</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-es-2.html b/toolkit/components/translations/tests/browser/translations-tester-es-2.html new file mode 100644 index 0000000000..abf2d42c62 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-es-2.html @@ -0,0 +1,36 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p>Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-es.html b/toolkit/components/translations/tests/browser/translations-tester-es.html new file mode 100644 index 0000000000..abf2d42c62 --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-es.html @@ -0,0 +1,36 @@ +<!DOCTYPE html> +<html lang="es"> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p>Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/tests/browser/translations-tester-no-tag.html b/toolkit/components/translations/tests/browser/translations-tester-no-tag.html new file mode 100644 index 0000000000..7ef29cb70b --- /dev/null +++ b/toolkit/components/translations/tests/browser/translations-tester-no-tag.html @@ -0,0 +1,35 @@ +<!DOCTYPE html> +<head> + <meta charset="utf-8" /> + <title>Translations Test</title> + <style> + div { + margin: 10px auto; + width: 300px + } + p { + margin: 47px 0; + font-size: 21px; + line-height: 2; + } + </style> +</head> +<body> + <div> + <header lang="en">The following is an excerpt from Don Quijote de la Mancha, which is in the public domain</header> + <h1>Don Quijote de La Mancha</h1> + <h2>Capítulo VIII.</h2> + <p>Del buen suceso que el valeroso don Quijote tuvo en la espantable y jamás imaginada aventura de los molinos de viento, con otros sucesos dignos de felice recordación</p> + <p>En esto, descubrieron treinta o cuarenta molinos de viento que hay en aquel campo; y, así como don Quijote los vio, dijo a su escudero:</p> + <p>— La ventura va guiando nuestras cosas mejor de lo que acertáramos a desear, porque ves allí, amigo Sancho Panza, donde se descubren treinta, o pocos más, desaforados gigantes, con quien pienso hacer batalla y quitarles a todos las vidas, con cuyos despojos comenzaremos a enriquecer; que ésta es buena guerra, y es gran servicio de Dios quitar tan mala simiente de sobre la faz de la tierra.</p> + <p>— ¿Qué gigantes? —dijo Sancho Panza.</p> + <p>— Aquellos que allí ves —respondió su amo— de los brazos largos, que los suelen tener algunos de casi dos leguas.</p> + <p>— Mire vuestra merced —respondió Sancho— que aquellos que allí se parecen no son gigantes, sino molinos de viento, y lo que en ellos parecen brazos son las aspas, que, volteadas del viento, hacen andar la piedra del molino.</p> + <p>— Bien parece —respondió don Quijote— que no estás cursado en esto de las aventuras: ellos son gigantes; y si tienes miedo, quítate de ahí, y ponte en oración en el espacio que yo voy a entrar con ellos en fiera y desigual batalla.</p> + <p>Y, diciendo esto, dio de espuelas a su caballo Rocinante, sin atender a las voces que su escudero Sancho le daba, advirtiéndole que, sin duda alguna, eran molinos de viento, y no gigantes, aquellos que iba a acometer. Pero él iba tan puesto en que eran gigantes, que ni oía las voces de su escudero Sancho ni echaba de ver, aunque estaba ya bien cerca, lo que eran; antes, iba diciendo en voces altas:</p> + <p>— Non fuyades, cobardes y viles criaturas, que un solo caballero es el que os acomete.</p> + <p>Levantóse en esto un poco de viento y las grandes aspas comenzaron a moverse, lo cual visto por don Quijote, dijo:</p> + <p>— Pues, aunque mováis más brazos que los del gigante Briareo, me lo habéis de pagar.</p> + </div> +</body> +</html> diff --git a/toolkit/components/translations/translations.d.ts b/toolkit/components/translations/translations.d.ts new file mode 100644 index 0000000000..3d6e0f5fd4 --- /dev/null +++ b/toolkit/components/translations/translations.d.ts @@ -0,0 +1,302 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains the shared types for the translations component. The intended use + * is for defining types to be used in JSDoc. They are used in a form that the TypeScript + * language server can read them, and provide code hints. + */ + +/** + * For Remote Settings, the JSON details about the attachment. + */ +export interface Attachment { + // e.g. "2f7c0f7bbc...ca79f0850c4de", + hash: string; + // e.g. 5047568, + size: string; + // e.g. "lex.50.50.deen.s2t.bin", + filename: string; + // e.g. "main-workspace/translations-models/316ebb3a-0682-42cc-8e73-a3ba4bbb280f.bin", + location: string; + // e.g. "application/octet-stream" + mimetype: string; +} + +/** + * The JSON that is synced from Remote Settings for the language-id models. + */ +export interface LanguageIdModelRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The full model name, e.g. "lid.176.ftz" + name: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + +/** + * The JSON that is synced from Remote Settings for the translation models. + */ +export interface TranslationModelRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The full model name, e.g. "lex.50.50.deen.s2t.bin" + name: string; + // The BCP 47 language tag, e.g. "de" + fromLang: string; + // The BCP 47 language tag, e.g. "en" + toLang: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // e.g. "lex" + fileType: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673023100578 + schema: number; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + +/** + * The JSON that is synced from Remote Settings for the wasm binaries. + */ +export interface WasmRecord { + // e.g. "0d4db293-a17c-4085-9bd8-e2e146c85000" + id: string; + // The name of the project, e.g. "bergamot-translator" + name: string; + // The human readable identifier for the release. e.g. "v0.4.4" + release: string; + // The commit hash for the project that generated the wasm. + revision: string; + // The license of the wasm, as a https://spdx.org/licenses/ + license: string; + // The semver number, used for handling future format changes. e.g. 1.0 + version: string; + // The file attachment for this record + attachment: Attachment; + // e.g. 1673455932527 + last_modified: string; + // A JEXL expression to determine whether this record should be pulled from Remote Settings + // See: https://remote-settings.readthedocs.io/en/latest/target-filters.html#filter-expressions + filter_expression: string; +} + +/** + * The following are the types that are provided by the Bergamot wasm library. + * + * See: https://github.com/mozilla/bergamot-translator/tree/main/wasm/bindings + */ +export namespace Bergamot { + /** + * The main module that is returned from bergamot-translator.js. + */ + export interface ModuleExport { + BlockingService: typeof BlockingService; + AlignedMemoryList: typeof AlignedMemoryList; + TranslationModel: typeof TranslationModel; + AlignedMemory: typeof AlignedMemory; + VectorResponseOptions: typeof VectorResponseOptions; + VectorString: typeof VectorString; + } + + /** + * This class represents a C++ std::vector. The implementations will extend from it. + */ + export class Vector<T> { + size(): number; + get(index: number): T; + push_back(item: T); + } + + export class VectorResponse extends Vector<Response> {} + export class VectorString extends Vector<string> {} + export class VectorResponseOptions extends Vector<ResponseOptions> {} + export class AlignedMemoryList extends Vector<AlignedMemory> {} + + /** + * A blocking (e.g. non-threaded) translation service, via Bergamot. + */ + export class BlockingService { + /** + * Translate multiple messages in a single synchronous API call using a single model. + */ + translate( + translationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + + /** + * Translate by pivoting between two models + * + * For example to translate "fr" to "es", pivot using "en": + * "fr" to "en" + * "en" to "es" + * + * See https://github.com/mozilla/bergamot-translator/blob/5ae1b1ebb3fa9a3eabed8a64ca6798154bd486eb/src/translator/service.h#L80 + */ + translateViaPivoting( + first: TranslationModel, + second: TranslationModel, + vectorSourceText: VectorString, + vectorResponseOptions: VectorResponseOptions + ): VectorResponse; + } + + /** + * The actual translation model, which is passed into the `BlockingService` methods. + */ + export class TranslationModel {} + + /** + * The models need to be placed in the wasm memory space. This object represents + * aligned memory that was allocated on the wasm side of things. The memory contents + * can be set via the getByteArrayView method and the Uint8Array.prototype.set method. + */ + export class AlignedMemory { + constructor(size: number, alignment: number); + size(): number; + getByteArrayView(): Uint8Array; + } + + /** + * The response from the translation. This definition isn't complete, but just + * contains a subset of the available methods. + * + * See https://github.com/mozilla/bergamot-translator/blob/main/src/translator/response.h + */ + export class Response { + getOriginalText(): string; + getTranslatedText(): string; + } + + /** + * The options to configure a translation response. + * + * See https://github.com/mozilla/bergamot-translator/blob/main/src/translator/response_options.h + */ + export class ResponseOptions { + // Include the quality estimations. + qualityScores: boolean; + // Include the alignments. + alignment: boolean; + // Remove HTML tags from text and insert it back into the output. + html: boolean; + // Whether to include sentenceMappings or not. Alignments require + // sentenceMappings and are available irrespective of this option if + // `alignment=true`. + sentenceMappings: boolean + } +} + + +/** + * The client to interact with RemoteSettings. + * See services/settings/RemoteSettingsClient.jsm + */ +interface RemoteSettingsClient { + on: Function, + get: Function, + attachments: any, +} + +/** + * A single language model file. + */ +interface LanguageTranslationModelFile { + buffer: ArrayBuffer, + record: TranslationModelRecord, +} + +/** + * The files necessary to run the translations, these will be sent to the Bergamot + * translation engine. + */ +interface LanguageTranslationModelFiles { + // The machine learning language model. + model: LanguageTranslationModelFile, + // The lexical shortlist that limits possible output of the decoder and makes + // inference faster. + lex: LanguageTranslationModelFile, + // A model that can generate a translation quality estimation. + qualityModel?: LanguageTranslationModelFile, + + // There is either a single vocab file: + vocab?: LanguageTranslationModelFile, + + // Or there are two: + srcvocab?: LanguageTranslationModelFile, + trgvocab?: LanguageTranslationModelFile, +}; + +/** + * This is the type that is generated when the models are loaded into wasm aligned memory. + */ +type LanguageTranslationModelFilesAligned = { + [K in keyof LanguageTranslationModelFiles]: AlignedMemory +}; + +/** + * These are the files that are downloaded from Remote Settings that are necessary + * to start the translations engine. These may not be available if running in tests, + * and so the engine will be mocked. + */ +interface TranslationsEnginePayload { + bergamotWasmArrayBuffer: ArrayBuffer, + languageModelFiles: LanguageTranslationModelFiles[] + isMocked: boolean, +} + +/** + * These are the files that are downloaded from Remote Settings that are necessary + * to start the language-identification engine. These may not be available if running + * in tests. + */ +interface LanguageIdEnginePayload { + wasmBuffer: ArrayBuffer, + modelBuffer: ArrayBuffer, + mockedConfidence: null | number, + mockedLangTag: null | string, +} + +/** + * Nodes that are being translated are given priority according to their visibility. + */ +export type NodeVisibility = "in-viewport" | "out-of-viewport" | "hidden"; + +/** + * Used to decide how to translate a page for full page translations. + */ +export interface LangTags { + isDocLangTagSupported: boolean, + docLangTag: string | null, + userLangTag: string | null, +} + +export interface LanguagePair { fromLang: string, toLang: string }; + +/** + * A structure that contains all of the information needed to render dropdowns + * for translation language selection. + */ +export interface SupportedLanguages { + languagePairs: LanguagePair[], + fromLanguages: Array<{ langTag: string, isBeta: boolean, displayName: string, }>, + toLanguages: Array<{ langTag: string, isBeta: boolean, displayName: string }>, +} + +export type TranslationErrors = "engine-load-error"; |
