diff options
Diffstat (limited to 'browser/components/translation')
21 files changed, 3922 insertions, 0 deletions
diff --git a/browser/components/translation/BingTranslator.jsm b/browser/components/translation/BingTranslator.jsm new file mode 100644 index 0000000000..b457e16324 --- /dev/null +++ b/browser/components/translation/BingTranslator.jsm @@ -0,0 +1,488 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["BingTranslator"]; + +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); +const { Async } = ChromeUtils.importESModule( + "resource://services-common/async.sys.mjs" +); +const { httpRequest } = ChromeUtils.importESModule( + "resource://gre/modules/Http.sys.mjs" +); + +// The maximum amount of net data allowed per request on Bing's API. +const MAX_REQUEST_DATA = 5000; // Documentation says 10000 but anywhere +// close to that is refused by the service. + +// The maximum number of chunks allowed to be translated in a single +// request. +const MAX_REQUEST_CHUNKS = 1000; // Documentation says 2000. + +// Self-imposed limit of 15 requests. This means that a page that would need +// to be broken in more than 15 requests won't be fully translated. +// The maximum amount of data that we will translate for a single page +// is MAX_REQUESTS * MAX_REQUEST_DATA. +const MAX_REQUESTS = 15; + +/** + * Translates a webpage using Bing's Translation API. + * + * @param translationDocument The TranslationDocument object that represents + * the webpage to be translated + * @param sourceLanguage The source language of the document + * @param targetLanguage The target language for the translation + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ +var BingTranslator = function ( + translationDocument, + sourceLanguage, + targetLanguage +) { + this.translationDocument = translationDocument; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this._pendingRequests = 0; + this._partialSuccess = false; + this._serviceUnavailable = false; + this._translatedCharacterCount = 0; +}; + +BingTranslator.prototype = { + /** + * Performs the translation, splitting the document into several chunks + * respecting the data limits of the API. + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ + translate() { + return (async () => { + let currentIndex = 0; + this._onFinishedDeferred = PromiseUtils.defer(); + + // Let's split the document into various requests to be sent to + // Bing's Translation API. + for (let requestCount = 0; requestCount < MAX_REQUESTS; requestCount++) { + // Generating the text for each request can be expensive, so + // let's take the opportunity of the chunkification process to + // allow for the event loop to attend other pending events + // before we continue. + await Async.promiseYield(); + + // Determine the data for the next request. + let request = this._generateNextTranslationRequest(currentIndex); + + // Create a real request to the server, and put it on the + // pending requests list. + let bingRequest = new BingRequest( + request.data, + this.sourceLanguage, + this.targetLanguage + ); + this._pendingRequests++; + bingRequest + .fireRequest() + .then(this._chunkCompleted.bind(this), this._chunkFailed.bind(this)); + + currentIndex = request.lastIndex; + if (request.finished) { + break; + } + } + + return this._onFinishedDeferred.promise; + })(); + }, + + /** + * Resets the expiration time of the current token, in order to + * force the token manager to ask for a new token during the next request. + */ + _resetToken() { + // Force the token manager to get update token + BingTokenManager._currentExpiryTime = 0; + }, + + /** + * Function called when a request sent to the server completed successfully. + * This function handles calling the function to parse the result and the + * function to resolve the promise returned by the public `translate()` + * method when there's no pending request left. + * + * @param request The BingRequest sent to the server. + */ + _chunkCompleted(bingRequest) { + if (this._parseChunkResult(bingRequest)) { + this._partialSuccess = true; + // Count the number of characters successfully translated. + this._translatedCharacterCount += bingRequest.characterCount; + } + + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has failed. + * This function handles deciding if the error is transient or means the + * service is unavailable (zero balance on the key or request credentials are + * not in an active state) and calling the function to resolve the promise + * returned by the public `translate()` method when there's no pending. + * request left. + * + * @param aError [optional] The XHR object of the request that failed. + */ + _chunkFailed(aError) { + if ( + XMLHttpRequest.isInstance(aError) && + [400, 401].includes(aError.status) + ) { + let body = aError.responseText; + if ( + body && + body.includes("TranslateApiException") && + (body.includes("balance") || body.includes("active state")) + ) { + this._serviceUnavailable = true; + } + } + + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has completed. + * This function handles resolving the promise + * returned by the public `translate()` method when all chunks are completed. + */ + _checkIfFinished() { + // Check if all pending requests have been + // completed and then resolves the promise. + // If at least one chunk was successful, the + // promise will be resolved positively which will + // display the "Success" state for the infobar. Otherwise, + // the "Error" state will appear. + if (--this._pendingRequests == 0) { + if (this._partialSuccess) { + this._onFinishedDeferred.resolve({ + characterCount: this._translatedCharacterCount, + }); + } else { + let error = this._serviceUnavailable ? "unavailable" : "failure"; + this._onFinishedDeferred.reject(error); + } + } + }, + + /** + * This function parses the result returned by Bing's Http.svc API, + * which is a XML file that contains a number of elements. To our + * particular interest, the only part of the response that matters + * are the <TranslatedText> nodes, which contains the resulting + * items that were sent to be translated. + * + * @param request The request sent to the server. + * @returns boolean True if parsing of this chunk was successful. + */ + _parseChunkResult(bingRequest) { + let results; + try { + let doc = bingRequest.networkRequest.responseXML; + results = doc.querySelectorAll("TranslatedText"); + } catch (e) { + return false; + } + + let len = results.length; + if (len != bingRequest.translationData.length) { + // This should never happen, but if the service returns a different number + // of items (from the number of items submitted), we can't use this chunk + // because all items would be paired incorrectly. + return false; + } + + let error = false; + for (let i = 0; i < len; i++) { + try { + let result = results[i].firstChild.nodeValue; + let root = bingRequest.translationData[i][0]; + + if (root.isSimpleRoot) { + // Workaround for Bing's service problem in which "&" chars in + // plain-text TranslationItems are double-escaped. + result = result.replace(/&/g, "&"); + } + + root.parseResult(result); + } catch (e) { + error = true; + } + } + + return !error; + }, + + /** + * This function will determine what is the data to be used for + * the Nth request we are generating, based on the input params. + * + * @param startIndex What is the index, in the roots list, that the + * chunk should start. + */ + _generateNextTranslationRequest(startIndex) { + let currentDataSize = 0; + let currentChunks = 0; + let output = []; + let rootsList = this.translationDocument.roots; + + for (let i = startIndex; i < rootsList.length; i++) { + let root = rootsList[i]; + let text = this.translationDocument.generateTextForItem(root); + if (!text) { + continue; + } + + text = escapeXML(text); + let newCurSize = currentDataSize + text.length; + let newChunks = currentChunks + 1; + + if (newCurSize > MAX_REQUEST_DATA || newChunks > MAX_REQUEST_CHUNKS) { + // If we've reached the API limits, let's stop accumulating data + // for this request and return. We return information useful for + // the caller to pass back on the next call, so that the function + // can keep working from where it stopped. + return { + data: output, + finished: false, + lastIndex: i, + }; + } + + currentDataSize = newCurSize; + currentChunks = newChunks; + output.push([root, text]); + } + + return { + data: output, + finished: true, + lastIndex: 0, + }; + }, +}; + +/** + * Represents a request (for 1 chunk) sent off to Bing's service. + * + * @params translationData The data to be used for this translation, + * generated by the generateNextTranslationRequest... + * function. + * @param sourceLanguage The source language of the document. + * @param targetLanguage The target language for the translation. + * + */ +function BingRequest(translationData, sourceLanguage, targetLanguage) { + this.translationData = translationData; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this.characterCount = 0; +} + +BingRequest.prototype = { + /** + * Initiates the request + */ + fireRequest() { + return (async () => { + // Prepare authentication. + let token = await BingTokenManager.getToken(); + let auth = "Bearer " + token; + + // Prepare URL. + let url = getUrlParam( + "https://api.microsofttranslator.com/v2/Http.svc/TranslateArray", + "browser.translation.bing.translateArrayURL" + ); + + // Prepare request headers. + let headers = [ + ["Content-type", "text/xml"], + ["Authorization", auth], + ]; + + // Prepare the request body. + let requestString = + "<TranslateArrayRequest>" + + "<AppId/>" + + "<From>" + + this.sourceLanguage + + "</From>" + + "<Options>" + + '<ContentType xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2">text/html</ContentType>' + + '<ReservedFlags xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" />' + + "</Options>" + + '<Texts xmlns:s="http://schemas.microsoft.com/2003/10/Serialization/Arrays">'; + + for (let [, text] of this.translationData) { + requestString += "<s:string>" + text + "</s:string>"; + this.characterCount += text.length; + } + + requestString += + "</Texts>" + + "<To>" + + this.targetLanguage + + "</To>" + + "</TranslateArrayRequest>"; + + // Set up request options. + return new Promise((resolve, reject) => { + let options = { + onLoad: (responseText, xhr) => { + resolve(this); + }, + onError(e, responseText, xhr) { + reject(xhr); + }, + postData: requestString, + headers, + }; + + // Fire the request. + let request = httpRequest(url, options); + + // Override the response MIME type. + request.overrideMimeType("text/xml"); + this.networkRequest = request; + }); + })(); + }, +}; + +/** + * Authentication Token manager for the API + */ +var BingTokenManager = { + _currentToken: null, + _currentExpiryTime: 0, + _pendingRequest: null, + + /** + * Get a valid, non-expired token to be used for the API calls. + * + * @returns {Promise} A promise that resolves with the token + * string once it is obtained. The token returned + * can be the same one used in the past if it is still + * valid. + */ + getToken() { + if (this._pendingRequest) { + return this._pendingRequest; + } + + let remainingMs = this._currentExpiryTime - new Date(); + // Our existing token is still good for more than a minute, let's use it. + if (remainingMs > 60 * 1000) { + return Promise.resolve(this._currentToken); + } + + return this._getNewToken(); + }, + + /** + * Generates a new token from the server. + * + * @returns {Promise} A promise that resolves with the token + * string once it is obtained. + */ + _getNewToken() { + let url = getUrlParam( + "https://datamarket.accesscontrol.windows.net/v2/OAuth2-13", + "browser.translation.bing.authURL" + ); + let params = [ + ["grant_type", "client_credentials"], + ["scope", "http://api.microsofttranslator.com"], + [ + "client_id", + getUrlParam( + "%BING_API_CLIENTID%", + "browser.translation.bing.clientIdOverride" + ), + ], + [ + "client_secret", + getUrlParam( + "%BING_API_KEY%", + "browser.translation.bing.apiKeyOverride" + ), + ], + ]; + + this._pendingRequest = new Promise((resolve, reject) => { + let options = { + onLoad(responseText, xhr) { + BingTokenManager._pendingRequest = null; + try { + let json = JSON.parse(responseText); + + if (json.error) { + reject(json.error); + return; + } + + let token = json.access_token; + let expires_in = json.expires_in; + BingTokenManager._currentToken = token; + BingTokenManager._currentExpiryTime = new Date( + Date.now() + expires_in * 1000 + ); + resolve(token); + } catch (e) { + reject(e); + } + }, + onError(e, responseText, xhr) { + BingTokenManager._pendingRequest = null; + reject(e); + }, + postData: params, + }; + + httpRequest(url, options); + }); + return this._pendingRequest; + }, +}; + +/** + * Escape a string to be valid XML content. + */ +function escapeXML(aStr) { + return aStr + .toString() + .replace(/&/g, "&") + .replace(/\"/g, """) + .replace(/\'/g, "'") + .replace(/</g, "<") + .replace(/>/g, ">"); +} + +/** + * Fetch an auth token (clientID or client secret), which may be overridden by + * a pref if it's set. + */ +function getUrlParam(paramValue, prefName) { + if (Services.prefs.getPrefType(prefName)) { + paramValue = Services.prefs.getCharPref(prefName); + } + paramValue = Services.urlFormatter.formatURL(paramValue); + return paramValue; +} diff --git a/browser/components/translation/GoogleTranslator.jsm b/browser/components/translation/GoogleTranslator.jsm new file mode 100644 index 0000000000..dd5da98b8b --- /dev/null +++ b/browser/components/translation/GoogleTranslator.jsm @@ -0,0 +1,306 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["GoogleTranslator"]; + +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); +const { httpRequest } = ChromeUtils.importESModule( + "resource://gre/modules/Http.sys.mjs" +); + +// The maximum amount of net data allowed per request on Google's API. +const MAX_REQUEST_DATA = 5000; // XXX This is the Bing value + +// The maximum number of chunks allowed to be translated in a single +// request. +const MAX_REQUEST_CHUNKS = 128; // Undocumented, but the de facto upper limit. + +// Self-imposed limit of 15 requests. This means that a page that would need +// to be broken in more than 15 requests won't be fully translated. +// The maximum amount of data that we will translate for a single page +// is MAX_REQUESTS * MAX_REQUEST_DATA. +const MAX_REQUESTS = 15; + +const URL = "https://translation.googleapis.com/language/translate/v2"; + +/** + * Translates a webpage using Google's Translation API. + * + * @param translationDocument The TranslationDocument object that represents + * the webpage to be translated + * @param sourceLanguage The source language of the document + * @param targetLanguage The target language for the translation + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ +var GoogleTranslator = function ( + translationDocument, + sourceLanguage, + targetLanguage +) { + this.translationDocument = translationDocument; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this._pendingRequests = 0; + this._partialSuccess = false; + this._translatedCharacterCount = 0; +}; + +GoogleTranslator.prototype = { + /** + * Performs the translation, splitting the document into several chunks + * respecting the data limits of the API. + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ + async translate() { + let currentIndex = 0; + this._onFinishedDeferred = PromiseUtils.defer(); + + // Let's split the document into various requests to be sent to + // Google's Translation API. + for (let requestCount = 0; requestCount < MAX_REQUESTS; requestCount++) { + // Generating the text for each request can be expensive, so + // let's take the opportunity of the chunkification process to + // allow for the event loop to attend other pending events + // before we continue. + await new Promise(resolve => Services.tm.dispatchToMainThread(resolve)); + + // Determine the data for the next request. + let request = this._generateNextTranslationRequest(currentIndex); + + // Create a real request to the server, and put it on the + // pending requests list. + let googleRequest = new GoogleRequest( + request.data, + this.sourceLanguage, + this.targetLanguage + ); + this._pendingRequests++; + googleRequest + .fireRequest() + .then(this._chunkCompleted.bind(this), this._chunkFailed.bind(this)); + + currentIndex = request.lastIndex; + if (request.finished) { + break; + } + } + + return this._onFinishedDeferred.promise; + }, + + /** + * Function called when a request sent to the server completed successfully. + * This function handles calling the function to parse the result and the + * function to resolve the promise returned by the public `translate()` + * method when there's no pending request left. + * + * @param request The GoogleRequest sent to the server. + */ + _chunkCompleted(googleRequest) { + if (this._parseChunkResult(googleRequest)) { + this._partialSuccess = true; + // Count the number of characters successfully translated. + this._translatedCharacterCount += googleRequest.characterCount; + } + + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has failed. + * This function handles deciding if the error is transient or means the + * service is unavailable (zero balance on the key or request credentials are + * not in an active state) and calling the function to resolve the promise + * returned by the public `translate()` method when there's no pending. + * request left. + * + * @param aError [optional] The XHR object of the request that failed. + */ + _chunkFailed(aError) { + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has completed. + * This function handles resolving the promise + * returned by the public `translate()` method when all chunks are completed. + */ + _checkIfFinished() { + // Check if all pending requests have been + // completed and then resolves the promise. + // If at least one chunk was successful, the + // promise will be resolved positively which will + // display the "Success" state for the infobar. Otherwise, + // the "Error" state will appear. + if (--this._pendingRequests == 0) { + if (this._partialSuccess) { + this._onFinishedDeferred.resolve({ + characterCount: this._translatedCharacterCount, + }); + } else { + this._onFinishedDeferred.reject("failure"); + } + } + }, + + /** + * This function parses the result returned by Bing's Http.svc API, + * which is a XML file that contains a number of elements. To our + * particular interest, the only part of the response that matters + * are the <TranslatedText> nodes, which contains the resulting + * items that were sent to be translated. + * + * @param request The request sent to the server. + * @returns boolean True if parsing of this chunk was successful. + */ + _parseChunkResult(googleRequest) { + let results; + try { + let response = googleRequest.networkRequest.response; + results = JSON.parse(response).data.translations; + } catch (e) { + return false; + } + let len = results.length; + if (len != googleRequest.translationData.length) { + // This should never happen, but if the service returns a different number + // of items (from the number of items submitted), we can't use this chunk + // because all items would be paired incorrectly. + return false; + } + + let error = false; + for (let i = 0; i < len; i++) { + try { + let result = results[i].translatedText; + let root = googleRequest.translationData[i][0]; + if (root.isSimpleRoot && result.includes("&")) { + // If the result contains HTML entities, we need to convert them as + // simple roots expect a plain text result. + let doc = new DOMParser().parseFromString(result, "text/html"); + result = doc.body.firstChild.nodeValue; + } + root.parseResult(result); + } catch (e) { + error = true; + } + } + + return !error; + }, + + /** + * This function will determine what is the data to be used for + * the Nth request we are generating, based on the input params. + * + * @param startIndex What is the index, in the roots list, that the + * chunk should start. + */ + _generateNextTranslationRequest(startIndex) { + let currentDataSize = 0; + let currentChunks = 0; + let output = []; + let rootsList = this.translationDocument.roots; + + for (let i = startIndex; i < rootsList.length; i++) { + let root = rootsList[i]; + let text = this.translationDocument.generateTextForItem(root); + if (!text) { + continue; + } + + let newCurSize = currentDataSize + text.length; + let newChunks = currentChunks + 1; + + if (newCurSize > MAX_REQUEST_DATA || newChunks > MAX_REQUEST_CHUNKS) { + // If we've reached the API limits, let's stop accumulating data + // for this request and return. We return information useful for + // the caller to pass back on the next call, so that the function + // can keep working from where it stopped. + return { + data: output, + finished: false, + lastIndex: i, + }; + } + + currentDataSize = newCurSize; + currentChunks = newChunks; + output.push([root, text]); + } + + return { + data: output, + finished: true, + lastIndex: 0, + }; + }, +}; + +/** + * Represents a request (for 1 chunk) sent off to Google's service. + * + * @params translationData The data to be used for this translation, + * generated by the generateNextTranslationRequest... + * function. + * @param sourceLanguage The source language of the document. + * @param targetLanguage The target language for the translation. + * + */ +function GoogleRequest(translationData, sourceLanguage, targetLanguage) { + this.translationData = translationData; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this.characterCount = 0; +} + +GoogleRequest.prototype = { + /** + * Initiates the request + */ + fireRequest() { + let key = + Services.cpmm.sharedData.get("translationKey") || + Services.prefs.getStringPref("browser.translation.google.apiKey", ""); + if (!key) { + return Promise.reject("no API key"); + } + + // Prepare the request body. + let postData = [ + ["key", key], + ["source", this.sourceLanguage], + ["target", this.targetLanguage], + ]; + + for (let [, text] of this.translationData) { + postData.push(["q", text]); + this.characterCount += text.length; + } + + // Set up request options. + return new Promise((resolve, reject) => { + let options = { + onLoad: (responseText, xhr) => { + resolve(this); + }, + onError(e, responseText, xhr) { + reject(xhr); + }, + postData, + }; + + // Fire the request. + this.networkRequest = httpRequest(URL, options); + }); + }, +}; diff --git a/browser/components/translation/TranslationChild.jsm b/browser/components/translation/TranslationChild.jsm new file mode 100644 index 0000000000..80a4c48661 --- /dev/null +++ b/browser/components/translation/TranslationChild.jsm @@ -0,0 +1,163 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TranslationChild"]; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + LanguageDetector: + "resource://gre/modules/translation/LanguageDetector.sys.mjs", +}); + +const STATE_OFFER = 0; +const STATE_TRANSLATED = 2; +const STATE_ERROR = 3; + +class TranslationChild extends JSWindowActorChild { + handleEvent(aEvent) { + switch (aEvent.type) { + case "pageshow": + // We are only listening to pageshow events. + let content = this.contentWindow; + if (!content.detectedLanguage) { + return; + } + + let data = {}; + let trDoc = content.translationDocument; + if (trDoc) { + data.state = trDoc.translationError ? STATE_ERROR : STATE_TRANSLATED; + data.translatedFrom = trDoc.translatedFrom; + data.translatedTo = trDoc.translatedTo; + data.originalShown = trDoc.originalShown; + } else { + data.state = STATE_OFFER; + data.originalShown = true; + } + data.detectedLanguage = content.detectedLanguage; + + this.sendAsyncMessage("Translation:DocumentState", data); + break; + + case "load": + this.checkForTranslation(); + break; + } + } + + checkForTranslation() { + let url = String(this.document.location); + if (!url.startsWith("http://") && !url.startsWith("https://")) { + return; + } + + let content = this.contentWindow; + if (content.detectedLanguage) { + return; + } + + // Grab a 60k sample of text from the page. + let encoder = Cu.createDocumentEncoder("text/plain"); + encoder.init(this.document, "text/plain", encoder.SkipInvisibleContent); + let string = encoder.encodeToStringWithMaxLength(60 * 1024); + + // Language detection isn't reliable on very short strings. + if (string.length < 100) { + return; + } + + lazy.LanguageDetector.detectLanguage(string).then(result => { + // Bail if we're not confident. + if (!result.confident) { + return; + } + + // The window might be gone by now. + if (Cu.isDeadWrapper(content)) { + return; + } + + content.detectedLanguage = result.language; + + let data = { + state: STATE_OFFER, + originalShown: true, + detectedLanguage: result.language, + }; + this.sendAsyncMessage("Translation:DocumentState", data); + }); + } + + async doTranslation(aFrom, aTo) { + var { TranslationDocument } = ChromeUtils.import( + "resource:///modules/translation/TranslationDocument.jsm" + ); + + // If a TranslationDocument already exists for this document, it should + // be used instead of creating a new one so that we can use the original + // content of the page for the new translation instead of the newly + // translated text. + let translationDocument = + this.contentWindow.translationDocument || + new TranslationDocument(this.document); + + let engine = Services.prefs.getCharPref("browser.translation.engine"); + let importScope = ChromeUtils.import( + `resource:///modules/translation/${engine}Translator.jsm` + ); + let translator = new importScope[engine + "Translator"]( + translationDocument, + aFrom, + aTo + ); + + this.contentWindow.translationDocument = translationDocument; + translationDocument.translatedFrom = aFrom; + translationDocument.translatedTo = aTo; + translationDocument.translationError = false; + + let result; + try { + let translateResult = await translator.translate(); + + result = { + characterCount: translateResult.characterCount, + from: aFrom, + to: aTo, + success: true, + }; + + translationDocument.showTranslation(); + return result; + } catch (ex) { + translationDocument.translationError = true; + result = { success: false }; + if (ex == "unavailable") { + result.unavailable = true; + } + } + + return result; + } + + receiveMessage(aMessage) { + switch (aMessage.name) { + case "Translation:TranslateDocument": { + return this.doTranslation(aMessage.data.from, aMessage.data.to); + } + + case "Translation:ShowOriginal": + this.contentWindow.translationDocument.showOriginal(); + break; + + case "Translation:ShowTranslation": + this.contentWindow.translationDocument.showTranslation(); + break; + } + + return undefined; + } +} diff --git a/browser/components/translation/TranslationDocument.jsm b/browser/components/translation/TranslationDocument.jsm new file mode 100644 index 0000000000..d5ea99f377 --- /dev/null +++ b/browser/components/translation/TranslationDocument.jsm @@ -0,0 +1,679 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TranslationDocument"]; + +const { Async } = ChromeUtils.importESModule( + "resource://services-common/async.sys.mjs" +); + +/** + * This class represents a document that is being translated, + * and it is responsible for parsing the document, + * generating the data structures translation (the list of + * translation items and roots), and managing the original + * and translated texts on the translation items. + * + * @param document The document to be translated + */ +var TranslationDocument = function (document) { + this.itemsMap = new Map(); + this.roots = []; + this._init(document); +}; + +TranslationDocument.prototype = { + translatedFrom: "", + translatedTo: "", + translationError: false, + originalShown: true, + + /** + * Initializes the object and populates + * the roots lists. + * + * @param document The document to be translated + */ + _init(document) { + let winUtils = document.defaultView.windowUtils; + + // Get all the translation nodes in the document's body: + // a translation node is a node from the document which + // contains useful content for translation, and therefore + // must be included in the translation process. + let nodeList = winUtils.getTranslationNodes(document.body); + + let length = nodeList.length; + + for (let i = 0; i < length; i++) { + let node = nodeList.item(i); + let isRoot = nodeList.isTranslationRootAtIndex(i); + + // Create a TranslationItem object for this node. + // This function will also add it to the this.roots array. + this._createItemForNode(node, i, isRoot); + } + + // At first all roots are stored in the roots list, and only after + // the process has finished we're able to determine which roots are + // simple, and which ones are not. + + // A simple root is defined by a root with no children items, which + // basically represents an element from a page with only text content + // inside. + + // This distinction is useful for optimization purposes: we treat a + // simple root as plain-text in the translation process and with that + // we are able to reduce their data payload sent to the translation service. + + for (let root of this.roots) { + if (!root.children.length && root.nodeRef.childElementCount == 0) { + root.isSimpleRoot = true; + } + } + }, + + /** + * Creates a TranslationItem object, which should be called + * for each node returned by getTranslationNodes. + * + * @param node The DOM node for this item. + * @param id A unique, numeric id for this item. + * @parem isRoot A boolean saying whether this item is a root. + * + * @returns A TranslationItem object. + */ + _createItemForNode(node, id, isRoot) { + if (this.itemsMap.has(node)) { + return this.itemsMap.get(node); + } + + let item = new TranslationItem(node, id, isRoot); + + if (isRoot) { + // Root items do not have a parent item. + this.roots.push(item); + } else { + let parentItem = this.itemsMap.get(node.parentNode); + if (parentItem) { + parentItem.children.push(item); + } + } + + this.itemsMap.set(node, item); + return item; + }, + + /** + * Generate the text string that represents a TranslationItem object. + * Besides generating the string, it's also stored in the "original" + * field of the TranslationItem object, which needs to be stored for + * later to be used in the "Show Original" functionality. + * If this function had already been called for the given item (determined + * by the presence of the "original" array in the item), the text will + * be regenerated from the "original" data instead of from the related + * DOM nodes (because the nodes might contain translated data). + * + * @param item A TranslationItem object + * + * @returns A string representation of the TranslationItem. + */ + generateTextForItem(item) { + if (item.original) { + return regenerateTextFromOriginalHelper(item); + } + + if (item.isSimpleRoot) { + let text = item.nodeRef.firstChild.nodeValue.trim(); + item.original = [text]; + return text; + } + + let str = ""; + item.original = []; + let wasLastItemPlaceholder = false; + + for (let child of item.nodeRef.childNodes) { + if (child.nodeType == child.TEXT_NODE) { + let x = child.nodeValue.trim(); + if (x != "") { + item.original.push(x); + str += x; + wasLastItemPlaceholder = false; + } + continue; + } + + let objInMap = this.itemsMap.get(child); + if (objInMap && !objInMap.isRoot) { + // If this childNode is present in the itemsMap, it means + // it's a translation node: it has useful content for translation. + // In this case, we need to stringify this node. + // However, if this item is a root, we should skip it here in this + // object's child list (and just add a placeholder for it), because + // it will be stringfied separately for being a root. + item.original.push(objInMap); + str += this.generateTextForItem(objInMap); + wasLastItemPlaceholder = false; + } else if (!wasLastItemPlaceholder) { + // Otherwise, if this node doesn't contain any useful content, + // or if it is a root itself, we can replace it with a placeholder node. + // We can't simply eliminate this node from our string representation + // because that could change the HTML structure (e.g., it would + // probably merge two separate text nodes). + // It's not necessary to add more than one placeholder in sequence; + // we can optimize them away. + item.original.push(TranslationItem_NodePlaceholder); + str += "<br>"; + wasLastItemPlaceholder = true; + } + } + + return generateTranslationHtmlForItem(item, str); + }, + + /** + * Changes the document to display its translated + * content. + */ + showTranslation() { + this.originalShown = false; + this._swapDocumentContent("translation"); + }, + + /** + * Changes the document to display its original + * content. + */ + showOriginal() { + this.originalShown = true; + this._swapDocumentContent("original"); + }, + + /** + * Swap the document with the resulting translation, + * or back with the original content. + * + * @param target A string that is either "translation" + * or "original". + */ + _swapDocumentContent(target) { + (async () => { + // Let the event loop breath on every 100 nodes + // that are replaced. + const YIELD_INTERVAL = 100; + await Async.yieldingForEach( + this.roots, + root => root.swapText(target), + YIELD_INTERVAL + ); + })(); + }, +}; + +/** + * This class represents an item for translation. It's basically our + * wrapper class around a node returned by getTranslationNode, with + * more data and structural information on it. + * + * At the end of the translation process, besides the properties below, + * a TranslationItem will contain two other properties: one called "original" + * and one called "translation". They are twin objects, one which reflect + * the structure of that node in its original state, and the other in its + * translated state. + * + * The "original" array is generated in the generateTextForItem function, + * and the "translation" array is generated when the translation results + * are parsed. + * + * They are both arrays, which contain a mix of strings and references to + * child TranslationItems. The references in both arrays point to the * same * + * TranslationItem object, but they might appear in different orders between the + * "original" and "translation" arrays. + * + * An example: + * + * English: <div id="n1">Welcome to <b id="n2">Mozilla's</b> website</div> + * Portuguese: <div id="n1">Bem vindo a pagina <b id="n2">da Mozilla</b></div> + * + * TranslationItem n1 = { + * id: 1, + * original: ["Welcome to", ptr to n2, "website"] + * translation: ["Bem vindo a pagina", ptr to n2] + * } + * + * TranslationItem n2 = { + * id: 2, + * original: ["Mozilla's"], + * translation: ["da Mozilla"] + * } + */ +function TranslationItem(node, id, isRoot) { + this.nodeRef = node; + this.id = id; + this.isRoot = isRoot; + this.children = []; +} + +TranslationItem.prototype = { + isRoot: false, + isSimpleRoot: false, + + toString() { + let rootType = ""; + if (this.isRoot) { + if (this.isSimpleRoot) { + rootType = " (simple root)"; + } else { + rootType = " (non simple root)"; + } + } + return ( + "[object TranslationItem: <" + + this.nodeRef.localName + + ">" + + rootType + + "]" + ); + }, + + /** + * This function will parse the result of the translation of one translation + * item. If this item was a simple root, all we sent was a plain-text version + * of it, so the result is also straightforward text. + * + * For non-simple roots, we sent a simplified HTML representation of that + * node, and we'll first parse that into an HTML doc and then call the + * parseResultNode helper function to parse it. + * + * While parsing, the result is stored in the "translation" field of the + * TranslationItem, which will be used to display the final translation when + * all items are finished. It remains stored too to allow back-and-forth + * switching between the "Show Original" and "Show Translation" functions. + * + * @param result A string with the textual result received from the server, + * which can be plain-text or a serialized HTML doc. + */ + parseResult(result) { + if (this.isSimpleRoot) { + this.translation = [result]; + return; + } + + let domParser = new DOMParser(); + + let doc = domParser.parseFromString(result, "text/html"); + parseResultNode(this, doc.body.firstChild); + }, + + /** + * This function finds a child TranslationItem + * with the given id. + * @param id The id to look for, in the format "n#" + * @returns A TranslationItem with the given id, or null if + * it was not found. + */ + getChildById(id) { + for (let child of this.children) { + if ("n" + child.id == id) { + return child; + } + } + return null; + }, + + /** + * Swap the text of this TranslationItem between + * its original and translated states. + * + * @param target A string that is either "translation" + * or "original". + */ + swapText(target) { + swapTextForItem(this, target); + }, +}; + +/** + * This object represents a placeholder item for translation. It's similar to + * the TranslationItem class, but it represents nodes that have no meaningful + * content for translation. These nodes will be replaced by "<br>" in a + * translation request. It's necessary to keep them to use it as a mark + * for correct positioning and spliting of text nodes. + */ +const TranslationItem_NodePlaceholder = { + toString() { + return "[object TranslationItem_NodePlaceholder]"; + }, +}; + +/** + * Generate the outer HTML representation for a given item. + * + * @param item A TranslationItem object. + * param content The inner content for this item. + * @returns string The outer HTML needed for translation + * of this item. + */ +function generateTranslationHtmlForItem(item, content) { + let localName = item.isRoot ? "div" : "b"; + return ( + "<" + localName + " id=n" + item.id + ">" + content + "</" + localName + ">" + ); +} + +/** + * Regenerate the text string that represents a TranslationItem object, + * with data from its "original" array. The array must have already + * been created by TranslationDocument.generateTextForItem(). + * + * @param item A TranslationItem object + * + * @returns A string representation of the TranslationItem. + */ +function regenerateTextFromOriginalHelper(item) { + if (item.isSimpleRoot) { + return item.original[0]; + } + + let str = ""; + for (let child of item.original) { + if (child instanceof TranslationItem) { + str += regenerateTextFromOriginalHelper(child); + } else if (child === TranslationItem_NodePlaceholder) { + str += "<br>"; + } else { + str += child; + } + } + + return generateTranslationHtmlForItem(item, str); +} + +/** + * Helper function to parse a HTML doc result. + * How it works: + * + * An example result string is: + * + * <div id="n1">Hello <b id="n2">World</b> of Mozilla.</div> + * + * For an element node, we look at its id and find the corresponding + * TranslationItem that was associated with this node, and then we + * walk down it repeating the process. + * + * For text nodes we simply add it as a string. + */ +function parseResultNode(item, node) { + item.translation = []; + for (let child of node.childNodes) { + if (child.nodeType == child.TEXT_NODE) { + item.translation.push(child.nodeValue); + } else if (child.localName == "br") { + item.translation.push(TranslationItem_NodePlaceholder); + } else { + let translationItemChild = item.getChildById(child.id); + + if (translationItemChild) { + item.translation.push(translationItemChild); + parseResultNode(translationItemChild, child); + } + } + } +} + +/** + * Helper function to swap the text of a TranslationItem + * between its original and translated states. + * How it works: + * + * The function iterates through the target array (either the `original` or + * `translation` array from the TranslationItem), while also keeping a pointer + * to a current position in the child nodes from the actual DOM node that we + * are modifying. This pointer is moved forward after each item of the array + * is translated. If, at any given time, the pointer doesn't match the expected + * node that was supposed to be seen, it means that the original and translated + * contents have a different ordering, and thus we need to adjust that. + * + * A full example of the reordering process, swapping from Original to + * Translation: + * + * Original (en): <div>I <em>miss</em> <b>you</b></div> + * + * Translation (fr): <div><b>Tu</b> me <em>manques</em></div> + * + * Step 1: + * pointer points to firstChild of the DOM node, textnode "I " + * first item in item.translation is [object TranslationItem <b>] + * + * pointer does not match the expected element, <b>. So let's move <b> to the + * pointer position. + * + * Current state of the DOM: + * <div><b>you</b>I <em>miss</em> </div> + * + * Step 2: + * pointer moves forward to nextSibling, textnode "I " again. + * second item in item.translation is the string " me " + * + * pointer points to a text node, and we were expecting a text node. Match! + * just replace the text content. + * + * Current state of the DOM: + * <div><b>you</b> me <em>miss</em> </div> + * + * Step 3: + * pointer moves forward to nextSibling, <em>miss</em> + * third item in item.translation is [object TranslationItem <em>] + * + * pointer points to the expected node. Match! Nothing to do. + * + * Step 4: + * all items in this item.translation were transformed. The remaining + * text nodes are cleared to "", and domNode.normalize() removes them. + * + * Current state of the DOM: + * <div><b>you</b> me <em>miss</em></div> + * + * Further steps: + * After that, the function will visit the child items (from the visitStack), + * and the text inside the <b> and <em> nodes will be swapped as well, + * yielding the final result: + * + * <div><b>Tu</b> me <em>manques</em></div> + * + * + * @param item A TranslationItem object + * @param target A string that is either "translation" + * or "original". + */ +function swapTextForItem(item, target) { + // visitStack is the stack of items that we still need to visit. + // Let's start the process by adding the root item. + let visitStack = [item]; + + while (visitStack.length) { + let curItem = visitStack.shift(); + + let domNode = curItem.nodeRef; + if (!domNode) { + // Skipping this item due to a missing node. + continue; + } + + if (!curItem[target]) { + // Translation not found for this item. This could be due to + // an error in the server response. For example, if a translation + // was broken in various chunks, and one of the chunks failed, + // the items from that chunk will be missing its "translation" + // field. + continue; + } + + domNode.normalize(); + + // curNode points to the child nodes of the DOM node that we are + // modifying. During most of the process, while the target array is + // being iterated (in the for loop below), it should walk together with + // the array and be pointing to the correct node that needs to modified. + // If it's not pointing to it, that means some sort of node reordering + // will be necessary to produce the correct translation. + // Note that text nodes don't need to be reordered, as we can just replace + // the content of one text node with another. + // + // curNode starts in the firstChild... + let curNode = domNode.firstChild; + + // ... actually, let's make curNode start at the first useful node (either + // a non-blank text node or something else). This is not strictly necessary, + // as the reordering algorithm would correctly handle this case. However, + // this better aligns the resulting translation with the DOM content of the + // page, avoiding cases that would need to be unecessarily reordered. + // + // An example of how this helps: + // + // ---- Original: <div> <b>Hello </b> world.</div> + // ^textnode 1 ^item 1 ^textnode 2 + // + // - Translation: <div><b>Hallo </b> Welt.</div> + // + // Transformation process without this optimization: + // 1 - start pointer at textnode 1 + // 2 - move item 1 to first position inside the <div> + // + // Node now looks like: <div><b>Hello </b>[ ][ world.]</div> + // textnode 1^ ^textnode 2 + // + // 3 - replace textnode 1 with " Welt." + // 4 - clear remaining text nodes (in this case, textnode 2) + // + // Transformation process with this optimization: + // 1 - start pointer at item 1 + // 2 - item 1 is already in position + // 3 - replace textnode 2 with " Welt." + // + // which completely avoids any node reordering, and requires only one + // text change instead of two (while also leaving the page closer to + // its original state). + while ( + curNode && + curNode.nodeType == curNode.TEXT_NODE && + curNode.nodeValue.trim() == "" + ) { + curNode = curNode.nextSibling; + } + + // Now let's walk through all items in the `target` array of the + // TranslationItem. This means either the TranslationItem.original or + // TranslationItem.translation array. + for (let targetItem of curItem[target]) { + if (targetItem instanceof TranslationItem) { + // If the array element is another TranslationItem object, let's + // add it to the stack to be visited. + visitStack.push(targetItem); + + let targetNode = targetItem.nodeRef; + + // If the node is not in the expected position, let's reorder + // it into position... + if ( + curNode != targetNode && + // ...unless the page has reparented this node under a totally + // different node (or removed it). In this case, all bets are off + // on being able to do anything correctly, so it's better not to + // bring back the node to this parent. + targetNode.parentNode == domNode + ) { + // We don't need to null-check curNode because insertBefore(..., null) + // does what we need in that case: reorder this node to the end + // of child nodes. + domNode.insertBefore(targetNode, curNode); + curNode = targetNode; + } + + // Move pointer forward. Since we do not add empty text nodes to the + // list of translation items, we must skip them here too while + // traversing the DOM in order to get better alignment between the + // text nodes and the translation items. + if (curNode) { + curNode = getNextSiblingSkippingEmptyTextNodes(curNode); + } + } else if (targetItem === TranslationItem_NodePlaceholder) { + // If the current item is a placeholder node, we need to move + // our pointer "past" it, jumping from one side of a block of + // elements + empty text nodes to the other side. Even if + // non-placeholder elements exists inside the jumped block, + // they will be pulled correctly later in the process when the + // targetItem for those nodes are handled. + + while ( + curNode && + (curNode.nodeType != curNode.TEXT_NODE || + curNode.nodeValue.trim() == "") + ) { + curNode = curNode.nextSibling; + } + } else { + // Finally, if it's a text item, we just need to find the next + // text node to use. Text nodes don't need to be reordered, so + // the first one found can be used. + while (curNode && curNode.nodeType != curNode.TEXT_NODE) { + curNode = curNode.nextSibling; + } + + // If none was found and we reached the end of the child nodes, + // let's create a new one. + if (!curNode) { + // We don't know if the original content had a space or not, + // so the best bet is to create the text node with " " which + // will add one space at the beginning and one at the end. + curNode = domNode.appendChild( + domNode.ownerDocument.createTextNode(" ") + ); + } + + // A trailing and a leading space must be preserved because + // they are meaningful in HTML. + let preSpace = /^\s/.test(curNode.nodeValue) ? " " : ""; + let endSpace = /\s$/.test(curNode.nodeValue) ? " " : ""; + + curNode.nodeValue = preSpace + targetItem + endSpace; + curNode = getNextSiblingSkippingEmptyTextNodes(curNode); + } + } + + // The translated version of a node might have less text nodes than its + // original version. If that's the case, let's clear the remaining nodes. + if (curNode) { + clearRemainingNonEmptyTextNodesFromElement(curNode); + } + + // And remove any garbage "" nodes left after clearing. + domNode.normalize(); + } +} + +function getNextSiblingSkippingEmptyTextNodes(startSibling) { + let item = startSibling.nextSibling; + while ( + item && + item.nodeType == item.TEXT_NODE && + item.nodeValue.trim() == "" + ) { + item = item.nextSibling; + } + return item; +} + +function clearRemainingNonEmptyTextNodesFromElement(startSibling) { + let item = startSibling; + while (item) { + if (item.nodeType == item.TEXT_NODE && item.nodeValue != "") { + item.nodeValue = ""; + } + item = item.nextSibling; + } +} diff --git a/browser/components/translation/TranslationParent.jsm b/browser/components/translation/TranslationParent.jsm new file mode 100644 index 0000000000..8baf3b2779 --- /dev/null +++ b/browser/components/translation/TranslationParent.jsm @@ -0,0 +1,455 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = [ + "Translation", + "TranslationParent", + "TranslationTelemetry", +]; + +const TRANSLATION_PREF_SHOWUI = "browser.translation.ui.show"; +const TRANSLATION_PREF_DETECT_LANG = "browser.translation.detectLanguage"; + +var Translation = { + STATE_OFFER: 0, + STATE_TRANSLATING: 1, + STATE_TRANSLATED: 2, + STATE_ERROR: 3, + STATE_UNAVAILABLE: 4, + + translationListener: null, + + serviceUnavailable: false, + + supportedSourceLanguages: [ + "bg", + "cs", + "de", + "en", + "es", + "fr", + "ja", + "ko", + "nl", + "no", + "pl", + "pt", + "ru", + "tr", + "vi", + "zh", + ], + supportedTargetLanguages: [ + "bg", + "cs", + "de", + "en", + "es", + "fr", + "ja", + "ko", + "nl", + "no", + "pl", + "pt", + "ru", + "tr", + "vi", + "zh", + ], + + setListenerForTests(listener) { + this.translationListener = listener; + }, + + _defaultTargetLanguage: "", + get defaultTargetLanguage() { + if (!this._defaultTargetLanguage) { + this._defaultTargetLanguage = + Services.locale.appLocaleAsBCP47.split("-")[0]; + } + return this._defaultTargetLanguage; + }, + + openProviderAttribution() { + let attribution = this.supportedEngines[this.translationEngine]; + const { BrowserWindowTracker } = ChromeUtils.import( + "resource:///modules/BrowserWindowTracker.jsm" + ); + BrowserWindowTracker.getTopWindow().openWebLinkIn(attribution, "tab"); + }, + + /** + * The list of translation engines and their attributions. + */ + supportedEngines: { + Google: "", + Bing: "http://aka.ms/MicrosoftTranslatorAttribution", + Yandex: "http://translate.yandex.com/", + }, + + /** + * Fallback engine (currently Google) if the preferences seem confusing. + */ + get defaultEngine() { + return Object.keys(this.supportedEngines)[0]; + }, + + /** + * Returns the name of the preferred translation engine. + */ + get translationEngine() { + let engine = Services.prefs.getCharPref("browser.translation.engine"); + return !Object.keys(this.supportedEngines).includes(engine) + ? this.defaultEngine + : engine; + }, +}; + +/* Translation objects keep the information related to translation for + * a specific browser. The properties exposed to the infobar are: + * - detectedLanguage, code of the language detected on the web page. + * - state, the state in which the infobar should be displayed + * - translatedFrom, if already translated, source language code. + * - translatedTo, if already translated, target language code. + * - translate, method starting the translation of the current page. + * - showOriginalContent, method showing the original page content. + * - showTranslatedContent, method showing the translation for an + * already translated page whose original content is shown. + * - originalShown, boolean indicating if the original or translated + * version of the page is shown. + */ +class TranslationParent extends JSWindowActorParent { + actorCreated() { + this._state = 0; + this.originalShown = true; + } + + get browser() { + return this.browsingContext.top.embedderElement; + } + + receiveMessage(aMessage) { + switch (aMessage.name) { + case "Translation:DocumentState": + this.documentStateReceived(aMessage.data); + break; + } + } + + documentStateReceived(aData) { + if (aData.state == Translation.STATE_OFFER) { + if (aData.detectedLanguage == Translation.defaultTargetLanguage) { + // Detected language is the same as the user's locale. + return; + } + + if ( + !Translation.supportedTargetLanguages.includes(aData.detectedLanguage) + ) { + // Detected language is not part of the supported languages. + TranslationTelemetry.recordMissedTranslationOpportunity( + aData.detectedLanguage + ); + return; + } + + TranslationTelemetry.recordTranslationOpportunity(aData.detectedLanguage); + } + + if (!Services.prefs.getBoolPref(TRANSLATION_PREF_SHOWUI)) { + return; + } + + // Set all values before showing a new translation infobar. + this._state = Translation.serviceUnavailable + ? Translation.STATE_UNAVAILABLE + : aData.state; + this.detectedLanguage = aData.detectedLanguage; + this.translatedFrom = aData.translatedFrom; + this.translatedTo = aData.translatedTo; + this.originalShown = aData.originalShown; + + this.showURLBarIcon(); + } + + translate(aFrom, aTo) { + if ( + aFrom == aTo || + (this.state == Translation.STATE_TRANSLATED && + this.translatedFrom == aFrom && + this.translatedTo == aTo) + ) { + // Nothing to do. + return; + } + + if (this.state == Translation.STATE_OFFER) { + if (this.detectedLanguage != aFrom) { + TranslationTelemetry.recordDetectedLanguageChange(true); + } + } else { + if (this.translatedFrom != aFrom) { + TranslationTelemetry.recordDetectedLanguageChange(false); + } + if (this.translatedTo != aTo) { + TranslationTelemetry.recordTargetLanguageChange(); + } + } + + this.state = Translation.STATE_TRANSLATING; + this.translatedFrom = aFrom; + this.translatedTo = aTo; + + this.sendQuery("Translation:TranslateDocument", { + from: aFrom, + to: aTo, + }).then( + result => { + this.translationFinished(result); + }, + () => {} + ); + } + + showURLBarIcon() { + let chromeWin = this.browser.ownerGlobal; + let PopupNotifications = chromeWin.PopupNotifications; + let removeId = this.originalShown ? "translated" : "translate"; + let notification = PopupNotifications.getNotification( + removeId, + this.browser + ); + if (notification) { + PopupNotifications.remove(notification); + } + + let callback = (aTopic, aNewBrowser) => { + if (aTopic == "swapping") { + return true; + } + + if (aTopic != "showing") { + return false; + } + let translationNotification = + this.notificationBox.getNotificationWithValue("translation"); + if (translationNotification) { + translationNotification.close(); + } + return true; + }; + + let addId = this.originalShown ? "translate" : "translated"; + PopupNotifications.show( + this.browser, + addId, + null, + addId + "-notification-icon", + null, + null, + { dismissed: true, eventCallback: callback } + ); + } + + get state() { + return this._state; + } + + set state(val) { + let notif = this.notificationBox.getNotificationWithValue("translation"); + if (notif) { + notif.state = val; + } + this._state = val; + } + + showOriginalContent() { + this.originalShown = true; + this.showURLBarIcon(); + this.sendAsyncMessage("Translation:ShowOriginal"); + TranslationTelemetry.recordShowOriginalContent(); + } + + showTranslatedContent() { + this.originalShown = false; + this.showURLBarIcon(); + this.sendAsyncMessage("Translation:ShowTranslation"); + } + + get notificationBox() { + return this.browser.ownerGlobal.gBrowser.getNotificationBox(this.browser); + } + + translationFinished(result) { + if (result.success) { + this.originalShown = false; + this.state = Translation.STATE_TRANSLATED; + this.showURLBarIcon(); + + // Record the number of characters translated. + TranslationTelemetry.recordTranslation( + result.from, + result.to, + result.characterCount + ); + } else if (result.unavailable) { + Translation.serviceUnavailable = true; + this.state = Translation.STATE_UNAVAILABLE; + } else { + this.state = Translation.STATE_ERROR; + } + + if (Translation.translationListener) { + Translation.translationListener(); + } + } + + infobarClosed() { + if (this.state == Translation.STATE_OFFER) { + TranslationTelemetry.recordDeniedTranslationOffer(); + } + } +} + +/** + * Uses telemetry histograms for collecting statistics on the usage of the + * translation component. + * + * NOTE: Metrics are only recorded if the user enabled the telemetry option. + */ +var TranslationTelemetry = { + init() { + // Constructing histograms. + const plain = id => Services.telemetry.getHistogramById(id); + const keyed = id => Services.telemetry.getKeyedHistogramById(id); + this.HISTOGRAMS = { + OPPORTUNITIES: () => plain("TRANSLATION_OPPORTUNITIES"), + OPPORTUNITIES_BY_LANG: () => + keyed("TRANSLATION_OPPORTUNITIES_BY_LANGUAGE"), + PAGES: () => plain("TRANSLATED_PAGES"), + PAGES_BY_LANG: () => keyed("TRANSLATED_PAGES_BY_LANGUAGE"), + CHARACTERS: () => plain("TRANSLATED_CHARACTERS"), + DENIED: () => plain("DENIED_TRANSLATION_OFFERS"), + AUTO_REJECTED: () => plain("AUTO_REJECTED_TRANSLATION_OFFERS"), + SHOW_ORIGINAL: () => plain("REQUESTS_OF_ORIGINAL_CONTENT"), + TARGET_CHANGES: () => plain("CHANGES_OF_TARGET_LANGUAGE"), + DETECTION_CHANGES: () => plain("CHANGES_OF_DETECTED_LANGUAGE"), + SHOW_UI: () => plain("SHOULD_TRANSLATION_UI_APPEAR"), + DETECT_LANG: () => plain("SHOULD_AUTO_DETECT_LANGUAGE"), + }; + + // Capturing the values of flags at the startup. + this.recordPreferences(); + }, + + /** + * Record a translation opportunity in the health report. + * @param language + * The language of the page. + */ + recordTranslationOpportunity(language) { + return this._recordOpportunity(language, true); + }, + + /** + * Record a missed translation opportunity in the health report. + * A missed opportunity is when the language detected is not part + * of the supported languages. + * @param language + * The language of the page. + */ + recordMissedTranslationOpportunity(language) { + return this._recordOpportunity(language, false); + }, + + /** + * Record an automatically rejected translation offer in the health + * report. A translation offer is automatically rejected when a user + * has previously clicked "Never translate this language" or "Never + * translate this site", which results in the infobar not being shown for + * the translation opportunity. + * + * These translation opportunities should still be recorded in addition to + * recording the automatic rejection of the offer. + */ + recordAutoRejectedTranslationOffer() { + this.HISTOGRAMS.AUTO_REJECTED().add(); + }, + + /** + * Record a translation in the health report. + * @param langFrom + * The language of the page. + * @param langTo + * The language translated to + * @param numCharacters + * The number of characters that were translated + */ + recordTranslation(langFrom, langTo, numCharacters) { + this.HISTOGRAMS.PAGES().add(); + this.HISTOGRAMS.PAGES_BY_LANG().add(langFrom + " -> " + langTo); + this.HISTOGRAMS.CHARACTERS().add(numCharacters); + }, + + /** + * Record a change of the detected language in the health report. This should + * only be called when actually executing a translation, not every time the + * user changes in the language in the UI. + * + * @param beforeFirstTranslation + * A boolean indicating if we are recording a change of detected + * language before translating the page for the first time. If we + * have already translated the page from the detected language and + * the user has manually adjusted the detected language false should + * be passed. + */ + recordDetectedLanguageChange(beforeFirstTranslation) { + this.HISTOGRAMS.DETECTION_CHANGES().add(beforeFirstTranslation); + }, + + /** + * Record a change of the target language in the health report. This should + * only be called when actually executing a translation, not every time the + * user changes in the language in the UI. + */ + recordTargetLanguageChange() { + this.HISTOGRAMS.TARGET_CHANGES().add(); + }, + + /** + * Record a denied translation offer. + */ + recordDeniedTranslationOffer() { + this.HISTOGRAMS.DENIED().add(); + }, + + /** + * Record a "Show Original" command use. + */ + recordShowOriginalContent() { + this.HISTOGRAMS.SHOW_ORIGINAL().add(); + }, + + /** + * Record the state of translation preferences. + */ + recordPreferences() { + if (Services.prefs.getBoolPref(TRANSLATION_PREF_SHOWUI)) { + this.HISTOGRAMS.SHOW_UI().add(1); + } + if (Services.prefs.getBoolPref(TRANSLATION_PREF_DETECT_LANG)) { + this.HISTOGRAMS.DETECT_LANG().add(1); + } + }, + + _recordOpportunity(language, success) { + this.HISTOGRAMS.OPPORTUNITIES().add(success); + this.HISTOGRAMS.OPPORTUNITIES_BY_LANG().add(language, success); + }, +}; + +TranslationTelemetry.init(); diff --git a/browser/components/translation/YandexTranslator.jsm b/browser/components/translation/YandexTranslator.jsm new file mode 100644 index 0000000000..c44283733d --- /dev/null +++ b/browser/components/translation/YandexTranslator.jsm @@ -0,0 +1,357 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["YandexTranslator"]; + +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); +const { Async } = ChromeUtils.importESModule( + "resource://services-common/async.sys.mjs" +); +const { httpRequest } = ChromeUtils.importESModule( + "resource://gre/modules/Http.sys.mjs" +); + +// The maximum amount of net data allowed per request on Bing's API. +const MAX_REQUEST_DATA = 5000; // Documentation says 10000 but anywhere +// close to that is refused by the service. + +// The maximum number of chunks allowed to be translated in a single +// request. +const MAX_REQUEST_CHUNKS = 1000; // Documentation says 2000. + +// Self-imposed limit of 15 requests. This means that a page that would need +// to be broken in more than 15 requests won't be fully translated. +// The maximum amount of data that we will translate for a single page +// is MAX_REQUESTS * MAX_REQUEST_DATA. +const MAX_REQUESTS = 15; + +const YANDEX_ERR_KEY_INVALID = 401; // Invalid API key +const YANDEX_ERR_KEY_BLOCKED = 402; // This API key has been blocked +const YANDEX_ERR_DAILY_REQ_LIMIT_EXCEEDED = 403; // Daily limit for requests reached +const YANDEX_ERR_DAILY_CHAR_LIMIT_EXCEEDED = 404; // Daily limit of chars reached +// const YANDEX_ERR_TEXT_TOO_LONG = 413; // The text size exceeds the maximum +// const YANDEX_ERR_UNPROCESSABLE_TEXT = 422; // The text could not be translated +// const YANDEX_ERR_LANG_NOT_SUPPORTED = 501; // The specified translation direction is not supported + +// Errors that should activate the service unavailable handling +const YANDEX_PERMANENT_ERRORS = [ + YANDEX_ERR_KEY_INVALID, + YANDEX_ERR_KEY_BLOCKED, + YANDEX_ERR_DAILY_REQ_LIMIT_EXCEEDED, + YANDEX_ERR_DAILY_CHAR_LIMIT_EXCEEDED, +]; + +/** + * Translates a webpage using Yandex's Translation API. + * + * @param translationDocument The TranslationDocument object that represents + * the webpage to be translated + * @param sourceLanguage The source language of the document + * @param targetLanguage The target language for the translation + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ +var YandexTranslator = function ( + translationDocument, + sourceLanguage, + targetLanguage +) { + this.translationDocument = translationDocument; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this._pendingRequests = 0; + this._partialSuccess = false; + this._serviceUnavailable = false; + this._translatedCharacterCount = 0; +}; + +YandexTranslator.prototype = { + /** + * Performs the translation, splitting the document into several chunks + * respecting the data limits of the API. + * + * @returns {Promise} A promise that will resolve when the translation + * task is finished. + */ + translate() { + return (async () => { + let currentIndex = 0; + this._onFinishedDeferred = PromiseUtils.defer(); + + // Let's split the document into various requests to be sent to + // Yandex's Translation API. + for (let requestCount = 0; requestCount < MAX_REQUESTS; requestCount++) { + // Generating the text for each request can be expensive, so + // let's take the opportunity of the chunkification process to + // allow for the event loop to attend other pending events + // before we continue. + await Async.promiseYield(); + + // Determine the data for the next request. + let request = this._generateNextTranslationRequest(currentIndex); + + // Create a real request to the server, and put it on the + // pending requests list. + let yandexRequest = new YandexRequest( + request.data, + this.sourceLanguage, + this.targetLanguage + ); + this._pendingRequests++; + yandexRequest + .fireRequest() + .then(this._chunkCompleted.bind(this), this._chunkFailed.bind(this)); + + currentIndex = request.lastIndex; + if (request.finished) { + break; + } + } + + return this._onFinishedDeferred.promise; + })(); + }, + + /** + * Function called when a request sent to the server completed successfully. + * This function handles calling the function to parse the result and the + * function to resolve the promise returned by the public `translate()` + * method when there are no pending requests left. + * + * @param request The YandexRequest sent to the server + */ + _chunkCompleted(yandexRequest) { + if (this._parseChunkResult(yandexRequest)) { + this._partialSuccess = true; + // Count the number of characters successfully translated. + this._translatedCharacterCount += yandexRequest.characterCount; + } + + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has failed. + * This function handles deciding if the error is transient or means the + * service is unavailable (zero balance on the key or request credentials are + * not in an active state) and calling the function to resolve the promise + * returned by the public `translate()` method when there are no pending + * requests left. + * + * @param aError [optional] The XHR object of the request that failed. + */ + _chunkFailed(aError) { + if (XMLHttpRequest.isInstance(aError)) { + let body = aError.responseText; + let json = { code: 0 }; + try { + json = JSON.parse(body); + } catch (e) {} + + if (json.code && YANDEX_PERMANENT_ERRORS.includes(json.code)) { + this._serviceUnavailable = true; + } + } + + this._checkIfFinished(); + }, + + /** + * Function called when a request sent to the server has completed. + * This function handles resolving the promise + * returned by the public `translate()` method when all chunks are completed. + */ + _checkIfFinished() { + // Check if all pending requests have been + // completed and then resolves the promise. + // If at least one chunk was successful, the + // promise will be resolved positively which will + // display the "Success" state for the infobar. Otherwise, + // the "Error" state will appear. + if (--this._pendingRequests == 0) { + if (this._partialSuccess) { + this._onFinishedDeferred.resolve({ + characterCount: this._translatedCharacterCount, + }); + } else { + let error = this._serviceUnavailable ? "unavailable" : "failure"; + this._onFinishedDeferred.reject(error); + } + } + }, + + /** + * This function parses the result returned by Yandex's Translation API, + * which returns a JSON result that contains a number of elements. The + * API is documented here: + * http://api.yandex.com/translate/doc/dg/reference/translate.xml + * + * @param request The request sent to the server. + * @returns boolean True if parsing of this chunk was successful. + */ + _parseChunkResult(yandexRequest) { + let results; + try { + let result = JSON.parse(yandexRequest.networkRequest.responseText); + if (result.code != 200) { + Services.console.logStringMessage( + "YandexTranslator: Result is " + result.code + ); + return false; + } + results = result.text; + } catch (e) { + return false; + } + + let len = results.length; + if (len != yandexRequest.translationData.length) { + // This should never happen, but if the service returns a different number + // of items (from the number of items submitted), we can't use this chunk + // because all items would be paired incorrectly. + return false; + } + + let error = false; + for (let i = 0; i < len; i++) { + try { + let result = results[i]; + let root = yandexRequest.translationData[i][0]; + root.parseResult(result); + } catch (e) { + error = true; + } + } + + return !error; + }, + + /** + * This function will determine what is the data to be used for + * the Nth request we are generating, based on the input params. + * + * @param startIndex What is the index, in the roots list, that the + * chunk should start. + */ + _generateNextTranslationRequest(startIndex) { + let currentDataSize = 0; + let currentChunks = 0; + let output = []; + let rootsList = this.translationDocument.roots; + + for (let i = startIndex; i < rootsList.length; i++) { + let root = rootsList[i]; + let text = this.translationDocument.generateTextForItem(root); + if (!text) { + continue; + } + + let newCurSize = currentDataSize + text.length; + let newChunks = currentChunks + 1; + + if (newCurSize > MAX_REQUEST_DATA || newChunks > MAX_REQUEST_CHUNKS) { + // If we've reached the API limits, let's stop accumulating data + // for this request and return. We return information useful for + // the caller to pass back on the next call, so that the function + // can keep working from where it stopped. + return { + data: output, + finished: false, + lastIndex: i, + }; + } + + currentDataSize = newCurSize; + currentChunks = newChunks; + output.push([root, text]); + } + + return { + data: output, + finished: true, + lastIndex: 0, + }; + }, +}; + +/** + * Represents a request (for 1 chunk) sent off to Yandex's service. + * + * @params translationData The data to be used for this translation, + * generated by the generateNextTranslationRequest... + * function. + * @param sourceLanguage The source language of the document. + * @param targetLanguage The target language for the translation. + * + */ +function YandexRequest(translationData, sourceLanguage, targetLanguage) { + this.translationData = translationData; + this.sourceLanguage = sourceLanguage; + this.targetLanguage = targetLanguage; + this.characterCount = 0; +} + +YandexRequest.prototype = { + /** + * Initiates the request + */ + fireRequest() { + return (async () => { + // Prepare URL. + let url = getUrlParam( + "https://translate.yandex.net/api/v1.5/tr.json/translate", + "browser.translation.yandex.translateURLOverride" + ); + + // Prepare the request body. + let apiKey = getUrlParam( + "%YANDEX_API_KEY%", + "browser.translation.yandex.apiKeyOverride" + ); + let params = [ + ["key", apiKey], + ["format", "html"], + ["lang", this.sourceLanguage + "-" + this.targetLanguage], + ]; + + for (let [, text] of this.translationData) { + params.push(["text", text]); + this.characterCount += text.length; + } + + // Set up request options. + return new Promise((resolve, reject) => { + let options = { + onLoad: (responseText, xhr) => { + resolve(this); + }, + onError(e, responseText, xhr) { + reject(xhr); + }, + postData: params, + }; + + // Fire the request. + this.networkRequest = httpRequest(url, options); + }); + })(); + }, +}; + +/** + * Fetch an auth token (clientID or client secret), which may be overridden by + * a pref if it's set. + */ +function getUrlParam(paramValue, prefName) { + if (Services.prefs.getPrefType(prefName)) { + paramValue = Services.prefs.getCharPref(prefName); + } + paramValue = Services.urlFormatter.formatURL(paramValue); + return paramValue; +} diff --git a/browser/components/translation/content/.eslintrc.js b/browser/components/translation/content/.eslintrc.js new file mode 100644 index 0000000000..43ab18578d --- /dev/null +++ b/browser/components/translation/content/.eslintrc.js @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + "mozilla/browser-window": true, + }, + + plugins: ["mozilla"], +}; diff --git a/browser/components/translation/content/jar.mn b/browser/components/translation/content/jar.mn new file mode 100644 index 0000000000..046645bcb7 --- /dev/null +++ b/browser/components/translation/content/jar.mn @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +browser.jar: + content/browser/microsoft-translator-attribution.png diff --git a/browser/components/translation/content/microsoft-translator-attribution.png b/browser/components/translation/content/microsoft-translator-attribution.png Binary files differnew file mode 100644 index 0000000000..bd96cf2036 --- /dev/null +++ b/browser/components/translation/content/microsoft-translator-attribution.png diff --git a/browser/components/translation/content/moz.build b/browser/components/translation/content/moz.build new file mode 100644 index 0000000000..d988c0ff9b --- /dev/null +++ b/browser/components/translation/content/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +JAR_MANIFESTS += ["jar.mn"] diff --git a/browser/components/translation/moz.build b/browser/components/translation/moz.build new file mode 100644 index 0000000000..02968f17a9 --- /dev/null +++ b/browser/components/translation/moz.build @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + "content", +] + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Translation") + +EXTRA_JS_MODULES.translation = [ + "BingTranslator.jsm", + "GoogleTranslator.jsm", + "TranslationChild.jsm", + "TranslationDocument.jsm", + "TranslationParent.jsm", + "YandexTranslator.jsm", +] + +BROWSER_CHROME_MANIFESTS += ["test/browser.ini"] diff --git a/browser/components/translation/test/bing.sjs b/browser/components/translation/test/bing.sjs new file mode 100644 index 0000000000..3769517e36 --- /dev/null +++ b/browser/components/translation/test/bing.sjs @@ -0,0 +1,250 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const CC = Components.Constructor; +const BinaryInputStream = CC( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); +Cu.importGlobalProperties(["DOMParser", "TextEncoder"]); + +function handleRequest(req, res) { + try { + reallyHandleRequest(req, res); + } catch (ex) { + res.setStatusLine("1.0", 200, "AlmostOK"); + let msg = "Error handling request: " + ex + "\n" + ex.stack; + log(msg); + res.write(msg); + } +} + +function log(msg) { + // dump("BING-SERVER-MOCK: " + msg + "\n"); +} + +const statusCodes = { + 400: "Bad Request", + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 500: "Internal Server Error", + 501: "Not Implemented", + 503: "Service Unavailable", +}; + +function HTTPError(code = 500, message) { + this.code = code; + this.name = statusCodes[code] || "HTTPError"; + this.message = message || this.name; +} +HTTPError.prototype = new Error(); +HTTPError.prototype.constructor = HTTPError; + +function sendError(res, err) { + if (!(err instanceof HTTPError)) { + err = new HTTPError( + typeof err == "number" ? err : 500, + err.message || typeof err == "string" ? err : "" + ); + } + res.setStatusLine("1.1", err.code, err.name); + res.write(err.message); +} + +function parseQuery(query) { + let ret = {}; + for (let param of query.replace(/^[?&]/, "").split("&")) { + param = param.split("="); + if (!param[0]) { + continue; + } + ret[unescape(param[0])] = unescape(param[1]); + } + return ret; +} + +function getRequestBody(req) { + let avail; + let bytes = []; + let body = new BinaryInputStream(req.bodyInputStream); + + while ((avail = body.available()) > 0) { + Array.prototype.push.apply(bytes, body.readByteArray(avail)); + } + + return String.fromCharCode.apply(null, bytes); +} + +function sha1(str) { + // `data` is an array of bytes. + let data = new TextEncoder().encode(str); + let ch = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + ch.init(ch.SHA1); + ch.update(data, data.length); + let hash = ch.finish(false); + + // Return the two-digit hexadecimal code for a byte. + function toHexString(charCode) { + return ("0" + charCode.toString(16)).slice(-2); + } + + // Convert the binary hash data to a hex string. + return Array.from(hash, (c, i) => toHexString(hash.charCodeAt(i))).join(""); +} + +function parseXml(body) { + let parser = new DOMParser(); + let xml = parser.parseFromString(body, "text/xml"); + if (xml.documentElement.localName == "parsererror") { + throw new Error("Invalid XML"); + } + return xml; +} + +function getInputStream(path) { + let file = Services.dirsvc.get("CurWorkD", Ci.nsIFile); + for (let part of path.split("/")) { + file.append(part); + } + let fileStream = Cc[ + "@mozilla.org/network/file-input-stream;1" + ].createInstance(Ci.nsIFileInputStream); + fileStream.init(file, 1, 0, false); + return fileStream; +} + +function checkAuth(req) { + let err = new Error("Authorization failed"); + err.code = 401; + + if (!req.hasHeader("Authorization")) { + throw new HTTPError(401, "No Authorization header provided."); + } + + let auth = req.getHeader("Authorization"); + if (!auth.startsWith("Bearer ")) { + throw new HTTPError( + 401, + "Invalid Authorization header content: '" + auth + "'" + ); + } + + // Rejecting inactive subscriptions. + if (auth.includes("inactive")) { + const INACTIVE_STATE_RESPONSE = + "<html><body><h1>TranslateApiException</h1><p>Method: TranslateArray()</p><p>Message: The Azure Market Place Translator Subscription associated with the request credentials is not in an active state.</p><code></code><p>message id=5641.V2_Rest.TranslateArray.48CC6470</p></body></html>"; + throw new HTTPError(401, INACTIVE_STATE_RESPONSE); + } +} + +function reallyHandleRequest(req, res) { + log("method: " + req.method); + if (req.method != "POST") { + sendError( + res, + "Bing only deals with POST requests, not '" + req.method + "'." + ); + return; + } + + let body = getRequestBody(req); + log("body: " + body); + + // First, we'll see if we're dealing with an XML body: + let contentType = req.hasHeader("Content-Type") + ? req.getHeader("Content-Type") + : null; + log("contentType: " + contentType); + + if (contentType.startsWith("text/xml")) { + try { + // For all these requests the client needs to supply the correct + // authentication headers. + checkAuth(req); + + let xml = parseXml(body); + let method = xml.documentElement.localName; + log("invoking method: " + method); + // If the requested method is supported, delegate it to its handler. + if (methodHandlers[method]) { + methodHandlers[method](res, xml); + } else { + throw new HTTPError(501); + } + } catch (ex) { + sendError(res, ex, ex.code); + } + } else { + // Not XML, so it must be a query-string. + let params = parseQuery(body); + + // Delegate an authentication request to the correct handler. + if ("grant_type" in params && params.grant_type == "client_credentials") { + methodHandlers.authenticate(res, params); + } else { + sendError(res, 501); + } + } +} + +const methodHandlers = { + authenticate(res, params) { + // Validate a few required parameters. + if (params.scope != "http://api.microsofttranslator.com") { + sendError(res, "Invalid scope."); + return; + } + if (!params.client_id) { + sendError(res, "Missing client_id param."); + return; + } + if (!params.client_secret) { + sendError(res, "Missing client_secret param."); + return; + } + + // Defines the tokens for certain client ids. + const TOKEN_MAP = { + testInactive: "inactive", + testClient: "test", + }; + let token = "test"; // Default token. + if (params.client_id in TOKEN_MAP) { + token = TOKEN_MAP[params.client_id]; + } + let content = JSON.stringify({ + access_token: token, + expires_in: 600, + }); + + res.setStatusLine("1.1", 200, "OK"); + res.setHeader("Content-Length", String(content.length)); + res.setHeader("Content-Type", "application/json"); + res.write(content); + }, + + TranslateArrayRequest(res, xml, body) { + let from = xml.querySelector("From").firstChild.nodeValue; + let to = xml.querySelector("To").firstChild.nodeValue; + log("translating from '" + from + "' to '" + to + "'"); + + res.setStatusLine("1.1", 200, "OK"); + res.setHeader("Content-Type", "text/xml"); + + let hash = sha1(body).substr(0, 10); + log("SHA1 hash of content: " + hash); + let inputStream = getInputStream( + "browser/browser/components/translation/test/fixtures/result-" + + hash + + ".txt" + ); + res.bodyOutputStream.writeFrom(inputStream, inputStream.available()); + inputStream.close(); + }, +}; diff --git a/browser/components/translation/test/browser.ini b/browser/components/translation/test/browser.ini new file mode 100644 index 0000000000..2c09e9992a --- /dev/null +++ b/browser/components/translation/test/browser.ini @@ -0,0 +1,13 @@ +[DEFAULT] +support-files = + bing.sjs + yandex.sjs + fixtures/bug1022725-fr.html + fixtures/result-da39a3ee5e.txt + fixtures/result-yandex-d448894848.json + +[browser_translation_bing.js] +[browser_translation_exceptions.js] +https_first_disabled = true +[browser_translation_yandex.js] +[browser_translations_settings.js] diff --git a/browser/components/translation/test/browser_translation_bing.js b/browser/components/translation/test/browser_translation_bing.js new file mode 100644 index 0000000000..8b4620f059 --- /dev/null +++ b/browser/components/translation/test/browser_translation_bing.js @@ -0,0 +1,160 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Test the Bing Translator client against a mock Bing service, bing.sjs. + +"use strict"; + +const kClientIdPref = "browser.translation.bing.clientIdOverride"; +const kClientSecretPref = "browser.translation.bing.apiKeyOverride"; + +const { BingTranslator } = ChromeUtils.import( + "resource:///modules/translation/BingTranslator.jsm" +); +const { TranslationDocument } = ChromeUtils.import( + "resource:///modules/translation/TranslationDocument.jsm" +); + +add_setup(async function () { + Services.prefs.setCharPref(kClientIdPref, "testClient"); + Services.prefs.setCharPref(kClientSecretPref, "testSecret"); + + registerCleanupFunction(function () { + Services.prefs.clearUserPref(kClientIdPref); + Services.prefs.clearUserPref(kClientSecretPref); + }); +}); + +/** + * Checks if the translation is happening. + */ +add_task(async function test_bing_translation() { + // Ensure the correct client id is used for authentication. + Services.prefs.setCharPref(kClientIdPref, "testClient"); + + // Loading the fixture page. + let url = constructFixtureURL("bug1022725-fr.html"); + let tab = await promiseTestPageLoad(url); + + // Translating the contents of the loaded tab. + gBrowser.selectedTab = tab; + let browser = tab.linkedBrowser; + + await SpecialPowers.spawn(browser, [], async function () { + // eslint-disable-next-line no-shadow + const { BingTranslator } = ChromeUtils.import( + "resource:///modules/translation/BingTranslator.jsm" + ); + // eslint-disable-next-line no-shadow + const { TranslationDocument } = ChromeUtils.import( + "resource:///modules/translation/TranslationDocument.jsm" + ); + + let client = new BingTranslator( + new TranslationDocument(content.document), + "fr", + "en" + ); + let result = await client.translate(); + + // XXXmikedeboer; here you would continue the test/ content inspection. + Assert.ok(result, "There should be a result"); + }); + + gBrowser.removeTab(tab); +}); + +/** + * Ensures that the BingTranslator handles out-of-valid-key response + * correctly. Sometimes Bing Translate replies with + * "request credentials is not in an active state" error. BingTranslator + * should catch this error and classify it as Service Unavailable. + * + */ +add_task(async function test_handling_out_of_valid_key_error() { + // Simulating request from inactive subscription. + Services.prefs.setCharPref(kClientIdPref, "testInactive"); + + // Loading the fixture page. + let url = constructFixtureURL("bug1022725-fr.html"); + let tab = await promiseTestPageLoad(url); + + // Translating the contents of the loaded tab. + gBrowser.selectedTab = tab; + let browser = tab.linkedBrowser; + + await SpecialPowers.spawn(browser, [], async function () { + // eslint-disable-next-line no-shadow + const { BingTranslator } = ChromeUtils.import( + "resource:///modules/translation/BingTranslator.jsm" + ); + // eslint-disable-next-line no-shadow + const { TranslationDocument } = ChromeUtils.import( + "resource:///modules/translation/TranslationDocument.jsm" + ); + + let client = new BingTranslator( + new TranslationDocument(content.document), + "fr", + "en" + ); + client._resetToken(); + try { + await client.translate(); + } catch (ex) { + // It is alright that the translation fails. + } + client._resetToken(); + + // Checking if the client detected service and unavailable. + Assert.ok( + client._serviceUnavailable, + "Service should be detected unavailable." + ); + }); + + // Cleaning up. + Services.prefs.setCharPref(kClientIdPref, "testClient"); + gBrowser.removeTab(tab); +}); + +/** + * A helper function for constructing a URL to a page stored in the + * local fixture folder. + * + * @param filename Name of a fixture file. + */ +function constructFixtureURL(filename) { + // Deduce the Mochitest server address in use from a pref that was pre-processed. + let server = Services.prefs + .getCharPref("browser.translation.bing.authURL") + .replace("http://", ""); + server = server.substr(0, server.indexOf("/")); + let url = + "http://" + + server + + "/browser/browser/components/translation/test/fixtures/" + + filename; + return url; +} + +/** + * A helper function to open a new tab and wait for its content to load. + * + * @param String url A URL to be loaded in the new tab. + */ +function promiseTestPageLoad(url) { + return new Promise(resolve => { + let tab = (gBrowser.selectedTab = BrowserTestUtils.addTab(gBrowser, url)); + let browser = gBrowser.selectedBrowser; + BrowserTestUtils.browserLoaded( + browser, + false, + loadurl => loadurl != "about:blank" + ).then(() => { + info("Page loaded: " + browser.currentURI.spec); + resolve(tab); + }); + }); +} diff --git a/browser/components/translation/test/browser_translation_exceptions.js b/browser/components/translation/test/browser_translation_exceptions.js new file mode 100644 index 0000000000..c0d325c6f7 --- /dev/null +++ b/browser/components/translation/test/browser_translation_exceptions.js @@ -0,0 +1,239 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// TODO (Bug 1817084) Remove this file when we disable the extension +// tests the translation infobar, using a fake 'Translation' implementation. + +const { PermissionTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PermissionTestUtils.sys.mjs" +); + +const kLanguagesPref = "browser.translation.neverForLanguages"; +const kShowUIPref = "browser.translation.ui.show"; +const kEnableTranslationPref = "browser.translation.detectLanguage"; + +function test() { + waitForExplicitFinish(); + + Services.prefs.setBoolPref(kShowUIPref, true); + Services.prefs.setBoolPref(kEnableTranslationPref, true); + let tab = BrowserTestUtils.addTab(gBrowser); + gBrowser.selectedTab = tab; + registerCleanupFunction(function () { + gBrowser.removeTab(tab); + Services.prefs.clearUserPref(kShowUIPref); + Services.prefs.clearUserPref(kEnableTranslationPref); + }); + BrowserTestUtils.browserLoaded(tab.linkedBrowser).then(() => { + (async function () { + for (let testCase of gTests) { + info(testCase.desc); + await testCase.run(); + } + })().then(finish, ex => { + ok(false, "Unexpected Exception: " + ex); + finish(); + }); + }); + + BrowserTestUtils.loadURIString( + gBrowser.selectedBrowser, + "http://example.com/" + ); +} + +function getLanguageExceptions() { + let langs = Services.prefs.getCharPref(kLanguagesPref); + return langs ? langs.split(",") : []; +} + +function getDomainExceptions() { + let results = []; + for (let perm of Services.perms.all) { + if ( + perm.type == "translate" && + perm.capability == Services.perms.DENY_ACTION + ) { + results.push(perm.principal); + } + } + + return results; +} + +function openPopup(aPopup) { + return new Promise(resolve => { + aPopup.addEventListener( + "popupshown", + function () { + TestUtils.executeSoon(resolve); + }, + { once: true } + ); + + aPopup.focus(); + // One down event to open the popup. + EventUtils.synthesizeKey("VK_DOWN", { + altKey: !navigator.platform.includes("Mac"), + }); + }); +} + +function waitForWindowLoad(aWin) { + return new Promise(resolve => { + aWin.addEventListener( + "load", + function () { + TestUtils.executeSoon(resolve); + }, + { capture: true, once: true } + ); + }); +} + +var gTests = [ + { + desc: "clean exception lists at startup", + run: function checkNeverForLanguage() { + is( + getLanguageExceptions().length, + 0, + "we start with an empty list of languages to never translate" + ); + is( + getDomainExceptions().length, + 0, + "we start with an empty list of sites to never translate" + ); + }, + }, + + { + desc: "language exception list", + run: async function checkLanguageExceptions() { + // Put 2 languages in the pref before opening the window to check + // the list is displayed on load. + Services.prefs.setCharPref(kLanguagesPref, "fr,de"); + + // Open the translation exceptions dialog. + let win = openDialog( + "chrome://browser/content/preferences/dialogs/translationExceptions.xhtml", + "Browser:TranslationExceptions", + "", + null + ); + await waitForWindowLoad(win); + + // Check that the list of language exceptions is loaded. + let getById = win.document.getElementById.bind(win.document); + let tree = getById("languagesTree"); + let remove = getById("removeLanguage"); + let removeAll = getById("removeAllLanguages"); + is(tree.view.rowCount, 2, "The language exceptions list has 2 items"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Select the first item. + tree.view.selection.select(0); + ok(!remove.disabled, "The 'Remove Language' button is enabled"); + + // Click the 'Remove' button. + remove.click(); + is(tree.view.rowCount, 1, "The language exceptions now contains 1 item"); + is(getLanguageExceptions().length, 1, "One exception in the pref"); + + // Clear the pref, and check the last item is removed from the display. + Services.prefs.setCharPref(kLanguagesPref, ""); + is(tree.view.rowCount, 0, "The language exceptions list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + + // Add an item and check it appears. + Services.prefs.setCharPref(kLanguagesPref, "fr"); + is(tree.view.rowCount, 1, "The language exceptions list has 1 item"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Click the 'Remove All' button. + removeAll.click(); + is(tree.view.rowCount, 0, "The language exceptions list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + is(Services.prefs.getCharPref(kLanguagesPref), "", "The pref is empty"); + + win.close(); + }, + }, + + { + desc: "domains exception list", + run: async function checkDomainExceptions() { + // Put 2 exceptions before opening the window to check the list is + // displayed on load. + PermissionTestUtils.add( + "http://example.org", + "translate", + Services.perms.DENY_ACTION + ); + PermissionTestUtils.add( + "http://example.com", + "translate", + Services.perms.DENY_ACTION + ); + + // Open the translation exceptions dialog. + let win = openDialog( + "chrome://browser/content/preferences/dialogs/translationExceptions.xhtml", + "Browser:TranslationExceptions", + "", + null + ); + await waitForWindowLoad(win); + + // Check that the list of language exceptions is loaded. + let getById = win.document.getElementById.bind(win.document); + let tree = getById("sitesTree"); + let remove = getById("removeSite"); + let removeAll = getById("removeAllSites"); + is(tree.view.rowCount, 2, "The sites exceptions list has 2 items"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Sites' button is enabled"); + + // Select the first item. + tree.view.selection.select(0); + ok(!remove.disabled, "The 'Remove Site' button is enabled"); + + // Click the 'Remove' button. + remove.click(); + is(tree.view.rowCount, 1, "The site exceptions now contains 1 item"); + is(getDomainExceptions().length, 1, "One exception in the permissions"); + + // Clear the permissions, and check the last item is removed from the display. + PermissionTestUtils.remove("http://example.org", "translate"); + PermissionTestUtils.remove("http://example.com", "translate"); + is(tree.view.rowCount, 0, "The site exceptions list is empty"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Site' button is disabled"); + + // Add an item and check it appears. + PermissionTestUtils.add( + "http://example.com", + "translate", + Services.perms.DENY_ACTION + ); + is(tree.view.rowCount, 1, "The site exceptions list has 1 item"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Sites' button is enabled"); + + // Click the 'Remove All' button. + removeAll.click(); + is(tree.view.rowCount, 0, "The site exceptions list is empty"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Sites' button is disabled"); + is(getDomainExceptions().length, 0, "No exceptions in the permissions"); + + win.close(); + }, + }, +]; diff --git a/browser/components/translation/test/browser_translation_yandex.js b/browser/components/translation/test/browser_translation_yandex.js new file mode 100644 index 0000000000..c6c97571de --- /dev/null +++ b/browser/components/translation/test/browser_translation_yandex.js @@ -0,0 +1,134 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Test the Yandex Translator client against a mock Yandex service, yandex.sjs. + +"use strict"; + +// The folllowing rejection is left unhandled in some cases. This bug should be +// fixed, but for the moment this file allows a class of rejections. +// +// NOTE: Allowing a whole class of rejections should be avoided. Normally you +// should use "expectUncaughtRejection" to flag individual failures. +const { PromiseTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PromiseTestUtils.sys.mjs" +); +PromiseTestUtils.allowMatchingRejectionsGlobally(/NS_ERROR_ILLEGAL_VALUE/); + +const kEnginePref = "browser.translation.engine"; +const kApiKeyPref = "browser.translation.yandex.apiKeyOverride"; +const kDetectLanguagePref = "browser.translation.detectLanguage"; +const kShowUIPref = "browser.translation.ui.show"; + +add_setup(async function () { + await SpecialPowers.pushPrefEnv({ + set: [ + [kEnginePref, "Yandex"], + [kApiKeyPref, "yandexValidKey"], + [kDetectLanguagePref, true], + [kShowUIPref, true], + ], + }); +}); + +/** + * Ensure that the translation engine behaives as expected when translating + * a sample page. + */ +add_task(async function test_yandex_translation() { + // Loading the fixture page. + let url = constructFixtureURL("bug1022725-fr.html"); + let tab = await promiseTestPageLoad(url); + + // Translating the contents of the loaded tab. + gBrowser.selectedTab = tab; + let browser = tab.linkedBrowser; + + await SpecialPowers.spawn(browser, [], async function () { + const { TranslationDocument } = ChromeUtils.import( + "resource:///modules/translation/TranslationDocument.jsm" + ); + const { YandexTranslator } = ChromeUtils.import( + "resource:///modules/translation/YandexTranslator.jsm" + ); + + let client = new YandexTranslator( + new TranslationDocument(content.document), + "fr", + "en" + ); + let result = await client.translate(); + + Assert.ok(result, "There should be a result."); + }); + + gBrowser.removeTab(tab); +}); + +add_task(async function test_preference_attribution() { + let prefUrl = "about:preferences#general"; + let waitPrefLoaded = TestUtils.topicObserved("sync-pane-loaded", () => true); + let tab = await promiseTestPageLoad(prefUrl); + await waitPrefLoaded; + let browser = gBrowser.getBrowserForTab(tab); + let win = browser.contentWindow; + let bingAttribution = win.document.getElementById("bingAttribution"); + ok(bingAttribution, "Bing attribution should exist."); + ok(bingAttribution.hidden, "Bing attribution should be hidden."); + + gBrowser.removeTab(tab); +}); + +/** + * A helper function for constructing a URL to a page stored in the + * local fixture folder. + * + * @param filename Name of a fixture file. + */ +function constructFixtureURL(filename) { + // Deduce the Mochitest server address in use from a pref that was pre-processed. + let server = Services.prefs + .getCharPref("browser.translation.yandex.translateURLOverride") + .replace("http://", ""); + server = server.substr(0, server.indexOf("/")); + let url = + "http://" + + server + + "/browser/browser/components/translation/test/fixtures/" + + filename; + return url; +} + +/** + * A helper function to open a new tab and wait for its content to load. + * + * @param String url A URL to be loaded in the new tab. + */ +function promiseTestPageLoad(url) { + return new Promise(resolve => { + let tab = (gBrowser.selectedTab = BrowserTestUtils.addTab(gBrowser, url)); + let browser = gBrowser.selectedBrowser; + BrowserTestUtils.browserLoaded( + browser, + false, + loadurl => loadurl != "about:blank" + ).then(() => { + info("Page loaded: " + browser.currentURI.spec); + resolve(tab); + }); + }); +} + +function showTranslationUI(tab, aDetectedLanguage) { + let browser = gBrowser.selectedBrowser; + let actor = + browser.browsingContext.currentWindowGlobal.getActor("Translation"); + actor.documentStateReceived({ + state: Translation.STATE_OFFER, + originalShown: true, + detectedLanguage: aDetectedLanguage, + }); + + return actor.notificationBox.getNotificationWithValue("translation"); +} diff --git a/browser/components/translation/test/browser_translations_settings.js b/browser/components/translation/test/browser_translations_settings.js new file mode 100644 index 0000000000..87860ad9bf --- /dev/null +++ b/browser/components/translation/test/browser_translations_settings.js @@ -0,0 +1,384 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// tests the translation infobar, using a fake 'Translation' implementation. + +const { PermissionTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PermissionTestUtils.sys.mjs" +); + +const TRANSLATIONS_PERMISSION = "translations"; +const ENABLE_TRANSLATIONS_PREF = "browser.translations.enable"; +const ALWAYS_TRANSLATE_LANGS_PREF = + "browser.translations.alwaysTranslateLanguages"; +const NEVER_TRANSLATE_LANGS_PREF = + "browser.translations.neverTranslateLanguages"; + +function test() { + waitForExplicitFinish(); + Services.prefs.setBoolPref(ENABLE_TRANSLATIONS_PREF, true); + Services.prefs.setCharPref(ALWAYS_TRANSLATE_LANGS_PREF, ""); + Services.prefs.setCharPref(NEVER_TRANSLATE_LANGS_PREF, ""); + let tab = BrowserTestUtils.addTab(gBrowser); + gBrowser.selectedTab = tab; + registerCleanupFunction(function () { + gBrowser.removeTab(tab); + Services.prefs.clearUserPref(ENABLE_TRANSLATIONS_PREF); + Services.prefs.clearUserPref(ALWAYS_TRANSLATE_LANGS_PREF); + Services.prefs.clearUserPref(NEVER_TRANSLATE_LANGS_PREF); + }); + BrowserTestUtils.browserLoaded(tab.linkedBrowser).then(() => { + (async function () { + for (let testCase of gTests) { + info(testCase.desc); + await testCase.run(); + } + })().then(finish, ex => { + ok(false, "Unexpected Exception: " + ex); + finish(); + }); + }); + + BrowserTestUtils.loadURIString( + gBrowser.selectedBrowser, + "https://example.com/" + ); +} + +/** + * Retrieves the always-translate language list as an array. + * @returns {Array<string>} + */ +function getAlwaysTranslateLanguages() { + let langs = Services.prefs.getCharPref(ALWAYS_TRANSLATE_LANGS_PREF); + return langs ? langs.split(",") : []; +} + +/** + * Retrieves the always-translate language list as an array. + * @returns {Array<string>} + */ +function getNeverTranslateLanguages() { + let langs = Services.prefs.getCharPref(NEVER_TRANSLATE_LANGS_PREF); + return langs ? langs.split(",") : []; +} + +/** + * Retrieves the always-translate site list as an array. + * @returns {Array<string>} + */ +function getNeverTranslateSites() { + let results = []; + for (let perm of Services.perms.all) { + if ( + perm.type == TRANSLATIONS_PERMISSION && + perm.capability == Services.perms.DENY_ACTION + ) { + results.push(perm.principal); + } + } + + return results; +} + +function openPopup(aPopup) { + return new Promise(resolve => { + aPopup.addEventListener( + "popupshown", + function () { + TestUtils.executeSoon(resolve); + }, + { once: true } + ); + + aPopup.focus(); + // One down event to open the popup. + EventUtils.synthesizeKey("VK_DOWN"); + }); +} + +function waitForWindowLoad(aWin) { + return new Promise(resolve => { + aWin.addEventListener( + "load", + function () { + TestUtils.executeSoon(resolve); + }, + { capture: true, once: true } + ); + }); +} + +var gTests = [ + { + desc: "ensure lists are empty on startup", + run: function checkPreferencesAreEmpty() { + is( + getAlwaysTranslateLanguages().length, + 0, + "we start with an empty list of languages to always translate" + ); + is( + getNeverTranslateLanguages().length, + 0, + "we start with an empty list of languages to never translate" + ); + is( + getNeverTranslateSites().length, + 0, + "we start with an empty list of sites to never translate" + ); + }, + }, + + { + desc: "ensure always-translate languages function correctly", + run: async function testAlwaysTranslateLanguages() { + // Put 2 languages in the pref before opening the window to check + // the list is displayed on load. + Services.prefs.setCharPref(ALWAYS_TRANSLATE_LANGS_PREF, "fr,de"); + + // Open the translations settings dialog. + let win = openDialog( + "chrome://browser/content/preferences/dialogs/translations.xhtml", + "Browser:TranslationsPreferences", + "", + null + ); + await waitForWindowLoad(win); + + // Check that the list of always-translate languages is loaded. + let getById = win.document.getElementById.bind(win.document); + let tree = getById("alwaysTranslateLanguagesTree"); + let remove = getById("removeAlwaysTranslateLanguage"); + let removeAll = getById("removeAllAlwaysTranslateLanguages"); + is( + tree.view.rowCount, + 2, + "The always-translate languages list has 2 items" + ); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Select the first item. + tree.view.selection.select(0); + ok(!remove.disabled, "The 'Remove Language' button is enabled"); + + // Click the 'Remove' button. + remove.click(); + is( + tree.view.rowCount, + 1, + "The always-translate languages list now contains 1 item" + ); + is( + getAlwaysTranslateLanguages().length, + 1, + "One language tag in the pref" + ); + + // Clear the pref, and check the last item is removed from the display. + Services.prefs.setCharPref(ALWAYS_TRANSLATE_LANGS_PREF, ""); + is(tree.view.rowCount, 0, "The always-translate languages list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + + // Add an item and check it appears. + Services.prefs.setCharPref(ALWAYS_TRANSLATE_LANGS_PREF, "fr,en,es"); + is( + tree.view.rowCount, + 3, + "The always-translate languages list has 3 items" + ); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Click the 'Remove All' button. + removeAll.click(); + is(tree.view.rowCount, 0, "The always-translate languages list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + is( + Services.prefs.getCharPref(ALWAYS_TRANSLATE_LANGS_PREF), + "", + "The pref is empty" + ); + + win.close(); + }, + }, + + { + desc: "ensure never-translate languages function correctly", + run: async function testNeverTranslateLanguages() { + // Put 2 languages in the pref before opening the window to check + // the list is displayed on load. + Services.prefs.setCharPref(NEVER_TRANSLATE_LANGS_PREF, "fr,de"); + + // Open the translations settings dialog. + let win = openDialog( + "chrome://browser/content/preferences/dialogs/translations.xhtml", + "Browser:TranslationsPreferences", + "", + null + ); + await waitForWindowLoad(win); + + // Check that the list of never-translate languages is loaded. + let getById = win.document.getElementById.bind(win.document); + let tree = getById("neverTranslateLanguagesTree"); + let remove = getById("removeNeverTranslateLanguage"); + let removeAll = getById("removeAllNeverTranslateLanguages"); + is( + tree.view.rowCount, + 2, + "The never-translate languages list has 2 items" + ); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Select the first item. + tree.view.selection.select(0); + ok(!remove.disabled, "The 'Remove Language' button is enabled"); + + // Click the 'Remove' button. + remove.click(); + is( + tree.view.rowCount, + 1, + "The never-translate language list now contains 1 item" + ); + is(getNeverTranslateLanguages().length, 1, "One langtag in the pref"); + + // Clear the pref, and check the last item is removed from the display. + Services.prefs.setCharPref(NEVER_TRANSLATE_LANGS_PREF, ""); + is(tree.view.rowCount, 0, "The never-translate languages list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + + // Add an item and check it appears. + Services.prefs.setCharPref(NEVER_TRANSLATE_LANGS_PREF, "fr,en,es"); + is( + tree.view.rowCount, + 3, + "The never-translate languages list has 3 items" + ); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Languages' button is enabled"); + + // Click the 'Remove All' button. + removeAll.click(); + is(tree.view.rowCount, 0, "The never-translate languages list is empty"); + ok(remove.disabled, "The 'Remove Language' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Languages' button is disabled"); + is( + Services.prefs.getCharPref(NEVER_TRANSLATE_LANGS_PREF), + "", + "The pref is empty" + ); + + win.close(); + }, + }, + + { + desc: "ensure never-translate sites function correctly", + run: async function testNeverTranslateSites() { + // Add two deny permissions before opening the window to + // check the list is displayed on load. + PermissionTestUtils.add( + "https://example.org", + TRANSLATIONS_PERMISSION, + Services.perms.DENY_ACTION + ); + PermissionTestUtils.add( + "https://example.com", + TRANSLATIONS_PERMISSION, + Services.perms.DENY_ACTION + ); + + // Open the translations settings dialog. + let win = openDialog( + "chrome://browser/content/preferences/dialogs/translations.xhtml", + "Browser:TranslationsPreferences", + "", + null + ); + await waitForWindowLoad(win); + + // Check that the list of never-translate sites is loaded. + let getById = win.document.getElementById.bind(win.document); + let tree = getById("neverTranslateSitesTree"); + let remove = getById("removeNeverTranslateSite"); + let removeAll = getById("removeAllNeverTranslateSites"); + is(tree.view.rowCount, 2, "The never-translate sites list has 2 items"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Sites' button is enabled"); + + // Select the first item. + tree.view.selection.select(0); + ok(!remove.disabled, "The 'Remove Site' button is enabled"); + + // Click the 'Remove' button. + remove.click(); + is( + tree.view.rowCount, + 1, + "The never-translate sites list now contains 1 item" + ); + is( + getNeverTranslateSites().length, + 1, + "One domain in the site permissions" + ); + + // Clear the permissions, and check the last item is removed from the display. + PermissionTestUtils.remove( + "https://example.org", + TRANSLATIONS_PERMISSION + ); + PermissionTestUtils.remove( + "https://example.com", + TRANSLATIONS_PERMISSION + ); + is(tree.view.rowCount, 0, "The never-translate sites list is empty"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Site' button is disabled"); + + // Add items back and check that they appear + PermissionTestUtils.add( + "https://example.org", + TRANSLATIONS_PERMISSION, + Services.perms.DENY_ACTION + ); + PermissionTestUtils.add( + "https://example.com", + TRANSLATIONS_PERMISSION, + Services.perms.DENY_ACTION + ); + PermissionTestUtils.add( + "https://example.net", + TRANSLATIONS_PERMISSION, + Services.perms.DENY_ACTION + ); + + is(tree.view.rowCount, 3, "The never-translate sites list has 3 item"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(!removeAll.disabled, "The 'Remove All Sites' button is enabled"); + + // Click the 'Remove All' button. + removeAll.click(); + is(tree.view.rowCount, 0, "The never-translate sites list is empty"); + ok(remove.disabled, "The 'Remove Site' button is disabled"); + ok(removeAll.disabled, "The 'Remove All Sites' button is disabled"); + is( + getNeverTranslateSites().length, + 0, + "No domains in the site permissions" + ); + + win.close(); + }, + }, +]; diff --git a/browser/components/translation/test/fixtures/bug1022725-fr.html b/browser/components/translation/test/fixtures/bug1022725-fr.html new file mode 100644 index 0000000000..f30edf52eb --- /dev/null +++ b/browser/components/translation/test/fixtures/bug1022725-fr.html @@ -0,0 +1,15 @@ +<!doctype html> +<html lang="fr"> + <head> + <!-- + - Text retrieved from http://fr.wikipedia.org/wiki/Coupe_du_monde_de_football_de_2014 + - at 06/13/2014, Creative Commons Attribution-ShareAlike License. + --> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>test</title> + </head> + <body> + <h1>Coupe du monde de football de 2014</h1> + <div>La Coupe du monde de football de 2014 est la 20e édition de la Coupe du monde de football, compétition organisée par la FIFA et qui réunit les trente-deux meilleures sélections nationales. Sa phase finale a lieu à l'été 2014 au Brésil. Avec le pays organisateur, toutes les équipes championnes du monde depuis 1930 (Uruguay, Italie, Allemagne, Angleterre, Argentine, France et Espagne) se sont qualifiées pour cette compétition. Elle est aussi la première compétition internationale de la Bosnie-Herzégovine.</div> + </body> +</html> diff --git a/browser/components/translation/test/fixtures/result-da39a3ee5e.txt b/browser/components/translation/test/fixtures/result-da39a3ee5e.txt new file mode 100644 index 0000000000..d2d14c7885 --- /dev/null +++ b/browser/components/translation/test/fixtures/result-da39a3ee5e.txt @@ -0,0 +1,22 @@ +<ArrayOfTranslateArrayResponse xmlns="http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2" xmlns:i="http://www.w3.org/2001/XMLSchema-instance"> + <TranslateArrayResponse> + <From>fr</From> + <OriginalTextSentenceLengths xmlns:a="http://schemas.microsoft.com/2003/10/Serialization/Arrays"> + <a:int>34</a:int> + </OriginalTextSentenceLengths> + <TranslatedText>Football's 2014 World Cup</TranslatedText> + <TranslatedTextSentenceLengths xmlns:a="http://schemas.microsoft.com/2003/10/Serialization/Arrays"> + <a:int>25</a:int> + </TranslatedTextSentenceLengths> + </TranslateArrayResponse> + <TranslateArrayResponse> + <From>fr</From> + <OriginalTextSentenceLengths xmlns:a="http://schemas.microsoft.com/2003/10/Serialization/Arrays"> + <a:int>508</a:int> + </OriginalTextSentenceLengths> + <TranslatedText>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus diam sem, porttitor eget neque sit amet, ultricies posuere metus. Cras placerat rutrum risus, nec dignissim magna dictum vitae. Fusce eleifend fermentum lacinia. Nulla sagittis cursus nibh. Praesent adipiscing, elit at pulvinar dapibus, neque massa tincidunt sapien, eu consectetur lectus metus sit amet odio. Proin blandit consequat porttitor. Pellentesque vehicula justo sed luctus vestibulum. Donec metus.</TranslatedText> + <TranslatedTextSentenceLengths xmlns:a="http://schemas.microsoft.com/2003/10/Serialization/Arrays"> + <a:int>475</a:int> + </TranslatedTextSentenceLengths> + </TranslateArrayResponse> +</ArrayOfTranslateArrayResponse> diff --git a/browser/components/translation/test/fixtures/result-yandex-d448894848.json b/browser/components/translation/test/fixtures/result-yandex-d448894848.json new file mode 100644 index 0000000000..9287611105 --- /dev/null +++ b/browser/components/translation/test/fixtures/result-yandex-d448894848.json @@ -0,0 +1,8 @@ +{ + "code": 200, + "lang": "fr-en", + "text": [ + "Football's 2014 World Cup", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus diam sem, porttitor eget neque sit amet, ultricies posuere metus. Cras placerat rutrum risus, nec dignissim magna dictum vitae. Fusce eleifend fermentum lacinia. Nulla sagittis cursus nibh. Praesent adipiscing, elit at pulvinar dapibus, neque massa tincidunt sapien, eu consectetur lectus metus sit amet odio. Proin blandit consequat porttitor. Pellentesque vehicula justo sed luctus vestibulum. Donec metus." + ] +} diff --git a/browser/components/translation/test/yandex.sjs b/browser/components/translation/test/yandex.sjs new file mode 100644 index 0000000000..24dd0b6e7c --- /dev/null +++ b/browser/components/translation/test/yandex.sjs @@ -0,0 +1,203 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const CC = Components.Constructor; +const BinaryInputStream = CC( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); +Cu.importGlobalProperties(["TextEncoder"]); + +function handleRequest(req, res) { + try { + reallyHandleRequest(req, res); + } catch (ex) { + res.setStatusLine("1.0", 200, "AlmostOK"); + let msg = "Error handling request: " + ex + "\n" + ex.stack; + log(msg); + res.write(msg); + } +} + +function log(msg) { + dump("YANDEX-SERVER-MOCK: " + msg + "\n"); +} + +const statusCodes = { + 400: "Bad Request", + 401: "Invalid API key", + 402: "This API key has been blocked", + 403: "Daily limit for requests reached", + 404: "Daily limit for chars reached", + 413: "The text size exceeds the maximum", + 422: "The text could not be translated", + 500: "Internal Server Error", + 501: "The specified translation direction is not supported", + 503: "Service Unavailable", +}; + +function HTTPError(code = 500, message) { + this.code = code; + this.name = statusCodes[code] || "HTTPError"; + this.message = message || this.name; +} +HTTPError.prototype = new Error(); +HTTPError.prototype.constructor = HTTPError; + +function sendError(res, err) { + if (!(err instanceof HTTPError)) { + err = new HTTPError( + typeof err == "number" ? err : 500, + err.message || typeof err == "string" ? err : "" + ); + } + res.setStatusLine("1.1", err.code, err.name); + res.write(err.message); +} + +// Based on the code borrowed from: +// http://stackoverflow.com/questions/901115/how-can-i-get-query-string-values-in-javascript +function parseQuery(query) { + let match, + params = {}, + pl = /\+/g, + search = /([^&=]+)=?([^&]*)/g, + decode = function (s) { + return decodeURIComponent(s.replace(pl, " ")); + }; + + while ((match = search.exec(query))) { + let k = decode(match[1]), + v = decode(match[2]); + if (k in params) { + if (params[k] instanceof Array) { + params[k].push(v); + } else { + params[k] = [params[k], v]; + } + } else { + params[k] = v; + } + } + + return params; +} + +function sha1(str) { + // `data` is an array of bytes. + let data = new TextEncoder().encode(str); + let ch = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + ch.init(ch.SHA1); + ch.update(data, data.length); + let hash = ch.finish(false); + + // Return the two-digit hexadecimal code for a byte. + function toHexString(charCode) { + return ("0" + charCode.toString(16)).slice(-2); + } + + // Convert the binary hash data to a hex string. + return Array.from(hash, (c, i) => toHexString(hash.charCodeAt(i))).join(""); +} + +function getRequestBody(req) { + let avail; + let bytes = []; + let body = new BinaryInputStream(req.bodyInputStream); + + while ((avail = body.available()) > 0) { + Array.prototype.push.apply(bytes, body.readByteArray(avail)); + } + + return String.fromCharCode.apply(null, bytes); +} + +function getInputStream(path) { + let file = Services.dirsvc.get("CurWorkD", Ci.nsIFile); + for (let part of path.split("/")) { + file.append(part); + } + let fileStream = Cc[ + "@mozilla.org/network/file-input-stream;1" + ].createInstance(Ci.nsIFileInputStream); + fileStream.init(file, 1, 0, false); + return fileStream; +} + +/** + * Yandex Requests have to be signed with an API Key. This mock server + * supports the following keys: + * + * yandexValidKey Always passes the authentication, + * yandexInvalidKey Never passes authentication and fails with 401 code, + * yandexBlockedKey Never passes authentication and fails with 402 code, + * yandexOutOfRequestsKey Never passes authentication and fails with 403 code, + * yandexOutOfCharsKey Never passes authentication and fails with 404 code. + * + * If any other key is used the server reponds with 401 error code. + */ +function checkAuth(params) { + if (!("key" in params)) { + throw new HTTPError(400); + } + + let key = params.key; + if (key === "yandexValidKey") { + return true; + } + + let invalidKeys = { + yandexInvalidKey: 401, + yandexBlockedKey: 402, + yandexOutOfRequestsKey: 403, + yandexOutOfCharsKey: 404, + }; + + if (key in invalidKeys) { + throw new HTTPError(invalidKeys[key]); + } + + throw new HTTPError(401); +} + +function reallyHandleRequest(req, res) { + try { + // Preparing the query parameters. + let params = {}; + if (req.method == "POST") { + params = parseQuery(getRequestBody(req)); + } + + // Extracting the API key and attempting to authenticate the request. + log(JSON.stringify(params)); + + checkAuth(params); + methodHandlers.translate(res, params); + } catch (ex) { + sendError(res, ex, ex.code); + } +} + +const methodHandlers = { + translate(res, params) { + res.setStatusLine("1.1", 200, "OK"); + res.setHeader("Content-Type", "application/json"); + + let hash = sha1(JSON.stringify(params)).substr(0, 10); + log("SHA1 hash of content: " + hash); + + let fixture = + "browser/browser/components/translation/test/fixtures/result-yandex-" + + hash + + ".json"; + log("PATH: " + fixture); + + let inputStream = getInputStream(fixture); + res.bodyOutputStream.writeFrom(inputStream, inputStream.available()); + inputStream.close(); + }, +}; |
