diff options
Diffstat (limited to '')
258 files changed, 37715 insertions, 0 deletions
diff --git a/toolkit/components/search/.eslintrc.js b/toolkit/components/search/.eslintrc.js new file mode 100644 index 0000000000..9aafb4a214 --- /dev/null +++ b/toolkit/components/search/.eslintrc.js @@ -0,0 +1,9 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + extends: ["plugin:mozilla/require-jsdoc"], +}; diff --git a/toolkit/components/search/AddonSearchEngine.sys.mjs b/toolkit/components/search/AddonSearchEngine.sys.mjs new file mode 100644 index 0000000000..b5ee486c82 --- /dev/null +++ b/toolkit/components/search/AddonSearchEngine.sys.mjs @@ -0,0 +1,455 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { SearchEngine } from "resource://gre/modules/SearchEngine.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + ExtensionParent: "resource://gre/modules/ExtensionParent.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "AddonSearchEngine", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +/** + * AddonSearchEngine represents a search engine defined by an add-on. + */ +export class AddonSearchEngine extends SearchEngine { + // Whether the engine is provided by the application. + #isAppProvided = false; + // The extension ID if added by an extension. + _extensionID = null; + // The locale, or "DEFAULT", if required. + _locale = null; + + /** + * Creates a AddonSearchEngine. + * + * @param {object} options + * The options object + * @param {boolean} options.isAppProvided + * Indicates whether the engine is provided by Firefox, either + * shipped in omni.ja or via Normandy. If it is, it will + * be treated as read-only. + * @param {object} [options.details] + * An object that simulates the manifest object from a WebExtension. + * @param {object} [options.json] + * An object that represents the saved JSON settings for the engine. + */ + constructor({ isAppProvided, details, json } = {}) { + let extensionId = + details?.extensionID ?? json.extensionID ?? json._extensionID; + let id = extensionId + (details?.locale ?? json._locale); + + super({ + loadPath: "[addon]" + extensionId, + id, + }); + + this._extensionID = extensionId; + this.#isAppProvided = isAppProvided; + + if (json) { + this._initWithJSON(json); + } + } + + _initWithJSON(json) { + super._initWithJSON(json); + this._extensionID = json.extensionID || json._extensionID || null; + this._locale = json.extensionLocale || json._locale || null; + } + + /** + * Call to initalise an instance with extension details. Does not need to be + * called if json has been passed to the constructor. + * + * @param {object} options + * The options object. + * @param {Extension} options.extension + * The extension object representing the add-on. + * @param {object} options.locale + * The locale to use from the extension for getting details of the search + * engine. + * @param {object} [options.config] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + * @param {object} [options.settings] + * The saved settings for the user. + */ + async init({ extension, locale, config, settings }) { + let { baseURI, manifest } = await this.#getExtensionDetailsForLocale( + extension, + locale + ); + + this.#initFromManifest(baseURI, manifest, locale, config); + this._loadSettings(settings); + } + + /** + * Manages updates to this engine. + * + * @param {object} options + * The options object. + * @param {object} [options.configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + * @param {object} [options.extension] + * The extension associated with this search engine, if known. + * @param {string} [options.locale] + * The locale to use from the extension for getting details of the search + * engine. + */ + async update({ configuration, extension, locale } = {}) { + let { baseURI, manifest } = await this.#getExtensionDetailsForLocale( + extension, + locale + ); + + let originalName = this.name; + let name = manifest.chrome_settings_overrides.search_provider.name.trim(); + if (originalName != name && Services.search.getEngineByName(name)) { + throw new Error("Can't upgrade to the same name as an existing engine"); + } + + this.#updateFromManifest(baseURI, manifest, locale, configuration); + } + + /** + * This will update the add-on search engine if there is no name change. + * + * @param {object} options + * The options object. + * @param {object} [options.configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + * @param {string} [options.locale] + * The locale to use from the extension for getting details of the search + * engine. + * @returns {boolean} + * Returns true if the engine was updated, false otherwise. + */ + async updateIfNoNameChange({ configuration, locale }) { + let { baseURI, manifest } = await this.#getExtensionDetailsForLocale( + null, + locale + ); + + if ( + this.name != + manifest.chrome_settings_overrides.search_provider.name.trim() + ) { + return false; + } + + this.#updateFromManifest(baseURI, manifest, locale, configuration); + return true; + } + + /** + * Whether or not this engine is provided by the application, e.g. it is + * in the list of configured search engines. Overrides the definition in + * `SearchEngine`. + * + * @returns {boolean} + */ + get isAppProvided() { + if (lazy.SearchUtils.newSearchConfigEnabled) { + return false; + } + + return this.#isAppProvided; + } + + /** + * Whether or not this engine is an in-memory only search engine. + * These engines are typically application provided or policy engines, + * where they are loaded every time on SearchService initialization + * using the policy JSON or the extension manifest. Minimal details of the + * in-memory engines are saved to disk, but they are never loaded + * from the user's saved settings file. + * + * @returns {boolean} + * Only returns true for application provided engines. + */ + get inMemory() { + return this.#isAppProvided; + } + + get isGeneralPurposeEngine() { + return !!( + this._extensionID && + lazy.SearchUtils.GENERAL_SEARCH_ENGINE_IDS.has(this._extensionID) + ); + } + + /** + * Creates a JavaScript object that represents this engine. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + // For built-in engines we don't want to store all their data in the settings + // file so just store the relevant metadata. + if (this.#isAppProvided) { + return { + id: this.id, + _name: this.name, + _isAppProvided: true, + _metaData: this._metaData, + }; + } + let json = super.toJSON(); + json._extensionID = this._extensionID; + json._locale = this._locale; + return json; + } + + /** + * Checks to see if this engine's settings are in sync with what the add-on + * manager has, and reports the results to telemetry. + */ + async checkAndReportIfSettingsValid() { + let addon = await lazy.AddonManager.getAddonByID(this._extensionID); + + if (!addon) { + lazy.logConsole.debug( + `Add-on ${this._extensionID} for search engine ${this.name} is not installed!` + ); + Services.telemetry.keyedScalarSet( + "browser.searchinit.engine_invalid_webextension", + this._extensionID, + 1 + ); + } else if (!addon.isActive) { + lazy.logConsole.debug( + `Add-on ${this._extensionID} for search engine ${this.name} is not active!` + ); + Services.telemetry.keyedScalarSet( + "browser.searchinit.engine_invalid_webextension", + this._extensionID, + 2 + ); + } else { + let policy = await AddonSearchEngine.getWebExtensionPolicy( + this._extensionID + ); + let providerSettings = + policy.extension.manifest?.chrome_settings_overrides?.search_provider; + + if (!providerSettings) { + lazy.logConsole.debug( + `Add-on ${this._extensionID} for search engine ${this.name} no longer has an engine defined` + ); + Services.telemetry.keyedScalarSet( + "browser.searchinit.engine_invalid_webextension", + this._extensionID, + 4 + ); + } else if (this.name != providerSettings.name) { + lazy.logConsole.debug( + `Add-on ${this._extensionID} for search engine ${this.name} has a different name!` + ); + Services.telemetry.keyedScalarSet( + "browser.searchinit.engine_invalid_webextension", + this._extensionID, + 5 + ); + } else if (!this.checkSearchUrlMatchesManifest(providerSettings)) { + lazy.logConsole.debug( + `Add-on ${this._extensionID} for search engine ${this.name} has out-of-date manifest!` + ); + Services.telemetry.keyedScalarSet( + "browser.searchinit.engine_invalid_webextension", + this._extensionID, + 6 + ); + } + } + } + + /** + * Initializes the engine based on the manifest and other values. + * + * @param {string} extensionBaseURI + * The Base URI of the WebExtension. + * @param {object} manifest + * An object representing the WebExtensions' manifest. + * @param {string} locale + * The locale that is being used for the WebExtension. + * @param {object} [configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + */ + #initFromManifest(extensionBaseURI, manifest, locale, configuration = {}) { + let searchProvider = manifest.chrome_settings_overrides.search_provider; + + this._locale = locale; + + // We only set _telemetryId for app-provided engines. See also telemetryId + // getter. + if (this.#isAppProvided) { + if (configuration.telemetryId) { + this._telemetryId = configuration.telemetryId; + } else { + let telemetryId = this._extensionID.split("@")[0]; + if (locale != lazy.SearchUtils.DEFAULT_TAG) { + telemetryId += "-" + locale; + } + this._telemetryId = telemetryId; + } + } + + // Set the main icon URL for the engine. + let iconURL = searchProvider.favicon_url; + + if (!iconURL) { + iconURL = + manifest.icons && + extensionBaseURI.resolve( + lazy.ExtensionParent.IconDetails.getPreferredIcon(manifest.icons).icon + ); + } + + // Record other icons that the WebExtension has. + if (manifest.icons) { + let iconList = Object.entries(manifest.icons).map(icon => { + return { + width: icon[0], + height: icon[0], + url: extensionBaseURI.resolve(icon[1]), + }; + }); + for (let icon of iconList) { + this._addIconToMap(icon.size, icon.size, icon.url); + } + } + + // Filter out any untranslated parameters, the extension has to list all + // possible mozParams for each engine where a 'locale' may only provide + // actual values for some (or none). + if (searchProvider.params) { + searchProvider.params = searchProvider.params.filter(param => { + return !(param.value && param.value.startsWith("__MSG_")); + }); + } + + this._initWithDetails( + { ...searchProvider, iconURL, description: manifest.description }, + configuration + ); + } + + /** + * Update this engine based on new manifest, used during + * webextension upgrades. + * + * @param {string} extensionBaseURI + * The Base URI of the WebExtension. + * @param {object} manifest + * An object representing the WebExtensions' manifest. + * @param {string} locale + * The locale that is being used for the WebExtension. + * @param {object} [configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + */ + #updateFromManifest(extensionBaseURI, manifest, locale, configuration = {}) { + this._urls = []; + this._iconMapObj = null; + this.#initFromManifest(extensionBaseURI, manifest, locale, configuration); + lazy.SearchUtils.notifyAction(this, lazy.SearchUtils.MODIFIED_TYPE.CHANGED); + } + + /** + * Get the localized manifest from the WebExtension for the given locale or + * manifest default locale. + * + * The search service configuration overloads the add-on manager concepts of + * locales, and forces particular locales within the WebExtension to be used, + * ignoring the user's current locale. The user's current locale is taken into + * account within the configuration, just not in the WebExtension. + * + * @param {object} [extension] + * The extension to get the manifest from. + * @param {string} locale + * The locale to load from the WebExtension. If this is `DEFAULT_TAG`, then + * the default locale is loaded. + * @returns {object} + * The loaded manifest. + */ + async #getExtensionDetailsForLocale(extension, locale) { + // If we haven't been passed an extension object, then go and find it. + if (!extension) { + extension = ( + await AddonSearchEngine.getWebExtensionPolicy(this._extensionID) + ).extension; + } + + let manifest = extension.manifest; + + let localeToLoad; + if (this.#isAppProvided) { + // If the locale we want from the WebExtension is the extension's default + // then we get that from the manifest here. We do this because if we + // are reloading due to the locale change, the add-on manager might not + // have updated the WebExtension's manifest to the new version by the + // time we hit this code. + localeToLoad = + locale == lazy.SearchUtils.DEFAULT_TAG + ? manifest.default_locale + : locale; + } else { + // For user installed add-ons, we have to simulate the add-on manager + // code for loading the correct locale. + // We do this, as in the case of a live language switch, the add-on manager + // may not have yet reloaded the extension, and there's no way for us to + // listen for that reload to complete. + // See also https://bugzilla.mozilla.org/show_bug.cgi?id=1781768#c3 for + // more background. + localeToLoad = Services.locale.negotiateLanguages( + Services.locale.appLocalesAsBCP47, + [...extension.localeData.locales.keys()], + manifest.default_locale + )[0]; + } + + if (localeToLoad) { + manifest = await extension.getLocalizedManifest(localeToLoad); + } + return { baseURI: extension.baseURI, manifest }; + } + + /** + * Gets the WebExtensionPolicy for an add-on. + * + * @param {string} id + * The WebExtension id. + * @returns {WebExtensionPolicy} + */ + static async getWebExtensionPolicy(id) { + let policy = WebExtensionPolicy.getByID(id); + if (!policy) { + let idPrefix = id.split("@")[0]; + let path = `resource://search-extensions/${idPrefix}/`; + await lazy.AddonManager.installBuiltinAddon(path); + policy = WebExtensionPolicy.getByID(id); + } + // On startup the extension may have not finished parsing the + // manifest, wait for that here. + await policy.readyPromise; + return policy; + } +} diff --git a/toolkit/components/search/AppProvidedSearchEngine.sys.mjs b/toolkit/components/search/AppProvidedSearchEngine.sys.mjs new file mode 100644 index 0000000000..a8db801ac4 --- /dev/null +++ b/toolkit/components/search/AppProvidedSearchEngine.sys.mjs @@ -0,0 +1,291 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { + SearchEngine, + EngineURL, +} from "resource://gre/modules/SearchEngine.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +/** + * AppProvidedSearchEngine represents a search engine defined by the + * search configuration. + */ +export class AppProvidedSearchEngine extends SearchEngine { + static URL_TYPE_MAP = new Map([ + ["search", lazy.SearchUtils.URL_TYPE.SEARCH], + ["suggestions", lazy.SearchUtils.URL_TYPE.SUGGEST_JSON], + ["trending", lazy.SearchUtils.URL_TYPE.TRENDING_JSON], + ]); + + /** + * @param {object} options + * The options for this search engine. + * @param {object} options.config + * The engine config from Remote Settings. + * @param {object} [options.settings] + * The saved settings for the user. + */ + constructor({ config, settings }) { + // TODO Bug 1875912 - Remove the webextension.id and webextension.locale when + // we're ready to remove old search-config and use search-config-v2 for all + // clients. The id in appProvidedSearchEngine should be changed to + // engine.identifier. + let extensionId = config.webExtension.id; + let id = config.webExtension.id + config.webExtension.locale; + + super({ + loadPath: "[app]" + extensionId, + isAppProvided: true, + id, + }); + + this._extensionID = extensionId; + this._locale = config.webExtension.locale; + + this.#init(config); + + this._loadSettings(settings); + } + + /** + * Update this engine based on new config, used during + * config upgrades. + + * @param {object} options + * The options object. + * + * @param {object} options.locale + * The locale that is being used for the engine. + * @param {object} options.configuration + * The search engine configuration for application provided engines. + */ + update({ locale, configuration } = {}) { + this._urls = []; + this._iconMapObj = null; + this.#init(configuration); + lazy.SearchUtils.notifyAction(this, lazy.SearchUtils.MODIFIED_TYPE.CHANGED); + } + + /** + * This will update the application provided search engine if there is no + * name change. + * + * @param {object} options + * The options object. + * @param {object} [options.configuration] + * The search engine configuration for application provided engines. + * @param {string} [options.locale] + * The locale to use for getting details of the search engine. + * @returns {boolean} + * Returns true if the engine was updated, false otherwise. + */ + async updateIfNoNameChange({ configuration, locale }) { + let newName; + if (locale != "default") { + newName = configuration.webExtension.searchProvider[locale].name; + } else if ( + locale == "default" && + configuration.webExtension.default_locale + ) { + newName = + configuration.webExtension.searchProvider[ + configuration.webExtension.default_locale + ].name; + } else { + newName = configuration.webExtension.name; + } + + if (this.name != newName.trim()) { + return false; + } + this.update({ locale, configuration }); + return true; + } + + /** + * Whether or not this engine is provided by the application, e.g. it is + * in the list of configured search engines. Overrides the definition in + * `SearchEngine`. + * + * @returns {boolean} + */ + get isAppProvided() { + return true; + } + + /** + * Whether or not this engine is an in-memory only search engine. + * These engines are typically application provided or policy engines, + * where they are loaded every time on SearchService initialization + * using the policy JSON or the extension manifest. Minimal details of the + * in-memory engines are saved to disk, but they are never loaded + * from the user's saved settings file. + * + * @returns {boolean} + * Only returns true for application provided engines. + */ + get inMemory() { + return true; + } + + get isGeneralPurposeEngine() { + return !!( + this._extensionID && + lazy.SearchUtils.GENERAL_SEARCH_ENGINE_IDS.has(this._extensionID) + ); + } + + /** + * Creates a JavaScript object that represents this engine. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + // For applicaiton provided engines we don't want to store all their data in + // the settings file so just store the relevant metadata. + return { + id: this.id, + _name: this.name, + _isAppProvided: true, + _metaData: this._metaData, + }; + } + + /** + * Initializes the engine. + * + * @param {object} [engineConfig] + * The search engine configuration for application provided engines. + */ + #init(engineConfig) { + this._orderHint = engineConfig.orderHint; + this._telemetryId = engineConfig.identifier; + + if (engineConfig.telemetrySuffix) { + this._telemetryId += `-${engineConfig.telemetrySuffix}`; + } + + // Set the main icon URL for the engine. + // let iconURL = searchProvider.favicon_url; + + // if (!iconURL) { + // iconURL = + // manifest.icons && + // extensionBaseURI.resolve( + // lazy.ExtensionParent.IconDetails.getPreferredIcon(manifest.icons).icon + // ); + // } + + // // Record other icons that the WebExtension has. + // if (manifest.icons) { + // let iconList = Object.entries(manifest.icons).map(icon => { + // return { + // width: icon[0], + // height: icon[0], + // url: extensionBaseURI.resolve(icon[1]), + // }; + // }); + // for (let icon of iconList) { + // this._addIconToMap(icon.size, icon.size, icon.url); + // } + // } + + // this._initWithDetails(config); + + // this._sendAttributionRequest = config.sendAttributionRequest ?? false; // TODO check if we need to this? + // if (details.iconURL) { + // this._setIcon(details.iconURL, true); + // } + + this._name = engineConfig.name.trim(); + this._definedAliases = + engineConfig.aliases?.map(alias => `@${alias}`) ?? []; + + for (const [type, urlData] of Object.entries(engineConfig.urls)) { + this.#setUrl(type, urlData, engineConfig.partnerCode); + } + } + + /** + * This sets the urls for the search engine based on the supplied parameters. + * + * @param {string} type + * The type of url. This could be a url for search, suggestions, or trending. + * @param {object} urlData + * The url data contains the template/base url and url params. + * @param {string} partnerCode + * The partner code associated with the search engine. + */ + #setUrl(type, urlData, partnerCode) { + let urlType = AppProvidedSearchEngine.URL_TYPE_MAP.get(type); + + if (!urlType) { + console.warn("unexpected engine url type.", type); + return; + } + + let engineURL = new EngineURL( + urlType, + urlData.method || "GET", + urlData.base + ); + + if (urlData.params) { + for (const param of urlData.params) { + switch (true) { + case "value" in param: + engineURL.addParam( + param.name, + param.value == "{partnerCode}" ? partnerCode : param.value + ); + break; + case "experimentConfig" in param: + engineURL._addMozParam({ + name: param.name, + pref: param.experimentConfig, + condition: "pref", + }); + break; + case "searchAccessPoint" in param: + for (const [key, value] of Object.entries( + param.searchAccessPoint + )) { + engineURL.addParam( + param.name, + value, + key == "addressbar" ? "keyword" : key + ); + } + break; + } + } + } + + if ( + !("searchTermParamName" in urlData) && + !urlData.base.includes("{searchTerms}") && + !urlType.includes("trending") + ) { + throw new Error("Search terms missing from engine URL."); + } + + if ("searchTermParamName" in urlData) { + // The search term parameter is always added last, which will add it to the + // end of the URL. This is because in the past we have seen users trying to + // modify their searches by altering the end of the URL. + engineURL.addParam(urlData.searchTermParamName, "{searchTerms}"); + } + + this._urls.push(engineURL); + } +} diff --git a/toolkit/components/search/OpenSearchEngine.sys.mjs b/toolkit/components/search/OpenSearchEngine.sys.mjs new file mode 100644 index 0000000000..e1b9b976fb --- /dev/null +++ b/toolkit/components/search/OpenSearchEngine.sys.mjs @@ -0,0 +1,257 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { + EngineURL, + SearchEngine, +} from "resource://gre/modules/SearchEngine.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + loadAndParseOpenSearchEngine: + "resource://gre/modules/OpenSearchLoader.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "OpenSearchEngine", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +// The default engine update interval, in days. This is only used if an engine +// specifies an updateURL, but not an updateInterval. +const OPENSEARCH_DEFAULT_UPDATE_INTERVAL = 7; + +/** + * OpenSearchEngine represents an OpenSearch base search engine. + */ +export class OpenSearchEngine extends SearchEngine { + // The data describing the engine, in the form of an XML document element. + _data = null; + // The number of days between update checks for new versions + _updateInterval = null; + // The url to check at for a new update + _updateURL = null; + // The url to check for a new icon + _iconUpdateURL = null; + + /** + * Creates a OpenSearchEngine. + * + * @param {object} [options] + * The options object + * @param {object} [options.json] + * An object that represents the saved JSON settings for the engine. + * @param {OpenSearchProperties} [options.engineData] + * The engine data for this search engine that will have been loaded via + * `OpenSearchLoader`. + */ + constructor(options = {}) { + super({ + loadPath: + options.json?._loadPath ?? + OpenSearchEngine.getAnonymizedLoadPath( + lazy.SearchUtils.sanitizeName(options.engineData.name), + options.engineData.installURL + ), + }); + + if (options.engineData) { + this.#setEngineData(options.engineData); + + // As this is a new engine, we must set the verification hash for the load + // path set in the constructor. + this.setAttr( + "loadPathHash", + lazy.SearchUtils.getVerificationHash(this._loadPath) + ); + + if (this.hasUpdates) { + this.#setNextUpdateTime(); + } + } else { + this._initWithJSON(options.json); + this._updateInterval = options.json._updateInterval ?? null; + this._updateURL = options.json._updateURL ?? null; + this._iconUpdateURL = options.json._iconUpdateURL ?? null; + } + } + + /** + * Creates a JavaScript object that represents this engine. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + let json = super.toJSON(); + json._updateInterval = this._updateInterval; + json._updateURL = this._updateURL; + json._iconUpdateURL = this._iconUpdateURL; + return json; + } + + /** + * Determines if this search engine has updates url. + * + * @returns {boolean} + * Returns true if this search engine may update itself. + */ + get hasUpdates() { + // Whether or not the engine has an update URL + let selfURL = this._getURLOfType( + lazy.SearchUtils.URL_TYPE.OPENSEARCH, + "self" + ); + return !!(this._updateURL || this._iconUpdateURL || selfURL); + } + + /** + * Returns the engine's updateURI if it exists and returns null otherwise + * + * @returns {?string} + */ + get updateURI() { + let updateURL = this._getURLOfType(lazy.SearchUtils.URL_TYPE.OPENSEARCH); + let updateURI = + updateURL && updateURL._hasRelation("self") + ? updateURL.getSubmission("", this).uri + : lazy.SearchUtils.makeURI(this._updateURL); + return updateURI; + } + + /** + * Considers if this engine needs to be updated, and updates it if necessary. + */ + async maybeUpdate() { + if (!this.hasUpdates) { + return; + } + + let currentTime = Date.now(); + + let expireTime = this.getAttr("updateexpir"); + + if (!expireTime || !(expireTime <= currentTime)) { + lazy.logConsole.debug(this.name, "Skipping update, not expired yet."); + return; + } + + await this.#update(); + + this.#setNextUpdateTime(); + } + + /** + * Updates the OpenSearch engine details from the server. + */ + async #update() { + let updateURI = this.updateURI; + if (updateURI) { + let data = await lazy.loadAndParseOpenSearchEngine( + updateURI, + this.getAttr("updatelastmodified") + ); + + this.#setEngineData(data); + + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + + // Keep track of the last modified date, so that we can make conditional + // server requests for future updates. + this.setAttr("updatelastmodified", new Date().toUTCString()); + } + + if (this._iconUpdateURL) { + // Force update of the icon from the icon URL. + this._setIcon(this._iconUpdateURL, true); + } + } + + /** + * Sets the data for this engine based on the OpenSearch properties. + * + * @param {OpenSearchProperties} data + * The OpenSearch data. + */ + #setEngineData(data) { + let name = data.name.trim(); + if (!this._engineToUpdate) { + if (Services.search.getEngineByName(name)) { + throw Components.Exception( + "Found a duplicate engine", + Ci.nsISearchService.ERROR_DUPLICATE_ENGINE + ); + } + } + + this._name = name; + this._description = data.description ?? ""; + this._searchForm = data.searchForm ?? ""; + this._queryCharset = data.queryCharset ?? "UTF-8"; + + for (let url of data.urls) { + let engineURL; + try { + engineURL = new EngineURL(url.type, url.method, url.template); + } catch (ex) { + throw Components.Exception( + `Failed to add ${url.template} as an Engine URL`, + Cr.NS_ERROR_FAILURE + ); + } + + if (url.rels.length) { + engineURL.rels = url.rels; + } + + for (let param of url.params) { + try { + engineURL.addParam(param.name, param.value); + } catch (ex) { + // Ignore failure + lazy.logConsole.error("OpenSearch url has an invalid param", param); + } + } + + this._urls.push(engineURL); + } + + for (let image of data.images) { + this._setIcon(image.url, image.isPrefered, image.width, image.height); + } + } + + /** + * Sets the next update time for this engine. + */ + #setNextUpdateTime() { + var interval = this._updateInterval || OPENSEARCH_DEFAULT_UPDATE_INTERVAL; + var milliseconds = interval * 86400000; // |interval| is in days + this.setAttr("updateexpir", Date.now() + milliseconds); + } + + /** + * This indicates where we found the .xml file to load the engine, + * and attempts to hide user-identifiable data (such as username). + * + * @param {string} sanitizedName + * The sanitized name of the engine. + * @param {nsIURI} uri + * The uri the engine was loaded from. + * @returns {string} + * A load path with reduced data. + */ + static getAnonymizedLoadPath(sanitizedName, uri) { + return `[${uri.scheme}]${uri.host}/${sanitizedName}.xml`; + } +} diff --git a/toolkit/components/search/OpenSearchLoader.sys.mjs b/toolkit/components/search/OpenSearchLoader.sys.mjs new file mode 100644 index 0000000000..e1d820b463 --- /dev/null +++ b/toolkit/components/search/OpenSearchLoader.sys.mjs @@ -0,0 +1,393 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * OpenSearchLoader is used for loading OpenSearch definitions from content. + */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "OpenSearchLoader", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +// The namespaces from the specification at +// https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md#namespace +const OPENSEARCH_NS_10 = "http://a9.com/-/spec/opensearch/1.0/"; +const OPENSEARCH_NS_11 = "http://a9.com/-/spec/opensearch/1.1/"; + +// Although the specification at gives the namespace names defined above, many +// existing OpenSearch engines are using the following versions. We therefore +// allow any one of these. +const OPENSEARCH_NAMESPACES = [ + OPENSEARCH_NS_11, + OPENSEARCH_NS_10, + "http://a9.com/-/spec/opensearchdescription/1.1/", + "http://a9.com/-/spec/opensearchdescription/1.0/", +]; + +// The name of the element defining the OpenSearch definition. +const OPENSEARCH_LOCALNAME = "OpenSearchDescription"; + +// These were OpenSearch definitions for engines used internally by Mozilla. +// It may be possible to deprecate/remove these in future. +const MOZSEARCH_NS_10 = "http://www.mozilla.org/2006/browser/search/"; +const MOZSEARCH_LOCALNAME = "SearchPlugin"; + +/** + * @typedef {object} OpenSearchProperties + * @property {string} name + * The display name of the engine. + * @property {nsIURI} installURL + * The URL that the engine was initially loaded from. + * @property {string} [description] + * The description of the engine. + * @property {string} [queryCharset] + * The character set to use for encoding query values. + * @property {string} [searchForm] + * Non-standard. The search form URL. + * @property {string} [UpdateUrl] + * Non-standard. The update URL for the engine. + * @property {number} [UpdateInterval] + * Non-standard. The update interval for the engine. + * @property {string} [IconUpdateUrl] + * Non-standard. The update URL for the icon. + * @property {OpenSearchURL[]} urls + * An array of URLs associated with the engine. + * @property {OpenSearchImage[]} images + * An array of images assocaiated with the engine. + */ + +/** + * @typedef {object} OpenSearchURL + * @property {string} type + * The OpenSearch based type of the URL see SearchUtils.URL_TYPE. + * @property {string} method + * The method of submission for the URL: GET or POST. + * @property {string} template + * The template for the URL. + * @property {object[]} params + * An array of additional properties of name/value pairs. These are not part + * of the OpenSearch specification, but were used in Firefox prior to Firefox 78. + * @property {string[]} rels + * An array of strings that define the relationship of this URL. + * + * @see SearchUtils.URL_TYPE + * @see https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md#url-rel-values + */ + +/** + * @typedef {object} OpenSearchImage + * @property {string} url + * The source URL of the image. + * @property {boolean} isPrefered + * If this image is of the preferred 16x16 size. + * @property {width} width + * The reported width of the image. + * @property {height} height + * The reported height of the image. + */ + +/** + * Retrieves the engine data from a URI and returns it. + * + * @param {nsIURI} sourceURI + * The uri from which to load the OpenSearch engine data. + * @param {string} [lastModified] + * The UTC date when the engine was last updated, if any. + * @returns {OpenSearchProperties} + * The properties of the loaded OpenSearch engine. + */ +export async function loadAndParseOpenSearchEngine(sourceURI, lastModified) { + if (!sourceURI) { + throw Components.Exception( + sourceURI, + "Must have URI when calling _install!", + Cr.NS_ERROR_UNEXPECTED + ); + } + if (!/^https?$/i.test(sourceURI.scheme)) { + throw Components.Exception( + "Invalid URI passed to SearchEngine constructor", + Cr.NS_ERROR_INVALID_ARG + ); + } + + lazy.logConsole.debug("Downloading OpenSearch engine from:", sourceURI.spec); + + let xmlData = await loadEngineXML(sourceURI, lastModified); + let xmlDocument = await parseXML(xmlData); + + lazy.logConsole.debug("Loading search plugin"); + + let engineData; + try { + engineData = processXMLDocument(xmlDocument); + } catch (ex) { + lazy.logConsole.error("parseData: Failed to init engine!", ex); + + if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) { + throw Components.Exception( + "", + Ci.nsISearchService.ERROR_ENGINE_CORRUPTED + ); + } + throw Components.Exception("", Ci.nsISearchService.ERROR_DOWNLOAD_FAILURE); + } + + engineData.installURL = sourceURI; + return engineData; +} + +/** + * Loads the engine XML from the given URI. + * + * @param {nsIURI} sourceURI + * The uri from which to load the OpenSearch engine data. + * @param {string} [lastModified] + * The UTC date when the engine was last updated, if any. + * @returns {Promise} + * A promise that is resolved with the data if the engine is successfully loaded + * and rejected otherwise. + */ +function loadEngineXML(sourceURI, lastModified) { + var chan = lazy.SearchUtils.makeChannel( + sourceURI, + // OpenSearchEngine is loading a definition file for a search engine, + // TYPE_DOCUMENT captures that load best. + Ci.nsIContentPolicy.TYPE_DOCUMENT + ); + + if (lastModified && chan instanceof Ci.nsIHttpChannel) { + chan.setRequestHeader("If-Modified-Since", lastModified, false); + } + let loadPromise = Promise.withResolvers(); + + let loadHandler = data => { + if (!data) { + loadPromise.reject( + Components.Exception("", Ci.nsISearchService.ERROR_DOWNLOAD_FAILURE) + ); + return; + } + loadPromise.resolve(data); + }; + + var listener = new lazy.SearchUtils.LoadListener( + chan, + /(^text\/|xml$)/, + loadHandler + ); + chan.notificationCallbacks = listener; + chan.asyncOpen(listener); + + return loadPromise.promise; +} + +/** + * Parses an engines XML data into a document element. + * + * @param {number[]} xmlData + * The loaded search engine data. + * @returns {Element} + * A document element containing the parsed data. + */ +function parseXML(xmlData) { + var parser = new DOMParser(); + var doc = parser.parseFromBuffer(xmlData, "text/xml"); + + if (!doc?.documentElement) { + throw Components.Exception( + "Could not parse file", + Ci.nsISearchService.ERROR_ENGINE_CORRUPTED + ); + } + + if (!hasExpectedNamspeace(doc.documentElement)) { + throw Components.Exception( + "Not a valid OpenSearch xml file", + Ci.nsISearchService.ERROR_ENGINE_CORRUPTED + ); + } + return doc.documentElement; +} + +/** + * Extract search engine information from the given document into a form that + * can be passed to an OpenSearchEngine. + * + * @param {Element} xmlDocument + * The document to examine. + * @returns {OpenSearchProperties} + * The properties of the OpenSearch engine. + */ +function processXMLDocument(xmlDocument) { + let result = { urls: [], images: [] }; + + for (let i = 0; i < xmlDocument.children.length; ++i) { + var child = xmlDocument.children[i]; + switch (child.localName) { + case "ShortName": + result.name = child.textContent; + break; + case "Description": + result.description = child.textContent; + break; + case "Url": + try { + result.urls.push(parseURL(child)); + } catch (ex) { + // Parsing of the element failed, just skip it. + lazy.logConsole.error("Failed to parse URL child:", ex); + } + break; + case "Image": { + let imageData = parseImage(child); + if (imageData) { + result.images.push(imageData); + } + break; + } + case "InputEncoding": + // If this is not specified we fallback to the SearchEngine constructor + // which currently uses SearchUtils.DEFAULT_QUERY_CHARSET which is + // UTF-8 - the same as for OpenSearch. + result.queryCharset = child.textContent; + break; + + // Non-OpenSearch elements + case "SearchForm": + result.searchForm = child.textContent; + break; + case "UpdateUrl": + result.updateURL = child.textContent; + break; + case "UpdateInterval": + result.updateInterval = parseInt(child.textContent); + break; + case "IconUpdateUrl": + result.iconUpdateURL = child.textContent; + break; + } + } + if (!result.name || !result.urls.length) { + throw Components.Exception( + "_parse: No name, or missing URL!", + Cr.NS_ERROR_FAILURE + ); + } + if (!result.urls.find(url => url.type == lazy.SearchUtils.URL_TYPE.SEARCH)) { + throw Components.Exception( + "_parse: No text/html result type!", + Cr.NS_ERROR_FAILURE + ); + } + return result; +} + +/** + * Extracts data from an OpenSearch URL element and creates an object which can + * be used to create an OpenSearchEngine's URL. + * + * @param {Element} element + * The OpenSearch URL element. + * @returns {OpenSearchURL} + * The extracted URL data. + * @throws NS_ERROR_FAILURE if a URL object could not be created. + * + * @see https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md#the-url-element + */ +function parseURL(element) { + var type = element.getAttribute("type"); + // According to the spec, method is optional, defaulting to "GET" if not + // specified. + var method = element.getAttribute("method") || "GET"; + var template = element.getAttribute("template"); + + let rels = []; + if (element.hasAttribute("rel")) { + rels = element.getAttribute("rel").toLowerCase().split(/\s+/); + } + + // Support an alternate suggestion type, see bug 1425827 for details. + if (type == "application/json" && rels.includes("suggestions")) { + type = lazy.SearchUtils.URL_TYPE.SUGGEST_JSON; + } + + let url = { + type, + method, + template, + params: [], + rels, + }; + + // Non-standard. Used to be for Mozilla search engine files. + for (var i = 0; i < element.children.length; ++i) { + var param = element.children[i]; + if (param.localName == "Param") { + url.params.push({ + name: param.getAttribute("name"), + value: param.getAttribute("value"), + }); + } + } + + return url; +} + +/** + * Extracts an icon from an OpenSearch Image element. + * + * @param {Element} element + * The OpenSearch URL element. + * @returns {OpenSearchImage} + * The properties of the image. + * @see https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md#the-image-element + */ +function parseImage(element) { + let width = parseInt(element.getAttribute("width"), 10); + let height = parseInt(element.getAttribute("height"), 10); + let isPrefered = width == 16 && height == 16; + + if (isNaN(width) || isNaN(height) || width <= 0 || height <= 0) { + lazy.logConsole.warn( + "OpenSearch image element must have positive width and height." + ); + return null; + } + + return { + url: element.textContent, + isPrefered, + width, + height, + }; +} + +/** + * Confirms if the document has the expected namespace. + * + * @param {DOMElement} element + * The document to check. + * @returns {boolean} + * True if the document matches the namespace. + */ +function hasExpectedNamspeace(element) { + return ( + (element.localName == MOZSEARCH_LOCALNAME && + element.namespaceURI == MOZSEARCH_NS_10) || + (element.localName == OPENSEARCH_LOCALNAME && + OPENSEARCH_NAMESPACES.includes(element.namespaceURI)) + ); +} diff --git a/toolkit/components/search/PolicySearchEngine.sys.mjs b/toolkit/components/search/PolicySearchEngine.sys.mjs new file mode 100644 index 0000000000..c026d66af4 --- /dev/null +++ b/toolkit/components/search/PolicySearchEngine.sys.mjs @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { SearchEngine } from "resource://gre/modules/SearchEngine.sys.mjs"; + +/** + * PolicySearchEngine represents a search engine defined by an enterprise + * policy. + */ +export class PolicySearchEngine extends SearchEngine { + /** + * Creates a PolicySearchEngine. + * + * @param {object} options + * The options for this search engine. + * @param {object} options.details + * An object that matches the `SearchEngines` policy schema. + * @param {object} [options.settings] + * The saved settings for the user. + * + * @see browser/components/enterprisepolicies/schemas/policies-schema.json + */ + constructor(options = {}) { + let id = "policy-" + options.details.Name; + + super({ + loadPath: "[policy]", + id, + }); + + // Map the data to look like a WebExtension manifest, as that is what + // _initWithDetails requires. + let details = { + description: options.details.Description, + iconURL: options.details.IconURL ? options.details.IconURL.href : null, + name: options.details.Name, + // If the encoding is not specified or is falsy, we will fall back to + // the default encoding. + encoding: options.details.Encoding, + search_url: encodeURI(options.details.URLTemplate), + keyword: options.details.Alias, + search_url_post_params: + options.details.Method == "POST" ? options.details.PostData : undefined, + suggest_url: options.details.SuggestURLTemplate, + }; + this._initWithDetails(details); + + this._loadSettings(options.settings); + } + + /** + * Whether or not this engine is an in-memory only search engine. + * These engines are typically application provided or policy engines, + * where they are loaded every time on SearchService initialization + * using the policy JSON or the extension manifest. Minimal details of the + * in-memory engines are saved to disk, but they are never loaded + * from the user's saved settings file. + * + * @returns {boolean} + * All policy engines are in-memory, so this always returns true. + */ + get inMemory() { + return true; + } + + /** + * Returns the appropriate identifier to use for telemetry. + * + * @returns {string} + */ + get telemetryId() { + return `other-${this.name}`; + } + + /** + * Creates a JavaScript object that represents this engine. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + // For policy engines, we load them at every startup and we don't want to + // store all their data in the settings file so just return the relevant + // metadata. + let json = super.toJSON(); + + // We only want to return a sub-set of fields, as the details for this engine + // are loaded on each startup from the enterprise policies. + return { + id: json.id, + _name: json._name, + // Load path is included so that we know this is an enterprise engine. + _loadPath: json._loadPath, + _metaData: json._metaData, + }; + } +} diff --git a/toolkit/components/search/SearchEngine.sys.mjs b/toolkit/components/search/SearchEngine.sys.mjs new file mode 100644 index 0000000000..a043c7d4ca --- /dev/null +++ b/toolkit/components/search/SearchEngine.sys.mjs @@ -0,0 +1,1808 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", + OpenSearchEngine: "resource://gre/modules/OpenSearchEngine.sys.mjs", +}); + +const BinaryInputStream = Components.Constructor( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchEngine", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +const USER_DEFINED = "searchTerms"; + +// Supported OpenSearch parameters +// See http://opensearch.a9.com/spec/1.1/querysyntax/#core +const OS_PARAM_INPUT_ENCODING = "inputEncoding"; +const OS_PARAM_LANGUAGE = "language"; +const OS_PARAM_OUTPUT_ENCODING = "outputEncoding"; + +// Default values +const OS_PARAM_LANGUAGE_DEF = "*"; +const OS_PARAM_OUTPUT_ENCODING_DEF = "UTF-8"; + +// "Unsupported" OpenSearch parameters. For example, we don't support +// page-based results, so if the engine requires that we send the "page index" +// parameter, we'll always send "1". +const OS_PARAM_COUNT = "count"; +const OS_PARAM_START_INDEX = "startIndex"; +const OS_PARAM_START_PAGE = "startPage"; + +// Default values +const OS_PARAM_COUNT_DEF = "20"; // 20 results +const OS_PARAM_START_INDEX_DEF = "1"; // start at 1st result +const OS_PARAM_START_PAGE_DEF = "1"; // 1st page + +// A array of arrays containing parameters that we don't fully support, and +// their default values. We will only send values for these parameters if +// required, since our values are just really arbitrary "guesses" that should +// give us the output we want. +var OS_UNSUPPORTED_PARAMS = [ + [OS_PARAM_COUNT, OS_PARAM_COUNT_DEF], + [OS_PARAM_START_INDEX, OS_PARAM_START_INDEX_DEF], + [OS_PARAM_START_PAGE, OS_PARAM_START_PAGE_DEF], +]; + +// An array of attributes that are saved in the engines `_metaData` object. +// Attributes not in this array are considered as system attributes. +const USER_ATTRIBUTES = ["alias", "order", "hideOneOffButton"]; + +/** + * Truncates big blobs of (data-)URIs to console-friendly sizes + * + * @param {string} str + * String to tone down + * @param {number} len + * Maximum length of the string to return. Defaults to the length of a tweet. + * @returns {string} + * The shortend string. + */ +function limitURILength(str, len) { + len = len || 140; + if (str.length > len) { + return str.slice(0, len) + "..."; + } + return str; +} + +/** + * Tries to rescale an icon to a given size. + * + * @param {Array} byteArray + * Byte array containing the icon payload. + * @param {string} contentType + * Mime type of the payload. + * @param {number} [size] + * Desired icon size. + * @returns {Array} + * An array of two elements - an array of integers and a string for the content + * type. + * @throws if the icon cannot be rescaled or the rescaled icon is too big. + */ +function rescaleIcon(byteArray, contentType, size = 32) { + if (contentType == "image/svg+xml") { + throw new Error("Cannot rescale SVG image"); + } + + let imgTools = Cc["@mozilla.org/image/tools;1"].getService(Ci.imgITools); + let arrayBuffer = new Int8Array(byteArray).buffer; + let container = imgTools.decodeImageFromArrayBuffer(arrayBuffer, contentType); + let stream = imgTools.encodeScaledImage(container, "image/png", size, size); + let streamSize = stream.available(); + if (streamSize > lazy.SearchUtils.MAX_ICON_SIZE) { + throw new Error("Icon is too big"); + } + let bis = new BinaryInputStream(stream); + return [bis.readByteArray(streamSize), "image/png"]; +} + +/** + * A simple class to handle caching of preferences that may be read from + * parameters. + */ +const ParamPreferenceCache = { + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + initCache() { + // Preference params are normally only on the default branch to avoid these being easily changed. + // We allow them on the normal branch in nightly builds to make testing easier. + let branchFetcher = AppConstants.NIGHTLY_BUILD + ? "getBranch" + : "getDefaultBranch"; + this.branch = Services.prefs[branchFetcher]( + lazy.SearchUtils.BROWSER_SEARCH_PREF + "param." + ); + this.cache = new Map(); + this.nimbusCache = new Map(); + for (let prefName of this.branch.getChildList("")) { + this.cache.set(prefName, this.branch.getCharPref(prefName, null)); + } + this.branch.addObserver("", this, true); + + this.onNimbusUpdate = this.onNimbusUpdate.bind(this); + this.onNimbusUpdate(); + lazy.NimbusFeatures.search.onUpdate(this.onNimbusUpdate); + lazy.NimbusFeatures.search.ready().then(this.onNimbusUpdate); + }, + + observe(subject, topic, data) { + this.cache.set(data, this.branch.getCharPref(data, null)); + }, + + onNimbusUpdate() { + let extraParams = + lazy.NimbusFeatures.search.getVariable("extraParams") || []; + this.nimbusCache.clear(); + // The try catch ensures that if the params were incorrect for some reason, + // the search service can still startup properly. + try { + for (const { key, value } of extraParams) { + this.nimbusCache.set(key, value); + } + } catch (ex) { + console.error("Failed to load nimbus variables for extraParams:", ex); + } + }, + + getPref(prefName) { + if (!this.cache) { + this.initCache(); + } + return this.nimbusCache.has(prefName) + ? this.nimbusCache.get(prefName) + : this.cache.get(prefName); + }, +}; + +/** + * Represents a name/value pair for a parameter + */ +class QueryParameter { + /** + * @param {string} name + * The parameter's name. Must not be null. + * @param {string} value + * The value of the parameter. May be an empty string, must not be null or + * undefined. + * @param {string} purpose + * The search purpose for which matches when this parameter should be + * applied, e.g. "searchbar", "contextmenu". + */ + constructor(name, value, purpose = null) { + if (!name || value == null) { + throw Components.Exception( + "missing name or value for QueryParameter!", + Cr.NS_ERROR_INVALID_ARG + ); + } + + this.name = name; + this._value = value; + this.purpose = purpose; + } + + get value() { + return this._value; + } + + toJSON() { + const result = { + name: this.name, + value: this.value, + }; + if (this.purpose) { + result.purpose = this.purpose; + } + return result; + } +} + +/** + * Represents a special paramater that can be set by preferences. The + * value is read from the 'browser.search.param.*' default preference + * branch. + */ +class QueryPreferenceParameter extends QueryParameter { + /** + * @param {string} name + * The name of the parameter as injected into the query string. + * @param {string} prefName + * The name of the preference to read from the branch. + * @param {string} purpose + * The search purpose for which matches when this parameter should be + * applied, e.g. `searchbar`, `contextmenu`. + */ + constructor(name, prefName, purpose) { + super(name, prefName, purpose); + } + + get value() { + const prefValue = ParamPreferenceCache.getPref(this._value); + return prefValue ? encodeURIComponent(prefValue) : null; + } + + /** + * Converts the object to json. This object is converted with a mozparam flag + * as it gets written to the cache and hence we then know what type it is + * when reading it back. + * + * @returns {object} + */ + toJSON() { + const result = { + condition: "pref", + mozparam: true, + name: this.name, + pref: this._value, + }; + if (this.purpose) { + result.purpose = this.purpose; + } + return result; + } +} + +/** + * Perform OpenSearch parameter substitution on aParamValue. + * + * @see http://opensearch.a9.com/spec/1.1/querysyntax/#core + * + * @param {string} paramValue + * The OpenSearch search parameters. + * @param {string} searchTerms + * The user-provided search terms. This string will inserted into + * paramValue as the value of the OS_PARAM_USER_DEFINED parameter. + * This value must already be escaped appropriately - it is inserted + * as-is. + * @param {nsISearchEngine} engine + * The engine which owns the string being acted on. + * @returns {string} + * An updated parameter string. + */ +function ParamSubstitution(paramValue, searchTerms, engine) { + const PARAM_REGEXP = /\{((?:\w+:)?\w+)(\??)\}/g; + return paramValue.replace(PARAM_REGEXP, function (match, name, optional) { + // {searchTerms} is by far the most common param so handle it first. + if (name == USER_DEFINED) { + return searchTerms; + } + + // {inputEncoding} is the second most common param. + if (name == OS_PARAM_INPUT_ENCODING) { + return engine.queryCharset; + } + + // moz: parameters are only available for default search engines. + if (engine.isAppProvided && name == lazy.SearchUtils.MOZ_PARAM.LOCALE) { + return Services.locale.requestedLocale; + } + + // Handle the less common OpenSearch parameters we're confident about. + if (name == OS_PARAM_LANGUAGE) { + return Services.locale.requestedLocale || OS_PARAM_LANGUAGE_DEF; + } + if (name == OS_PARAM_OUTPUT_ENCODING) { + return OS_PARAM_OUTPUT_ENCODING_DEF; + } + + // At this point, if a parameter is optional, just omit it. + if (optional) { + return ""; + } + + // Replace unsupported parameters that only have hardcoded default values. + for (let param of OS_UNSUPPORTED_PARAMS) { + if (name == param[0]) { + return param[1]; + } + } + + // Don't replace unknown non-optional parameters. + return match; + }); +} + +/** + * EngineURL holds a query URL and all associated parameters. + */ +export class EngineURL { + params = []; + rels = []; + + /** + * Creates an EngineURL. + * + * @param {string} mimeType + * The name of the MIME type of the search results returned by this URL. + * @param {string} requestMethod + * The HTTP request method. Must be a case insensitive value of either + * "GET" or "POST". + * @param {string} template + * The URL to which search queries should be sent. For GET requests, + * must contain the string "{searchTerms}", to indicate where the user + * entered search terms should be inserted. + * + * @see http://opensearch.a9.com/spec/1.1/querysyntax/#urltag + * + * @throws NS_ERROR_NOT_IMPLEMENTED if aType is unsupported. + */ + constructor(mimeType, requestMethod, template) { + if (!mimeType || !requestMethod || !template) { + throw Components.Exception( + "missing mimeType, method or template for EngineURL!", + Cr.NS_ERROR_INVALID_ARG + ); + } + + var method = requestMethod.toUpperCase(); + var type = mimeType.toLowerCase(); + + if (method != "GET" && method != "POST") { + throw Components.Exception( + 'method passed to EngineURL must be "GET" or "POST"', + Cr.NS_ERROR_INVALID_ARG + ); + } + + this.type = type; + this.method = method; + + var templateURI = lazy.SearchUtils.makeURI(template); + if (!templateURI) { + throw Components.Exception( + "new EngineURL: template is not a valid URI!", + Cr.NS_ERROR_FAILURE + ); + } + + switch (templateURI.scheme) { + case "http": + case "https": + this.template = template; + break; + default: + throw Components.Exception( + "new EngineURL: template uses invalid scheme!", + Cr.NS_ERROR_FAILURE + ); + } + + this.templateHost = templateURI.host; + } + + addParam(name, value, purpose) { + this.params.push(new QueryParameter(name, value, purpose)); + } + + /** + * Adds a MozParam to the parameters list for this URL. For purpose based params + * these are saved as standard parameters, for preference based we save them + * as a special type. + * + * @param {object} param + * The parameter to add. + * @param {string} param.name + * The name of the parameter to add to the url. + * @param {string} [param.condition] + * The type of parameter this is, e.g. "pref" for a preference parameter, + * or "purpose" for a value-based parameter with a specific purpose. The + * default is "purpose". + * @param {string} [param.value] + * The value if it is a "purpose" parameter. + * @param {string} [param.purpose] + * The purpose of the parameter for when it is applied, e.g. for `searchbar` + * searches. + * @param {string} [param.pref] + * The preference name of the parameter, that gets appended to + * `browser.search.param.`. + */ + _addMozParam(param) { + const purpose = param.purpose || undefined; + if (param.condition && param.condition == "pref") { + this.params.push( + new QueryPreferenceParameter(param.name, param.pref, purpose) + ); + } else { + this.addParam(param.name, param.value || undefined, purpose); + } + } + + getSubmission(searchTerms, engine, purpose) { + var url = ParamSubstitution(this.template, searchTerms, engine); + // Default to searchbar if the purpose is not provided + var requestPurpose = purpose || "searchbar"; + + // If a particular purpose isn't defined in the plugin, fallback to 'searchbar'. + if ( + requestPurpose != "searchbar" && + !this.params.some(p => p.purpose && p.purpose == requestPurpose) + ) { + requestPurpose = "searchbar"; + } + + // Create an application/x-www-form-urlencoded representation of our params + // (name=value&name=value&name=value) + let dataArray = []; + for (var i = 0; i < this.params.length; ++i) { + var param = this.params[i]; + + // If this parameter has a purpose, only add it if the purpose matches + if (param.purpose && param.purpose != requestPurpose) { + continue; + } + + // Preference MozParams might not have a preferenced saved, or a valid value. + if (param.value != null) { + var value = ParamSubstitution(param.value, searchTerms, engine); + + dataArray.push(param.name + "=" + value); + } + } + let dataString = dataArray.join("&"); + + var postData = null; + if (this.method == "GET") { + // GET method requests have no post data, and append the encoded + // query string to the url... + if (dataString) { + if (url.includes("?")) { + url = `${url}&${dataString}`; + } else { + url = `${url}?${dataString}`; + } + } + } else if (this.method == "POST") { + // POST method requests must wrap the encoded text in a MIME + // stream and supply that as POSTDATA. + var stringStream = Cc[ + "@mozilla.org/io/string-input-stream;1" + ].createInstance(Ci.nsIStringInputStream); + stringStream.data = dataString; + + postData = Cc["@mozilla.org/network/mime-input-stream;1"].createInstance( + Ci.nsIMIMEInputStream + ); + postData.addHeader("Content-Type", "application/x-www-form-urlencoded"); + postData.setData(stringStream); + } + + return new Submission(Services.io.newURI(url), postData); + } + + _getTermsParameterName() { + let searchTerms = "{" + USER_DEFINED + "}"; + let paramName = this.params.find(p => p.value == searchTerms)?.name; + // Some query params might not be added to this.params + // in the engine construction process, so try checking the URL + // template for the existence of the query parameter value. + if (!paramName) { + let urlParms = new URL(this.template).searchParams; + for (let [name, value] of urlParms.entries()) { + if (value == searchTerms) { + paramName = name; + break; + } + } + } + return paramName ?? ""; + } + + _hasRelation(rel) { + return this.rels.some(e => e == rel.toLowerCase()); + } + + _initWithJSON(json) { + if (!json.params) { + return; + } + + this.rels = json.rels; + + for (let i = 0; i < json.params.length; ++i) { + let param = json.params[i]; + // mozparam and purpose are only supported for app-provided engines. + // Since we do not store the details for those engines, we don't want + // to handle it here. + if (!param.mozparam && !param.purpose) { + this.addParam(param.name, param.value); + } + } + } + + /** + * Creates a JavaScript object that represents this URL. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + var json = { + params: this.params, + rels: this.rels, + template: this.template, + }; + + if (this.type != lazy.SearchUtils.URL_TYPE.SEARCH) { + json.type = this.type; + } + if (this.method != "GET") { + json.method = this.method; + } + + return json; + } +} + +/** + * SearchEngine represents WebExtension based search engines. + */ +export class SearchEngine { + QueryInterface = ChromeUtils.generateQI(["nsISearchEngine"]); + // Data set by the user. + _metaData = {}; + // Anonymized path of where we initially loaded the engine from. + // This will stay null for engines installed in the profile before we moved + // to a JSON storage. + _loadPath = null; + // The engine's description + _description = ""; + // Set to true if the engine has a preferred icon (an icon that should not be + // overridden by a non-preferred icon). + _hasPreferredIcon = null; + // The engine's name. + _name = null; + // The name of the charset used to submit the search terms. + _queryCharset = null; + // The engine's raw SearchForm value (URL string pointing to a search form). + #cachedSearchForm = null; + // The order hint from the configuration (if any). + _orderHint = null; + // The telemetry id from the configuration (if any). + _telemetryId = null; + // Set to true once the engine has been added to the store, and the initial + // notification sent. This allows to skip sending notifications during + // initialization. + _engineAddedToStore = false; + // The aliases coming from the engine definition (via webextension + // keyword field for example). + _definedAliases = []; + // The urls associated with this engine. + _urls = []; + // The query parameter name of the search url, cached in memory to avoid + // repeated look-ups. + _searchUrlQueryParamName = null; + // The known public suffix of the search url, cached in memory to avoid + // repeated look-ups. + _searchUrlPublicSuffix = null; + /** + * The unique id of the Search Engine. + * The id is an UUID. + * + * @type {string} + */ + #id; + + /** + * Creates a Search Engine. + * + * @param {object} options + * The options for this search engine. + * @param {string} [options.id] + * The identifier to use for this engine, if none is specified a random + * uuid is created. + * @param {string} options.loadPath + * The path of the engine was originally loaded from. Should be anonymized. + */ + constructor(options = {}) { + this.#id = options.id ?? this.#uuid(); + if (!("loadPath" in options)) { + throw new Error("loadPath missing from options."); + } + this._loadPath = options.loadPath; + } + + get _searchForm() { + return this.#cachedSearchForm; + } + set _searchForm(value) { + if (/^https?:/i.test(value)) { + this.#cachedSearchForm = value; + } else { + lazy.logConsole.debug( + "_searchForm: Invalid URL dropped for", + this._name || "the current engine" + ); + } + } + + /** + * Attempts to find an EngineURL object in the set of EngineURLs for + * this Engine that has the given type string. (This corresponds to the + * "type" attribute in the "Url" node in the OpenSearch spec.) + * + * @param {string} type + * The type to match the EngineURL's type attribute. + * @param {string} [rel] + * Only return URLs that with this rel value. + * @returns {EngineURL|null} + * Returns the first matching URL found, null otherwise. + */ + _getURLOfType(type, rel) { + for (let url of this._urls) { + if (url.type == type && (!rel || url._hasRelation(rel))) { + return url; + } + } + + return null; + } + + /** + * Creates a key by serializing an object that contains the icon's width + * and height. + * + * @param {number} width + * Width of the icon. + * @param {number} height + * Height of the icon. + * @returns {string} + * Key string. + */ + _getIconKey(width, height) { + let keyObj = { + width, + height, + }; + + return JSON.stringify(keyObj); + } + + /** + * Add an icon to the icon map used by getIconURL(). + * + * @param {number} width + * Width of the icon. + * @param {number} height + * Height of the icon. + * @param {string} uriSpec + * String with the icon's URI. + */ + _addIconToMap(width, height, uriSpec) { + if (width == 16 && height == 16) { + // The 16x16 icon is stored in _iconURL, we don't need to store it twice. + return; + } + + // Use an object instead of a Map() because it needs to be serializable. + this._iconMapObj = this._iconMapObj || {}; + let key = this._getIconKey(width, height); + this._iconMapObj[key] = uriSpec; + } + + /** + * Sets the .iconURI property of the engine. If both aWidth and aHeight are + * provided an entry will be added to _iconMapObj that will enable accessing + * icon's data through getIconURL() APIs. + * + * @param {string} iconURL + * A URI string pointing to the engine's icon. Must have a http[s] + * or data scheme. Icons with HTTP[S] schemes will be + * downloaded and converted to data URIs for storage in the engine + * XML files, if the engine is not built-in. + * @param {boolean} isPreferred + * Whether or not this icon is to be preferred. Preferred icons can + * override non-preferred icons. + * @param {number} [width] + * Width of the icon. + * @param {number} [height] + * Height of the icon. + */ + _setIcon(iconURL, isPreferred, width, height) { + var uri = lazy.SearchUtils.makeURI(iconURL); + + // Ignore bad URIs + if (!uri) { + return; + } + + lazy.logConsole.debug( + "_setIcon: Setting icon url for", + this.name, + "to", + limitURILength(uri.spec) + ); + // Only accept remote icons from http[s] + switch (uri.scheme) { + // Fall through to the data case + case "moz-extension": + case "data": + if (!this._hasPreferredIcon || isPreferred) { + this._iconURI = uri; + + this._hasPreferredIcon = isPreferred; + } + + if (width && height) { + this._addIconToMap(width, height, iconURL); + } + break; + case "http": + case "https": + let iconLoadCallback = function (byteArray, contentType) { + // This callback may run after we've already set a preferred icon, + // so check again. + if (this._hasPreferredIcon && !isPreferred) { + return; + } + + if (!byteArray) { + lazy.logConsole.warn("iconLoadCallback: load failed"); + return; + } + + if (byteArray.length > lazy.SearchUtils.MAX_ICON_SIZE) { + try { + lazy.logConsole.debug("iconLoadCallback: rescaling icon"); + [byteArray, contentType] = rescaleIcon(byteArray, contentType); + } catch (ex) { + lazy.logConsole.error( + "Unable to set icon for the search engine:", + ex + ); + return; + } + } + + let dataURL = + "data:" + + contentType + + ";base64," + + btoa(String.fromCharCode.apply(null, byteArray)); + + this._iconURI = lazy.SearchUtils.makeURI(dataURL); + + if (width && height) { + this._addIconToMap(width, height, dataURL); + } + + if (this._engineAddedToStore) { + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + this._hasPreferredIcon = isPreferred; + }; + + let chan = lazy.SearchUtils.makeChannel( + uri, + Ci.nsIContentPolicy.TYPE_IMAGE + ); + let listener = new lazy.SearchUtils.LoadListener( + chan, + /^image\//, + iconLoadCallback.bind(this) + ); + chan.notificationCallbacks = listener; + chan.asyncOpen(listener); + break; + } + } + + /** + * Initialize an EngineURL object from metadata. + * + * @param {string} type + * The url type. + * @param {object} params + * The URL parameters. + * @param {string | Array} [params.getParams] + * Any parameters for a GET method. This is either a query string, or + * an array of objects which have name/value pairs. + * @param {string} [params.method] + * The type of method, defaults to GET. + * @param {string} [params.mozParams] + * Any special Mozilla Parameters. + * @param {string | Array} [params.postParams] + * Any parameters for a POST method. This is either a query string, or + * an array of objects which have name/value pairs. + * @param {string} params.template + * The url template. + * @returns {EngineURL} + * The newly created EngineURL. + */ + _getEngineURLFromMetaData(type, params) { + let url = new EngineURL(type, params.method || "GET", params.template); + + // Do the MozParams first, so that we are more likely to get the query + // on the end of the URL, rather than the MozParams (xref bug 1484232). + if (params.mozParams) { + for (let p of params.mozParams) { + if ((p.condition || p.purpose) && !this.isAppProvided) { + continue; + } + url._addMozParam(p); + } + } + if (params.postParams) { + if (Array.isArray(params.postParams)) { + for (let { name, value } of params.postParams) { + url.addParam(name, value); + } + } else { + for (let [name, value] of new URLSearchParams(params.postParams)) { + url.addParam(name, value); + } + } + } + + if (params.getParams) { + if (Array.isArray(params.getParams)) { + for (let { name, value } of params.getParams) { + url.addParam(name, value); + } + } else { + for (let [name, value] of new URLSearchParams(params.getParams)) { + url.addParam(name, value); + } + } + } + + return url; + } + + /** + * Initialize this engine object. + * + * @param {object} details + * The details of the engine. + * @param {string} details.name + * The name of the engine. + * @param {string} details.keyword + * The keyword for the engine. + * @param {string} details.iconURL + * The url to use for the icon of the engine. + * @param {string} details.search_url + * The search url template for the engine. + * @param {string} [details.search_url_get_params] + * The search url parameters for use with the GET method. + * @param {string} [details.search_url_post_params] + * The search url parameters for use with the POST method. + * @param {object} [details.params] + * Any special Mozilla parameters. + * @param {string} [details.suggest_url] + * The suggestion url template for the engine. + * @param {string} [details.suggest_url_get_params] + * The suggestion url parameters for use with the GET method. + * @param {string} [details.suggest_url_post_params] + * The suggestion url parameters for use with the POST method. + * @param {string} [details.encoding] + * The encoding to use for the engine. + * @param {string} [details.search_form] + * THe search form url for the engine. + * @param {object} [configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + */ + _initWithDetails(details, configuration = {}) { + this._orderHint = configuration.orderHint; + this._name = details.name.trim(); + + this._definedAliases = []; + if (Array.isArray(details.keyword)) { + this._definedAliases = details.keyword.map(k => k.trim()); + } else if (details.keyword?.trim()) { + this._definedAliases = [details.keyword?.trim()]; + } + + this._description = details.description; + if (details.iconURL) { + this._setIcon(details.iconURL, true); + } + this._setUrls(details, configuration); + } + + /** + * This sets the urls for the search engine based on the supplied parameters. + * If you add anything here, please consider if it needs to be handled in the + * overrideWithEngine / removeExtensionOverride functions as well. + * + * @param {object} details + * The details of the engine. + * @param {string} details.search_url + * The search url template for the engine. + * @param {string} [details.search_url_get_params] + * The search url parameters for use with the GET method. + * @param {string} [details.search_url_post_params] + * The search url parameters for use with the POST method. + * @param {object} [details.params] + * Any special Mozilla parameters. + * @param {string} [details.suggest_url] + * The suggestion url template for the engine. + * @param {string} [details.suggest_url_get_params] + * The suggestion url parameters for use with the GET method. + * @param {string} [details.suggest_url_post_params] + * The suggestion url parameters for use with the POST method. + * @param {string} [details.encoding] + * The encoding to use for the engine. + * @param {string} [details.search_form] + * THe search form url for the engine. + * @param {object} [configuration] + * The search engine configuration for application provided engines, that + * may be overriding some of the WebExtension's settings. + */ + _setUrls(details, configuration = {}) { + let postParams = + configuration.params?.searchUrlPostParams || + details.search_url_post_params || + ""; + let url = this._getEngineURLFromMetaData(lazy.SearchUtils.URL_TYPE.SEARCH, { + method: (postParams && "POST") || "GET", + // AddonManager will sometimes encode the URL via `new URL()`. We want + // to ensure we're always dealing with decoded urls. + template: decodeURI(details.search_url), + getParams: + configuration.params?.searchUrlGetParams || + details.search_url_get_params || + "", + postParams, + mozParams: configuration.extraParams || details.params || [], + }); + + this._urls.push(url); + + if (details.suggest_url) { + let suggestPostParams = + configuration.params?.suggestUrlPostParams || + details.suggest_url_post_params || + ""; + url = this._getEngineURLFromMetaData( + lazy.SearchUtils.URL_TYPE.SUGGEST_JSON, + { + method: (suggestPostParams && "POST") || "GET", + // suggest_url doesn't currently get encoded. + template: details.suggest_url, + getParams: + configuration.params?.suggestUrlGetParams || + details.suggest_url_get_params || + "", + postParams: suggestPostParams, + mozParams: configuration.suggestExtraParams || [], + } + ); + + this._urls.push(url); + } + + if (configuration?.urls?.trending) { + let trending = this._getEngineURLFromMetaData( + lazy.SearchUtils.URL_TYPE.TRENDING_JSON, + { + method: "GET", + template: decodeURI(configuration.urls.trending.fullPath), + getParams: configuration.urls.trending.query, + } + ); + this._urls.push(trending); + } + + if (configuration.clickUrl) { + this.clickUrl = configuration.clickUrl; + } + + if (details.encoding) { + this._queryCharset = details.encoding; + } + this.#cachedSearchForm = details.search_form; + } + + checkSearchUrlMatchesManifest(details) { + let existingUrl = this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH); + + let newUrl = this._getEngineURLFromMetaData( + lazy.SearchUtils.URL_TYPE.SEARCH, + { + method: (details.search_url_post_params && "POST") || "GET", + // AddonManager will sometimes encode the URL via `new URL()`. We want + // to ensure we're always dealing with decoded urls. + template: decodeURI(details.search_url), + getParams: details.search_url_get_params || "", + postParams: details.search_url_post_params || "", + } + ); + + let existingSubmission = existingUrl.getSubmission("", this); + let newSubmission = newUrl.getSubmission("", this); + + return ( + existingSubmission.uri.equals(newSubmission.uri) && + existingSubmission.postData?.data.data == + newSubmission.postData?.data.data + ); + } + + /** + * Overrides the urls/parameters with those of the provided engine or extension. + * The url parameters are not saved to the search settings - the code handling + * the extension should set these on every restart, this avoids potential + * third party modifications and means that we can verify the WebExtension is + * still in the allow list. + * + * @param {string} options + * The options for this function. + * @param {AddonSearchEngine|OpenSearchEngine} [options.engine] + * The search engine to override with this engine. If not specified, `manifest` + * must be provided. + * @param {object} [options.extension] + * An object representing the WebExtensions. If not specified, + * `engine` must be provided + */ + overrideWithEngine({ engine, extension }) { + this._overriddenData = { + urls: this._urls, + queryCharset: this._queryCharset, + searchForm: this.#cachedSearchForm, + }; + if (engine) { + // Copy any saved user data (alias, order etc). + this.copyUserSettingsFrom(engine); + + this._urls = engine._urls; + this.setAttr("overriddenBy", engine._extensionID ?? engine.id); + if (engine instanceof lazy.OpenSearchEngine) { + this.setAttr("overriddenByOpenSearch", engine.toJSON()); + } + } else { + this._urls = []; + this.setAttr("overriddenBy", extension.id); + this._setUrls( + extension.manifest.chrome_settings_overrides.search_provider + ); + } + lazy.SearchUtils.notifyAction(this, lazy.SearchUtils.MODIFIED_TYPE.CHANGED); + } + + /** + * Resets the overrides for the engine if it has been overridden. + */ + removeExtensionOverride() { + if (this.getAttr("overriddenBy")) { + // If the attribute is set, but there is no data, skip it. Worst case, + // the urls will be reset on a restart. + if (this._overriddenData) { + this._urls = this._overriddenData.urls; + this._queryCharset = this._overriddenData.queryCharset; + this.#cachedSearchForm = this._overriddenData.searchForm; + delete this._overriddenData; + } else { + lazy.logConsole.error( + `${this._name} had overriddenBy set, but no _overriddenData` + ); + } + this.clearAttr("overriddenBy"); + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + } + + /** + * Copies settings from the supplied search engine. Typically used for + * restoring settings when removing an override. + * + * @param {SearchEngine|object} engine + * The engine to copy the settings from, or the engine settings from + * the user's saved settings. + */ + copyUserSettingsFrom(engine) { + for (let attribute of USER_ATTRIBUTES) { + if (attribute in engine._metaData) { + this._metaData[attribute] = engine._metaData[attribute]; + } + } + } + + /** + * Init from a JSON record. + * + * @param {object} json + * The json record to use. + */ + _initWithJSON(json) { + this.#id = json.id ?? this.#id; + this._name = json._name; + this._description = json.description; + this._hasPreferredIcon = json._hasPreferredIcon == undefined; + this._queryCharset = + json.queryCharset || lazy.SearchUtils.DEFAULT_QUERY_CHARSET; + this.#cachedSearchForm = json.__searchForm; + this._iconURI = lazy.SearchUtils.makeURI(json._iconURL); + this._iconMapObj = json._iconMapObj || null; + this._metaData = json._metaData || {}; + this._orderHint = json._orderHint || null; + this._definedAliases = json._definedAliases || []; + // These changed keys in Firefox 80, maintain the old keys + // for backwards compatibility. + if (json._definedAlias) { + this._definedAliases.push(json._definedAlias); + } + this._filePath = json.filePath || json._filePath || null; + + for (let i = 0; i < json._urls.length; ++i) { + let url = json._urls[i]; + let engineURL = new EngineURL( + url.type || lazy.SearchUtils.URL_TYPE.SEARCH, + url.method || "GET", + url.template + ); + engineURL._initWithJSON(url); + this._urls.push(engineURL); + } + } + + /** + * Creates a JavaScript object that represents this engine. + * + * @returns {object} + * An object suitable for serialization as JSON. + */ + toJSON() { + const fieldsToCopy = [ + "id", + "_name", + "_loadPath", + "description", + "_iconURL", + "_iconMapObj", + "_metaData", + "_urls", + "_orderHint", + "_telemetryId", + "_filePath", + "_definedAliases", + ]; + + let json = {}; + for (const field of fieldsToCopy) { + if (field in this) { + json[field] = this[field]; + } + } + + if (this.#cachedSearchForm) { + json.__searchForm = this.#cachedSearchForm; + } + if (!this._hasPreferredIcon) { + json._hasPreferredIcon = this._hasPreferredIcon; + } + if (this.queryCharset != lazy.SearchUtils.DEFAULT_QUERY_CHARSET) { + json.queryCharset = this.queryCharset; + } + + return json; + } + + setAttr(name, val) { + this._metaData[name] = val; + } + + getAttr(name) { + return this._metaData[name] || undefined; + } + + clearAttr(name) { + delete this._metaData[name]; + } + + /** + * Loads engine settings (_metaData) from the list of settings, finding + * the appropriate details for this engine. + * + * @param {object} [settings] + * The saved settings for the user. + */ + _loadSettings(settings) { + if (!settings) { + return; + } + + let engineSettings; + if (settings.version <= 6) { + engineSettings = settings.engines?.find(e => e._name == this.name); + } else { + engineSettings = settings.engines?.find(e => e.id == this.id); + } + if (engineSettings?._metaData) { + this._metaData = structuredClone(engineSettings._metaData); + } + } + + /** + * Get the user-defined alias. + * + * @type {string} + */ + get alias() { + return this.getAttr("alias") || ""; + } + + set alias(val) { + var value = val ? val.trim() : ""; + if (value != this.alias) { + this.setAttr("alias", value); + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + } + + /** + * Returns a list of aliases, including a user defined alias and + * a list defined by webextension keywords. + * + * @returns {Array} + */ + get aliases() { + return [ + ...(this.getAttr("alias") ? [this.getAttr("alias")] : []), + ...this._definedAliases, + ]; + } + + /** + * Returns the appropriate identifier to use for telemetry. It is based on + * the following order: + * + * - telemetryId: The telemetry id from the configuration, or derived from + * the WebExtension name. + * - other-<name>: The engine name prefixed by `other-` for non-app-provided + * engines. + * + * @returns {string} + */ + get telemetryId() { + let telemetryId = this._telemetryId || `other-${this.name}`; + if (this.getAttr("overriddenBy")) { + return telemetryId + "-addon"; + } + return telemetryId; + } + + /** + * Return the built-in identifier of app-provided engines. + * + * @returns {string|null} + * Returns a valid if this is a built-in engine, null otherwise. + */ + get identifier() { + // No identifier if If the engine isn't app-provided + return this.isAppProvided ? this._telemetryId : null; + } + + get description() { + return this._description; + } + + get hidden() { + return this.getAttr("hidden") || false; + } + set hidden(val) { + var value = !!val; + if (value != this.hidden) { + this.setAttr("hidden", value); + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + } + + get hideOneOffButton() { + return this.getAttr("hideOneOffButton") || false; + } + set hideOneOffButton(val) { + const value = !!val; + if (value != this.hideOneOffButton) { + this.setAttr("hideOneOffButton", value); + lazy.SearchUtils.notifyAction( + this, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + } + + get _iconURL() { + if (!this._iconURI) { + return ""; + } + return this._iconURI.spec; + } + + // Where the engine is being loaded from: will return the URI's spec if the + // engine is being downloaded and does not yet have a file. This is only used + // for logging and error messages. + get _location() { + if (this._uri) { + return this._uri.spec; + } + + return this._loadPath; + } + + /** + * Whether or not this engine is provided by the application, e.g. it is + * in the list of configured search engines. + * + * @returns {boolean} + * This returns false for most engines, but may be overridden by particular + * engine types, such as add-on engines which are used by the application. + */ + get isAppProvided() { + return false; + } + + /** + * Whether or not this engine is an in-memory only search engine. + * These engines are typically application provided or policy engines, + * where they are loaded every time on SearchService initialization + * using the policy JSON or the extension manifest. Minimal details of the + * in-memory engines are saved to disk, but they are never loaded + * from the user's saved settings file. + * + * @returns {boolean} + * This results false for most engines, but may be overridden by particular + * engine types, such as add-on engines and policy engines. + */ + get inMemory() { + return false; + } + + get isGeneralPurposeEngine() { + return false; + } + + get _hasUpdates() { + return false; + } + + get name() { + return this._name; + } + + /** + * The searchForm URL points to the engine's organic search page. This should + * not contain neither search term parameters nor partner codes, but may + * contain parameters which set the engine in the correct way. + * + * This URL is typically the prePath and filePath of the search submission URI, + * but may vary for different engines. For example, some engines may use a + * different domain, e.g. https://sub.example.com for the search URI but + * https://example.org/ for the organic search page. + * + * @returns {string} + */ + get searchForm() { + // First look for a <Url rel="searchform"> + var searchFormURL = this._getURLOfType( + lazy.SearchUtils.URL_TYPE.SEARCH, + "searchform" + ); + if (searchFormURL) { + let submission = searchFormURL.getSubmission("", this); + + // If the rel=searchform URL is not type="get" (i.e. has postData), + // ignore it, since we can only return a URL. + if (!submission.postData) { + return submission.uri.spec; + } + } + + if (!this._searchForm) { + // No SearchForm specified in the engine definition file, use the prePath + // (e.g. https://foo.com for https://foo.com/search.php?q=bar). + var htmlUrl = this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH); + if (!htmlUrl) { + throw Components.Exception( + "Engine has no HTML URL!", + Cr.NS_ERROR_UNEXPECTED + ); + } + this._searchForm = lazy.SearchUtils.makeURI(htmlUrl.template).prePath; + } + + return ParamSubstitution(this._searchForm, "", this); + } + + get queryCharset() { + return this._queryCharset || lazy.SearchUtils.DEFAULT_QUERY_CHARSET; + } + + /** + * Gets an object that contains information about what to send to the search + * engine, for a request. This will be a URI and may also include data for POST + * requests. + * + * @param {string} searchTerms + * The search term(s) for the submission. + * Note: If an empty data string is supplied, the search form of the search + * engine will be returned. This is intentional, as in some cases on the current + * UI an empty search is intended to open the search engine's home/search page. + * @param {lazy.SearchUtils.URL_TYPE} [responseType] + * The MIME type that we'd like to receive in response + * to this submission. If null, will default to "text/html". + * @param {string} [purpose] + * A string that indicates the context of the search request. This may then + * be used to provide different submission data depending on the context. + * @returns {nsISearchSubmission|null} + * The submission data. If no appropriate submission can be determined for + * the request type, this may be null. + */ + getSubmission(searchTerms, responseType, purpose) { + // We can't use a default parameter as that doesn't work correctly with + // the idl interfaces. + if (!responseType) { + responseType = lazy.SearchUtils.URL_TYPE.SEARCH; + } + + var url = this._getURLOfType(responseType); + + if (!url) { + return null; + } + + if ( + !searchTerms && + responseType != lazy.SearchUtils.URL_TYPE.TRENDING_JSON + ) { + // Return a dummy submission object with our searchForm attribute + return new Submission(lazy.SearchUtils.makeURI(this.searchForm)); + } + + var submissionData = ""; + try { + submissionData = Services.textToSubURI.ConvertAndEscape( + this.queryCharset, + searchTerms + ); + } catch (ex) { + lazy.logConsole.warn( + "getSubmission: Falling back to default queryCharset!" + ); + submissionData = Services.textToSubURI.ConvertAndEscape( + lazy.SearchUtils.DEFAULT_QUERY_CHARSET, + searchTerms + ); + } + return url.getSubmission(submissionData, this, purpose); + } + + /** + * Returns a search URL with no search terms. This is typically used for + * purposes where we want to check something on the URL, but not use it for + * an actual submission to the search engine. + * + * Note: getSubmission cannot be used for this case, as that returns the + * search form when passed an empty string. + * + * @returns {nsIURI} + */ + get searchURLWithNoTerms() { + return this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH).getSubmission( + "", + this + ).uri; + } + + /** + * Returns the search term of a possible search result URI if and only if: + * - The URI has the same scheme, host, and path as the engine. + * - All query parameters of the URI have a matching name and value in the engine. + * - An exception to the equality check is the engine's termsParameterName + * value, which contains a placeholder, i.e. {searchTerms}. + * - If an engine has query parameters with "null" values, they will be ignored. + * + * @param {nsIURI} uri + * A URI that may or may not be from a search result matching the engine. + * + * @returns {string} + * A string representing the termsParameterName value of the URI, + * or an empty string if the URI isn't matched to the engine. + */ + searchTermFromResult(uri) { + let url = this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH); + if (!url) { + return ""; + } + + // The engine URL and URI should have the same scheme, host, and path. + if ( + uri.spec.split("?")[0].toLowerCase() != + url.template.split("?")[0].toLowerCase() + ) { + return ""; + } + + let engineParams; + if (url.params.length) { + engineParams = new URLSearchParams(); + for (let { name, value } of url.params) { + // Some values might be null, so avoid adding + // them since the input is unlikely to have it too. + if (value) { + // Use append() rather than set() so multiple + // values of the same name can be stored. + engineParams.append(name, value); + } + } + } else { + // Try checking the template for the presence of query params. + engineParams = new URL(url.template).searchParams; + } + + let uriParams = new URLSearchParams(uri.query); + if ( + new Set([...uriParams.keys()]).size != + new Set([...engineParams.keys()]).size + ) { + return ""; + } + + let termsParameterName = this.getURLParsingInfo().termsParameterName; + for (let [name, value] of uriParams.entries()) { + // Don't check the name matching the search + // query because its value will differ. + if (name == termsParameterName) { + continue; + } + // All params of an input must have a matching + // key and value in the list of engine parameters. + if (!engineParams.getAll(name).includes(value)) { + return ""; + } + } + + // An engine can use a non UTF-8 charset, which URLSearchParams + // might not parse properly. Convert the terms parameter value + // from the original input using the appropriate charset. + if (this.queryCharset.toLowerCase() != "utf-8") { + let name = `${termsParameterName}=`; + let queryString = uri.query + .split("&") + .filter(str => str.startsWith(name)) + .pop(); + return Services.textToSubURI.UnEscapeAndConvert( + this.queryCharset, + queryString.substring(queryString.indexOf("=") + 1).replace(/\+/g, " ") + ); + } + + return uriParams.get(termsParameterName); + } + + get searchUrlQueryParamName() { + if (this._searchUrlQueryParamName != null) { + return this._searchUrlQueryParamName; + } + + let submission = this.getSubmission( + "{searchTerms}", + lazy.SearchUtils.URL_TYPE.SEARCH + ); + + if (submission.postData) { + console.error("searchUrlQueryParamName can't handle POST urls."); + return (this._searchUrlQueryParamName = ""); + } + + let queryParams = new URLSearchParams(submission.uri.query); + let searchUrlQueryParamName = ""; + for (let [key, value] of queryParams) { + if (value == "{searchTerms}") { + searchUrlQueryParamName = key; + } + } + + return (this._searchUrlQueryParamName = searchUrlQueryParamName); + } + + get searchUrlPublicSuffix() { + if (this._searchUrlPublicSuffix != null) { + return this._searchUrlPublicSuffix; + } + let searchURLPublicSuffix = Services.eTLD.getKnownPublicSuffix( + this.searchURLWithNoTerms + ); + return (this._searchUrlPublicSuffix = searchURLPublicSuffix); + } + + // from nsISearchEngine + supportsResponseType(type) { + return this._getURLOfType(type) != null; + } + + // from nsISearchEngine + get searchUrlDomain() { + let url = this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH); + if (url) { + return url.templateHost; + } + return ""; + } + + /** + * @returns {object} + * URL parsing properties used by _buildParseSubmissionMap. + */ + getURLParsingInfo() { + let url = this._getURLOfType(lazy.SearchUtils.URL_TYPE.SEARCH); + if (!url || url.method != "GET") { + return null; + } + + let termsParameterName = url._getTermsParameterName(); + if (!termsParameterName) { + return null; + } + + let templateUrl = Services.io.newURI(url.template); + return { + mainDomain: templateUrl.host, + path: templateUrl.filePath.toLowerCase(), + termsParameterName, + }; + } + + get wrappedJSObject() { + return this; + } + + /** + * Retrieves the icon URL for this search engine, if any. + * + * @param {number} preferredWidth + * Width of the requested icon. If not specified, it is assumed that + * 16x16 is desired. + * @returns {string|undefined} + */ + getIconURL(preferredWidth) { + // XPCOM interfaces pass optional number parameters as 0 and can't be + // handled in the same way. + if (!preferredWidth) { + preferredWidth = 16; + } + + if (preferredWidth == 16) { + return this._iconURL || undefined; + } + + if (!this._iconMapObj) { + return undefined; + } + + let key = this._getIconKey(preferredWidth, preferredWidth); + if (key in this._iconMapObj) { + return this._iconMapObj[key]; + } + return undefined; + } + + /** + * Opens a speculative connection to the engine's search URI + * (and suggest URI, if different) to reduce request latency + * + * @param {object} options + * The options object + * @param {DOMWindow} options.window + * The content window for the window performing the search. + * @param {object} options.originAttributes + * The originAttributes for performing the search + * @throws NS_ERROR_INVALID_ARG if options is omitted or lacks required + * elements + */ + speculativeConnect(options) { + if (!options || !options.window) { + console.error( + "invalid options arg passed to nsISearchEngine.speculativeConnect" + ); + throw Components.Exception("", Cr.NS_ERROR_INVALID_ARG); + } + let connector = Services.io.QueryInterface(Ci.nsISpeculativeConnect); + + let searchURI = this.searchURLWithNoTerms; + + let callbacks = options.window.docShell.QueryInterface(Ci.nsILoadContext); + + // Using the content principal which is constructed by the search URI + // and given originAttributes. If originAttributes are not given, we + // fallback to use the docShell's originAttributes. + let attrs = options.originAttributes; + + if (!attrs) { + attrs = options.window.docShell.getOriginAttributes(); + } + + let principal = Services.scriptSecurityManager.createContentPrincipal( + searchURI, + attrs + ); + + try { + connector.speculativeConnect(searchURI, principal, callbacks, false); + } catch (e) { + // Can't setup speculative connection for this url, just ignore it. + console.error(e); + } + + if (this.supportsResponseType(lazy.SearchUtils.URL_TYPE.SUGGEST_JSON)) { + let suggestURI = this.getSubmission( + "dummy", + lazy.SearchUtils.URL_TYPE.SUGGEST_JSON + ).uri; + if (suggestURI.prePath != searchURI.prePath) { + try { + connector.speculativeConnect(suggestURI, principal, callbacks, false); + } catch (e) { + // Can't setup speculative connection for this url, just ignore it. + console.error(e); + } + } + } + } + + get id() { + return this.#id; + } + + /** + * Generates an UUID. + * + * @returns {string} + * An UUID string, without leading or trailing braces. + */ + #uuid() { + let uuid = Services.uuid.generateUUID().toString(); + return uuid.slice(1, uuid.length - 1); + } +} + +/** + * Implements nsISearchSubmission. + */ +class Submission { + QueryInterface = ChromeUtils.generateQI(["nsISearchSubmission"]); + + constructor(uri, postData = null) { + this._uri = uri; + this._postData = postData; + } + + get uri() { + return this._uri; + } + get postData() { + return this._postData; + } +} diff --git a/toolkit/components/search/SearchEngineSelector.sys.mjs b/toolkit/components/search/SearchEngineSelector.sys.mjs new file mode 100644 index 0000000000..0c9fb7cb70 --- /dev/null +++ b/toolkit/components/search/SearchEngineSelector.sys.mjs @@ -0,0 +1,743 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchEngineSelector", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +/** + * SearchEngineSelector parses the JSON configuration for + * search engines and returns the applicable engines depending + * on their region + locale. + */ +export class SearchEngineSelector { + /** + * @param {Function} listener + * A listener for configuration update changes. + */ + constructor(listener) { + this._remoteConfig = lazy.RemoteSettings(lazy.SearchUtils.NEW_SETTINGS_KEY); + this._remoteConfigOverrides = lazy.RemoteSettings( + lazy.SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + ); + this._listenerAdded = false; + this._onConfigurationUpdated = this._onConfigurationUpdated.bind(this); + this._onConfigurationOverridesUpdated = + this._onConfigurationOverridesUpdated.bind(this); + this._changeListener = listener; + } + + /** + * Resets the remote settings listeners. + */ + reset() { + if (this._listenerAdded) { + this._remoteConfig.off("sync", this._onConfigurationUpdated); + this._remoteConfigOverrides.off( + "sync", + this._onConfigurationOverridesUpdated + ); + this._listenerAdded = false; + } + } + + /** + * Handles getting the configuration from remote settings. + */ + async getEngineConfiguration() { + if (this._getConfigurationPromise) { + return this._getConfigurationPromise; + } + + this._getConfigurationPromise = Promise.all([ + this._getConfiguration(), + this._getConfigurationOverrides(), + ]); + let remoteSettingsData = await this._getConfigurationPromise; + this._configuration = remoteSettingsData[0]; + this._configurationOverrides = remoteSettingsData[1]; + delete this._getConfigurationPromise; + + if (!this._configuration?.length) { + throw Components.Exception( + "Failed to get engine data from Remote Settings", + Cr.NS_ERROR_UNEXPECTED + ); + } + + if (!this._listenerAdded) { + this._remoteConfig.on("sync", this._onConfigurationUpdated); + this._remoteConfigOverrides.on( + "sync", + this._onConfigurationOverridesUpdated + ); + this._listenerAdded = true; + } + + return this._configuration; + } + + /** + * Used by tests to get the configuration overrides. + */ + async getEngineConfigurationOverrides() { + await this.getEngineConfiguration(); + return this._configurationOverrides; + } + + /** + * Obtains the configuration from remote settings. This includes + * verifying the signature of the record within the database. + * + * If the signature in the database is invalid, the database will be wiped + * and the stored dump will be used, until the settings next update. + * + * Note that this may cause a network check of the certificate, but that + * should generally be quick. + * + * @param {boolean} [firstTime] + * Internal boolean to indicate if this is the first time check or not. + * @returns {Array} + * An array of objects in the database, or an empty array if none + * could be obtained. + */ + async _getConfiguration(firstTime = true) { + let result = []; + let failed = false; + try { + result = await this._remoteConfig.get({ + order: "id", + }); + } catch (ex) { + lazy.logConsole.error(ex); + failed = true; + } + if (!result.length) { + lazy.logConsole.error("Received empty search configuration!"); + failed = true; + } + // If we failed, or the result is empty, try loading from the local dump. + if (firstTime && failed) { + await this._remoteConfig.db.clear(); + // Now call this again. + return this._getConfiguration(false); + } + return result; + } + + /** + * Handles updating of the configuration. Note that the search service is + * only updated after a period where the user is observed to be idle. + * + * @param {object} options + * The options object + * @param {object} options.data + * The data to update + * @param {Array} options.data.current + * The new configuration object + */ + _onConfigurationUpdated({ data: { current } }) { + this._configuration = current; + lazy.logConsole.debug("Search configuration updated remotely"); + if (this._changeListener) { + this._changeListener(); + } + } + + /** + * Handles updating of the configuration. Note that the search service is + * only updated after a period where the user is observed to be idle. + * + * @param {object} options + * The options object + * @param {object} options.data + * The data to update + * @param {Array} options.data.current + * The new configuration object + */ + _onConfigurationOverridesUpdated({ data: { current } }) { + this._configurationOverrides = current; + lazy.logConsole.debug("Search configuration overrides updated remotely"); + if (this._changeListener) { + this._changeListener(); + } + } + + /** + * Obtains the configuration overrides from remote settings. + * + * @returns {Array} + * An array of objects in the database, or an empty array if none + * could be obtained. + */ + async _getConfigurationOverrides() { + let result = []; + try { + result = await this._remoteConfigOverrides.get(); + } catch (ex) { + // This data is remote only, so we just return an empty array if it fails. + } + return result; + } + + /** + * @param {object} options + * The options object + * @param {string} options.locale + * Users locale. + * @param {string} options.region + * Users region. + * @param {string} [options.channel] + * The update channel the application is running on. + * @param {string} [options.distroID] + * The distribution ID of the application. + * @param {string} [options.experiment] + * Any associated experiment id. + * @param {string} [options.appName] + * The name of the application. + * @param {string} [options.version] + * The version of the application. + * @returns {object} + * An object with "engines" field, a sorted list of engines and + * optionally "privateDefault" which is an object containing the engine + * details for the engine which should be the default in Private Browsing mode. + */ + async fetchEngineConfiguration({ + locale, + region, + channel = "default", + distroID, + experiment, + appName = Services.appinfo.name ?? "", + version = Services.appinfo.version ?? "", + }) { + if (!this._configuration) { + await this.getEngineConfiguration(); + } + + lazy.logConsole.debug( + `fetchEngineConfiguration ${locale}:${region}:${channel}:${distroID}:${experiment}:${appName}:${version}` + ); + + appName = appName.toLowerCase(); + version = version.toLowerCase(); + locale = locale.toLowerCase(); + region = region.toLowerCase(); + + let engines = []; + let defaultsConfig; + let engineOrders; + let userEnv = { + appName, + version, + locale, + region, + channel, + distroID, + experiment, + }; + + for (let config of this._configuration) { + if (config.recordType == "defaultEngines") { + defaultsConfig = config; + } + + if (config.recordType == "engineOrders") { + engineOrders = config; + } + + if (config.recordType !== "engine") { + continue; + } + + let variants = + config.variants?.filter(variant => + this.#matchesUserEnvironment(variant, userEnv) + ) ?? []; + + if (!variants.length) { + continue; + } + + let engine = structuredClone(config.base); + engine.identifier = config.identifier; + + for (let variant of variants) { + engine = this.#deepCopyObject(engine, variant); + } + + for (let override of this._configurationOverrides) { + if (override.identifier == engine.identifier) { + engine = this.#deepCopyObject(engine, override); + } + } + + engines.push(engine); + } + + let { defaultEngine, privateDefault } = this.#defaultEngines( + engines, + defaultsConfig, + userEnv + ); + + for (const orderData of engineOrders.orders) { + let environment = orderData.environment; + + if (this.#matchesUserEnvironment({ environment }, userEnv)) { + this.#setEngineOrders(engines, orderData.order); + } + } + + engines.sort(this._sort.bind(this, defaultEngine, privateDefault)); + + let result = { engines }; + + if (privateDefault) { + result.privateDefault = privateDefault; + } + + if (lazy.SearchUtils.loggingEnabled) { + lazy.logConsole.debug( + "fetchEngineConfiguration: " + result.engines.map(e => e.identifier) + ); + } + return result; + } + + _sort(defaultEngine, defaultPrivateEngine, a, b) { + return ( + this._sortIndex(b, defaultEngine, defaultPrivateEngine) - + this._sortIndex(a, defaultEngine, defaultPrivateEngine) + ); + } + + /** + * Create an index order to ensure default (and backup default) + * engines are ordered correctly. + * + * @param {object} obj + * Object representing the engine configuration. + * @param {object} defaultEngine + * The default engine, for comparison to obj. + * @param {object} defaultPrivateEngine + * The default private engine, for comparison to obj. + * @returns {integer} + * Number indicating how this engine should be sorted. + */ + _sortIndex(obj, defaultEngine, defaultPrivateEngine) { + if (obj == defaultEngine) { + return Number.MAX_SAFE_INTEGER; + } + if (obj == defaultPrivateEngine) { + return Number.MAX_SAFE_INTEGER - 1; + } + return obj.orderHint || 0; + } + + /** + * Deep copies an object to the target object and ignores some keys. + * + * @param {object} target - Object to copy to. + * @param {object} source - Object to copy from. + * @returns {object} - The source object. + */ + #deepCopyObject(target, source) { + for (let key in source) { + if (["environment"].includes(key)) { + continue; + } + + if (typeof source[key] == "object" && !Array.isArray(source[key])) { + if (key in target) { + this.#deepCopyObject(target[key], source[key]); + } else { + target[key] = structuredClone(source[key]); + } + } else { + target[key] = structuredClone(source[key]); + } + } + + return target; + } + + /** + * Matches the user's environment against the engine config's environment. + * + * @param {object} config + * The config for the given base or variant engine. + * @param {object} user + * The user's environment we use to match with the engine's environment. + * @param {string} user.appName + * The name of the application. + * @param {string} user.version + * The version of the application. + * @param {string} user.locale + * The locale of the user. + * @param {string} user.region + * The region of the user. + * @param {string} user.channel + * The channel the application is running on. + * @param {string} user.distroID + * The distribution ID of the application. + * @param {string} user.experiment + * Any associated experiment id. + * @returns {boolean} + * True if the engine config's environment matches the user's environment. + */ + #matchesUserEnvironment(config, user = {}) { + if ("experiment" in config.environment) { + if (user.experiment != config.environment.experiment) { + return false; + } + } + + if ("excludedDistributions" in config.environment) { + if (config.environment.excludedDistributions.includes(user.distroID)) { + return false; + } + } + + // Skip the optional flag for Desktop, it's a feature only on Android. + if (config.optional) { + return false; + } + + return ( + this.#matchesRegionAndLocale( + user.region, + user.locale, + config.environment + ) && + this.#matchesDistribution( + user.distroID, + config.environment.distributions + ) && + this.#matchesVersions( + config.environment.minVersion, + config.environment.maxVersion, + user.version + ) && + this.#matchesChannel(config.environment.channels, user.channel) && + this.#matchesApplication(config.environment.applications, user.appName) + ); + } + + /** + * @param {string} userDistro + * The distribution from the user's environment. + * @param {Array} configDistro + * An array of distributions for the particular environment in the config. + * @returns {boolean} + * True if the user's distribution is included in the config distribution + * list. + */ + #matchesDistribution(userDistro, configDistro) { + // If there's no distribution for this engineConfig, ignore the check. + if (!configDistro) { + return true; + } + + return configDistro?.includes(userDistro); + } + + /** + * @param {string} min + * The minimum version supported. + * @param {string} max + * The maximum version supported. + * @param {string} userVersion + * The user's version. + * @returns {boolean} + * True if the user's version is within the range of the min and max versions + * supported. + */ + #matchesVersions(min, max, userVersion) { + // If there's no versions for this engineConfig, ignore the check. + if (!min && !max) { + return true; + } + + if (!userVersion) { + return false; + } + + if (min && !max) { + return this.#isAboveOrEqualMin(userVersion, min); + } + + if (!min && max) { + return this.#isBelowOrEqualMax(userVersion, max); + } + + return ( + this.#isAboveOrEqualMin(userVersion, min) && + this.#isBelowOrEqualMax(userVersion, max) + ); + } + + #isAboveOrEqualMin(userVersion, min) { + return Services.vc.compare(userVersion, min) >= 0; + } + + #isBelowOrEqualMax(userVersion, max) { + return Services.vc.compare(userVersion, max) <= 0; + } + + /** + * @param {Array} configChannels + * Release channels such as nightly, beta, release, esr. + * @param {string} userChannel + * The user's channel. + * @returns {boolean} + * True if the user's channel is included in the config channels. + */ + #matchesChannel(configChannels, userChannel) { + // If there's no channels for this engineConfig, ignore the check. + if (!configChannels) { + return true; + } + + return configChannels.includes(userChannel); + } + + /** + * @param {Array} configApps + * The applications such as firefox, firefox-android, firefox-ios, + * focus-android, and focus-ios. + * @param {string} userApp + * The user's application. + * @returns {boolean} + * True if the user's application is included in the config applications. + */ + #matchesApplication(configApps, userApp) { + // If there's no config Applications for this engineConfig, ignore the check. + if (!configApps) { + return true; + } + + return configApps.includes(userApp); + } + + /** + * Determines whether the region and locale constraints in the config + * environment applies to a user given what region and locale they are using. + * + * @param {string} region + * The region the user is in. + * @param {string} locale + * The language the user has configured. + * @param {object} configEnv + * The environment of the engine configuration. + * @returns {boolean} + * True if the user's region and locale matches the config's region and + * locale contraints. Otherwise false. + */ + #matchesRegionAndLocale(region, locale, configEnv) { + if ( + this.#doesConfigInclude(configEnv.excludedLocales, locale) || + this.#doesConfigInclude(configEnv.excludedRegions, region) + ) { + return false; + } + + if (configEnv.allRegionsAndLocales) { + return true; + } + + // When none of the regions and locales are set. This implies its available + // everywhere. + if ( + !Object.hasOwn(configEnv, "allRegionsAndLocales") && + !Object.hasOwn(configEnv, "regions") && + !Object.hasOwn(configEnv, "locales") + ) { + return true; + } + + if ( + this.#doesConfigInclude(configEnv?.locales, locale) && + this.#doesConfigInclude(configEnv?.regions, region) + ) { + return true; + } + + if ( + this.#doesConfigInclude(configEnv?.locales, locale) && + !Object.hasOwn(configEnv, "regions") + ) { + return true; + } + + if ( + this.#doesConfigInclude(configEnv?.regions, region) && + !Object.hasOwn(configEnv, "locales") + ) { + return true; + } + + return false; + } + + /** + * This function converts the characters in the config to lowercase and + * checks if the user's locale or region is included in config the + * environment. + * + * @param {Array} configArray + * An Array of locales or regions from the config environment. + * @param {string} compareItem + * The user's locale or region. + * @returns {boolean} + * True if user's region or locale is found in the config environment. + * Otherwise false. + */ + #doesConfigInclude(configArray, compareItem) { + if (!configArray) { + return false; + } + + return configArray.find( + configItem => configItem.toLowerCase() === compareItem + ); + } + + /** + * Gets the default engine and default private engine based on the user's + * environment. + * + * @param {Array} engines + * An array that contains the engines for the user environment. + * @param {object} defaultsConfig + * The defaultEngines record type from the search config. + * @param {object} userEnv + * The user's environment. + * @returns {object} + * An object with default engine and default private engine. + */ + #defaultEngines(engines, defaultsConfig, userEnv) { + let defaultEngine, privateDefault; + + for (let data of defaultsConfig.specificDefaults) { + let environment = data.environment; + + if (this.#matchesUserEnvironment({ environment }, userEnv)) { + defaultEngine = this.#findDefault(engines, data) ?? defaultEngine; + privateDefault = + this.#findDefault(engines, data, "private") ?? privateDefault; + } + } + + defaultEngine ??= this.#findGlobalDefault(engines, defaultsConfig); + privateDefault ??= this.#findGlobalDefault( + engines, + defaultsConfig, + "private" + ); + + return { defaultEngine, privateDefault }; + } + + /** + * Finds the global default engine or global default private engine. + * + * @param {Array} engines + * The engines for the user environment. + * @param {string} config + * The defaultEngines record from the config. + * @param {string} [engineType] + * A string to identify default or default private. + * @returns {object} + * The global default engine or global default private engine. + */ + #findGlobalDefault(engines, config, engineType = "default") { + let engine; + if (config.globalDefault && engineType == "default") { + engine = engines.find(e => e.identifier == config.globalDefault); + } + + if (config.globalDefaultPrivate && engineType == "private") { + engine = engines.find(e => e.identifier == config.globalDefaultPrivate); + } + + return engine; + } + + /** + * Finds the default engine or default private engine from the list of + * engines that match the user's environment. + * + * @param {Array} engines + * The engines for the user environment. + * @param {string} config + * The specific defaults record that contains the default engine or default + * private engine identifer for the environment. + * @param {string} [engineType] + * A string to identify default engine or default private engine. + * @returns {object|undefined} + * The default engine or default private engine. Undefined if none can be + * found. + */ + #findDefault(engines, config, engineType = "default") { + let defaultMatch = + engineType == "default" ? config.default : config.defaultPrivate; + + if (!defaultMatch) { + return undefined; + } + + return this.#findEngineWithMatch(engines, defaultMatch); + } + + /** + * Sets the orderHint number for the engines. + * + * @param {Array} engines + * The engines for the user environment. + * @param {Array} orderedEngines + * The ordering of engines. Engines in the beginning of the list get a higher + * orderHint number. + */ + #setEngineOrders(engines, orderedEngines) { + let orderNumber = orderedEngines.length; + + for (const engine of orderedEngines) { + let foundEngine = this.#findEngineWithMatch(engines, engine); + if (foundEngine) { + foundEngine.orderHint = orderNumber; + orderNumber -= 1; + } + } + } + + /** + * Finds an engine with the given match. + * + * @param {object[]} engines + * An array of search engine configurations. + * @param {string} match + * A string to match against the engine identifier. This will be an exact + * match, unless the string ends with `*`, in which case it will use a + * startsWith match. + * @returns {object|undefined} + */ + #findEngineWithMatch(engines, match) { + if (match.endsWith("*")) { + let matchNoStar = match.slice(0, -1); + return engines.find(e => e.identifier.startsWith(matchNoStar)); + } + return engines.find(e => e.identifier == match); + } +} diff --git a/toolkit/components/search/SearchEngineSelectorOld.sys.mjs b/toolkit/components/search/SearchEngineSelectorOld.sys.mjs new file mode 100644 index 0000000000..3dab73ee2f --- /dev/null +++ b/toolkit/components/search/SearchEngineSelectorOld.sys.mjs @@ -0,0 +1,533 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +const USER_LOCALE = "$USER_LOCALE"; +const USER_REGION = "$USER_REGION"; + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchEngineSelector", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +function hasAppKey(config, key) { + return "application" in config && key in config.application; +} + +function sectionExcludes(config, key, value) { + return hasAppKey(config, key) && !config.application[key].includes(value); +} + +function sectionIncludes(config, key, value) { + return hasAppKey(config, key) && config.application[key].includes(value); +} + +function isDistroExcluded(config, key, distroID) { + // Should be excluded when: + // - There's a distroID and that is not in the non-empty distroID list. + // - There's no distroID and the distroID list is not empty. + const appKey = hasAppKey(config, key); + if (!appKey) { + return false; + } + const distroList = config.application[key]; + if (distroID) { + return distroList.length && !distroList.includes(distroID); + } + return !!distroList.length; +} + +function belowMinVersion(config, version) { + return ( + hasAppKey(config, "minVersion") && + Services.vc.compare(version, config.application.minVersion) < 0 + ); +} + +function aboveMaxVersion(config, version) { + return ( + hasAppKey(config, "maxVersion") && + Services.vc.compare(version, config.application.maxVersion) > 0 + ); +} + +/** + * SearchEngineSelector parses the JSON configuration for + * search engines and returns the applicable engines depending + * on their region + locale. + */ +export class SearchEngineSelectorOld { + /** + * @param {Function} listener + * A listener for configuration update changes. + */ + constructor(listener) { + this.QueryInterface = ChromeUtils.generateQI(["nsIObserver"]); + this._remoteConfig = lazy.RemoteSettings(lazy.SearchUtils.OLD_SETTINGS_KEY); + this._remoteConfigOverrides = lazy.RemoteSettings( + lazy.SearchUtils.OLD_SETTINGS_OVERRIDES_KEY + ); + this._listenerAdded = false; + this._onConfigurationUpdated = this._onConfigurationUpdated.bind(this); + this._onConfigurationOverridesUpdated = + this._onConfigurationOverridesUpdated.bind(this); + this._changeListener = listener; + } + + /** + * Resets the remote settings listeners. + */ + reset() { + if (this._listenerAdded) { + this._remoteConfig.off("sync", this._onConfigurationUpdated); + this._remoteConfigOverrides.off( + "sync", + this._onConfigurationOverridesUpdated + ); + this._listenerAdded = false; + } + } + + /** + * Handles getting the configuration from remote settings. + */ + async getEngineConfiguration() { + if (this._getConfigurationPromise) { + return this._getConfigurationPromise; + } + + this._getConfigurationPromise = Promise.all([ + this._getConfiguration(), + this._getConfigurationOverrides(), + ]); + let remoteSettingsData = await this._getConfigurationPromise; + this._configuration = remoteSettingsData[0]; + this._configurationOverrides = remoteSettingsData[1]; + delete this._getConfigurationPromise; + + if (!this._configuration?.length) { + throw Components.Exception( + "Failed to get engine data from Remote Settings", + Cr.NS_ERROR_UNEXPECTED + ); + } + + if (!this._listenerAdded) { + this._remoteConfig.on("sync", this._onConfigurationUpdated); + this._remoteConfigOverrides.on( + "sync", + this._onConfigurationOverridesUpdated + ); + this._listenerAdded = true; + } + + return this._configuration; + } + + /** + * Used by tests to get the configuration overrides. + */ + async getEngineConfigurationOverrides() { + await this.getEngineConfiguration(); + return this._configurationOverrides; + } + + /** + * Obtains the configuration from remote settings. This includes + * verifying the signature of the record within the database. + * + * If the signature in the database is invalid, the database will be wiped + * and the stored dump will be used, until the settings next update. + * + * Note that this may cause a network check of the certificate, but that + * should generally be quick. + * + * @param {boolean} [firstTime] + * Internal boolean to indicate if this is the first time check or not. + * @returns {Array} + * An array of objects in the database, or an empty array if none + * could be obtained. + */ + async _getConfiguration(firstTime = true) { + let result = []; + let failed = false; + try { + result = await this._remoteConfig.get({ + order: "id", + }); + } catch (ex) { + lazy.logConsole.error(ex); + failed = true; + } + if (!result.length) { + lazy.logConsole.error("Received empty search configuration!"); + failed = true; + } + // If we failed, or the result is empty, try loading from the local dump. + if (firstTime && failed) { + await this._remoteConfig.db.clear(); + // Now call this again. + return this._getConfiguration(false); + } + return result; + } + + /** + * Handles updating of the configuration. Note that the search service is + * only updated after a period where the user is observed to be idle. + * + * @param {object} options + * The options object + * @param {object} options.data + * The data to update + * @param {Array} options.data.current + * The new configuration object + */ + _onConfigurationUpdated({ data: { current } }) { + this._configuration = current; + lazy.logConsole.debug("Search configuration updated remotely"); + if (this._changeListener) { + this._changeListener(); + } + } + + /** + * Handles updating of the configuration. Note that the search service is + * only updated after a period where the user is observed to be idle. + * + * @param {object} options + * The options object + * @param {object} options.data + * The data to update + * @param {Array} options.data.current + * The new configuration object + */ + _onConfigurationOverridesUpdated({ data: { current } }) { + this._configurationOverrides = current; + lazy.logConsole.debug("Search configuration overrides updated remotely"); + if (this._changeListener) { + this._changeListener(); + } + } + + /** + * Obtains the configuration overrides from remote settings. + * + * @returns {Array} + * An array of objects in the database, or an empty array if none + * could be obtained. + */ + async _getConfigurationOverrides() { + let result = []; + try { + result = await this._remoteConfigOverrides.get(); + } catch (ex) { + // This data is remote only, so we just return an empty array if it fails. + } + return result; + } + + /** + * @param {object} options + * The options object + * @param {string} options.locale + * Users locale. + * @param {string} options.region + * Users region. + * @param {string} [options.channel] + * The update channel the application is running on. + * @param {string} [options.distroID] + * The distribution ID of the application. + * @param {string} [options.experiment] + * Any associated experiment id. + * @param {string} [options.name] + * The name of the application. + * @param {string} [options.version] + * The version of the application. + * @returns {object} + * An object with "engines" field, a sorted list of engines and + * optionally "privateDefault" which is an object containing the engine + * details for the engine which should be the default in Private Browsing mode. + */ + async fetchEngineConfiguration({ + locale, + region, + channel = "default", + distroID, + experiment, + name = Services.appinfo.name ?? "", + version = Services.appinfo.version ?? "", + }) { + if (!this._configuration) { + await this.getEngineConfiguration(); + } + lazy.logConsole.debug( + `fetchEngineConfiguration ${locale}:${region}:${channel}:${distroID}:${experiment}:${name}:${version}` + ); + let engines = []; + const lcName = name.toLowerCase(); + const lcVersion = version.toLowerCase(); + const lcLocale = locale.toLowerCase(); + const lcRegion = region.toLowerCase(); + for (let config of this._configuration) { + const appliesTo = config.appliesTo || []; + const applies = appliesTo.filter(section => { + if ("experiment" in section) { + if (experiment != section.experiment) { + return false; + } + if (section.override) { + return true; + } + } + + let shouldInclude = () => { + let included = + "included" in section && + this._isInSection(lcRegion, lcLocale, section.included); + let excluded = + "excluded" in section && + this._isInSection(lcRegion, lcLocale, section.excluded); + return included && !excluded; + }; + + const distroExcluded = + (distroID && + sectionIncludes(section, "excludedDistributions", distroID)) || + isDistroExcluded(section, "distributions", distroID); + + if (distroID && !distroExcluded && section.override) { + if ("included" in section || "excluded" in section) { + return shouldInclude(); + } + return true; + } + + if ( + sectionExcludes(section, "channel", channel) || + sectionExcludes(section, "name", lcName) || + distroExcluded || + belowMinVersion(section, lcVersion) || + aboveMaxVersion(section, lcVersion) + ) { + return false; + } + return shouldInclude(); + }); + + let baseConfig = this._copyObject({}, config); + + // Don't include any engines if every section is an override + // entry, these are only supposed to override otherwise + // included engine configurations. + let allOverrides = applies.every(e => "override" in e && e.override); + // Loop through all the appliedTo sections that apply to + // this configuration. + if (applies.length && !allOverrides) { + for (let section of applies) { + this._copyObject(baseConfig, section); + } + + if ( + "webExtension" in baseConfig && + "locales" in baseConfig.webExtension + ) { + for (const webExtensionLocale of baseConfig.webExtension.locales) { + const engine = { ...baseConfig }; + engine.webExtension = { ...baseConfig.webExtension }; + delete engine.webExtension.locales; + engine.webExtension.locale = webExtensionLocale + .replace(USER_LOCALE, locale) + .replace(USER_REGION, lcRegion); + engines.push(engine); + } + } else { + const engine = { ...baseConfig }; + (engine.webExtension = engine.webExtension || {}).locale = + lazy.SearchUtils.DEFAULT_TAG; + engines.push(engine); + } + } + } + + let defaultEngine; + let privateEngine; + + function shouldPrefer(setting, hasCurrentDefault, currentDefaultSetting) { + if ( + setting == "yes" && + (!hasCurrentDefault || currentDefaultSetting == "yes-if-no-other") + ) { + return true; + } + return setting == "yes-if-no-other" && !hasCurrentDefault; + } + + for (const engine of engines) { + engine.telemetryId = engine.telemetryId + ?.replace(USER_LOCALE, locale) + .replace(USER_REGION, lcRegion); + if ( + "default" in engine && + shouldPrefer( + engine.default, + !!defaultEngine, + defaultEngine && defaultEngine.default + ) + ) { + defaultEngine = engine; + } + if (engine.telemetryId) { + for (let override of this._configurationOverrides) { + if (override.telemetryId == engine.telemetryId) { + engine.params = this._copyObject( + engine.params || {}, + override.params + ); + if (override.clickUrl) { + engine.clickUrl = override.clickUrl; + } + if (override.telemetrySuffix) { + engine.telemetryId += `-${override.telemetrySuffix}`; + } + } + } + } + if ( + "defaultPrivate" in engine && + shouldPrefer( + engine.defaultPrivate, + !!privateEngine, + privateEngine && privateEngine.defaultPrivate + ) + ) { + privateEngine = engine; + } + } + + engines.sort(this._sort.bind(this, defaultEngine, privateEngine)); + + let result = { engines }; + + if (privateEngine) { + result.privateDefault = privateEngine; + } + + if (lazy.SearchUtils.loggingEnabled) { + lazy.logConsole.debug( + "fetchEngineConfiguration: " + + result.engines.map(e => e.webExtension.id) + ); + } + return result; + } + + _sort(defaultEngine, privateEngine, a, b) { + return ( + this._sortIndex(b, defaultEngine, privateEngine) - + this._sortIndex(a, defaultEngine, privateEngine) + ); + } + + /** + * Create an index order to ensure default (and backup default) + * engines are ordered correctly. + * + * @param {object} obj + * Object representing the engine configation. + * @param {object} defaultEngine + * The default engine, for comparison to obj. + * @param {object} privateEngine + * The private engine, for comparison to obj. + * @returns {integer} + * Number indicating how this engine should be sorted. + */ + _sortIndex(obj, defaultEngine, privateEngine) { + if (obj == defaultEngine) { + return Number.MAX_SAFE_INTEGER; + } + if (obj == privateEngine) { + return Number.MAX_SAFE_INTEGER - 1; + } + return obj.orderHint || 0; + } + + /** + * Is the engine marked to be the default search engine. + * + * @param {object} obj - Object representing the engine configation. + * @returns {boolean} - Whether the engine should be default. + */ + _isDefault(obj) { + return "default" in obj && obj.default === "yes"; + } + + /** + * Object.assign but ignore some keys + * + * @param {object} target - Object to copy to. + * @param {object} source - Object to copy from. + * @returns {object} - The source object. + */ + _copyObject(target, source) { + for (let key in source) { + if (["included", "excluded", "appliesTo"].includes(key)) { + continue; + } + if (key == "webExtension") { + if (key in target) { + this._copyObject(target[key], source[key]); + } else { + target[key] = { ...source[key] }; + } + } else { + target[key] = source[key]; + } + } + return target; + } + + /** + * Determines wether the section of the config applies to a user + * given what region + locale they are using. + * + * @param {string} region - The region the user is in. + * @param {string} locale - The language the user has configured. + * @param {object} config - Section of configuration. + * @returns {boolean} - Does the section apply for the region + locale. + */ + _isInSection(region, locale, config) { + if (!config) { + return false; + } + if (config.everywhere) { + return true; + } + let locales = config.locales || {}; + let inLocales = + "matches" in locales && + !!locales.matches.find(e => e.toLowerCase() == locale); + let inRegions = + "regions" in config && + !!config.regions.find(e => e.toLowerCase() == region); + if ( + locales.startsWith && + locales.startsWith.some(key => locale.startsWith(key)) + ) { + inLocales = true; + } + if (config.locales && config.regions) { + return inLocales && inRegions; + } + return inLocales || inRegions; + } +} diff --git a/toolkit/components/search/SearchService.sys.mjs b/toolkit/components/search/SearchService.sys.mjs new file mode 100644 index 0000000000..4e1a99671f --- /dev/null +++ b/toolkit/components/search/SearchService.sys.mjs @@ -0,0 +1,4068 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AppProvidedSearchEngine: + "resource://gre/modules/AppProvidedSearchEngine.sys.mjs", + AddonSearchEngine: "resource://gre/modules/AddonSearchEngine.sys.mjs", + IgnoreLists: "resource://gre/modules/IgnoreLists.sys.mjs", + loadAndParseOpenSearchEngine: + "resource://gre/modules/OpenSearchLoader.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + OpenSearchEngine: "resource://gre/modules/OpenSearchEngine.sys.mjs", + PolicySearchEngine: "resource://gre/modules/PolicySearchEngine.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchEngine: "resource://gre/modules/SearchEngine.sys.mjs", + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", + SearchSettings: "resource://gre/modules/SearchSettings.sys.mjs", + SearchStaticData: "resource://gre/modules/SearchStaticData.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", + UserSearchEngine: "resource://gre/modules/UserSearchEngine.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchService", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "timerManager", + "@mozilla.org/updates/timer-manager;1", + "nsIUpdateTimerManager" +); + +/** + * A reference to the handler for the default override allowlist. + * + * @type {SearchDefaultOverrideAllowlistHandler} + */ +ChromeUtils.defineLazyGetter(lazy, "defaultOverrideAllowlist", () => { + return new SearchDefaultOverrideAllowlistHandler(); +}); + +// Exported to tests for not splitting ids when building webextension ids. +export const NON_SPLIT_ENGINE_IDS = [ + "allegro-pl", + "bok-NO", + "daum-kr", + "faclair-beag", + "gulesider-NO", + "mapy-cz", + "naver-kr", + "prisjakt-sv-SE", + "seznam-cz", + "tyda-sv-SE", + "wolnelektury-pl", + "yahoo-jp", + "yahoo-jp-auctions", +]; + +const TOPIC_LOCALES_CHANGE = "intl:app-locales-changed"; +const QUIT_APPLICATION_TOPIC = "quit-application"; + +// The update timer for OpenSearch engines checks in once a day. +const OPENSEARCH_UPDATE_TIMER_TOPIC = "search-engine-update-timer"; +const OPENSEARCH_UPDATE_TIMER_INTERVAL = 60 * 60 * 24; + +// This is the amount of time we'll be idle for before applying any configuration +// changes. +const RECONFIG_IDLE_TIME_SEC = 5 * 60; + +/** + * A reason that is used in the change of default search engine event telemetry. + * These are mutally exclusive. + */ +const REASON_CHANGE_MAP = new Map([ + // The cause of the change is unknown. + [Ci.nsISearchService.CHANGE_REASON_UNKNOWN, "unknown"], + // The user changed the default search engine via the options in the + // preferences UI. + [Ci.nsISearchService.CHANGE_REASON_USER, "user"], + // The change resulted from the user toggling the "Use this search engine in + // Private Windows" option in the preferences UI. + [Ci.nsISearchService.CHANGE_REASON_USER_PRIVATE_SPLIT, "user_private_split"], + // The user changed the default via keys (cmd/ctrl-up/down) in the separate + // search bar. + [Ci.nsISearchService.CHANGE_REASON_USER_SEARCHBAR, "user_searchbar"], + // The user changed the default via context menu on the one-off buttons in the + // separate search bar. + [ + Ci.nsISearchService.CHANGE_REASON_USER_SEARCHBAR_CONTEXT, + "user_searchbar_context", + ], + // An add-on requested the change of default on install, which was either + // accepted automatically or by the user. + [Ci.nsISearchService.CHANGE_REASON_ADDON_INSTALL, "addon-install"], + // An add-on was uninstalled, which caused the engine to be uninstalled. + [Ci.nsISearchService.CHANGE_REASON_ADDON_UNINSTALL, "addon-uninstall"], + // A configuration update caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_CONFIG, "config"], + // A locale update caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_LOCALE, "locale"], + // A region update caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_REGION, "region"], + // Turning on/off an experiment caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_EXPERIMENT, "experiment"], + // An enterprise policy caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_ENTERPRISE, "enterprise"], + // The UI Tour caused a change of default. + [Ci.nsISearchService.CHANGE_REASON_UITOUR, "uitour"], + // The engine updated. + [Ci.nsISearchService.CHANGE_REASON_ENGINE_UPDATE, "engine-update"], +]); + +/** + * The ParseSubmissionResult contains getter methods that return attributes + * about the parsed submission url. + * + * @implements {nsIParseSubmissionResult} + */ +class ParseSubmissionResult { + constructor(engine, terms, termsParameterName) { + this.#engine = engine; + this.#terms = terms; + this.#termsParameterName = termsParameterName; + } + + get engine() { + return this.#engine; + } + + get terms() { + return this.#terms; + } + + get termsParameterName() { + return this.#termsParameterName; + } + + /** + * The search engine associated with the URL passed in to + * nsISearchEngine::parseSubmissionURL, or null if the URL does not represent + * a search submission. + * + * @type {nsISearchEngine|null} + */ + #engine; + + /** + * String containing the sought terms. This can be an empty string in case no + * terms were specified or the URL does not represent a search submission.* + * + * @type {string} + */ + #terms; + + /** + * The name of the query parameter used by `engine` for queries. E.g. "q". + * + * @type {string} + */ + #termsParameterName; + + QueryInterface = ChromeUtils.generateQI(["nsISearchParseSubmissionResult"]); +} + +const gEmptyParseSubmissionResult = Object.freeze( + new ParseSubmissionResult(null, "", "") +); + +/** + * The search service handles loading and maintaining of search engines. It will + * also work out the default lists for each locale/region. + * + * @implements {nsISearchService} + */ +export class SearchService { + constructor() { + // this._engines is prefixed with _ rather than # because it is called from + // a test. + this._engines = new Map(); + this._settings = new lazy.SearchSettings(this); + + this.#defineLazyPreferenceGetters(); + } + + classID = Components.ID("{7319788a-fe93-4db3-9f39-818cf08f4256}"); + + get defaultEngine() { + this.#ensureInitialized(); + return this._getEngineDefault(false); + } + + set defaultEngine(newEngine) { + this.#ensureInitialized(); + this.#setEngineDefault(false, newEngine); + } + + get defaultPrivateEngine() { + this.#ensureInitialized(); + return this._getEngineDefault(this.#separatePrivateDefault); + } + + set defaultPrivateEngine(newEngine) { + this.#ensureInitialized(); + if (!this._separatePrivateDefaultPrefValue) { + Services.prefs.setBoolPref( + lazy.SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + } + this.#setEngineDefault(this.#separatePrivateDefault, newEngine); + } + + async getDefault() { + await this.init(); + return this.defaultEngine; + } + + async setDefault(engine, changeSource) { + await this.init(); + this.#setEngineDefault(false, engine, changeSource); + } + + async getDefaultPrivate() { + await this.init(); + return this.defaultPrivateEngine; + } + + async setDefaultPrivate(engine, changeSource) { + await this.init(); + if (!this._separatePrivateDefaultPrefValue) { + Services.prefs.setBoolPref( + lazy.SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + } + this.#setEngineDefault(this.#separatePrivateDefault, engine, changeSource); + } + + /** + * @returns {SearchEngine} + * The engine that is the default for this locale/region, ignoring any + * user changes to the default engine. + */ + get appDefaultEngine() { + return this.#appDefaultEngine(); + } + + /** + * @returns {SearchEngine} + * The engine that is the default for this locale/region in private browsing + * mode, ignoring any user changes to the default engine. + * Note: if there is no default for this locale/region, then the non-private + * browsing engine will be returned. + */ + get appPrivateDefaultEngine() { + return this.#appDefaultEngine(this.#separatePrivateDefault); + } + + /** + * Determine whether initialization has been completed. + * + * Clients of the service can use this attribute to quickly determine whether + * initialization is complete, and decide to trigger some immediate treatment, + * to launch asynchronous initialization or to bailout. + * + * Note that this attribute does not indicate that initialization has + * succeeded, use hasSuccessfullyInitialized() for that. + * + * @returns {boolean} + * |true | if the search service has finished its attempt to initialize and + * we have an outcome. It could have failed or succeeded during this + * process. + * |false| if initialization has not been triggered yet or initialization is + * still ongoing. + */ + get isInitialized() { + return ( + this.#initializationStatus == "success" || + this.#initializationStatus == "failed" + ); + } + + /** + * Determine whether initialization has been successfully completed. + * + * @returns {boolean} + * |true | if the search service has succesfully initialized. + * |false| if initialization has not been started yet, initialization is + * still ongoing or initializaiton has failed. + */ + get hasSuccessfullyInitialized() { + return this.#initializationStatus == "success"; + } + + /** + * A promise that is resolved when initialization has finished. This does not + * trigger initialization to begin. + * + * @returns {Promise} + * Resolved when initalization has successfully finished, and rejected if it + * has failed. + */ + get promiseInitialized() { + return this.#initDeferredPromise.promise; + } + + getDefaultEngineInfo() { + let [telemetryId, defaultSearchEngineData] = this.#getEngineInfo( + this.defaultEngine + ); + const result = { + defaultSearchEngine: telemetryId, + defaultSearchEngineData, + }; + + if (this.#separatePrivateDefault) { + let [privateTelemetryId, defaultPrivateSearchEngineData] = + this.#getEngineInfo(this.defaultPrivateEngine); + result.defaultPrivateSearchEngine = privateTelemetryId; + result.defaultPrivateSearchEngineData = defaultPrivateSearchEngineData; + } + + return result; + } + + /** + * If possible, please call getEngineById() rather than getEngineByName() + * because engines are stored as { id: object } in this._engine Map. + * + * Returns the engine associated with the name. + * + * @param {string} engineName + * The name of the engine. + * @returns {SearchEngine} + * The associated engine if found, null otherwise. + */ + getEngineByName(engineName) { + this.#ensureInitialized(); + return this.#getEngineByName(engineName); + } + + /** + * Returns the engine associated with the name without initialization checks. + * + * @param {string} engineName + * The name of the engine. + * @returns {SearchEngine} + * The associated engine if found, null otherwise. + */ + #getEngineByName(engineName) { + for (let engine of this._engines.values()) { + if (engine.name == engineName) { + return engine; + } + } + + return null; + } + + /** + * Returns the engine associated with the id. + * + * @param {string} engineId + * The id of the engine. + * @returns {SearchEngine} + * The associated engine if found, null otherwise. + */ + getEngineById(engineId) { + this.#ensureInitialized(); + return this._engines.get(engineId) || null; + } + + async getEngineByAlias(alias) { + await this.init(); + for (var engine of this._engines.values()) { + if (engine && engine.aliases.includes(alias)) { + return engine; + } + } + return null; + } + + async getEngines() { + await this.init(); + lazy.logConsole.debug("getEngines: getting all engines"); + return this.#sortedEngines; + } + + async getVisibleEngines() { + await this.init(true); + lazy.logConsole.debug("getVisibleEngines: getting all visible engines"); + return this.#sortedVisibleEngines; + } + + async getAppProvidedEngines() { + await this.init(); + + return this._sortEnginesByDefaults( + this.#sortedEngines.filter(e => e.isAppProvided) + ); + } + + async getEnginesByExtensionID(extensionID) { + await this.init(); + return this.#getEnginesByExtensionID(extensionID); + } + + /** + * This function calls #init to start initialization when it has not been + * started yet. Otherwise, it returns the pending promise. + * + * @returns {Promise} + * Returns the pending Promise when #init has started but not yet finished. + * | Resolved | when initialization has successfully finished. + * | Rejected | when initialization has failed. + * + */ + async init() { + if (["started", "success", "failed"].includes(this.#initializationStatus)) { + return this.promiseInitialized; + } + this.#initializationStatus = "started"; + return this.#init(); + } + + /** + * Runs background checks for the search service. This is called from + * BrowserGlue and may be run once per session if the user is idle for + * long enough. + */ + async runBackgroundChecks() { + await this.init(); + await this.#migrateLegacyEngines(); + await this.#checkWebExtensionEngines(); + await this.#addOpenSearchTelemetry(); + } + + /** + * Test only - reset SearchService data. Ideally this should be replaced + */ + reset() { + this.#initializationStatus = "not initialized"; + this.#initDeferredPromise = Promise.withResolvers(); + this.#startupExtensions = new Set(); + this._engines.clear(); + this._cachedSortedEngines = null; + this.#currentEngine = null; + this.#currentPrivateEngine = null; + this._searchDefault = null; + this.#searchPrivateDefault = null; + this.#maybeReloadDebounce = false; + this._settings._batchTask?.disarm(); + if (this.#engineSelector) { + this.#engineSelector.reset(); + this.#engineSelector = null; + } + } + + // Test-only function to set SearchService initialization status + forceInitializationStatusForTests(status) { + this.#initializationStatus = status; + } + + /** + * Test only variable to indicate an error should occur during + * search service initialization. + * + * @type {string} + */ + errorToThrowInTest = null; + + // Test-only function to reset just the engine selector so that it can + // load a different configuration. + resetEngineSelector() { + if (lazy.SearchUtils.newSearchConfigEnabled) { + this.#engineSelector = new lazy.SearchEngineSelector( + this.#handleConfigurationUpdated.bind(this) + ); + } else { + this.#engineSelector = new lazy.SearchEngineSelectorOld( + this.#handleConfigurationUpdated.bind(this) + ); + } + } + + resetToAppDefaultEngine() { + let appDefaultEngine = this.appDefaultEngine; + appDefaultEngine.hidden = false; + this.defaultEngine = appDefaultEngine; + } + + async maybeSetAndOverrideDefault(extension) { + let searchProvider = + extension.manifest.chrome_settings_overrides.search_provider; + let engine = this.getEngineByName(searchProvider.name); + if (!engine || !engine.isAppProvided || engine.hidden) { + // If the engine is not application provided, then we shouldn't simply + // set default to it. + // If the engine is application provided, but hidden, then we don't + // switch to it, nor do we try to install it. + return { + canChangeToAppProvided: false, + canInstallEngine: !engine?.hidden, + }; + } + + if ( + extension.startupReason === "ADDON_INSTALL" || + extension.startupReason === "ADDON_ENABLE" + ) { + // Don't allow an extension to set the default if it is already the default. + if (this.defaultEngine.name == searchProvider.name) { + return { + canChangeToAppProvided: false, + canInstallEngine: false, + }; + } + if ( + !(await lazy.defaultOverrideAllowlist.canOverride( + extension, + engine._extensionID + )) + ) { + lazy.logConsole.debug( + "Allowing default engine to be set to app-provided.", + extension.id + ); + // We don't allow overriding the engine in this case, but we can allow + // the extension to change the default engine. + return { + canChangeToAppProvided: true, + canInstallEngine: false, + }; + } + // We're ok to override. + engine.overrideWithEngine({ extension }); + lazy.logConsole.debug( + "Allowing default engine to be set to app-provided and overridden.", + extension.id + ); + return { + canChangeToAppProvided: true, + canInstallEngine: false, + }; + } + + if ( + engine.getAttr("overriddenBy") == extension.id && + (await lazy.defaultOverrideAllowlist.canOverride( + extension, + engine._extensionID + )) + ) { + engine.overrideWithEngine({ extension }); + lazy.logConsole.debug( + "Re-enabling overriding of core extension by", + extension.id + ); + return { + canChangeToAppProvided: true, + canInstallEngine: false, + }; + } + + return { + canChangeToAppProvided: false, + canInstallEngine: false, + }; + } + + /** + * Adds a search engine that is specified from enterprise policies. + * + * @param {object} details + * An object that matches the `SearchEngines` policy schema. + * @param {object} [settings] + * The saved settings for the user. + * @see browser/components/enterprisepolicies/schemas/policies-schema.json + */ + async #addPolicyEngine(details, settings) { + let newEngine = new lazy.PolicySearchEngine({ details, settings }); + lazy.logConsole.debug("Adding Policy Engine:", newEngine.name); + this.#addEngineToStore(newEngine); + } + + /** + * Adds a search engine that is specified by the user. + * + * @param {string} name + * The name of the search engine + * @param {string} url + * The url that the search engine uses for searches + * @param {string} alias + * An alias for the search engine + */ + async addUserEngine(name, url, alias) { + await this.init(); + + let newEngine = new lazy.UserSearchEngine({ + details: { name, url, alias }, + }); + lazy.logConsole.debug(`Adding ${newEngine.name}`); + this.#addEngineToStore(newEngine); + } + + /** + * Called from the AddonManager when it either installs a new + * extension containing a search engine definition or an upgrade + * to an existing one. + * + * @param {object} extension + * An Extension object containing data about the extension. + */ + async addEnginesFromExtension(extension) { + lazy.logConsole.debug("addEnginesFromExtension: " + extension.id); + // Treat add-on upgrade and downgrades the same - either way, the search + // engine gets updated, not added. Generally, we don't expect a downgrade, + // but just in case... + if ( + extension.startupReason == "ADDON_UPGRADE" || + extension.startupReason == "ADDON_DOWNGRADE" + ) { + // Bug 1679861 An a upgrade or downgrade could be adding a search engine + // that was not in a prior version, or the addon may have been blocklisted. + // In either case, there will not be an existing engine. + let existing = await this.#upgradeExtensionEngine(extension); + if (existing?.length) { + return existing; + } + } + + if (extension.isAppProvided) { + // If we are in the middle of initialization or reloading engines, + // don't add the engine here. This has been called as the result + // of _makeEngineFromConfig installing the extension, and that is already + // handling the addition of the engine. + if (this.isInitialized && !this._reloadingEngines) { + let { engines } = await this._fetchEngineSelectorEngines(); + let inConfig = engines.filter(el => el.webExtension.id == extension.id); + if (inConfig.length) { + return this.#installExtensionEngine( + extension, + inConfig.map(el => el.webExtension.locale) + ); + } + } + lazy.logConsole.debug( + "addEnginesFromExtension: Ignoring builtIn engine." + ); + return []; + } + + // If we havent started SearchService yet, store this extension + // to install in SearchService.init(). + if (!this.isInitialized) { + this.#startupExtensions.add(extension); + return []; + } + + return this.#installExtensionEngine(extension, [ + lazy.SearchUtils.DEFAULT_TAG, + ]); + } + + async addOpenSearchEngine(engineURL, iconURL) { + lazy.logConsole.debug("addOpenSearchEngine: Adding", engineURL); + await this.init(); + let engine; + try { + let engineData = await lazy.loadAndParseOpenSearchEngine( + Services.io.newURI(engineURL) + ); + engine = new lazy.OpenSearchEngine({ engineData }); + engine._setIcon(iconURL, false); + } catch (ex) { + throw Components.Exception( + "addEngine: Error adding engine:\n" + ex, + ex.result || Cr.NS_ERROR_FAILURE + ); + } + this.#addEngineToStore(engine); + this.#maybeStartOpenSearchUpdateTimer(); + return engine; + } + + async removeWebExtensionEngine(id) { + if (!this.isInitialized) { + lazy.logConsole.debug( + "Delaying removing extension engine on startup:", + id + ); + this.#startupRemovedExtensions.add(id); + return; + } + + lazy.logConsole.debug("removeWebExtensionEngine:", id); + for (let engine of this.#getEnginesByExtensionID(id)) { + await this.removeEngine(engine); + } + } + + async removeEngine(engine) { + await this.init(); + if (!engine) { + throw Components.Exception( + "no engine passed to removeEngine!", + Cr.NS_ERROR_INVALID_ARG + ); + } + + var engineToRemove = null; + for (var e of this._engines.values()) { + if (engine.wrappedJSObject == e) { + engineToRemove = e; + } + } + + if (!engineToRemove) { + throw Components.Exception( + "removeEngine: Can't find engine to remove!", + Cr.NS_ERROR_FILE_NOT_FOUND + ); + } + + engineToRemove.pendingRemoval = true; + + if (engineToRemove == this.defaultEngine) { + this.#findAndSetNewDefaultEngine({ + privateMode: false, + }); + } + + // Bug 1575649 - We can't just check the default private engine here when + // we're not using separate, as that re-checks the normal default, and + // triggers update of the default search engine, which messes up various + // tests. Really, removeEngine should always commit to updating any + // changed defaults. + if ( + this.#separatePrivateDefault && + engineToRemove == this.defaultPrivateEngine + ) { + this.#findAndSetNewDefaultEngine({ + privateMode: true, + }); + } + + if (engineToRemove.inMemory) { + // Just hide it (the "hidden" setter will notify) and remove its alias to + // avoid future conflicts with other engines. + engineToRemove.hidden = true; + engineToRemove.alias = null; + engineToRemove.pendingRemoval = false; + } else { + // Remove the engine file from disk if we had a legacy file in the profile. + if (engineToRemove._filePath) { + let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile); + file.persistentDescriptor = engineToRemove._filePath; + if (file.exists()) { + file.remove(false); + } + engineToRemove._filePath = null; + } + this.#internalRemoveEngine(engineToRemove); + + // Since we removed an engine, we may need to update the preferences. + if (!this.#dontSetUseSavedOrder) { + this.#saveSortedEngineList(); + } + } + lazy.SearchUtils.notifyAction( + engineToRemove, + lazy.SearchUtils.MODIFIED_TYPE.REMOVED + ); + } + + async moveEngine(engine, newIndex) { + await this.init(); + if (newIndex > this.#sortedEngines.length || newIndex < 0) { + throw Components.Exception( + "moveEngine: Index out of bounds!", + Cr.NS_ERROR_INVALID_ARG + ); + } + if ( + !(engine instanceof Ci.nsISearchEngine) && + !(engine instanceof lazy.SearchEngine) + ) { + throw Components.Exception( + "moveEngine: Invalid engine passed to moveEngine!", + Cr.NS_ERROR_INVALID_ARG + ); + } + if (engine.hidden) { + throw Components.Exception( + "moveEngine: Can't move a hidden engine!", + Cr.NS_ERROR_FAILURE + ); + } + + engine = engine.wrappedJSObject; + + var currentIndex = this.#sortedEngines.indexOf(engine); + if (currentIndex == -1) { + throw Components.Exception( + "moveEngine: Can't find engine to move!", + Cr.NS_ERROR_UNEXPECTED + ); + } + + // Our callers only take into account non-hidden engines when calculating + // newIndex, but we need to move it in the array of all engines, so we + // need to adjust newIndex accordingly. To do this, we count the number + // of hidden engines in the list before the engine that we're taking the + // place of. We do this by first finding newIndexEngine (the engine that + // we were supposed to replace) and then iterating through the complete + // engine list until we reach it, increasing newIndex for each hidden + // engine we find on our way there. + // + // This could be further simplified by having our caller pass in + // newIndexEngine directly instead of newIndex. + var newIndexEngine = this.#sortedVisibleEngines[newIndex]; + if (!newIndexEngine) { + throw Components.Exception( + "moveEngine: Can't find engine to replace!", + Cr.NS_ERROR_UNEXPECTED + ); + } + + for (var i = 0; i < this.#sortedEngines.length; ++i) { + if (newIndexEngine == this.#sortedEngines[i]) { + break; + } + if (this.#sortedEngines[i].hidden) { + newIndex++; + } + } + + if (currentIndex == newIndex) { + return; + } // nothing to do! + + // Move the engine + var movedEngine = this._cachedSortedEngines.splice(currentIndex, 1)[0]; + this._cachedSortedEngines.splice(newIndex, 0, movedEngine); + + lazy.SearchUtils.notifyAction( + engine, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + + // Since we moved an engine, we need to update the preferences. + this.#saveSortedEngineList(); + } + + restoreDefaultEngines() { + this.#ensureInitialized(); + for (let e of this._engines.values()) { + // Unhide all default engines + if (e.hidden && e.isAppProvided) { + e.hidden = false; + } + } + } + + parseSubmissionURL(url) { + if (!this.hasSuccessfullyInitialized) { + // If search is not initialized or failed initializing, do nothing. + // This allows us to use this function early in telemetry. + // The only other consumer of this (places) uses it much later. + return gEmptyParseSubmissionResult; + } + + if (!this.#parseSubmissionMap) { + this.#buildParseSubmissionMap(); + } + + // Extract the elements of the provided URL first. + let soughtKey, soughtQuery; + try { + let soughtUrl = Services.io.newURI(url); + + // Exclude any URL that is not HTTP or HTTPS from the beginning. + if (soughtUrl.schemeIs("http") && soughtUrl.schemeIs("https")) { + return gEmptyParseSubmissionResult; + } + + // Reading these URL properties may fail and raise an exception. + soughtKey = soughtUrl.host + soughtUrl.filePath.toLowerCase(); + soughtQuery = soughtUrl.query; + } catch (ex) { + // Errors while parsing the URL or accessing the properties are not fatal. + return gEmptyParseSubmissionResult; + } + + // Look up the domain and path in the map to identify the search engine. + let mapEntry = this.#parseSubmissionMap.get(soughtKey); + if (!mapEntry) { + return gEmptyParseSubmissionResult; + } + + // Extract the search terms from the parameter, for example "caff%C3%A8" + // from the URL "https://www.google.com/search?q=caff%C3%A8&client=firefox". + // We cannot use `URLSearchParams` here as the terms might not be + // encoded in UTF-8. + let encodedTerms = null; + for (let param of soughtQuery.split("&")) { + let equalPos = param.indexOf("="); + if ( + equalPos != -1 && + param.substr(0, equalPos) == mapEntry.termsParameterName + ) { + // This is the parameter we are looking for. + encodedTerms = param.substr(equalPos + 1); + break; + } + } + if (encodedTerms === null) { + return gEmptyParseSubmissionResult; + } + + // Decode the terms using the charset defined in the search engine. + let terms; + try { + terms = Services.textToSubURI.UnEscapeAndConvert( + mapEntry.engine.queryCharset, + encodedTerms.replace(/\+/g, " ") + ); + } catch (ex) { + // Decoding errors will cause this match to be ignored. + return gEmptyParseSubmissionResult; + } + + return new ParseSubmissionResult( + mapEntry.engine, + terms, + mapEntry.termsParameterName + ); + } + + /** + * This is a nsITimerCallback for the timerManager notification that is + * registered for handling updates to search engines. Only OpenSearch engines + * have these updates and hence, only those are handled here. + */ + async notify() { + lazy.logConsole.debug("notify: checking for updates"); + + // Walk the engine list, looking for engines whose update time has expired. + for (let engine of this._engines.values()) { + if (!(engine instanceof lazy.OpenSearchEngine)) { + continue; + } + await engine.maybeUpdate(); + } + } + + #currentEngine; + #currentPrivateEngine; + #queuedIdle; + + /** + * A deferred promise that is resolved when initialization has finished. + * + * @type {Promise} + * Resolved when initalization has successfully finished, and rejected if it + * has failed. + */ + #initDeferredPromise = Promise.withResolvers(); + + /** + * Indicates if initialization has started, failed, succeeded or has not + * started yet. + * + * These are the statuses: + * "not initialized" - The SearchService has not started initialization. + * "started" - The SearchService has started initializaiton. + * "success" - The SearchService successfully completed initialization. + * "failed" - The SearchService failed during initialization. + * + * @type {string} + */ + #initializationStatus = "not initialized"; + + /** + * Indicates if we're already waiting for maybeReloadEngines to be called. + * + * @type {boolean} + */ + #maybeReloadDebounce = false; + + /** + * Indicates if we're currently in maybeReloadEngines. + * + * This is prefixed with _ rather than # because it is + * called in a test. + * + * @type {boolean} + */ + _reloadingEngines = false; + + /** + * The engine selector singleton that is managing the engine configuration. + * + * @type {SearchEngineSelector|null} + */ + #engineSelector = null; + + /** + * Various search engines may be ignored if their submission urls contain a + * string that is in the list. The list is controlled via remote settings. + * + * @type {Array} + */ + #submissionURLIgnoreList = []; + + /** + * Various search engines may be ignored if their load path is contained + * in this list. The list is controlled via remote settings. + * + * @type {Array} + */ + #loadPathIgnoreList = []; + + /** + * A map of engine display names to `SearchEngine`. + * + * @type {Map<string, object>|null} + */ + _engines = null; + + /** + * An array of engine short names sorted into display order. + * + * @type {Array} + */ + _cachedSortedEngines = null; + + /** + * A flag to prevent setting of useSavedOrder when there's non-user + * activity happening. + * + * @type {boolean} + */ + #dontSetUseSavedOrder = false; + + /** + * An object containing the {id, locale} of the WebExtension for the default + * engine, as suggested by the configuration. + * For the legacy configuration, this is the user visible name. + * + * @type {object} + * + * This is prefixed with _ rather than # because it is + * called in a test. + */ + _searchDefault = null; + + /** + * An object containing the {id, locale} of the WebExtension for the default + * engine for private browsing mode, as suggested by the configuration. + * For the legacy configuration, this is the user visible name. + * + * @type {object} + */ + #searchPrivateDefault = null; + + /** + * A Set of installed search extensions reported by AddonManager + * startup before SearchSevice has started. Will be installed + * during init(). + * + * @type {Set<object>} + */ + #startupExtensions = new Set(); + + /** + * A Set of removed search extensions reported by AddonManager + * startup before SearchSevice has started. Will be removed + * during init(). + * + * @type {Set<object>} + */ + #startupRemovedExtensions = new Set(); + + /** + * This map is built lazily after the available search engines change. It + * allows quick parsing of an URL representing a search submission into the + * search engine name and original terms. + * + * The keys are strings containing the domain name and lowercase path of the + * engine submission, for example "www.google.com/search". + * + * The values are objects with these properties: + * { + * engine: The associated nsISearchEngine. + * termsParameterName: Name of the URL parameter containing the search + * terms, for example "q". + * } + */ + #parseSubmissionMap = null; + + /** + * Keep track of observers have been added. + * + * @type {boolean} + */ + #observersAdded = false; + + /** + * Keeps track to see if the OpenSearch update timer has been started or not. + * + * @type {boolean} + */ + #openSearchUpdateTimerStarted = false; + + get #sortedEngines() { + if (!this._cachedSortedEngines) { + return this.#buildSortedEngineList(); + } + return this._cachedSortedEngines; + } + /** + * This reflects the combined values of the prefs for enabling the separate + * private default UI, and for the user choosing a separate private engine. + * If either one is disabled, then we don't enable the separate private default. + * + * @returns {boolean} + */ + get #separatePrivateDefault() { + return ( + this._separatePrivateDefaultPrefValue && + this._separatePrivateDefaultEnabledPrefValue + ); + } + + #getEnginesByExtensionID(extensionID) { + lazy.logConsole.debug("getEngines: getting all engines for", extensionID); + var engines = this.#sortedEngines.filter(function (engine) { + return engine._extensionID == extensionID; + }); + return engines; + } + + /** + * Returns the engine associated with the WebExtension details. + * + * @param {object} details + * Details of the WebExtension. + * @param {string} details.id + * The WebExtension ID + * @param {string} details.locale + * The WebExtension locale + * @returns {nsISearchEngine|null} + * The found engine, or null if no engine matched. + */ + #getEngineByWebExtensionDetails(details) { + for (const engine of this._engines.values()) { + if ( + engine._extensionID == details.id && + engine._locale == details.locale + ) { + return engine; + } + } + return null; + } + + /** + * Helper function to get the current default engine. + * + * This is prefixed with _ rather than # because it is + * called in test_remove_engine_notification_box.js + * + * @param {boolean} privateMode + * If true, returns the default engine for private browsing mode, otherwise + * the default engine for the normal mode. Note, this function does not + * check the "separatePrivateDefault" preference - that is up to the caller. + * @returns {nsISearchEngine|null} + * The appropriate search engine, or null if one could not be determined. + */ + _getEngineDefault(privateMode) { + let currentEngine = privateMode + ? this.#currentPrivateEngine + : this.#currentEngine; + + if (currentEngine && !currentEngine.hidden) { + return currentEngine; + } + + // No default loaded, so find it from settings. + const attributeName = privateMode + ? "privateDefaultEngineId" + : "defaultEngineId"; + + let engineId = this._settings.getMetaDataAttribute(attributeName); + let engine = this._engines.get(engineId) || null; + if ( + engine && + this._settings.getVerifiedMetaDataAttribute( + attributeName, + engine.isAppProvided + ) + ) { + if (privateMode) { + this.#currentPrivateEngine = engine; + } else { + this.#currentEngine = engine; + } + } + if (!engineId) { + if (privateMode) { + this.#currentPrivateEngine = this.appPrivateDefaultEngine; + } else { + this.#currentEngine = this.appDefaultEngine; + } + } + + currentEngine = privateMode + ? this.#currentPrivateEngine + : this.#currentEngine; + if (currentEngine && !currentEngine.hidden) { + return currentEngine; + } + // No default in settings or it is hidden, so find the new default. + return this.#findAndSetNewDefaultEngine({ privateMode }); + } + + /** + * If initialization has not been completed yet, perform synchronous + * initialization. + * Throws in case of initialization error. + */ + #ensureInitialized() { + if (this.#initializationStatus === "success") { + return; + } + + if (this.#initializationStatus === "failed") { + throw new Error("SearchService failed while it was initializing."); + } + + let err = new Error( + "Something tried to use the search service before it finished " + + "initializing. Please examine the stack trace to figure out what and " + + "where to fix it:\n" + ); + err.message += err.stack; + throw err; + } + + /** + * Define lazy preference getters for separate private default engine in + * private browsing mode. + */ + #defineLazyPreferenceGetters() { + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_separatePrivateDefaultPrefValue", + lazy.SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false, + this.#onSeparateDefaultPrefChanged.bind(this) + ); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_separatePrivateDefaultEnabledPrefValue", + lazy.SearchUtils.BROWSER_SEARCH_PREF + + "separatePrivateDefault.ui.enabled", + false, + this.#onSeparateDefaultPrefChanged.bind(this) + ); + + XPCOMUtils.defineLazyPreferenceGetter( + this, + "separatePrivateDefaultUrlbarResultEnabled", + lazy.SearchUtils.BROWSER_SEARCH_PREF + + "separatePrivateDefault.urlbarResult.enabled", + false + ); + } + + /** + * This function adds observers, retrieves the search engine ignore list, and + * initializes the Search Engine Selector prior to doing the core tasks of + * search service initialization. + * + */ + #doPreInitWork() { + // We need to catch the region being updated during initialization so we + // start listening straight away. + Services.obs.addObserver(this, lazy.Region.REGION_TOPIC); + + this.#getIgnoreListAndSubscribe().catch(ex => + console.error(ex, "Search Service could not get the ignore list.") + ); + + if (lazy.SearchUtils.newSearchConfigEnabled) { + this.#engineSelector = new lazy.SearchEngineSelector( + this.#handleConfigurationUpdated.bind(this) + ); + } else { + this.#engineSelector = new lazy.SearchEngineSelectorOld( + this.#handleConfigurationUpdated.bind(this) + ); + } + } + + /** + * This function fetches information to load search engines and ensures the + * search service is in the correct state for external callers to interact + * with it. + * + * This function sets #initDeferredPromise to resolve or reject. + * | Resolved | when initalization has successfully finished. + * | Rejected | when initialization has failed. + */ + async #init() { + lazy.logConsole.debug("init"); + + const timerId = Glean.searchService.startupTime.start(); + + this.#doPreInitWork(); + + let initSection; + try { + initSection = "Settings"; + this.#maybeThrowErrorInTest(initSection); + const settings = await this._settings.get(); + + initSection = "FetchEngines"; + this.#maybeThrowErrorInTest(initSection); + const { engines, privateDefault } = + await this._fetchEngineSelectorEngines(); + + initSection = "LoadEngines"; + this.#maybeThrowErrorInTest(initSection); + await this.#loadEngines(settings, engines, privateDefault); + } catch (ex) { + Glean.searchService.initializationStatus[`failed${initSection}`].add(); + Glean.searchService.startupTime.cancel(timerId); + + lazy.logConsole.error("#init: failure initializing search:", ex); + this.#initializationStatus = "failed"; + this.#initDeferredPromise.reject(ex); + + throw ex; + } + + // If we've got this far, but the application is now shutting down, + // then we need to abandon any further work, especially not writing + // the settings. We do this, because the add-on manager has also + // started shutting down and as a result, we might have an incomplete + // picture of the installed search engines. Writing the settings at + // this stage would potentially mean the user would loose their engine + // data. + // We will however, rebuild the settings on next start up if we detect + // it is necessary. + if (Services.startup.shuttingDown) { + Glean.searchService.startupTime.cancel(timerId); + + let ex = Components.Exception( + "#init: abandoning init due to shutting down", + Cr.NS_ERROR_ABORT + ); + + this.#initializationStatus = "failed"; + this.#initDeferredPromise.reject(ex); + throw ex; + } + + this.#initializationStatus = "success"; + Glean.searchService.initializationStatus.success.add(); + this.#initDeferredPromise.resolve(); + this.#addObservers(); + + Glean.searchService.startupTime.stopAndAccumulate(timerId); + + this.#recordTelemetryData(); + + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "init-complete" + ); + + lazy.logConsole.debug("Completed #init"); + this.#doPostInitWork(); + } + + /** + * This function records telemetry, checks experiment updates, sets up a timer + * for opensearch, removes any necessary Add-on engines immediately after the + * search service has successfully initialized. + * + */ + #doPostInitWork() { + // It is possible that Nimbus could have called onUpdate before + // we started listening, so do a check on startup. + Services.tm.dispatchToMainThread(async () => { + await lazy.NimbusFeatures.searchConfiguration.ready(); + this.#checkNimbusPrefs(true); + }); + + this.#maybeStartOpenSearchUpdateTimer(); + + if (this.#startupRemovedExtensions.size) { + Services.tm.dispatchToMainThread(async () => { + // Now that init() has successfully finished, we remove any engines + // that have had their add-ons removed by the add-on manager. + // We do this after init() has complete, as that allows us to use + // removeEngine to look after any default engine changes as well. + // This could cause a slight flicker on startup, but it should be + // a rare action. + lazy.logConsole.debug("Removing delayed extension engines"); + for (let id of this.#startupRemovedExtensions) { + for (let engine of this.#getEnginesByExtensionID(id)) { + // Only do this for non-application provided engines. We shouldn't + // ever get application provided engines removed here, but just in case. + if (!engine.isAppProvided) { + await this.removeEngine(engine); + } + } + } + this.#startupRemovedExtensions.clear(); + }); + } + } + + /** + * Obtains the ignore list from remote settings. This should only be + * called from init(). Any subsequent updates to the remote settings are + * handled via a sync listener. + * + */ + async #getIgnoreListAndSubscribe() { + let listener = this.#handleIgnoreListUpdated.bind(this); + const current = await lazy.IgnoreLists.getAndSubscribe(listener); + + // Only save the listener after the subscribe, otherwise for tests it might + // not be fully set up by the time we remove it again. + this.ignoreListListener = listener; + + await this.#handleIgnoreListUpdated({ data: { current } }); + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "settings-update-complete" + ); + } + + /** + * This handles updating of the ignore list settings, and removing any ignored + * engines. + * + * @param {object} eventData + * The event in the format received from RemoteSettings. + */ + async #handleIgnoreListUpdated(eventData) { + lazy.logConsole.debug("#handleIgnoreListUpdated"); + const { + data: { current }, + } = eventData; + + for (const entry of current) { + if (entry.id == "load-paths") { + this.#loadPathIgnoreList = [...entry.matches]; + } else if (entry.id == "submission-urls") { + this.#submissionURLIgnoreList = [...entry.matches]; + } + } + + try { + await this.promiseInitialized; + } catch (ex) { + // If there's a problem with initialization return early to allow + // search service to continue in a limited mode without engines. + return; + } + + // We try to remove engines manually, as this should be more efficient and + // we don't really want to cause a re-init as this upsets unit tests. + let engineRemoved = false; + for (let engine of this._engines.values()) { + if (this.#engineMatchesIgnoreLists(engine)) { + await this.removeEngine(engine); + engineRemoved = true; + } + } + // If we've removed an engine, and we don't have any left, we need to + // reload the engines - it is possible the settings just had one engine in it, + // and that is now empty, so we need to load from our main list. + if (engineRemoved && !this._engines.size) { + this._maybeReloadEngines().catch(console.error); + } + } + + /** + * Determines if a given engine matches the ignorelists or not. + * + * @param {Engine} engine + * The engine to check against the ignorelists. + * @returns {boolean} + * Returns true if the engine matches a ignorelists entry. + */ + #engineMatchesIgnoreLists(engine) { + if (this.#loadPathIgnoreList.includes(engine._loadPath)) { + return true; + } + let url = engine.searchURLWithNoTerms.spec.toLowerCase(); + if ( + this.#submissionURLIgnoreList.some(code => + url.includes(code.toLowerCase()) + ) + ) { + return true; + } + return false; + } + + /** + * Handles the search configuration being - adds a wait on the user + * being idle, before the search engine update gets handled. + */ + #handleConfigurationUpdated() { + if (this.#queuedIdle) { + return; + } + + this.#queuedIdle = true; + + this.idleService.addIdleObserver(this, RECONFIG_IDLE_TIME_SEC); + } + + /** + * Returns the engine that is the default for this locale/region, ignoring any + * user changes to the default engine. + * + * @param {boolean} privateMode + * Set to true to return the default engine in private mode, + * false for normal mode. + * @returns {SearchEngine} + * The engine that is default. + */ + #appDefaultEngine(privateMode = false) { + let defaultEngine = this.#getEngineByWebExtensionDetails( + privateMode && this.#searchPrivateDefault + ? this.#searchPrivateDefault + : this._searchDefault + ); + + if (Services.policies?.status == Ci.nsIEnterprisePolicies.ACTIVE) { + let activePolicies = Services.policies.getActivePolicies(); + if (activePolicies.SearchEngines) { + if (activePolicies.SearchEngines.Default) { + return this.#getEngineByName(activePolicies.SearchEngines.Default); + } + if (activePolicies.SearchEngines.Remove?.includes(defaultEngine.name)) { + defaultEngine = null; + } + } + } + + if (defaultEngine) { + return defaultEngine; + } + + if (privateMode) { + // If for some reason we can't find the private mode engine, fall back + // to the non-private one. + return this.#appDefaultEngine(false); + } + + // Something unexpected has happened. In order to recover the app default + // engine, use the first visible engine that is also a general purpose engine. + // Worst case, we just use the first visible engine. + defaultEngine = this.#sortedVisibleEngines.find( + e => e.isGeneralPurposeEngine + ); + return defaultEngine ? defaultEngine : this.#sortedVisibleEngines[0]; + } + + /** + * Loads engines asynchronously. + * + * @param {object} settings + * An object representing the search engine settings. + * @param {Array} engines + * An array containing the engines objects from remote settings. + * @param {object} privateDefault + * An object representing the private default search engine. + */ + async #loadEngines(settings, engines, privateDefault) { + // Get user's current settings and search engine before we load engines from + // config. These values will be compared after engines are loaded. + let prevMetaData = { ...settings?.metaData }; + let prevCurrentEngineId = prevMetaData.defaultEngineId; + let prevAppDefaultEngineId = prevMetaData?.appDefaultEngineId; + + lazy.logConsole.debug("#loadEngines: start"); + this.#setDefaultAndOrdersFromSelector(engines, privateDefault); + + // We've done what we can without the add-on manager, now ensure that + // it has finished starting before we continue. + if (!lazy.SearchUtils.newSearchConfigEnabled) { + await lazy.AddonManager.readyPromise; + } + + await this.#loadEnginesFromConfig(engines, settings); + + await this.#loadStartupEngines(settings); + + this.#loadEnginesFromPolicies(settings); + + // `loadEnginesFromSettings` loads the engines and their settings together. + // If loading the settings caused the default engine to change because of an + // override, then we don't want to show the notification box. + let skipDefaultChangedNotification = await this.#loadEnginesFromSettings( + settings + ); + + // If #loadEnginesFromSettings changed the default engine, then we don't + // need to call #checkOpenSearchOverrides as we know that the overrides have + // only just been applied. + skipDefaultChangedNotification ||= await this.#checkOpenSearchOverrides( + settings + ); + + // Settings file version 6 and below will need a migration to store the + // engine ids rather than engine names. + this._settings.migrateEngineIds(settings); + + lazy.logConsole.debug("#loadEngines: done"); + + let newCurrentEngine = this._getEngineDefault(false); + let newCurrentEngineId = newCurrentEngine?.id; + + this._settings.setMetaDataAttribute( + "appDefaultEngineId", + this.appDefaultEngine?.id + ); + + if ( + !skipDefaultChangedNotification && + this.#shouldDisplayRemovalOfEngineNotificationBox( + settings, + prevMetaData, + newCurrentEngineId, + prevCurrentEngineId, + prevAppDefaultEngineId + ) + ) { + let newCurrentEngineName = newCurrentEngine?.name; + + let [prevCurrentEngineName, prevAppDefaultEngineName] = [ + settings.engines.find(e => e.id == prevCurrentEngineId)?._name, + settings.engines.find(e => e.id == prevAppDefaultEngineId)?._name, + ]; + + this._showRemovalOfSearchEngineNotificationBox( + prevCurrentEngineName || prevAppDefaultEngineName, + newCurrentEngineName + ); + } + } + + /** + * Helper function to determine if the removal of search engine notification + * box should be displayed. + * + * @param { object } settings + * The user's search engine settings. + * @param { object } prevMetaData + * The user's previous search settings metadata. + * @param { object } newCurrentEngineId + * The user's new current default engine. + * @param { object } prevCurrentEngineId + * The user's previous default engine. + * @param { object } prevAppDefaultEngineId + * The user's previous app default engine. + * @returns { boolean } + * Return true if the previous default engine has been removed and + * notification box should be displayed. + */ + #shouldDisplayRemovalOfEngineNotificationBox( + settings, + prevMetaData, + newCurrentEngineId, + prevCurrentEngineId, + prevAppDefaultEngineId + ) { + if ( + !Services.prefs.getBoolPref("browser.search.removeEngineInfobar.enabled") + ) { + return false; + } + + // If for some reason we were unable to install any engines and hence no + // default engine, do not display the notification box + if (!newCurrentEngineId) { + return false; + } + + // If the previous engine is still available, don't show the notification + // box. + if (prevCurrentEngineId && this._engines.has(prevCurrentEngineId)) { + return false; + } + if (!prevCurrentEngineId && this._engines.has(prevAppDefaultEngineId)) { + return false; + } + + // Don't show the notification if the previous engine was an enterprise engine - + // the text doesn't quite make sense. + // let checkPolicyEngineId = prevCurrentEngineId ? prevCurrentEngineId : prevAppDefaultEngineId; + let checkPolicyEngineId = prevCurrentEngineId || prevAppDefaultEngineId; + if (checkPolicyEngineId) { + let engineSettings = settings.engines.find( + e => e.id == checkPolicyEngineId + ); + if (engineSettings?._loadPath?.startsWith("[policy]")) { + return false; + } + } + + // If the user's previous engine id is different than the new current + // engine id, or if the user was using the app default engine and the + // app default engine id is different than the new current engine id, + // we check if the user's settings metadata has been upddated. + if ( + (prevCurrentEngineId && prevCurrentEngineId !== newCurrentEngineId) || + (!prevCurrentEngineId && + prevAppDefaultEngineId && + prevAppDefaultEngineId !== newCurrentEngineId) + ) { + // Check settings metadata to detect an update to locale. Sometimes when + // the user changes their locale it causes a change in engines. + // If there is no update to settings metadata then the engine change was + // caused by an update to config rather than a user changing their locale. + if (!this.#didSettingsMetaDataUpdate(prevMetaData)) { + return true; + } + } + + return false; + } + + /** + * Loads engines as specified by the configuration. We only expect + * configured engines here, user engines should not be listed. + * + * @param {Array} engineConfigs + * An array of engines configurations based on the schema. + * @param {object} [settings] + * The saved settings for the user. + */ + async #loadEnginesFromConfig(engineConfigs, settings) { + lazy.logConsole.debug("#loadEnginesFromConfig"); + for (let config of engineConfigs) { + try { + let engine = await this._makeEngineFromConfig(config, settings); + this.#addEngineToStore(engine); + } catch (ex) { + console.error( + `Could not load engine ${ + "webExtension" in config ? config.webExtension.id : "unknown" + }: ${ex}` + ); + } + } + } + + /** + * Loads any engines that have been received from the AddonManager during + * startup and before we have finished initialising. + * + * @param {object} [settings] + * The saved settings for the user. + */ + async #loadStartupEngines(settings) { + if ( + this.#startupExtensions.size && + lazy.SearchUtils.newSearchConfigEnabled + ) { + await lazy.AddonManager.readyPromise; + } + + lazy.logConsole.debug( + "#loadEngines: loading", + this.#startupExtensions.size, + "engines reported by AddonManager startup" + ); + for (let extension of this.#startupExtensions) { + try { + await this.#installExtensionEngine( + extension, + [lazy.SearchUtils.DEFAULT_TAG], + settings, + true + ); + } catch (ex) { + lazy.logConsole.error( + `#installExtensionEngine failed for ${extension.id}`, + ex + ); + } + } + this.#startupExtensions.clear(); + } + + /** + * When starting up, check if any of the saved application provided engines + * are no longer required, previously were default and were overridden by + * an OpenSearch engine. + * + * Also check if any OpenSearch overrides need to be re-applied. + * + * Add-on search engines are handled separately. + * + * @param {object} settings + * The loaded settings for the user. + * @returns {boolean} + * Returns true if the default engine was changed. + */ + async #checkOpenSearchOverrides(settings) { + let defaultEngineChanged = false; + let savedDefaultEngineId = + settings.metaData.defaultEngineId || settings.metaData.appDefaultEngineId; + if (!savedDefaultEngineId) { + return false; + } + // First handle the case where the application provided engine was removed, + // and we need to restore the OpenSearch engine. + for (let engineSettings of settings.engines) { + if ( + !this._engines.get(engineSettings.id) && + engineSettings._isAppProvided && + engineSettings.id == savedDefaultEngineId && + engineSettings._metaData.overriddenByOpenSearch + ) { + let restoringEngine = new lazy.OpenSearchEngine({ + json: engineSettings._metaData.overriddenByOpenSearch, + }); + restoringEngine.copyUserSettingsFrom(engineSettings); + this.#addEngineToStore(restoringEngine, true); + + // We assume that the app provided engine was removed due to a + // configuration change, and therefore we have re-added the OpenSearch + // search engine. It is possible that it was actually due to a + // locale/region change, but that is harder to detect here. + this.#setEngineDefault( + false, + restoringEngine, + Ci.nsISearchService.CHANGE_REASON_CONFIG + ); + delete engineSettings._metaData.overriddenByOpenSearch; + } + } + // Now handle the case where the an application provided engine has been + // overridden by an OpenSearch engine, and we need to re-apply the override. + for (let engine of this._engines.values()) { + if ( + engine.isAppProvided && + engine.getAttr("overriddenByOpenSearch") && + engine.id == savedDefaultEngineId + ) { + let restoringEngine = new lazy.OpenSearchEngine({ + json: engine.getAttr("overriddenByOpenSearch"), + }); + if ( + await lazy.defaultOverrideAllowlist.canEngineOverride( + restoringEngine, + engine._extensionID + ) + ) { + engine.overrideWithEngine({ engine: restoringEngine }); + } + } + } + + return defaultEngineChanged; + } + + /** + * Reloads engines asynchronously, but only when + * the service has already been initialized. + * + * This is prefixed with _ rather than # because it is + * called in test_reload_engines.js + * + * @param {integer} changeReason + * The reason reload engines is being called, one of + * Ci.nsISearchService.CHANGE_REASON* + */ + async _maybeReloadEngines(changeReason) { + if (this.#maybeReloadDebounce) { + lazy.logConsole.debug("We're already waiting to reload engines."); + return; + } + + if (!this.isInitialized || this._reloadingEngines) { + this.#maybeReloadDebounce = true; + // Schedule a reload to happen at most 10 seconds after the current run. + Services.tm.idleDispatchToMainThread(() => { + if (!this.#maybeReloadDebounce) { + return; + } + this.#maybeReloadDebounce = false; + this._maybeReloadEngines(changeReason).catch(console.error); + }, 10000); + lazy.logConsole.debug( + "Post-poning maybeReloadEngines() as we're currently initializing." + ); + return; + } + + // Before entering `_reloadingEngines` get the settings which we'll need. + // This also ensures that any pending settings have finished being written, + // which could otherwise cause data loss. + let settings = await this._settings.get(); + + lazy.logConsole.debug("Running maybeReloadEngines"); + this._reloadingEngines = true; + + try { + await this._reloadEngines(settings, changeReason); + } catch (ex) { + lazy.logConsole.error("maybeReloadEngines failed", ex); + } + this._reloadingEngines = false; + lazy.logConsole.debug("maybeReloadEngines complete"); + } + + /** + * Manages reloading of the search engines when something in the user's + * environment or the configuration has changed. + * + * The order of work here is designed to avoid potential issues when updating + * the default engines, so that we're not removing active defaults or trying + * to set a default to something that hasn't been added yet. The order is: + * + * 1) Update exising engines that are in both the old and new configuration. + * 2) Add any new engines from the new configuration. + * 3) Check for changes needed to the default engines due to environment changes + * and potentially overriding engines as per the override allowlist. + * 4) Update the default engines. + * 5) Remove any old engines. + * + * This is prefixed with _ rather than # because it is called in + * test_remove_engine_notification_box.js + * + * @param {object} settings + * The user's current saved settings. + * @param {integer} changeReason + * The reason reload engines is being called, one of + * Ci.nsISearchService.CHANGE_REASON* + */ + async _reloadEngines(settings, changeReason) { + // Capture the current engine state, in case we need to notify below. + let prevCurrentEngine = this.#currentEngine; + let prevPrivateEngine = this.#currentPrivateEngine; + let prevMetaData = { ...settings?.metaData }; + + // Ensure that we don't set the useSavedOrder flag whilst we're doing this. + // This isn't a user action, so we shouldn't be switching it. + this.#dontSetUseSavedOrder = true; + + let { engines: appDefaultConfigEngines, privateDefault } = + await this._fetchEngineSelectorEngines(); + + let configEngines = [...appDefaultConfigEngines]; + let oldEngineList = [...this._engines.values()]; + + for (let engine of oldEngineList) { + if (!engine.isAppProvided) { + if (engine instanceof lazy.AddonSearchEngine) { + // If this is an add-on search engine, check to see if it needs + // an update. + await engine.update(); + } + continue; + } + + let index = configEngines.findIndex( + e => + e.webExtension.id == engine._extensionID && + e.webExtension.locale == engine._locale + ); + + if (index == -1) { + // No engines directly match on id and locale, however, check to see + // if we have a new entry that matches on id and name - we might just + // be swapping the in-use locale. + let replacementEngines = configEngines.filter( + e => e.webExtension.id == engine._extensionID + ); + // If there's no possible, or more than one, we treat these as distinct + // engines so we'll remove the existing engine and add new later if + // necessary. + if (replacementEngines.length != 1) { + engine.pendingRemoval = true; + continue; + } + + // Update the index so we can handle the updating below. + index = configEngines.findIndex( + e => + e.webExtension.id == replacementEngines[0].webExtension.id && + e.webExtension.locale == replacementEngines[0].webExtension.locale + ); + let locale = + replacementEngines[0].webExtension.locale || + lazy.SearchUtils.DEFAULT_TAG; + + // If the name is different, then we must treat the engine as different, + // and go through the remove and add cycle, rather than modifying the + // existing one. + let hasUpdated = await engine.updateIfNoNameChange({ + configuration: configEngines[index], + locale, + }); + if (!hasUpdated) { + // No matching name, so just remove it. + engine.pendingRemoval = true; + continue; + } + } else { + // This is an existing engine that we should update (we don't know if + // the configuration for this engine has changed or not). + await engine.update({ + configuration: configEngines[index], + locale: engine._locale, + }); + } + + configEngines.splice(index, 1); + } + + let existingDuplicateEngines = []; + + // Any remaining configuration engines are ones that we need to add. + for (let engine of configEngines) { + try { + let newAppEngine = await this._makeEngineFromConfig(engine, settings); + + // If this is a duplicate name, keep track of the old engine as we need + // to handle it later. + let duplicateEngine = this.#getEngineByName(newAppEngine.name); + if (duplicateEngine) { + existingDuplicateEngines.push({ + duplicateEngine, + newAppEngine, + }); + } + // We add our new engine to the store anyway, as we know it is an + // application provided engine which will take priority over the + // duplicate. + this.#addEngineToStore(newAppEngine, true); + } catch (ex) { + lazy.logConsole.warn( + `Could not load engine ${ + "webExtension" in engine ? engine.webExtension.id : "unknown" + }: ${ex}` + ); + } + } + + // Now set the sort out the default engines and notify as appropriate. + + // Clear the current values, so that we'll completely reset. + this.#currentEngine = null; + this.#currentPrivateEngine = null; + + // If the user's default is one of the private engines that is being removed, + // reset the stored setting, so that we correctly detect the change in + // in default. + if (prevCurrentEngine?.pendingRemoval) { + this._settings.setMetaDataAttribute("defaultEngineId", ""); + } + if (prevPrivateEngine?.pendingRemoval) { + this._settings.setMetaDataAttribute("privateDefaultEngineId", ""); + } + + this.#setDefaultAndOrdersFromSelector( + appDefaultConfigEngines, + privateDefault + ); + + let skipDefaultChangedNotification = false; + + for (let { duplicateEngine, newAppEngine } of existingDuplicateEngines) { + if (prevCurrentEngine && prevCurrentEngine == duplicateEngine) { + if ( + await lazy.defaultOverrideAllowlist.canEngineOverride( + duplicateEngine, + newAppEngine?._extensionID + ) + ) { + lazy.logConsole.log( + "Applying override from", + duplicateEngine.id, + "to application engine", + newAppEngine._extensionID, + "and setting app engine default" + ); + // This engine was default, and is allowed to override our application + // provided engines, so update the application engine and set it as + // default. + newAppEngine.overrideWithEngine({ + engine: duplicateEngine, + }); + + this.defaultEngine = newAppEngine; + // We're removing the old engine and we've changed the default, but this + // is intentional and effectively everything is the same for the user, so + // don't notify. + skipDefaultChangedNotification = true; + } + } + duplicateEngine.pendingRemoval = true; + } + + if (prevCurrentEngine && prevCurrentEngine.pendingRemoval) { + skipDefaultChangedNotification ||= + await this.#maybeRestoreEngineFromOverride(prevCurrentEngine); + } + + // If the defaultEngine has changed between the previous load and this one, + // dispatch the appropriate notifications. + if (prevCurrentEngine && this.defaultEngine !== prevCurrentEngine) { + this.#recordDefaultChangedEvent( + false, + prevCurrentEngine, + this.defaultEngine, + changeReason + ); + lazy.SearchUtils.notifyAction( + this.#currentEngine, + lazy.SearchUtils.MODIFIED_TYPE.DEFAULT + ); + // If we've not got a separate private active, notify update of the + // private so that the UI updates correctly. + if (!this.#separatePrivateDefault) { + lazy.SearchUtils.notifyAction( + this.#currentEngine, + lazy.SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + ); + } + + if ( + !skipDefaultChangedNotification && + prevMetaData && + settings.metaData && + !this.#didSettingsMetaDataUpdate(prevMetaData) && + prevCurrentEngine?.pendingRemoval && + Services.prefs.getBoolPref("browser.search.removeEngineInfobar.enabled") + ) { + this._showRemovalOfSearchEngineNotificationBox( + prevCurrentEngine.name, + this.defaultEngine.name + ); + } + } + + if ( + this.#separatePrivateDefault && + prevPrivateEngine && + this.defaultPrivateEngine !== prevPrivateEngine + ) { + this.#recordDefaultChangedEvent( + true, + prevPrivateEngine, + this.defaultPrivateEngine, + changeReason + ); + lazy.SearchUtils.notifyAction( + this.#currentPrivateEngine, + lazy.SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + ); + } + + // Finally, remove any engines that need removing. We do this after sorting + // out the new default, as otherwise this could cause multiple notifications + // and the wrong engine to be selected as default. + + for (let engine of this._engines.values()) { + if (!engine.pendingRemoval) { + continue; + } + + // If we have other engines that use the same extension ID, then + // we do not want to remove the add-on - only remove the engine itself. + let inUseEngines = [...this._engines.values()].filter( + e => e._extensionID == engine._extensionID + ); + + if (inUseEngines.length <= 1) { + if (inUseEngines.length == 1 && inUseEngines[0] == engine) { + // No other engines are using this extension ID. + + // The internal remove is done first to avoid a call to removeEngine + // which could adjust the sort order when we don't want it to. + this.#internalRemoveEngine(engine); + + // Only uninstall application provided engines. We don't want to + // remove third-party add-ons. Their search engine names might conflict, + // but we still allow the add-on to be installed. + if (engine.isAppProvided) { + let addon = await lazy.AddonManager.getAddonByID( + engine._extensionID + ); + if (addon) { + // AddonManager won't call removeEngine if an engine with the + // WebExtension id doesn't exist in the search service. + await addon.uninstall(); + } + } + } + // For the case where `inUseEngines[0] != engine`: + // This is a situation where there was an engine added earlier in this + // function with the same name. + // For example, eBay has the same name for both US and GB, but has + // a different domain and uses a different locale of the same + // WebExtension. + // The result of this is the earlier addition has already replaced + // the engine in `this._engines` (which is indexed by name), so all that + // needs to be done here is to pretend the old engine was removed + // which is notified below. + } else { + // More than one engine is using this extension ID, so we don't want to + // remove the add-on. + this.#internalRemoveEngine(engine); + } + lazy.SearchUtils.notifyAction( + engine, + lazy.SearchUtils.MODIFIED_TYPE.REMOVED + ); + } + + // Save app default engine to the user's settings metaData incase it has + // been updated + this._settings.setMetaDataAttribute( + "appDefaultEngineId", + this.appDefaultEngine?.id + ); + + // If we are leaving an experiment, and the default is the same as the + // application default, we reset the user's setting to blank, so that + // future changes of the application default engine may take effect. + if ( + prevMetaData.experiment && + !this._settings.getMetaDataAttribute("experiment") + ) { + if (this.defaultEngine == this.appDefaultEngine) { + this._settings.setVerifiedMetaDataAttribute("defaultEngineId", ""); + } + if ( + this.#separatePrivateDefault && + this.defaultPrivateEngine == this.appPrivateDefaultEngine + ) { + this._settings.setVerifiedMetaDataAttribute( + "privateDefaultEngineId", + "" + ); + } + } + + this.#dontSetUseSavedOrder = false; + // Clear out the sorted engines settings, so that we re-sort it if necessary. + this._cachedSortedEngines = null; + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "engines-reloaded" + ); + } + + /** + * Potentially restores an engine if it was previously overriding the app + * provided engine. + * + * @param {SearchEngine} prevCurrentEngine + * The previous current engine to check for override. + * @returns {boolean} + * True if an engine was restored. + */ + async #maybeRestoreEngineFromOverride(prevCurrentEngine) { + let overriddenBy = prevCurrentEngine.getAttr("overriddenBy"); + if (!overriddenBy) { + return false; + } + let overriddenByOpenSearch = prevCurrentEngine.getAttr( + "overriddenByOpenSearch" + ); + let engine; + if (overriddenByOpenSearch) { + engine = new lazy.OpenSearchEngine({ + json: overriddenByOpenSearch, + }); + } else { + // The previous application default engine is being removed, and it was + // overridden by another engine. We want to put the previous engine back, + // so that the user retains that engine as default. + engine = new lazy.AddonSearchEngine({ + isAppProvided: false, + details: { + extensionID: overriddenBy, + locale: lazy.SearchUtils.DEFAULT_TAG, + }, + }); + try { + await engine.init({ locale: lazy.SearchUtils.DEFAULT_TAG }); + } catch (ex) { + // If there is an error, the add-on may no longer be available, or + // there was some other issue with the settings. + lazy.logConsole.error( + "Error restoring overridden engine", + overriddenBy, + ex + ); + return false; + } + } + engine.copyUserSettingsFrom(prevCurrentEngine); + this.#addEngineToStore(engine, true); + + // Now set it back to default. + this.defaultEngine = engine; + return true; + } + + #addEngineToStore(engine, skipDuplicateCheck = false) { + if (this.#engineMatchesIgnoreLists(engine)) { + lazy.logConsole.debug("#addEngineToStore: Ignoring engine"); + return; + } + + lazy.logConsole.debug("#addEngineToStore: Adding engine:", engine.name); + + // See if there is an existing engine with the same name. + if (!skipDuplicateCheck && this.#getEngineByName(engine.name)) { + throw Components.Exception( + `#addEngineToStore: An engine called ${engine.name} already exists!`, + Cr.NS_ERROR_FILE_ALREADY_EXISTS + ); + } + + // Not an update, just add the new engine. + this._engines.set(engine.id, engine); + // Only add the engine to the list of sorted engines if the initial list + // has already been built (i.e. if this._cachedSortedEngines is non-null). If + // it hasn't, we're loading engines from disk and the sorted engine list + // will be built once we need it. + if (this._cachedSortedEngines && !this.#dontSetUseSavedOrder) { + this._cachedSortedEngines.push(engine); + this.#saveSortedEngineList(); + } + lazy.SearchUtils.notifyAction(engine, lazy.SearchUtils.MODIFIED_TYPE.ADDED); + + // Let the engine know it can start notifying new updates. + engine._engineAddedToStore = true; + } + + /** + * Loads any search engines specified by enterprise policies. + * + * @param {object} [settings] + * The saved settings for the user. + */ + #loadEnginesFromPolicies(settings) { + if (Services.policies?.status != Ci.nsIEnterprisePolicies.ACTIVE) { + return; + } + + let activePolicies = Services.policies.getActivePolicies(); + if (!activePolicies.SearchEngines) { + return; + } + for (let engineDetails of activePolicies.SearchEngines.Add ?? []) { + this.#addPolicyEngine(engineDetails, settings); + } + } + + /** + * Loads remaining user search engines from settings. + * + * @param {object} [settings] + * The saved settings for the user. + * @returns {boolean} + * Returns true if the default engine was changed. + */ + async #loadEnginesFromSettings(settings) { + if (!settings.engines) { + return false; + } + + lazy.logConsole.debug( + "#loadEnginesFromSettings: Loading", + settings.engines.length, + "engines from settings" + ); + + let defaultEngineChanged = false; + let skippedEngines = 0; + for (let engineJSON of settings.engines) { + // We renamed isBuiltin to isAppProvided in bug 1631898, + // keep checking isBuiltin for older settings. + if (engineJSON._isAppProvided || engineJSON._isBuiltin) { + ++skippedEngines; + continue; + } + + // Some OpenSearch type engines are now obsolete and no longer supported. + // These were application provided engines that used to use the OpenSearch + // format before gecko transitioned to WebExtensions. + // These will sometimes have been missed in migration due to various + // reasons, and due to how the settings saves everything. We therefore + // explicitly ignore them here to drop them, and let the rest of the code + // fallback to the application/distribution default if necessary. + let loadPath = engineJSON._loadPath?.toLowerCase(); + if ( + loadPath && + // Replaced by application provided in Firefox 79. + (loadPath.startsWith("[distribution]") || + // Langpack engines moved in-app in Firefox 62. + // Note: these may be prefixed by jar:, + loadPath.includes("[app]/extensions/langpack") || + loadPath.includes("[other]/langpack") || + loadPath.includes("[profile]/extensions/langpack") || + // Old omni.ja engines also moved to in-app in Firefox 62. + loadPath.startsWith("jar:[app]/omni.ja")) + ) { + continue; + } + + try { + let engine; + if (loadPath?.startsWith("[policy]")) { + skippedEngines++; + continue; + } else if (loadPath?.startsWith("[user]")) { + engine = new lazy.UserSearchEngine({ json: engineJSON }); + } else if (engineJSON.extensionID ?? engineJSON._extensionID) { + engine = new lazy.AddonSearchEngine({ + isAppProvided: false, + json: engineJSON, + }); + } else { + engine = new lazy.OpenSearchEngine({ + json: engineJSON, + }); + } + // Only check the override for Add-on or OpenSearch engines, and only + // if they are the default engine. + if ( + (engine instanceof lazy.OpenSearchEngine || + engine instanceof lazy.AddonSearchEngine) && + settings.metaData?.defaultEngineId == engine.id + ) { + defaultEngineChanged = await this.#maybeApplyOverride(engine); + if (defaultEngineChanged) { + continue; + } + } + this.#addEngineToStore(engine); + } catch (ex) { + lazy.logConsole.error( + "Failed to load", + engineJSON._name, + "from settings:", + ex, + engineJSON + ); + } + } + + if (skippedEngines) { + lazy.logConsole.debug( + "#loadEnginesFromSettings: skipped", + skippedEngines, + "built-in/policy engines." + ); + } + return defaultEngineChanged; + } + + /** + * Looks to see if an override may be applied to an application engine + * if the supplied engine is a duplicate of it. This should only be called + * in the case where the engine would become the default engine. + * + * @param {SearchEngine} engine + * The search engine to check to see if it should override an existing engine. + * @returns {boolean} + * True if the default engine was changed. + */ + async #maybeApplyOverride(engine) { + // If an engine with the same name already exists, we're not going to + // be allowed to add it - however, if it is default, and it + // matches an existing engine, then we might be allowed to + // override the application provided engine. + let existingEngine = this.#getEngineByName(engine.name); + if ( + existingEngine?.isAppProvided && + (await lazy.defaultOverrideAllowlist.canEngineOverride( + engine, + existingEngine?._extensionID + )) + ) { + existingEngine.overrideWithEngine({ + engine, + }); + this.#setEngineDefault( + false, + existingEngine, + // We assume that the application provided engine was added due + // to a configuration change. It is possible that it was actually + // due to a locale/region change, but that is harder to detect + // here. + Ci.nsISearchService.CHANGE_REASON_CONFIG + ); + return true; + } + return false; + } + + // This is prefixed with _ rather than # because it is + // called in test_remove_engine_notification_box.js + async _fetchEngineSelectorEngines() { + let searchEngineSelectorProperties = { + locale: Services.locale.appLocaleAsBCP47, + region: lazy.Region.home || "default", + channel: lazy.SearchUtils.MODIFIED_APP_CHANNEL, + experiment: + lazy.NimbusFeatures.searchConfiguration.getVariable("experiment") ?? "", + distroID: lazy.SearchUtils.distroID ?? "", + }; + + for (let [key, value] of Object.entries(searchEngineSelectorProperties)) { + this._settings.setMetaDataAttribute(key, value); + } + + let { engines, privateDefault } = + await this.#engineSelector.fetchEngineConfiguration( + searchEngineSelectorProperties + ); + + for (let e of engines) { + if (!e.webExtension) { + e.webExtension = {}; + } + e.webExtension.locale = + e.webExtension?.locale ?? lazy.SearchUtils.DEFAULT_TAG; + + // TODO Bug 1875912 - Remove the webextension.id and webextension.locale when + // we're ready to remove old search-config and use search-config-v2 for all + // clients. The id in appProvidedSearchEngine should be changed to + // engine.identifier. + if (lazy.SearchUtils.newSearchConfigEnabled) { + let identifierComponents = NON_SPLIT_ENGINE_IDS.includes(e.identifier) + ? [e.identifier] + : e.identifier.split("-"); + + if (e.identifier == "amazon-se") { + identifierComponents[1] = "sweden"; + } + + if (e.identifier == "amazon-es") { + identifierComponents[1] = "spain"; + } + + let locale = identifierComponents.slice(1).join("-") || "default"; + + e.webExtension.id = identifierComponents[0] + "@search.mozilla.org"; + e.webExtension.locale = locale; + } + } + + return { engines, privateDefault }; + } + + #setDefaultAndOrdersFromSelector(engines, privateDefault) { + const defaultEngine = engines[0]; + this._searchDefault = { + id: defaultEngine.webExtension.id, + locale: defaultEngine.webExtension.locale, + }; + if (privateDefault) { + this.#searchPrivateDefault = { + id: privateDefault.webExtension.id, + locale: privateDefault.webExtension.locale, + }; + } + } + + #saveSortedEngineList() { + lazy.logConsole.debug("#saveSortedEngineList"); + + // Set the useSavedOrder attribute to indicate that from now on we should + // use the user's order information stored in settings. + this._settings.setMetaDataAttribute("useSavedOrder", true); + + var engines = this.#sortedEngines; + + for (var i = 0; i < engines.length; ++i) { + engines[i].setAttr("order", i + 1); + } + } + + #buildSortedEngineList() { + // We must initialise _cachedSortedEngines here to avoid infinite recursion + // in the case of tests which don't define a default search engine. + // If there's no default defined, then we revert to the first item in the + // sorted list, but we can't do that if we don't have a list. + this._cachedSortedEngines = []; + + // If the user has specified a custom engine order, read the order + // information from the metadata instead of the default prefs. + if (this._settings.getMetaDataAttribute("useSavedOrder")) { + lazy.logConsole.debug("#buildSortedEngineList: using saved order"); + let addedEngines = {}; + + // Flag to keep track of whether or not we need to call #saveSortedEngineList. + let needToSaveEngineList = false; + + for (let engine of this._engines.values()) { + var orderNumber = engine.getAttr("order"); + + // Since the DB isn't regularly cleared, and engine files may disappear + // without us knowing, we may already have an engine in this slot. If + // that happens, we just skip it - it will be added later on as an + // unsorted engine. + if (orderNumber && !this._cachedSortedEngines[orderNumber - 1]) { + this._cachedSortedEngines[orderNumber - 1] = engine; + addedEngines[engine.name] = engine; + } else { + // We need to call #saveSortedEngineList so this gets sorted out. + needToSaveEngineList = true; + } + } + + // Filter out any nulls for engines that may have been removed + var filteredEngines = this._cachedSortedEngines.filter(function (a) { + return !!a; + }); + if (this._cachedSortedEngines.length != filteredEngines.length) { + needToSaveEngineList = true; + } + this._cachedSortedEngines = filteredEngines; + + if (needToSaveEngineList) { + this.#saveSortedEngineList(); + } + + // Array for the remaining engines, alphabetically sorted. + let alphaEngines = []; + + for (let engine of this._engines.values()) { + if (!(engine.name in addedEngines)) { + alphaEngines.push(engine); + } + } + + const collator = new Intl.Collator(); + alphaEngines.sort((a, b) => { + return collator.compare(a.name, b.name); + }); + return (this._cachedSortedEngines = + this._cachedSortedEngines.concat(alphaEngines)); + } + lazy.logConsole.debug("#buildSortedEngineList: using default orders"); + + return (this._cachedSortedEngines = this._sortEnginesByDefaults( + Array.from(this._engines.values()) + )); + } + + /** + * Sorts engines by the default settings (prefs, configuration values). + * + * @param {Array} engines + * An array of engine objects to sort. + * @returns {Array} + * The sorted array of engine objects. + * + * This is a private method with _ rather than # because it is + * called in a test. + */ + _sortEnginesByDefaults(engines) { + const sortedEngines = []; + const addedEngines = new Set(); + + function maybeAddEngineToSort(engine) { + if (!engine || addedEngines.has(engine.name)) { + return; + } + + sortedEngines.push(engine); + addedEngines.add(engine.name); + } + + // The app default engine should always be first in the list (except + // for distros, that we should respect). + const appDefault = this.appDefaultEngine; + maybeAddEngineToSort(appDefault); + + // If there's a private default, and it is different to the normal + // default, then it should be second in the list. + const appPrivateDefault = this.appPrivateDefaultEngine; + if (appPrivateDefault && appPrivateDefault != appDefault) { + maybeAddEngineToSort(appPrivateDefault); + } + + let remainingEngines; + const collator = new Intl.Collator(); + + remainingEngines = engines.filter(e => !addedEngines.has(e.name)); + + // We sort by highest orderHint first, then alphabetically by name. + remainingEngines.sort((a, b) => { + if (a._orderHint && b._orderHint) { + if (a._orderHint == b._orderHint) { + return collator.compare(a.name, b.name); + } + return b._orderHint - a._orderHint; + } + if (a._orderHint) { + return -1; + } + if (b._orderHint) { + return 1; + } + return collator.compare(a.name, b.name); + }); + + return [...sortedEngines, ...remainingEngines]; + } + + /** + * Get a sorted array of the visible engines. + * + * @returns {Array<SearchEngine>} + */ + + get #sortedVisibleEngines() { + return this.#sortedEngines.filter(engine => !engine.hidden); + } + + /** + * Migrates legacy add-ons which used the OpenSearch definitions to + * WebExtensions, if an equivalent WebExtension is installed. + * + * Run during the background checks. + */ + async #migrateLegacyEngines() { + lazy.logConsole.debug("Running migrate legacy engines"); + + const matchRegExp = /extensions\/(.*?)\.xpi!/i; + for (let engine of this._engines.values()) { + if ( + !engine.isAppProvided && + !engine._extensionID && + engine._loadPath.includes("[profile]/extensions/") + ) { + let match = engine._loadPath.match(matchRegExp); + if (match?.[1]) { + // There's a chance here that the WebExtension might not be + // installed any longer, even though the engine is. We'll deal + // with that in `checkWebExtensionEngines`. + let engines = await this.getEnginesByExtensionID(match[1]); + if (engines.length) { + lazy.logConsole.debug( + `Migrating ${engine.name} to WebExtension install` + ); + + if (this.defaultEngine == engine) { + this.defaultEngine = engines[0]; + } + await this.removeEngine(engine); + } + } + } + } + + lazy.logConsole.debug("Migrate legacy engines complete"); + } + + /** + * Checks if Search Engines associated with WebExtensions are valid and + * up-to-date, and reports them via telemetry if not. + * + * Run during the background checks. + */ + async #checkWebExtensionEngines() { + lazy.logConsole.debug("Running check on WebExtension engines"); + + for (let engine of this._engines.values()) { + if (engine instanceof lazy.AddonSearchEngine && !engine.isAppProvided) { + await engine.checkAndReportIfSettingsValid(); + } + } + lazy.logConsole.debug("WebExtension engine check complete"); + } + + /** + * Counts the number of secure, insecure, securely updated and insecurely + * updated OpenSearch engines the user has installed and reports those + * counts via telemetry. + * + * Run during the background checks. + */ + async #addOpenSearchTelemetry() { + let totalSecure = 0; + let totalInsecure = 0; + let totalWithSecureUpdates = 0; + let totalWithInsecureUpdates = 0; + + let engine; + let searchURI; + let updateURI; + for (let elem of this._engines) { + engine = elem[1]; + if (engine instanceof lazy.OpenSearchEngine) { + searchURI = engine.searchURLWithNoTerms; + updateURI = engine.updateURI; + + if (lazy.SearchUtils.isSecureURIForOpenSearch(searchURI)) { + totalSecure++; + } else { + totalInsecure++; + } + + if (updateURI && lazy.SearchUtils.isSecureURIForOpenSearch(updateURI)) { + totalWithSecureUpdates++; + } else if (updateURI) { + totalWithInsecureUpdates++; + } + } + } + + Services.telemetry.scalarSet( + "browser.searchinit.secure_opensearch_engine_count", + totalSecure + ); + Services.telemetry.scalarSet( + "browser.searchinit.insecure_opensearch_engine_count", + totalInsecure + ); + Services.telemetry.scalarSet( + "browser.searchinit.secure_opensearch_update_count", + totalWithSecureUpdates + ); + Services.telemetry.scalarSet( + "browser.searchinit.insecure_opensearch_update_count", + totalWithInsecureUpdates + ); + } + + /** + * Creates and adds a WebExtension based engine. + * + * @param {object} options + * Options for the engine. + * @param {Extension} options.extension + * An Extension object containing data about the extension. + * @param {string} [options.locale] + * The locale to use within the WebExtension. Defaults to the WebExtension's + * default locale. + * @param {object} [options.settings] + * The saved settings for the user. + * @param {initEngine} [options.initEngine] + * Set to true if this engine is being loaded during initialization. + */ + async _createAndAddEngine({ + extension, + locale = lazy.SearchUtils.DEFAULT_TAG, + settings, + initEngine = false, + }) { + // If we're in the startup cycle, and we've already loaded this engine, + // then we use the existing one rather than trying to start from scratch. + // This also avoids console errors. + if (extension.startupReason == "APP_STARTUP") { + let engine = this.#getEngineByWebExtensionDetails({ + id: extension.id, + locale, + }); + if (engine) { + lazy.logConsole.debug( + "Engine already loaded via settings, skipping due to APP_STARTUP:", + extension.id + ); + return engine; + } + } + + // We install search extensions during the init phase, both built in + // web extensions freshly installed (via addEnginesFromExtension) or + // user installed extensions being reenabled calling this directly. + if (!this.isInitialized && !extension.isAppProvided && !initEngine) { + await this.init(); + } + + let shouldSetAsDefault = false; + let changeReason = Ci.nsISearchService.CHANGE_REASON_UNKNOWN; + + for (let engine of this._engines.values()) { + if ( + !engine.extensionID && + engine._loadPath.startsWith(`jar:[profile]/extensions/${extension.id}`) + ) { + // This is a legacy extension engine that needs to be migrated to WebExtensions. + lazy.logConsole.debug("Migrating existing engine"); + shouldSetAsDefault = shouldSetAsDefault || this.defaultEngine == engine; + await this.removeEngine(engine); + } + } + + let newEngine = new lazy.AddonSearchEngine({ + isAppProvided: extension.isAppProvided, + details: { + extensionID: extension.id, + locale, + }, + }); + await newEngine.init({ + settings, + extension, + locale, + }); + + // If this extension is starting up, check to see if it previously overrode + // an application provided engine that has now been removed from the user's + // set-up. If the application provided engine has been removed and was + // default, then we should set this engine back to default and copy + // the settings across. + if (extension.startupReason == "APP_STARTUP") { + if (!settings) { + settings = await this._settings.get(); + } + // We check the saved settings for the overridden flag, because if the engine + // has been removed, we won't have that in _engines. + let previouslyOverridden = settings.engines?.find( + e => !!e._metaData.overriddenBy + ); + if (previouslyOverridden) { + let previousWebExtensionId = previouslyOverridden.id.endsWith("default") + ? previouslyOverridden.id.slice(0, -7) + : previouslyOverridden.id; + + // Only allow override if we were previously overriding and the + // engine is no longer installed, and the new engine still matches the + // override allow list. + if ( + previouslyOverridden._metaData.overriddenBy == extension.id && + !this._engines.get(previouslyOverridden.id) && + (await lazy.defaultOverrideAllowlist.canEngineOverride( + newEngine, + previousWebExtensionId + )) + ) { + shouldSetAsDefault = true; + // We assume that the app provided engine was removed due to a + // configuration change, and therefore we have re-added the add-on + // search engine. It is possible that it was actually due to a + // locale/region change, but that is harder to detect here. + changeReason = Ci.nsISearchService.CHANGE_REASON_CONFIG; + newEngine.copyUserSettingsFrom(previouslyOverridden); + } + } + } + + this.#addEngineToStore(newEngine); + if (shouldSetAsDefault) { + this.#setEngineDefault(false, newEngine, changeReason); + } + return newEngine; + } + + /** + * Called when we see an upgrade to an existing search extension. + * + * @param {object} extension + * An Extension object containing data about the extension. + */ + async #upgradeExtensionEngine(extension) { + let { engines } = await this._fetchEngineSelectorEngines(); + let extensionEngines = await this.getEnginesByExtensionID(extension.id); + + for (let engine of extensionEngines) { + let isDefault = engine == this.defaultEngine; + let isDefaultPrivate = engine == this.defaultPrivateEngine; + + let originalName = engine.name; + let locale = engine._locale || lazy.SearchUtils.DEFAULT_TAG; + let configuration = + engines.find( + e => + e.webExtension.id == extension.id && e.webExtension.locale == locale + ) ?? {}; + + await engine.update({ + configuration, + extension, + locale, + }); + + if (engine.name != originalName) { + if (isDefault) { + this._settings.setVerifiedMetaDataAttribute( + "defaultEngineId", + engine.id + ); + } + if (isDefaultPrivate) { + this._settings.setVerifiedMetaDataAttribute( + "privateDefaultEngineId", + engine.id + ); + } + this._cachedSortedEngines = null; + } + } + return extensionEngines; + } + + async #installExtensionEngine( + extension, + locales, + settings, + initEngine = false + ) { + lazy.logConsole.debug("installExtensionEngine:", extension.id); + + let installLocale = async locale => { + return this._createAndAddEngine({ + extension, + locale, + settings, + initEngine, + }); + }; + + let engines = []; + for (let locale of locales) { + lazy.logConsole.debug( + "addEnginesFromExtension: installing:", + extension.id, + ":", + locale + ); + engines.push(await installLocale(locale)); + } + return engines; + } + + #internalRemoveEngine(engine) { + // Remove the engine from _sortedEngines + if (this._cachedSortedEngines) { + var index = this._cachedSortedEngines.indexOf(engine); + if (index == -1) { + throw Components.Exception( + "Can't find engine to remove in _sortedEngines!", + Cr.NS_ERROR_FAILURE + ); + } + this._cachedSortedEngines.splice(index, 1); + } + + // Remove the engine from the internal store + this._engines.delete(engine.id); + } + + /** + * Helper function to find a new default engine and set it. This could + * be used if there is not default set yet, or if the current default is + * being removed. + * + * This function will not consider engines that have a `pendingRemoval` + * property set to true. + * + * The new default will be chosen from (in order): + * + * - Existing default from configuration, if it is not hidden. + * - The first non-hidden engine that is a general search engine. + * - If all other engines are hidden, unhide the default from the configuration. + * - If the default from the configuration is the one being removed, unhide + * the first general search engine, or first visible engine. + * + * @param {boolean} privateMode + * If true, returns the default engine for private browsing mode, otherwise + * the default engine for the normal mode. Note, this function does not + * check the "separatePrivateDefault" preference - that is up to the caller. + * @returns {nsISearchEngine|null} + * The appropriate search engine, or null if one could not be determined. + */ + #findAndSetNewDefaultEngine({ privateMode }) { + // First to the app default engine... + let newDefault = privateMode + ? this.appPrivateDefaultEngine + : this.appDefaultEngine; + + if (!newDefault || newDefault.hidden || newDefault.pendingRemoval) { + let sortedEngines = this.#sortedVisibleEngines; + let generalSearchEngines = sortedEngines.filter( + e => e.isGeneralPurposeEngine + ); + + // then to the first visible general search engine that isn't excluded... + let firstVisible = generalSearchEngines.find(e => !e.pendingRemoval); + if (firstVisible) { + newDefault = firstVisible; + } else if (newDefault) { + // then to the app default if it is not the one that is excluded... + if (!newDefault.pendingRemoval) { + newDefault.hidden = false; + } else { + newDefault = null; + } + } + + // and finally as a last resort we unhide the first engine + // even if the name is the same as the excluded one (should never happen). + if (!newDefault) { + if (!firstVisible) { + sortedEngines = this.#sortedEngines; + firstVisible = sortedEngines.find(e => e.isGeneralPurposeEngine); + if (!firstVisible) { + firstVisible = sortedEngines[0]; + } + } + if (firstVisible) { + firstVisible.hidden = false; + newDefault = firstVisible; + } + } + } + // We tried out best but something went very wrong. + if (!newDefault) { + lazy.logConsole.error("Could not find a replacement default engine."); + return null; + } + + // If the current engine wasn't set or was hidden, we used a fallback + // to pick a new current engine. As soon as we return it, this new + // current engine will become user-visible, so we should persist it. + // by calling the setter. + this.#setEngineDefault(privateMode, newDefault); + + return privateMode ? this.#currentPrivateEngine : this.#currentEngine; + } + + /** + * Helper function to set the current default engine. + * + * @param {boolean} privateMode + * If true, sets the default engine for private browsing mode, otherwise + * sets the default engine for the normal mode. Note, this function does not + * check the "separatePrivateDefault" preference - that is up to the caller. + * @param {nsISearchEngine} newEngine + * The search engine to select + * @param {SearchUtils.REASON_CHANGE_MAP} changeSource + * The source of the change of engine. + */ + #setEngineDefault(privateMode, newEngine, changeSource) { + // Sometimes we get wrapped nsISearchEngine objects (external XPCOM callers), + // and sometimes we get raw Engine JS objects (callers in this file), so + // handle both. + if ( + !(newEngine instanceof Ci.nsISearchEngine) && + !(newEngine instanceof lazy.SearchEngine) + ) { + throw Components.Exception( + "Invalid argument passed to defaultEngine setter", + Cr.NS_ERROR_INVALID_ARG + ); + } + + const newCurrentEngine = this._engines.get(newEngine.id); + if (!newCurrentEngine) { + throw Components.Exception( + "Can't find engine in store!", + Cr.NS_ERROR_UNEXPECTED + ); + } + + if (!newCurrentEngine.isAppProvided) { + // If a non default engine is being set as the current engine, ensure + // its loadPath has a verification hash. + if (!newCurrentEngine._loadPath) { + newCurrentEngine._loadPath = "[other]unknown"; + } + let loadPathHash = lazy.SearchUtils.getVerificationHash( + newCurrentEngine._loadPath + ); + let currentHash = newCurrentEngine.getAttr("loadPathHash"); + if (!currentHash || currentHash != loadPathHash) { + newCurrentEngine.setAttr("loadPathHash", loadPathHash); + lazy.SearchUtils.notifyAction( + newCurrentEngine, + lazy.SearchUtils.MODIFIED_TYPE.CHANGED + ); + } + } + + let currentEngine = privateMode + ? this.#currentPrivateEngine + : this.#currentEngine; + + if (newCurrentEngine == currentEngine) { + return; + } + + // Ensure that we reset an engine override if it was previously overridden. + currentEngine?.removeExtensionOverride(); + + if (privateMode) { + this.#currentPrivateEngine = newCurrentEngine; + } else { + this.#currentEngine = newCurrentEngine; + } + + // If we change the default engine in the future, that change should impact + // users who have switched away from and then back to the build's + // "app default" engine. So clear the user pref when the currentEngine is + // set to the build's app default engine, so that the currentEngine getter + // falls back to whatever the default is. + // However, we do not do this whilst we are running an experiment - an + // experiment must preseve the user's choice of default engine during it's + // runtime and when it ends. Once the experiment ends, we will reset the + // attribute elsewhere. + let newId = newCurrentEngine.id; + const appDefaultEngine = privateMode + ? this.appPrivateDefaultEngine + : this.appDefaultEngine; + if ( + newCurrentEngine == appDefaultEngine && + !lazy.NimbusFeatures.searchConfiguration.getVariable("experiment") + ) { + newId = ""; + } + + this._settings.setVerifiedMetaDataAttribute( + privateMode ? "privateDefaultEngineId" : "defaultEngineId", + newId + ); + + // Only do this if we're initialized though - this function can get called + // during initalization. + if (this.isInitialized) { + this.#recordDefaultChangedEvent( + privateMode, + currentEngine, + newCurrentEngine, + changeSource + ); + this.#recordTelemetryData(); + } + + lazy.SearchUtils.notifyAction( + newCurrentEngine, + lazy.SearchUtils.MODIFIED_TYPE[ + privateMode ? "DEFAULT_PRIVATE" : "DEFAULT" + ] + ); + // If we've not got a separate private active, notify update of the + // private so that the UI updates correctly. + if (!privateMode && !this.#separatePrivateDefault) { + lazy.SearchUtils.notifyAction( + newCurrentEngine, + lazy.SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + ); + } + } + + #onSeparateDefaultPrefChanged(prefName, previousValue, currentValue) { + // Clear out the sorted engines settings, so that we re-sort it if necessary. + this._cachedSortedEngines = null; + // We should notify if the normal default, and the currently saved private + // default are different. Otherwise, save the energy. + if (this.defaultEngine != this._getEngineDefault(true)) { + lazy.SearchUtils.notifyAction( + // Always notify with the new private engine, the function checks + // the preference value for us. + this.defaultPrivateEngine, + lazy.SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + ); + } + // Always notify about the change of status of private default if the user + // toggled the UI. + if ( + prefName == + lazy.SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault" + ) { + if (!previousValue && currentValue) { + this.#recordDefaultChangedEvent( + true, + null, + this._getEngineDefault(true), + Ci.nsISearchService.CHANGE_REASON_USER_PRIVATE_SPLIT + ); + } else { + this.#recordDefaultChangedEvent( + true, + this._getEngineDefault(true), + null, + Ci.nsISearchService.CHANGE_REASON_USER_PRIVATE_SPLIT + ); + } + } + // Update the telemetry data. + this.#recordTelemetryData(); + } + + #getEngineInfo(engine) { + if (!engine) { + // The defaultEngine getter will throw if there's no engine at all, + // which shouldn't happen unless an add-on or a test deleted all of them. + // Our preferences UI doesn't let users do that. + console.error("getDefaultEngineInfo: No default engine"); + return ["NONE", { name: "NONE" }]; + } + + const engineData = { + loadPath: engine._loadPath, + name: engine.name ? engine.name : "", + }; + + if (engine.isAppProvided) { + engineData.origin = "default"; + } else { + let currentHash = engine.getAttr("loadPathHash"); + if (!currentHash) { + engineData.origin = "unverified"; + } else { + let loadPathHash = lazy.SearchUtils.getVerificationHash( + engine._loadPath + ); + engineData.origin = + currentHash == loadPathHash ? "verified" : "invalid"; + } + } + + // For privacy, we only collect the submission URL for default engines... + let sendSubmissionURL = engine.isAppProvided; + + if (!sendSubmissionURL) { + // ... or engines that are the same domain as a default engine. + let engineHost = engine.searchUrlDomain; + for (let innerEngine of this._engines.values()) { + if (!innerEngine.isAppProvided) { + continue; + } + + if (innerEngine.searchUrlDomain == engineHost) { + sendSubmissionURL = true; + break; + } + } + + if (!sendSubmissionURL) { + // ... or well known search domains. + // + // Starts with: www.google., search.aol., yandex. + // or + // Ends with: search.yahoo.com, .ask.com, .bing.com, .startpage.com, baidu.com, duckduckgo.com + const urlTest = + /^(?:www\.google\.|search\.aol\.|yandex\.)|(?:search\.yahoo|\.ask|\.bing|\.startpage|\.baidu|duckduckgo)\.com$/; + sendSubmissionURL = urlTest.test(engineHost); + } + } + + if (sendSubmissionURL) { + let uri = engine.searchURLWithNoTerms; + uri = uri + .mutate() + .setUserPass("") // Avoid reporting a username or password. + .finalize(); + engineData.submissionURL = uri.spec; + } + + return [engine.telemetryId, engineData]; + } + + /** + * Records an event for where the default engine is changed. This is + * recorded to both Glean and Telemetry. + * + * The Glean GIFFT functionality is not used here because we use longer + * names in the extra arguments to the event. + * + * @param {boolean} isPrivate + * True if this is a event about a private engine. + * @param {SearchEngine} [previousEngine] + * The previously default search engine. + * @param {SearchEngine} [newEngine] + * The new default search engine. + * @param {string} changeSource + * The source of the change of default. + */ + #recordDefaultChangedEvent( + isPrivate, + previousEngine, + newEngine, + changeSource = Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ) { + changeSource = REASON_CHANGE_MAP.get(changeSource) ?? "unknown"; + Services.telemetry.setEventRecordingEnabled("search", true); + let telemetryId; + let engineInfo; + // If we are toggling the separate private browsing settings, we might not + // have an engine to record. + if (newEngine) { + [telemetryId, engineInfo] = this.#getEngineInfo(newEngine); + } else { + telemetryId = ""; + engineInfo = { + name: "", + loadPath: "", + submissionURL: "", + }; + } + + let submissionURL = engineInfo.submissionURL ?? ""; + Services.telemetry.recordEvent( + "search", + "engine", + isPrivate ? "change_private" : "change_default", + changeSource, + { + // In docshell tests, the previous engine does not exist, so we allow + // for the previousEngine to be undefined. + prev_id: previousEngine?.telemetryId ?? "", + new_id: telemetryId, + new_name: engineInfo.name, + new_load_path: engineInfo.loadPath, + // Telemetry has a limit of 80 characters. + new_sub_url: submissionURL.slice(0, 80), + } + ); + + let extraArgs = { + // In docshell tests, the previous engine does not exist, so we allow + // for the previousEngine to be undefined. + previous_engine_id: previousEngine?.telemetryId ?? "", + new_engine_id: telemetryId, + new_display_name: engineInfo.name, + new_load_path: engineInfo.loadPath, + // Glean has a limit of 100 characters. + new_submission_url: submissionURL.slice(0, 100), + change_source: changeSource, + }; + if (isPrivate) { + Glean.searchEnginePrivate.changed.record(extraArgs); + } else { + Glean.searchEngineDefault.changed.record(extraArgs); + } + } + + /** + * Records the user's current default engine (normal and private) data to + * telemetry. + */ + #recordTelemetryData() { + let info = this.getDefaultEngineInfo(); + + Glean.searchEngineDefault.engineId.set(info.defaultSearchEngine); + Glean.searchEngineDefault.displayName.set( + info.defaultSearchEngineData.name + ); + Glean.searchEngineDefault.loadPath.set( + info.defaultSearchEngineData.loadPath + ); + Glean.searchEngineDefault.submissionUrl.set( + info.defaultSearchEngineData.submissionURL ?? "blank:" + ); + Glean.searchEngineDefault.verified.set(info.defaultSearchEngineData.origin); + + Glean.searchEnginePrivate.engineId.set( + info.defaultPrivateSearchEngine ?? "" + ); + + if (info.defaultPrivateSearchEngineData) { + Glean.searchEnginePrivate.displayName.set( + info.defaultPrivateSearchEngineData.name + ); + Glean.searchEnginePrivate.loadPath.set( + info.defaultPrivateSearchEngineData.loadPath + ); + Glean.searchEnginePrivate.submissionUrl.set( + info.defaultPrivateSearchEngineData.submissionURL ?? "blank:" + ); + Glean.searchEnginePrivate.verified.set( + info.defaultPrivateSearchEngineData.origin + ); + } else { + Glean.searchEnginePrivate.displayName.set(""); + Glean.searchEnginePrivate.loadPath.set(""); + Glean.searchEnginePrivate.submissionUrl.set("blank:"); + Glean.searchEnginePrivate.verified.set(""); + } + } + + /** + * This function is called at the beginning of search service init. + * If the error type set in a test environment matches errorType + * passed to this function, we throw an error. + * + * @param {string} errorType + * The error that can occur during search service init. + * + */ + #maybeThrowErrorInTest(errorType) { + if ( + Services.env.exists("XPCSHELL_TEST_PROFILE_DIR") && + this.errorToThrowInTest === errorType + ) { + throw new Error( + `Fake ${errorType} error during search service initialization.` + ); + } + } + + #buildParseSubmissionMap() { + this.#parseSubmissionMap = new Map(); + + // Used only while building the map, indicates which entries do not refer to + // the main domain of the engine but to an alternate domain, for example + // "www.google.fr" for the "www.google.com" search engine. + let keysOfAlternates = new Set(); + + for (let engine of this.#sortedEngines) { + if (engine.hidden) { + continue; + } + + let urlParsingInfo = engine.getURLParsingInfo(); + if (!urlParsingInfo) { + continue; + } + + // Store the same object on each matching map key, as an optimization. + let mapValueForEngine = { + engine, + termsParameterName: urlParsingInfo.termsParameterName, + }; + + let processDomain = (domain, isAlternate) => { + let key = domain + urlParsingInfo.path; + + // Apply the logic for which main domains take priority over alternate + // domains, even if they are found later in the ordered engine list. + let existingEntry = this.#parseSubmissionMap.get(key); + if (!existingEntry) { + if (isAlternate) { + keysOfAlternates.add(key); + } + } else if (!isAlternate && keysOfAlternates.has(key)) { + keysOfAlternates.delete(key); + } else { + return; + } + + this.#parseSubmissionMap.set(key, mapValueForEngine); + }; + + processDomain(urlParsingInfo.mainDomain, false); + lazy.SearchStaticData.getAlternateDomains( + urlParsingInfo.mainDomain + ).forEach(d => processDomain(d, true)); + } + } + + #nimbusSearchUpdatedFun = null; + + async #nimbusSearchUpdated() { + this.#checkNimbusPrefs(); + Services.search.wrappedJSObject._maybeReloadEngines( + Ci.nsISearchService.CHANGE_REASON_EXPERIMENT + ); + } + + /** + * Check the prefs are correctly updated for users enrolled in a Nimbus experiment. + * + * @param {boolean} isStartup + * Whether this function was called as part of the startup flow. + */ + #checkNimbusPrefs(isStartup = false) { + // If we are in an experiment we may need to check the status on startup, otherwise + // ignore the call to check on startup so we do not reset users prefs when they are + // not an experiment. + if ( + isStartup && + !lazy.NimbusFeatures.searchConfiguration.getVariable("experiment") + ) { + return; + } + let nimbusPrivateDefaultUIEnabled = + lazy.NimbusFeatures.searchConfiguration.getVariable( + "seperatePrivateDefaultUIEnabled" + ); + let nimbusPrivateDefaultUrlbarResultEnabled = + lazy.NimbusFeatures.searchConfiguration.getVariable( + "seperatePrivateDefaultUrlbarResultEnabled" + ); + + let previousPrivateDefault = this.defaultPrivateEngine; + let uiWasEnabled = this._separatePrivateDefaultEnabledPrefValue; + if ( + this._separatePrivateDefaultEnabledPrefValue != + nimbusPrivateDefaultUIEnabled + ) { + Services.prefs.setBoolPref( + `${lazy.SearchUtils.BROWSER_SEARCH_PREF}separatePrivateDefault.ui.enabled`, + nimbusPrivateDefaultUIEnabled + ); + let newPrivateDefault = this.defaultPrivateEngine; + if (previousPrivateDefault != newPrivateDefault) { + if (!uiWasEnabled) { + this.#recordDefaultChangedEvent( + true, + null, + newPrivateDefault, + Ci.nsISearchService.CHANGE_REASON_EXPERIMENT + ); + } else { + this.#recordDefaultChangedEvent( + true, + previousPrivateDefault, + null, + Ci.nsISearchService.CHANGE_REASON_EXPERIMENT + ); + } + } + } + if ( + this.separatePrivateDefaultUrlbarResultEnabled != + nimbusPrivateDefaultUrlbarResultEnabled + ) { + Services.prefs.setBoolPref( + `${lazy.SearchUtils.BROWSER_SEARCH_PREF}separatePrivateDefault.urlbarResult.enabled`, + nimbusPrivateDefaultUrlbarResultEnabled + ); + } + } + + #addObservers() { + if (this.#observersAdded) { + // There might be a race between synchronous and asynchronous + // initialization for which we try to register the observers twice. + return; + } + this.#observersAdded = true; + + this.#nimbusSearchUpdatedFun = this.#nimbusSearchUpdated.bind(this); + lazy.NimbusFeatures.searchConfiguration.onUpdate( + this.#nimbusSearchUpdatedFun + ); + + Services.obs.addObserver(this, lazy.SearchUtils.TOPIC_ENGINE_MODIFIED); + Services.obs.addObserver(this, QUIT_APPLICATION_TOPIC); + Services.obs.addObserver(this, TOPIC_LOCALES_CHANGE); + + this._settings.addObservers(); + + // The current stage of shutdown. Used to help analyze crash + // signatures in case of shutdown timeout. + let shutdownState = { + step: "Not started", + latestError: { + message: undefined, + stack: undefined, + }, + }; + IOUtils.profileBeforeChange.addBlocker( + "Search service: shutting down", + () => + (async () => { + // If we are in initialization, then don't attempt to save the settings. + // It is likely that shutdown will have caused the add-on manager to + // stop, which can cause initialization to fail. + // Hence at that stage, we could have broken settings which we don't + // want to write. + // The good news is, that if we don't write the settings here, we'll + // detect the out-of-date settings on next state, and automatically + // rebuild it. + if (!this.isInitialized) { + lazy.logConsole.warn( + "not saving settings on shutdown due to initializing." + ); + return; + } + + try { + await this._settings.shutdown(shutdownState); + } catch (ex) { + // Ensure that error is reported and that it causes tests + // to fail, otherwise ignore it. + Promise.reject(ex); + } + })(), + + () => shutdownState + ); + } + + // This is prefixed with _ rather than # because it is + // called in a test. + _removeObservers() { + if (this.ignoreListListener) { + lazy.IgnoreLists.unsubscribe(this.ignoreListListener); + delete this.ignoreListListener; + } + if (this.#queuedIdle) { + this.idleService.removeIdleObserver(this, RECONFIG_IDLE_TIME_SEC); + this.#queuedIdle = false; + } + + this._settings.removeObservers(); + + lazy.NimbusFeatures.searchConfiguration.offUpdate( + this.#nimbusSearchUpdatedFun + ); + + Services.obs.removeObserver(this, lazy.SearchUtils.TOPIC_ENGINE_MODIFIED); + Services.obs.removeObserver(this, QUIT_APPLICATION_TOPIC); + Services.obs.removeObserver(this, TOPIC_LOCALES_CHANGE); + Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC); + } + + QueryInterface = ChromeUtils.generateQI([ + "nsISearchService", + "nsIObserver", + "nsITimerCallback", + ]); + + // nsIObserver + observe(engine, topic, verb) { + switch (topic) { + case lazy.SearchUtils.TOPIC_ENGINE_MODIFIED: + switch (verb) { + case lazy.SearchUtils.MODIFIED_TYPE.ADDED: + this.#parseSubmissionMap = null; + break; + case lazy.SearchUtils.MODIFIED_TYPE.CHANGED: + engine = engine.wrappedJSObject; + if ( + engine == this.defaultEngine || + engine == this.defaultPrivateEngine + ) { + this.#recordDefaultChangedEvent( + engine != this.defaultEngine, + engine, + engine, + Ci.nsISearchService.CHANGE_REASON_ENGINE_UPDATE + ); + } + this.#parseSubmissionMap = null; + break; + case lazy.SearchUtils.MODIFIED_TYPE.REMOVED: + // Invalidate the map used to parse URLs to search engines. + this.#parseSubmissionMap = null; + break; + } + break; + + case "idle": { + this.idleService.removeIdleObserver(this, RECONFIG_IDLE_TIME_SEC); + this.#queuedIdle = false; + lazy.logConsole.debug( + "Reloading engines after idle due to configuration change" + ); + this._maybeReloadEngines( + Ci.nsISearchService.CHANGE_REASON_CONFIG + ).catch(console.error); + break; + } + + case QUIT_APPLICATION_TOPIC: + this._removeObservers(); + break; + + case TOPIC_LOCALES_CHANGE: + // Locale changed. Re-init. We rely on observers, because we can't + // return this promise to anyone. + + // At the time of writing, when the user does a "Apply and Restart" for + // a new language the preferences code triggers the locales change and + // restart straight after, so we delay the check, which means we should + // be able to avoid the reload on shutdown, and we'll sort it out + // on next startup. + // This also helps to avoid issues with the add-on manager shutting + // down at the same time (see _reInit for more info). + Services.tm.dispatchToMainThread(() => { + if (!Services.startup.shuttingDown) { + this._maybeReloadEngines( + Ci.nsISearchService.CHANGE_REASON_LOCALE + ).catch(console.error); + } + }); + break; + case lazy.Region.REGION_TOPIC: + lazy.logConsole.debug("Region updated:", lazy.Region.home); + this._maybeReloadEngines( + Ci.nsISearchService.CHANGE_REASON_REGION + ).catch(console.error); + break; + } + } + + /** + * Create an engine object from the search configuration details. + * + * This method is prefixed with _ rather than # because it is + * called in a test. + * + * @param {object} config + * The configuration object that defines the details of the engine + * webExtensionId etc. + * @param {object} [settings] + * The saved settings for the user. + * @returns {nsISearchEngine} + * Returns the search engine object. + */ + async _makeEngineFromConfig(config, settings) { + lazy.logConsole.debug("_makeEngineFromConfig:", config); + + if (!lazy.SearchUtils.newSearchConfigEnabled) { + let locale = + "locale" in config.webExtension + ? config.webExtension.locale + : lazy.SearchUtils.DEFAULT_TAG; + + let engine = new lazy.AddonSearchEngine({ + isAppProvided: true, + details: { + extensionID: config.webExtension.id, + locale, + }, + }); + await engine.init({ + settings, + locale, + config, + }); + return engine; + } + + return new lazy.AppProvidedSearchEngine({ config, settings }); + } + + /** + * @param {object} metaData + * The metadata object that defines the details of the engine. + * @returns {boolean} + * Returns true if metaData has different property values than + * the cached _metaData. + */ + #didSettingsMetaDataUpdate(metaData) { + let metaDataProperties = [ + "locale", + "region", + "channel", + "experiment", + "distroID", + ]; + + return metaDataProperties.some(p => { + return metaData?.[p] !== this._settings.getMetaDataAttribute(p); + }); + } + + /** + * Shows an infobar to notify the user their default search engine has been + * removed and replaced by a new default search engine. + * + * This method is prefixed with _ rather than # because it is + * called in a test. + * + * @param {string} prevCurrentEngineName + * The name of the previous default engine that will be replaced. + * @param {string} newCurrentEngineName + * The name of the engine that will be the new default engine. + * + */ + _showRemovalOfSearchEngineNotificationBox( + prevCurrentEngineName, + newCurrentEngineName + ) { + let win = Services.wm.getMostRecentBrowserWindow(); + win.BrowserSearch.removalOfSearchEngineNotificationBox( + prevCurrentEngineName, + newCurrentEngineName + ); + } + + /** + * Maybe starts the timer for OpenSearch engine updates. This will be set + * only if updates are enabled and there are OpenSearch engines installed + * which have updates. + */ + #maybeStartOpenSearchUpdateTimer() { + if ( + this.#openSearchUpdateTimerStarted || + !Services.prefs.getBoolPref( + lazy.SearchUtils.BROWSER_SEARCH_PREF + "update", + true + ) + ) { + return; + } + + let engineWithUpdates = [...this._engines.values()].some( + engine => engine instanceof lazy.OpenSearchEngine && engine.hasUpdates + ); + + if (engineWithUpdates) { + lazy.logConsole.debug("Engine with updates found, setting update timer"); + lazy.timerManager.registerTimer( + OPENSEARCH_UPDATE_TIMER_TOPIC, + this, + OPENSEARCH_UPDATE_TIMER_INTERVAL, + true + ); + this.#openSearchUpdateTimerStarted = true; + } + } +} // end SearchService class + +XPCOMUtils.defineLazyServiceGetter( + SearchService.prototype, + "idleService", + "@mozilla.org/widget/useridleservice;1", + "nsIUserIdleService" +); + +/** + * Handles getting and checking extensions against the allow list. + */ +class SearchDefaultOverrideAllowlistHandler { + /** + * @param {Function} listener + * A listener for configuration update changes. + */ + constructor(listener) { + this._remoteConfig = lazy.RemoteSettings( + lazy.SearchUtils.SETTINGS_ALLOWLIST_KEY + ); + } + + /** + * Determines if a search engine extension can override a default one + * according to the allow list. + * + * @param {object} extension + * The extension object (from add-on manager) that will override the + * app provided search engine. + * @param {string} appProvidedExtensionId + * The id of the search engine that will be overriden. + * @returns {boolean} + * Returns true if the search engine extension may override the app provided + * instance. + */ + async canOverride(extension, appProvidedExtensionId) { + const overrideTable = await this._getAllowlist(); + + let entry = overrideTable.find(e => e.thirdPartyId == extension.id); + if (!entry) { + return false; + } + + if (appProvidedExtensionId != entry.overridesId) { + return false; + } + + let searchProvider = + extension.manifest.chrome_settings_overrides.search_provider; + + return entry.urls.some( + e => + searchProvider.search_url == e.search_url && + searchProvider.search_form == e.search_form && + searchProvider.search_url_get_params == e.search_url_get_params && + searchProvider.search_url_post_params == e.search_url_post_params + ); + } + + /** + * Determines if an existing search engine is allowed to override a default one + * according to the allow list. + * + * @param {SearchEngine} engine + * The existing search engine. + * @param {string} appProvidedEngineExtensionId + * The id of the search engine that will be overriden. + * @returns {boolean} + * Returns true if the existing search engine is allowed to override the + * app provided instance. + */ + async canEngineOverride(engine, appProvidedEngineExtensionId) { + const overrideEntries = await this._getAllowlist(); + + let entry; + + if (engine instanceof lazy.AddonSearchEngine) { + entry = overrideEntries.find(e => e.thirdPartyId == engine._extensionID); + } else if (engine instanceof lazy.OpenSearchEngine) { + entry = overrideEntries.find( + e => + e.thirdPartyId == "opensearch@search.mozilla.org" && + e.engineName == engine.name + ); + } + if (!entry) { + return false; + } + + if (appProvidedEngineExtensionId != entry.overridesId) { + return false; + } + + return entry.urls.some(urlSet => + engine.checkSearchUrlMatchesManifest(urlSet) + ); + } + + /** + * Obtains the configuration from remote settings. This includes + * verifying the signature of the record within the database. + * + * If the signature in the database is invalid, the database will be wiped + * and the stored dump will be used, until the settings next update. + * + * Note that this may cause a network check of the certificate, but that + * should generally be quick. + * + * @returns {Array} + * An array of objects in the database, or an empty array if none + * could be obtained. + */ + async _getAllowlist() { + let result = []; + try { + result = await this._remoteConfig.get(); + } catch (ex) { + // Don't throw an error just log it, just continue with no data, and hopefully + // a sync will fix things later on. + console.error(ex); + } + lazy.logConsole.debug("Allow list is:", result); + return result; + } +} diff --git a/toolkit/components/search/SearchSettings.sys.mjs b/toolkit/components/search/SearchSettings.sys.mjs new file mode 100644 index 0000000000..fed0dd1808 --- /dev/null +++ b/toolkit/components/search/SearchSettings.sys.mjs @@ -0,0 +1,633 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs", + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchSettings", + maxLogLevel: lazy.SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +const SETTINGS_FILENAME = "search.json.mozlz4"; + +/** + * This class manages the saves search settings. + * + * Global settings can be saved and obtained from this class via the + * `*Attribute` methods. + */ +export class SearchSettings { + constructor(searchService) { + this.#searchService = searchService; + + // Once the search service has initialized, schedule a write to ensure + // that any settings that may have changed or need updating are handled. + searchService.promiseInitialized.then(() => { + this._delayedWrite(); + }); + } + + QueryInterface = ChromeUtils.generateQI([Ci.nsIObserver]); + + // Delay for batching invalidation of the JSON settings (ms) + static SETTINGS_INVALIDATION_DELAY = 1000; + + /** + * A reference to the pending DeferredTask, if there is one. + */ + _batchTask = null; + + /** + * A reference to the search service so that we can save the engines list. + */ + #searchService = null; + + /* + * The user's settings file read from disk so we can persist metadata for + * engines that are default or hidden, the user's locale and region, hashes + * for the loadPath, and hashes for default and private default engines. + * This is the JSON we read from disk and save to disk when there's an update + * to the settings. + * + * Structure of settings: + * Object { version: <number>, + * engines: [...], + * metaData: {...}, + * } + * + * Settings metaData is the active metadata for setting and getting attributes. + * When a new metadata attribute is set, we save it to #settings.metaData and + * write #settings to disk. + * + * #settings.metaData attributes: + * @property {string} current + * The current user-set default engine. The associated hash is called + * 'hash'. + * @property {string} private + * The current user-set private engine. The associated hash is called + * 'privateHash'. + * The current and prviate objects have associated hash fields to validate + * the value is set by the application. + * @property {string} appDefaultEngine + * @property {string} channel + * Configuration is restricted to the specified channel. ESR is an example + * of a channel. + * @property {string} distroID + * Specifies which distribution the default engine is included in. + * @property {string} experiment + * Specifies if the application is running on an experiment. + * @property {string} locale + * @property {string} region + * @property {boolean} useSavedOrder + * True if the user's order information stored in settings is used. + * + */ + #settings = null; + + /** + * @type {object} A deep copy of #settings. + * #cachedSettings is updated when we read the settings from disk and when + * we write settings to disk. #cachedSettings is compared with #settings + * before we do a write to disk. If there's no change to the settings + * attributes, then we don't write the settings to disk. + */ + #cachedSettings = {}; + + addObservers() { + Services.obs.addObserver(this, lazy.SearchUtils.TOPIC_ENGINE_MODIFIED); + Services.obs.addObserver(this, lazy.SearchUtils.TOPIC_SEARCH_SERVICE); + } + + /** + * Cleans up, removing observers. + */ + removeObservers() { + Services.obs.removeObserver(this, lazy.SearchUtils.TOPIC_ENGINE_MODIFIED); + Services.obs.removeObserver(this, lazy.SearchUtils.TOPIC_SEARCH_SERVICE); + } + + /** + * Reads the settings file. + * + * @param {string} origin + * If this parameter is "test", then the settings will not be written. As + * some tests manipulate the settings directly, we allow turning off writing to + * avoid writing stale settings data. + * @returns {object} + * Returns the settings file data. + */ + async get(origin = "") { + let json; + await this._ensurePendingWritesCompleted(origin); + try { + let settingsFilePath = PathUtils.join( + PathUtils.profileDir, + SETTINGS_FILENAME + ); + json = await IOUtils.readJSON(settingsFilePath, { decompress: true }); + if (!json.engines || !json.engines.length) { + throw new Error("no engine in the file"); + } + } catch (ex) { + lazy.logConsole.debug("get: No settings file exists, new profile?", ex); + json = {}; + } + + this.#settings = json; + this.#cachedSettings = structuredClone(json); + + if (!this.#settings.metaData) { + this.#settings.metaData = {}; + } + + // Versions of gecko older than 82 stored the order flag as a preference. + // This was changed in version 6 of the settings file. + if ( + this.#settings.version < 6 || + !("useSavedOrder" in this.#settings.metaData) + ) { + const prefName = lazy.SearchUtils.BROWSER_SEARCH_PREF + "useDBForOrder"; + let useSavedOrder = Services.prefs.getBoolPref(prefName, false); + + this.setMetaDataAttribute("useSavedOrder", useSavedOrder); + + // Clear the old pref so it isn't lying around. + Services.prefs.clearUserPref(prefName); + } + + // Added in Firefox 110. + if (this.#settings.version < 8 && Array.isArray(this.#settings.engines)) { + this.#migrateTelemetryLoadPaths(); + } + + // Migration for hiddenOneOffs + if (this.#settings.version < 9 && this.#settings.engines) { + const hiddenOneOffsPrefs = Services.prefs.getStringPref( + "browser.search.hiddenOneOffs", + "" + ); + for (const engine of this.#settings.engines) { + engine._metaData.hideOneOffButton = hiddenOneOffsPrefs.includes( + engine._name + ); + } + Services.prefs.clearUserPref("browser.search.hiddenOneOffs"); + } + + return structuredClone(json); + } + + /** + * Queues writing the settings until after SETTINGS_INVALIDATION_DELAY. If there + * is a currently queued task then it will be restarted. + */ + _delayedWrite() { + if (this._batchTask) { + this._batchTask.disarm(); + } else { + let task = async () => { + if ( + !this.#searchService.isInitialized || + this.#searchService._reloadingEngines + ) { + // Re-arm the task as we don't want to save potentially incomplete + // information during the middle of (re-)initializing. + this._batchTask.arm(); + return; + } + lazy.logConsole.debug("batchTask: Invalidating engine settings"); + await this._write(); + }; + this._batchTask = new lazy.DeferredTask( + task, + SearchSettings.SETTINGS_INVALIDATION_DELAY + ); + } + this._batchTask.arm(); + } + + /** + * Ensures any pending writes of the settings are completed. + * + * @param {string} origin + * If this parameter is "test", then the settings will not be written. As + * some tests manipulate the settings directly, we allow turning off writing to + * avoid writing stale settings data. + */ + async _ensurePendingWritesCompleted(origin = "") { + // Before we read the settings file, first make sure all pending tasks are clear. + if (!this._batchTask) { + return; + } + lazy.logConsole.debug("finalizing batch task"); + let task = this._batchTask; + this._batchTask = null; + // Tests manipulate the settings directly, so let's not double-write with + // stale settings data here. + if (origin == "test") { + task.disarm(); + } else { + await task.finalize(); + } + } + + /** + * Writes the settings to disk (no delay). + */ + async _write() { + if (this._batchTask) { + this._batchTask.disarm(); + } + + let settings = {}; + + // Allows us to force a settings refresh should the settings format change. + settings.version = lazy.SearchUtils.SETTINGS_VERSION; + settings.engines = [...this.#searchService._engines.values()].map(engine => + JSON.parse(JSON.stringify(engine)) + ); + settings.metaData = this.#settings.metaData; + + // Persist metadata for AppProvided engines even if they aren't currently + // active, this means if they become active again their settings + // will be restored. + if (this.#settings?.engines) { + for (let engine of this.#settings.engines) { + let included = settings.engines.some(e => e._name == engine._name); + if (engine._isAppProvided && !included) { + settings.engines.push(engine); + } + } + } + + // Update the local copy. + this.#settings = settings; + + try { + if (!settings.engines.length) { + throw new Error("cannot write without any engine."); + } + + if (this.isCurrentAndCachedSettingsEqual()) { + lazy.logConsole.debug( + "_write: Settings unchanged. Did not write to disk." + ); + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "write-prevented-when-settings-unchanged" + ); + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "write-settings-to-disk-complete" + ); + + return; + } + + // At this point, the settings and cached settings are different. We + // write settings to disk and update #cachedSettings. + this.#cachedSettings = structuredClone(this.#settings); + + lazy.logConsole.debug("_write: Writing to settings file."); + let path = PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME); + await IOUtils.writeJSON(path, settings, { + compress: true, + tmpPath: path + ".tmp", + }); + lazy.logConsole.debug("_write: settings file written to disk."); + Services.obs.notifyObservers( + null, + lazy.SearchUtils.TOPIC_SEARCH_SERVICE, + "write-settings-to-disk-complete" + ); + } catch (ex) { + lazy.logConsole.error("_write: Could not write to settings file:", ex); + } + } + + /** + * Sets an attribute without verification. + * + * @param {string} name + * The name of the attribute to set. + * @param {*} val + * The value to set. + */ + setMetaDataAttribute(name, val) { + this.#settings.metaData[name] = val; + this._delayedWrite(); + } + + /** + * Sets a verified attribute. This will save an additional hash + * value, that can be verified when reading back. + * + * @param {string} name + * The name of the attribute to set. + * @param {*} val + * The value to set. + */ + setVerifiedMetaDataAttribute(name, val) { + this.#settings.metaData[name] = val; + this.#settings.metaData[this.getHashName(name)] = + lazy.SearchUtils.getVerificationHash(val); + this._delayedWrite(); + } + + /** + * Gets an attribute without verification. + * + * @param {string} name + * The name of the attribute to get. + * @returns {*} + * The value of the attribute, or undefined if not known. + */ + getMetaDataAttribute(name) { + return this.#settings.metaData[name] ?? undefined; + } + + /** + * Gets a copy of the settings metadata. + * + * @returns {*} + * A copy of the settings metadata object. + * + */ + getSettingsMetaData() { + return { ...this.#settings.metaData }; + } + + /** + * Gets a verified attribute. + * + * @param {string} name + * The name of the attribute to get. + * @param {boolean} isAppProvided + * |true| if the engine associated with the attribute is an application + * provided engine. + * @returns {*} + * The value of the attribute. + * We return undefined if the value of the attribute is not known or does + * not match the verification hash. + * + */ + getVerifiedMetaDataAttribute(name, isAppProvided) { + let attribute = this.getMetaDataAttribute(name); + + // If the selected engine is an application provided one, we can relax the + // verification hash check to reduce the annoyance for users who + // backup/sync their profile in custom ways. + if (isAppProvided) { + return attribute; + } + + if ( + attribute && + this.getMetaDataAttribute(this.getHashName(name)) != + lazy.SearchUtils.getVerificationHash(attribute) + ) { + lazy.logConsole.warn( + "getVerifiedMetaDataAttribute, invalid hash for", + name + ); + return undefined; + } + return attribute; + } + + /** + * Sets an attribute in #settings.engines._metaData + * + * @param {string} engineName + * The name of the engine. + * @param {string} property + * The name of the attribute to set. + * @param {*} value + * The value to set. + */ + setEngineMetaDataAttribute(engineName, property, value) { + let engines = [...this.#searchService._engines.values()]; + let engine = engines.find(engine => engine._name == engineName); + if (engine) { + engine._metaData[property] = value; + this._delayedWrite(); + } + } + + /** + * Gets an attribute from #settings.engines._metaData + * + * @param {string} engineName + * The name of the engine. + * @param {string} property + * The name of the attribute to get. + * @returns {*} + * The value of the attribute, or undefined if not known. + */ + getEngineMetaDataAttribute(engineName, property) { + let engine = this.#settings.engines.find( + engine => engine._name == engineName + ); + return engine._metaData[property] ?? undefined; + } + + /** + * Returns the name for the hash for a particular attribute. This is + * necessary because the default engine ID property is named `current` + * with its hash as `hash`. All other hashes are in the `<name>Hash` format. + * + * @param {string} name + * The name of the attribute to get the hash name for. + * @returns {string} + * The hash name to use. + */ + getHashName(name) { + // The "current" check remains here because we need to retrieve the + // "current" hash name for the migration of engine ids. After the migration, + // the "current" property is no longer used because we now store + // "defaultEngineId" instead. + if (name == "current") { + return "hash"; + } + return name + "Hash"; + } + + /** + * Handles shutdown; writing the settings if necessary. + * + * @param {object} state + * The shutdownState object that is used to help analyzing the shutdown + * state in case of a crash or shutdown timeout. + */ + async shutdown(state) { + if (!this._batchTask) { + return; + } + state.step = "Finalizing batched task"; + try { + await this._batchTask.finalize(); + state.step = "Batched task finalized"; + } catch (ex) { + state.step = "Batched task failed to finalize"; + + state.latestError.message = "" + ex; + if (ex && typeof ex == "object") { + state.latestError.stack = ex.stack || undefined; + } + } + } + + // nsIObserver + observe(engine, topic, verb) { + switch (topic) { + case lazy.SearchUtils.TOPIC_ENGINE_MODIFIED: + switch (verb) { + case lazy.SearchUtils.MODIFIED_TYPE.ADDED: + case lazy.SearchUtils.MODIFIED_TYPE.CHANGED: + case lazy.SearchUtils.MODIFIED_TYPE.REMOVED: + this._delayedWrite(); + break; + } + break; + case lazy.SearchUtils.TOPIC_SEARCH_SERVICE: + switch (verb) { + case "engines-reloaded": + this._delayedWrite(); + break; + } + break; + } + } + + /** + * Compares the #settings and #cachedSettings objects. + * + * @returns {boolean} + * True if the objects have the same property and values. + */ + isCurrentAndCachedSettingsEqual() { + return lazy.ObjectUtils.deepEqual(this.#settings, this.#cachedSettings); + } + + /** + * This function writes to settings versions 6 and below. It does two + * updates: + * 1) Store engine ids. + * 2) Store "defaultEngineId" and "privateDefaultEngineId" to replace + * "current" and "private" because we are no longer referencing the + * "current" and "private" attributes with engine names as their values. + * + * @param {object} clonedSettings + * The SearchService holds a deep copy of the settings file object. This + * clonedSettings is passed in as an argument from SearchService. + */ + migrateEngineIds(clonedSettings) { + if (clonedSettings.version <= 6) { + lazy.logConsole.debug("migrateEngineIds: start"); + + for (let engineSettings of clonedSettings.engines) { + let engine = this.#getEngineByName(engineSettings._name); + + if (engine) { + // Store the engine id + engineSettings.id = engine.id; + } + } + + let currentDefaultEngine = this.#getEngineByName( + clonedSettings.metaData.current + ); + let privateDefaultEngine = this.#getEngineByName( + clonedSettings.metaData.private + ); + + // As per SearchService._getEngineDefault, we relax the verification hash + // check for application provided engines to reduce the annoyance for + // users who backup/sync their profile in custom ways. + if ( + currentDefaultEngine && + (currentDefaultEngine.isAppProvided || + lazy.SearchUtils.getVerificationHash( + clonedSettings.metaData.current + ) == clonedSettings.metaData[this.getHashName("current")]) + ) { + // Store the defaultEngineId + this.setVerifiedMetaDataAttribute( + "defaultEngineId", + currentDefaultEngine.id + ); + } else { + this.setVerifiedMetaDataAttribute("defaultEngineId", ""); + } + + if ( + privateDefaultEngine && + (privateDefaultEngine.isAppProvided || + lazy.SearchUtils.getVerificationHash( + clonedSettings.metaData.private + ) == clonedSettings.metaData[this.getHashName("private")]) + ) { + // Store the privateDefaultEngineId + this.setVerifiedMetaDataAttribute( + "privateDefaultEngineId", + privateDefaultEngine.id + ); + } else { + this.setVerifiedMetaDataAttribute("privateDefaultEngineId", ""); + } + + lazy.logConsole.debug("migrateEngineIds: done"); + } + } + + /** + * Migrates telemetry load paths for versions of settings prior to v8. + */ + #migrateTelemetryLoadPaths() { + for (let engine of this.#settings.engines) { + if (!engine._loadPath) { + continue; + } + if (engine._loadPath.includes("set-via-policy")) { + engine._loadPath = "[policy]"; + } else if (engine._loadPath.includes("set-via-user")) { + engine._loadPath = "[user]"; + } else if (engine._loadPath.startsWith("[other]addEngineWithDetails:")) { + engine._loadPath = engine._loadPath.replace( + "[other]addEngineWithDetails:", + "[addon]" + ); + } + } + } + + /** + * Returns the engine associated with the name without SearchService + * initialization checks. + * + * @param {string} engineName + * The name of the engine. + * @returns {SearchEngine} + * The associated engine if found, null otherwise. + */ + #getEngineByName(engineName) { + for (let engine of this.#searchService._engines.values()) { + if (engine.name == engineName) { + return engine; + } + } + + return null; + } +} diff --git a/toolkit/components/search/SearchStaticData.sys.mjs b/toolkit/components/search/SearchStaticData.sys.mjs new file mode 100644 index 0000000000..1e3513f9de --- /dev/null +++ b/toolkit/components/search/SearchStaticData.sys.mjs @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This module contains additional data about default search engines that is the + * same across all languages. This information is defined outside of the actual + * search engine definition files, so that localizers don't need to update them + * when a change is made. + * + * This separate module is also easily overridable, in case a hotfix is needed. + * No high-level processing logic is applied here. + */ + +// To update this list of known alternate domains, just cut-and-paste from +// https://www.google.com/supported_domains +const gGoogleDomainsSource = + ".google.com .google.ad .google.ae .google.com.af .google.com.ag .google.com.ai .google.al .google.am .google.co.ao .google.com.ar .google.as .google.at .google.com.au .google.az .google.ba .google.com.bd .google.be .google.bf .google.bg .google.com.bh .google.bi .google.bj .google.com.bn .google.com.bo .google.com.br .google.bs .google.bt .google.co.bw .google.by .google.com.bz .google.ca .google.cd .google.cf .google.cg .google.ch .google.ci .google.co.ck .google.cl .google.cm .google.cn .google.com.co .google.co.cr .google.com.cu .google.cv .google.com.cy .google.cz .google.de .google.dj .google.dk .google.dm .google.com.do .google.dz .google.com.ec .google.ee .google.com.eg .google.es .google.com.et .google.fi .google.com.fj .google.fm .google.fr .google.ga .google.ge .google.gg .google.com.gh .google.com.gi .google.gl .google.gm .google.gp .google.gr .google.com.gt .google.gy .google.com.hk .google.hn .google.hr .google.ht .google.hu .google.co.id .google.ie .google.co.il .google.im .google.co.in .google.iq .google.is .google.it .google.je .google.com.jm .google.jo .google.co.jp .google.co.ke .google.com.kh .google.ki .google.kg .google.co.kr .google.com.kw .google.kz .google.la .google.com.lb .google.li .google.lk .google.co.ls .google.lt .google.lu .google.lv .google.com.ly .google.co.ma .google.md .google.me .google.mg .google.mk .google.ml .google.com.mm .google.mn .google.ms .google.com.mt .google.mu .google.mv .google.mw .google.com.mx .google.com.my .google.co.mz .google.com.na .google.com.nf .google.com.ng .google.com.ni .google.ne .google.nl .google.no .google.com.np .google.nr .google.nu .google.co.nz .google.com.om .google.com.pa .google.com.pe .google.com.pg .google.com.ph .google.com.pk .google.pl .google.pn .google.com.pr .google.ps .google.pt .google.com.py .google.com.qa .google.ro .google.ru .google.rw .google.com.sa .google.com.sb .google.sc .google.se .google.com.sg .google.sh .google.si .google.sk .google.com.sl .google.sn .google.so .google.sm .google.sr .google.st .google.com.sv .google.td .google.tg .google.co.th .google.com.tj .google.tk .google.tl .google.tm .google.tn .google.to .google.com.tr .google.tt .google.com.tw .google.co.tz .google.com.ua .google.co.ug .google.co.uk .google.com.uy .google.co.uz .google.com.vc .google.co.ve .google.vg .google.co.vi .google.com.vn .google.vu .google.ws .google.rs .google.co.za .google.co.zm .google.co.zw .google.cat"; +const gGoogleDomains = gGoogleDomainsSource.split(" ").map(d => "www" + d); + +export var SearchStaticData = { + /** + * Returns a list of alternate domains for a given search engine domain. + * + * @param {string} aDomain + * Lowercase host name to look up. For example, if this argument is + * "www.google.com" or "www.google.co.uk", the function returns the + * full list of supported Google domains. + * + * @returns {Array} + * Containing one entry for each alternate host name, or empty array + * if none is known. The returned array should not be modified. + */ + getAlternateDomains(aDomain) { + return !gGoogleDomains.includes(aDomain) ? [] : gGoogleDomains; + }, +}; diff --git a/toolkit/components/search/SearchSuggestionController.sys.mjs b/toolkit/components/search/SearchSuggestionController.sys.mjs new file mode 100644 index 0000000000..b528066d84 --- /dev/null +++ b/toolkit/components/search/SearchSuggestionController.sys.mjs @@ -0,0 +1,805 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + FormHistory: "resource://gre/modules/FormHistory.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", +}); + +const DEFAULT_FORM_HISTORY_PARAM = "searchbar-history"; +const HTTP_OK = 200; +const BROWSER_SUGGEST_PREF = "browser.search.suggest.enabled"; +const BROWSER_SUGGEST_PRIVATE_PREF = "browser.search.suggest.enabled.private"; +const BROWSER_RICH_SUGGEST_PREF = "browser.urlbar.richSuggestions.featureGate"; +const REMOTE_TIMEOUT_PREF = "browser.search.suggest.timeout"; +const REMOTE_TIMEOUT_DEFAULT = 500; // maximum time (ms) to wait before giving up on a remote suggestions + +const SEARCH_DATA_TRANSFERRED_SCALAR = "browser.search.data_transferred"; +const SEARCH_TELEMETRY_KEY_PREFIX = "sggt"; +const SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX = "pb"; + +const SEARCH_TELEMETRY_LATENCY = "SEARCH_SUGGESTIONS_LATENCY_MS"; + +/** + * Generates an UUID. + * + * @returns {string} + * An UUID string, without leading or trailing braces. + */ +function uuid() { + let uuid = Services.uuid.generateUUID().toString(); + return uuid.slice(1, uuid.length - 1); +} + +/** + * Represents a search suggestion. + * TODO: Support other Google tail fields: `a`, `dc`, `i`, `q`, `ansa`, + * `ansb`, `ansc`, `du`. See bug 1626897 comment 2. + */ +class SearchSuggestionEntry { + /** + * Creates an entry. + * + * @param {string} value + * The suggestion as a full-text string. Suitable for display directly to + * the user. + * @param {object} options + * An object with the following properties: + * @param {string} [options.matchPrefix] + * Represents the part of a tail suggestion that is already typed. For + * example, Google returns "…" as the match prefix to replace + * "what time is it in" in a tail suggestion for the query + * "what time is it in t". + * @param {string} [options.tail] + * Represents the suggested part of a tail suggestion. For example, Google + * might return "toronto" as the tail for the query "what time is it in t". + * @param {string} [options.icon] + * An icon representing the result in a data uri format. + * @param {string} [options.description] + * A description of the result. + * @param {boolean} [options.trending] + * Whether this is a trending suggestion. + */ + constructor(value, { matchPrefix, tail, icon, description, trending } = {}) { + this.#value = value; + this.#matchPrefix = matchPrefix; + this.#tail = tail; + this.#trending = trending; + this.#icon = icon; + this.#description = description; + } + + get value() { + return this.#value; + } + + get matchPrefix() { + return this.#matchPrefix; + } + + get tail() { + return this.#tail; + } + + get trending() { + return this.#trending; + } + + get icon() { + return this.#icon; + } + + get description() { + return this.#description; + } + + get tailOffsetIndex() { + if (!this.#tail) { + return -1; + } + + let offsetIndex = this.#value.lastIndexOf(this.#tail); + if (offsetIndex + this.#tail.length < this.#value.length) { + // We might have a tail suggestion that starts with a word contained in + // the full-text suggestion. e.g. "london sights in l" ... "london". + let lastWordIndex = this.#value.lastIndexOf(" "); + if (this.#tail.startsWith(this.#value.substring(lastWordIndex))) { + offsetIndex = lastWordIndex; + } else { + // Something's gone wrong. Consumers should not show this result. + offsetIndex = -1; + } + } + + return offsetIndex; + } + + /** + * Returns true if `otherEntry` is equivalent to this instance of + * SearchSuggestionEntry. + * + * @param {SearchSuggestionEntry} otherEntry The entry to compare to. + * @returns {boolean} + */ + equals(otherEntry) { + return otherEntry.value == this.value; + } + + #value; + #matchPrefix; + #tail; + #trending; + #icon; + #description; +} + +// Maps each engine name to a unique firstPartyDomain, so that requests to +// different engines are isolated from each other and from normal browsing. +// This is the same for all the controllers. +var gFirstPartyDomains = new Map(); + +/** + * + * The SearchSuggestionController class fetches search suggestions from two + * sources: a remote search engine and the user's previous searches stored + * locally in their profile (also called "form history"). + * + * The number of each suggestion type is configurable, and the controller will + * fetch and return both types at the same time. Instances of the class are + * reusable, but one instance should be used per input. The fetch() method is + * the main entry point. After creating an instance of the class, fetch() can + * be called many times to fetch suggestions. + * + */ +export class SearchSuggestionController { + /** + * Constructor + * + * @param {string} [formHistoryParam] + * The form history type to use with this controller. + */ + constructor(formHistoryParam = DEFAULT_FORM_HISTORY_PARAM) { + this.formHistoryParam = formHistoryParam; + } + + /** + * The maximum length of a value to be stored in search history. + * + * @type {number} + */ + static SEARCH_HISTORY_MAX_VALUE_LENGTH = 255; + + /** + * Maximum time (ms) to wait before giving up on remote suggestions + * + * @type {number} + */ + static REMOTE_TIMEOUT_DEFAULT = REMOTE_TIMEOUT_DEFAULT; + + /** + * Determines whether the given engine offers search suggestions. + * + * @param {nsISearchEngine} engine - The search engine + * @param {boolean} fetchTrending - Whether we should fetch trending suggestions. + * @returns {boolean} True if the engine offers suggestions and false otherwise. + */ + static engineOffersSuggestions(engine, fetchTrending) { + return engine.supportsResponseType( + fetchTrending + ? lazy.SearchUtils.URL_TYPE.TRENDING_JSON + : lazy.SearchUtils.URL_TYPE.SUGGEST_JSON + ); + } + + /** + * The maximum number of local form history results to return. This limit is + * only enforced if remote results are also returned. + * + * @type {number} + */ + maxLocalResults = 5; + + /** + * The maximum number of remote search engine results to return. + * We'll actually only display at most + * maxRemoteResults - <displayed local results count> remote results. + * + * @type {number} + */ + maxRemoteResults = 10; + + /** + * The additional parameter used when searching form history. + * + * @type {string} + */ + formHistoryParam = DEFAULT_FORM_HISTORY_PARAM; + + /** + * The last form history result used to improve the performance of + * subsequent searches. This shouldn't be used for any other purpose as it + * is never cleared and therefore could be stale. + * + * @type {object|null} + */ + formHistoryResult = null; + + /** + * Gets the firstPartyDomains Map, useful for tests. + * + * @returns {Map} firstPartyDomains mapped by engine names. + */ + get firstPartyDomains() { + return gFirstPartyDomains; + } + + /** + * @typedef {object} FetchResult + * @property {Array<SearchSuggestionEntry>} local + * Contains local search suggestions. + * @property {Array<SearchSuggestionEntry>} remote + * Contains remote search suggestions. + */ + + /** + * Fetch search suggestions from all of the providers. Fetches in progress + * will be stopped and results from them will not be provided. + * + * @param {string} searchTerm - the term to provide suggestions for + * @param {boolean} privateMode - whether the request is being made in the + * context of private browsing. + * @param {nsISearchEngine} engine - search engine for the suggestions. + * @param {int} userContextId - the userContextId of the selected tab. + * @param {boolean} restrictToEngine - whether to restrict local historical + * suggestions to the ones registered under the given engine. + * @param {boolean} dedupeRemoteAndLocal - whether to remove remote + * suggestions that dupe local suggestions + * @param {boolean} fetchTrending - Whether we should fetch trending suggestions. + * + * @returns {Promise<FetchResult>} + */ + fetch( + searchTerm, + privateMode, + engine, + userContextId = 0, + restrictToEngine = false, + dedupeRemoteAndLocal = true, + fetchTrending = false + ) { + // There is no smart filtering from previous results here (as there is when + // looking through history/form data) because the result set returned by the + // server is different for every typed value - e.g. "ocean breathes" does + // not return a subset of the results returned for "ocean". + + this.stop(); + + if (!Services.search.isInitialized) { + throw new Error("Search not initialized yet (how did you get here?)"); + } + if (typeof privateMode === "undefined") { + throw new Error( + "The privateMode argument is required to avoid unintentional privacy leaks" + ); + } + if (!engine.getSubmission) { + throw new Error("Invalid search engine"); + } + if (!this.maxLocalResults && !this.maxRemoteResults) { + throw new Error("Zero results expected, what are you trying to do?"); + } + if (this.maxLocalResults < 0 || this.maxRemoteResults < 0) { + throw new Error("Number of requested results must be positive"); + } + + // Array of promises to resolve before returning results. + let promises = []; + let context = (this.#context = { + awaitingLocalResults: false, + dedupeRemoteAndLocal, + engine, + engineId: engine?.identifier || "other", + fetchTrending, + privateMode, + request: null, + restrictToEngine, + searchString: searchTerm, + telemetryHandled: false, + timer: Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer), + userContextId, + }); + + // Fetch local results from Form History, if requested. + if (this.maxLocalResults && !fetchTrending) { + context.awaitingLocalResults = true; + promises.push(this.#fetchFormHistory(context)); + } + // Fetch remote results from Search Service, if requested. + if ( + (searchTerm || fetchTrending) && + this.suggestionsEnabled && + (!privateMode || this.suggestionsInPrivateBrowsingEnabled) && + this.maxRemoteResults && + SearchSuggestionController.engineOffersSuggestions(engine, fetchTrending) + ) { + promises.push(this.#fetchRemote(context)); + } + + function handleRejection(reason) { + if (reason == "HTTP request aborted") { + // Do nothing since this is normal. + return null; + } + console.error("SearchSuggestionController rejection:", reason); + return null; + } + return Promise.all(promises).then( + results => this.#dedupeAndReturnResults(context, results), + handleRejection + ); + } + + /** + * Stop pending fetches so no results are returned from them. + * + * Note: If there was no remote results fetched, the fetching cannot be + * stopped and local results will still be returned because stopping relies + * on aborting the XMLHTTPRequest to reject the promise for Promise.all. + */ + stop() { + if (this.#context) { + this.#context.abort = true; + this.#context.request?.abort(); + } + this.#context = null; + } + + #context; + + async #fetchFormHistory(context) { + // We don't cache these results as we assume that the in-memory SQL cache is + // good enough in performance. + let params = { + fieldname: this.formHistoryParam, + }; + + if (context.restrictToEngine) { + params.source = context.engine.name; + } + + let results = await lazy.FormHistory.getAutoCompleteResults( + context.searchString, + params + ); + + context.awaitingLocalResults = false; + + return { localResults: results }; + } + + /** + * Records per-engine telemetry after a search has finished. + * + * @param {object} context + * The search context. + */ + #reportTelemetryForEngine(context) { + this.#reportBandwidthForEngine(context); + + // Stop the latency stopwatch. + if (!context.telemetryHandled) { + if (context.abort) { + TelemetryStopwatch.cancelKeyed( + SEARCH_TELEMETRY_LATENCY, + context.engineId, + context + ); + } else { + TelemetryStopwatch.finishKeyed( + SEARCH_TELEMETRY_LATENCY, + context.engineId, + context + ); + } + context.telemetryHandled = true; + } + } + + /** + * Report bandwidth used by search activities. It only reports when it matches + * search provider information. + * + * @param {object} context + * The search context. + * @param {boolean} context.abort + * If the request should be aborted. + * @param {string} context.engineId + * The search engine identifier. + * @param {object} context.request + * Request information + * @param {boolean} context.privateMode + * Set to true if this is coming from a private browsing mode request. + */ + #reportBandwidthForEngine(context) { + if (context.abort || !context.request.channel) { + return; + } + + let channel = ChannelWrapper.get(context.request.channel); + let bytesTransferred = channel.requestSize + channel.responseSize; + if (bytesTransferred == 0) { + return; + } + + let telemetryKey = `${SEARCH_TELEMETRY_KEY_PREFIX}-${context.engineId}`; + if (context.privateMode) { + telemetryKey += `-${SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX}`; + } + + Services.telemetry.keyedScalarAdd( + SEARCH_DATA_TRANSFERRED_SCALAR, + telemetryKey, + bytesTransferred + ); + } + + /** + * Fetch suggestions from the search engine over the network. + * + * @param {object} context + * The search context. + * @returns {Promise} + * Returns a promise that is resolved when the response is received, or + * rejected if there is an error. + */ + #fetchRemote(context) { + let deferredResponse = Promise.withResolvers(); + let request = (context.request = new XMLHttpRequest()); + // Expect the response type to be JSON, so that the network layer will + // decode it for us. This will also ignore incorrect Mime Types, as we are + // dictating how we process it. + request.responseType = "json"; + + let submission = context.engine.getSubmission( + context.searchString, + context.searchString + ? lazy.SearchUtils.URL_TYPE.SUGGEST_JSON + : lazy.SearchUtils.URL_TYPE.TRENDING_JSON + ); + let method = submission.postData ? "POST" : "GET"; + request.open(method, submission.uri.spec, true); + // Don't set or store cookies or on-disk cache. + request.channel.loadFlags = + Ci.nsIChannel.LOAD_ANONYMOUS | Ci.nsIChannel.INHIBIT_PERSISTENT_CACHING; + // Use a unique first-party domain for each engine, to isolate the + // suggestions requests. + if (!gFirstPartyDomains.has(context.engine.name)) { + // Use the engine identifier, or an uuid when not available, because the + // domain cannot contain invalid chars and the engine name may not be + // suitable. When using an uuid the firstPartyDomain of the same engine + // will differ across restarts, but that's acceptable for now. + // TODO (Bug 1511339): use a persistent unique identifier per engine. + gFirstPartyDomains.set( + context.engine.name, + `${context.engine.identifier || uuid()}.search.suggestions.mozilla` + ); + } + let firstPartyDomain = gFirstPartyDomains.get(context.engine.name); + + request.setOriginAttributes({ + userContextId: context.userContextId, + privateBrowsingId: context.privateMode ? 1 : 0, + firstPartyDomain, + }); + + request.mozBackgroundRequest = true; // suppress dialogs and fail silently + + context.timer.initWithCallback( + () => { + // Abort if we already got local results. + if ( + request.readyState != 4 /* not complete */ && + !context.awaitingLocalResults + ) { + deferredResponse.resolve("HTTP request timeout"); + } + }, + this.remoteTimeout, + Ci.nsITimer.TYPE_ONE_SHOT + ); + + request.addEventListener("load", () => { + context.timer.cancel(); + this.#reportTelemetryForEngine(context); + if (!this.#context || context != this.#context || context.abort) { + deferredResponse.resolve( + "Got HTTP response after the request was cancelled" + ); + return; + } + this.#onRemoteLoaded(context, deferredResponse); + }); + + request.addEventListener("error", evt => { + this.#reportTelemetryForEngine(context); + deferredResponse.resolve("HTTP error"); + }); + + // Reject for an abort assuming it's always from .stop() in which case we + // shouldn't return local or remote results for existing searches. + request.addEventListener("abort", evt => { + context.timer.cancel(); + this.#reportTelemetryForEngine(context); + deferredResponse.reject("HTTP request aborted"); + }); + + if (submission.postData) { + request.sendInputStream(submission.postData); + } else { + request.send(); + } + + TelemetryStopwatch.startKeyed( + SEARCH_TELEMETRY_LATENCY, + context.engineId, + context + ); + + return deferredResponse.promise; + } + + /** + * Called when the request completed successfully (thought the HTTP status + * could be anything) so we can handle the response data. + * + * @param {object} context + * The search context. + * @param {Promise} deferredResponse + * The promise to resolve when a response is received. + * @private + */ + #onRemoteLoaded(context, deferredResponse) { + let status; + try { + status = context.request.status; + } catch (e) { + // The XMLHttpRequest can throw NS_ERROR_NOT_AVAILABLE. + deferredResponse.resolve("Unknown HTTP status: " + e); + return; + } + + if (status != HTTP_OK) { + deferredResponse.resolve( + "Non-200 status or empty HTTP response: " + status + ); + return; + } + + let serverResults = context.request.response; + + try { + if ( + !Array.isArray(serverResults) || + serverResults[0] == undefined || + (context.searchString.localeCompare(serverResults[0], undefined, { + sensitivity: "base", + }) && + // Some engines (e.g. Amazon) return a search string containing + // escaped Unicode sequences. Try decoding the remote search string + // and compare that with our typed search string. + context.searchString.localeCompare( + decodeURIComponent( + JSON.parse('"' + serverResults[0].replace(/\"/g, '\\"') + '"') + ), + undefined, + { + sensitivity: "base", + } + )) + ) { + // something is wrong here so drop remote results + deferredResponse.resolve( + "Unexpected response, searchString does not match remote response" + ); + return; + } + } catch (ex) { + deferredResponse.resolve( + `Failed to parse the remote response string: ${ex}` + ); + return; + } + + // Remove the search string from the server results since it is no longer + // needed. + let results = serverResults.slice(1) || []; + deferredResponse.resolve({ result: results }); + } + + /** + * @param {object} context + * The search context. + * @param {Array} suggestResults - an array of result objects from different + * sources (local or remote). + * @returns {object} + */ + #dedupeAndReturnResults(context, suggestResults) { + if (context.abort) { + return null; + } + + let results = { + term: context.searchString, + remote: [], + local: [], + }; + + for (let resultData of suggestResults) { + if (typeof resultData === "string") { + // Failure message + console.error( + "SearchSuggestionController found an unexpected string value:", + resultData + ); + } else if (resultData.localResults) { + results.formHistoryResults = resultData.localResults; + results.local = resultData.localResults.map( + s => new SearchSuggestionEntry(s.text) + ); + } else if (resultData.result) { + // Remote result + let richSuggestionData = this.#getRichSuggestionData(resultData.result); + let fullTextSuggestions = resultData.result[0]; + for (let i = 0; i < fullTextSuggestions.length; ++i) { + results.remote.push( + this.#newSearchSuggestionEntry( + fullTextSuggestions[i], + richSuggestionData?.[i], + context.fetchTrending + ) + ); + } + } + } + + // If we have remote results, cap the number of local results + if (results.remote.length) { + results.local = results.local.slice(0, this.maxLocalResults); + } + + // We don't want things to appear in both history and suggestions so remove + // entries from remote results that are already in local. + if ( + results.remote.length && + results.local.length && + context.dedupeRemoteAndLocal + ) { + for (let i = 0; i < results.local.length; ++i) { + let dupIndex = results.remote.findIndex(e => + e.equals(results.local[i]) + ); + if (dupIndex != -1) { + results.remote.splice(dupIndex, 1); + } + } + } + + // Trim the number of results to the maximum requested (now that we've pruned dupes). + let maxRemoteCount = this.maxRemoteResults; + if (context.dedupeRemoteAndLocal) { + maxRemoteCount -= results.local.length; + } + results.remote = results.remote.slice(0, maxRemoteCount); + + return results; + } + + /** + * Returns rich suggestion data from a remote fetch, if available. + * + * @param {Array} remoteResultData + * The results.remote array returned by SearchSuggestionsController.fetch. + * @returns {Array} + * An array of additional rich suggestion data. Each element should + * correspond to the array of text suggestions. + */ + #getRichSuggestionData(remoteResultData) { + if (!remoteResultData || !Array.isArray(remoteResultData)) { + return undefined; + } + + for (let entry of remoteResultData) { + if ( + typeof entry == "object" && + entry.hasOwnProperty("google:suggestdetail") + ) { + let richData = entry["google:suggestdetail"]; + if ( + Array.isArray(richData) && + richData.length == remoteResultData[0].length + ) { + return richData; + } + } + } + return undefined; + } + + /** + * Given a text suggestion and rich suggestion data, returns a + * SearchSuggestionEntry. + * + * @param {string} suggestion + * A suggestion string. + * @param {object} richSuggestionData + * Rich suggestion data returned by the engine. In Google's case, this is + * the corresponding entry at "google:suggestdetail". + * @param {boolean} trending + * Whether the suggestion is a trending suggestion. + * @returns {SearchSuggestionEntry} + */ + #newSearchSuggestionEntry(suggestion, richSuggestionData, trending) { + if (richSuggestionData && (!trending || this.richSuggestionsEnabled)) { + // We have valid rich suggestions. + let args = { trending }; + + // RichSuggestions come with icon and tail data, we only want one or the other + if (!richSuggestionData?.i) { + args.matchPrefix = richSuggestionData?.mp; + args.tail = richSuggestionData?.t; + } else if (this.richSuggestionsEnabled) { + args.icon = richSuggestionData?.i; + args.description = richSuggestionData?.a; + } + + return new SearchSuggestionEntry(suggestion, args); + } + // Return a regular suggestion. + return new SearchSuggestionEntry(suggestion, { trending }); + } +} + +/** + * The maximum time (ms) to wait before giving up on a remote suggestions. + */ +XPCOMUtils.defineLazyPreferenceGetter( + SearchSuggestionController.prototype, + "remoteTimeout", + REMOTE_TIMEOUT_PREF, + REMOTE_TIMEOUT_DEFAULT +); + +/** + * Whether or not remote suggestions are turned on. + */ +XPCOMUtils.defineLazyPreferenceGetter( + SearchSuggestionController.prototype, + "suggestionsEnabled", + BROWSER_SUGGEST_PREF, + true +); + +/** + * Whether or not remote suggestions are turned on in private browsing mode. + */ +XPCOMUtils.defineLazyPreferenceGetter( + SearchSuggestionController.prototype, + "suggestionsInPrivateBrowsingEnabled", + BROWSER_SUGGEST_PRIVATE_PREF, + false +); + +/** + * Whether or not rich suggestions are turned on. + */ +XPCOMUtils.defineLazyPreferenceGetter( + SearchSuggestionController.prototype, + "richSuggestionsEnabled", + BROWSER_RICH_SUGGEST_PREF, + false +); diff --git a/toolkit/components/search/SearchSuggestions.sys.mjs b/toolkit/components/search/SearchSuggestions.sys.mjs new file mode 100644 index 0000000000..32cd25a0cd --- /dev/null +++ b/toolkit/components/search/SearchSuggestions.sys.mjs @@ -0,0 +1,219 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + FormAutoCompleteResult: "resource://gre/modules/FormAutoComplete.sys.mjs", + FormHistoryClient: "resource://gre/modules/FormAutoComplete.sys.mjs", + + SearchSuggestionController: + "resource://gre/modules/SearchSuggestionController.sys.mjs", +}); + +/** + * SuggestAutoComplete is a base class that implements nsIAutoCompleteSearch + * and can collect results for a given search by using this.#suggestionController. + * We do it this way since the AutoCompleteController in Mozilla requires a + * unique XPCOM Service for every search provider, even if the logic for two + * providers is identical. + * + * @class + */ +class SuggestAutoComplete { + constructor() { + this.#suggestionController = new lazy.SearchSuggestionController(); + this.#suggestionController.maxLocalResults = this.#historyLimit; + } + + /** + * Notifies the front end of new results. + * + * @param {FormAutoCompleteResult} result + * Any previous form history result. + * @private + */ + onResultsReady(result) { + if (this.#listener) { + this.#listener.onSearchResult(this, result); + + // Null out listener to make sure we don't notify it twice + this.#listener = null; + } + } + + /** + * Initiates the search result gathering process. Part of + * nsIAutoCompleteSearch implementation. + * + * @param {string} searchString + * The user's query string. + * @param {string} searchParam + * unused, "an extra parameter"; even though this parameter and the + * next are unused, pass them through in case the form history + * service wants them + * @param {object} previousResult + * unused, a client-cached store of the previous generated resultset + * for faster searching. + * @param {object} listener + * object implementing nsIAutoCompleteObserver which we notify when + * results are ready. + */ + startSearch(searchString, searchParam, previousResult, listener) { + var formHistorySearchParam = searchParam.split("|")[0]; + + // Receive the information about the privacy mode of the window to which + // this search box belongs. The front-end's search.xml bindings passes this + // information in the searchParam parameter. The alternative would have + // been to modify nsIAutoCompleteSearch to add an argument to startSearch + // and patch all of autocomplete to be aware of this, but the searchParam + // argument is already an opaque argument, so this solution is hopefully + // less hackish (although still gross.) + var privacyMode = searchParam.split("|")[1] == "private"; + + // Start search immediately if possible, otherwise once the search + // service is initialized + if (Services.search.isInitialized) { + this.#triggerSearch( + searchString, + formHistorySearchParam, + listener, + privacyMode + ).catch(console.error); + return; + } + + Services.search + .init() + .then(() => { + this.#triggerSearch( + searchString, + formHistorySearchParam, + listener, + privacyMode + ).catch(console.error); + }) + .catch(result => + console.error( + "Could not initialize search service, bailing out:", + result + ) + ); + } + + /** + * Ends the search result gathering process. Part of nsIAutoCompleteSearch + * implementation. + */ + stopSearch() { + // Prevent reporting results of stopped search + this.#listener = null; + this.#suggestionController.stop(); + } + + #suggestionController; + + /** + * Maximum number of history items displayed. This is capped at 7 + * because the primary consumer (Firefox search bar) displays 10 rows + * by default, and so we want to leave some space for suggestions + * to be visible. + * + * @type {number} + */ + #historyLimit = 7; + + /** + * The object implementing nsIAutoCompleteObserver that we notify when + * we have found results. + * + * @type {object|null} + */ + #listener = null; + + /** + * Actual implementation of search. + * + * @param {string} searchString + * The user's query string. + * @param {string} searchParam + * unused + * @param {object} listener + * object implementing nsIAutoCompleteObserver which we notify when + * results are ready. + * @param {boolean} privacyMode + * True if the search was made from a private browsing mode context. + */ + async #triggerSearch(searchString, searchParam, listener, privacyMode) { + this.#listener = listener; + let results = await this.#suggestionController.fetch( + searchString, + privacyMode, + Services.search.defaultEngine + ); + + // Bug 1822297: This re-uses the wrappers from Satchel, to avoid re-writing + // our own nsIAutoCompleteSimpleResult implementation for now. However, + // we should do that at some stage to remove the dependency on satchel. + let client = new lazy.FormHistoryClient({ + formField: null, + inputName: this.#suggestionController.formHistoryParam, + }); + let formHistoryEntries = (results?.formHistoryResults ?? []).map( + historyEntry => ({ + // We supply the comments field so that autocomplete does not kick + // in the unescaping of the results for display which it uses for + // urls. + comment: historyEntry.text, + ...historyEntry, + }) + ); + let autoCompleteResult = new lazy.FormAutoCompleteResult( + client, + formHistoryEntries, + this.#suggestionController.formHistoryParam, + searchString + ); + + if (results?.remote?.length) { + // We shouldn't show tail suggestions in their full-text form. + // Suggestions are shown after form history results. + autoCompleteResult.fixedEntries = results.remote.reduce( + (results, item) => { + if (!item.matchPrefix && !item.tail) { + results.push({ + value: item.value, + label: item.value, + // We supply the comments field so that autocomplete does not kick + // in the unescaping of the results for display which it uses for + // urls. + comment: item.value, + }); + } + + return results; + }, + [] + ); + } + + // Notify the FE of our new results + this.onResultsReady(autoCompleteResult); + } + + QueryInterface = ChromeUtils.generateQI([ + "nsIAutoCompleteSearch", + "nsIAutoCompleteObserver", + ]); +} + +/** + * SearchSuggestAutoComplete is a service implementation that handles suggest + * results specific to web searches. + * + * @class + */ +export class SearchSuggestAutoComplete extends SuggestAutoComplete { + classID = Components.ID("{aa892eb4-ffbf-477d-9f9a-06c995ae9f27}"); + serviceURL = ""; +} diff --git a/toolkit/components/search/SearchUtils.sys.mjs b/toolkit/components/search/SearchUtils.sys.mjs new file mode 100644 index 0000000000..c1a956e56e --- /dev/null +++ b/toolkit/components/search/SearchUtils.sys.mjs @@ -0,0 +1,448 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineLazyGetter(lazy, "logConsole", () => { + return console.createInstance({ + prefix: "SearchUtils", + maxLogLevel: SearchUtils.loggingEnabled ? "Debug" : "Warn", + }); +}); + +const BROWSER_SEARCH_PREF = "browser.search."; + +/** + * Load listener + */ +class LoadListener { + _bytes = []; + _callback = null; + _channel = null; + _countRead = 0; + _expectedContentType = null; + _stream = null; + QueryInterface = ChromeUtils.generateQI([ + Ci.nsIRequestObserver, + Ci.nsIStreamListener, + Ci.nsIChannelEventSink, + Ci.nsIInterfaceRequestor, + Ci.nsIProgressEventSink, + ]); + + /** + * Constructor + * + * @param {nsIChannel} channel + * The initial channel to load from. + * @param {RegExp} expectedContentType + * A regular expression to match the expected content type to. + * @param {Function} callback + * A callback to receive the loaded data. The callback is passed the bytes + * (array) and the content type received. The bytes argument may be null if + * no data could be loaded. + */ + constructor(channel, expectedContentType, callback) { + this._channel = channel; + this._callback = callback; + this._expectedContentType = expectedContentType; + } + + // nsIRequestObserver + onStartRequest(request) { + lazy.logConsole.debug("loadListener: Starting request:", request.name); + this._stream = Cc["@mozilla.org/binaryinputstream;1"].createInstance( + Ci.nsIBinaryInputStream + ); + } + + onStopRequest(request, statusCode) { + lazy.logConsole.debug("loadListener: Stopping request:", request.name); + + var requestFailed = !Components.isSuccessCode(statusCode); + if (!requestFailed && request instanceof Ci.nsIHttpChannel) { + requestFailed = !request.requestSucceeded; + } + + if (requestFailed || this._countRead == 0) { + lazy.logConsole.warn("loadListener: request failed!"); + // send null so the callback can deal with the failure + this._bytes = null; + } else if (!this._expectedContentType.test(this._channel.contentType)) { + lazy.logConsole.warn( + "loadListener: Content type does not match expected", + this._channel.contentType + ); + this._bytes = null; + } + this._callback(this._bytes, this._bytes ? this._channel.contentType : ""); + this._channel = null; + } + + // nsIStreamListener + onDataAvailable(request, inputStream, offset, count) { + this._stream.setInputStream(inputStream); + + // Get a byte array of the data + this._bytes = this._bytes.concat(this._stream.readByteArray(count)); + this._countRead += count; + } + + // nsIChannelEventSink + asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) { + this._channel = newChannel; + callback.onRedirectVerifyCallback(Cr.NS_OK); + } + + // nsIInterfaceRequestor + getInterface(iid) { + return this.QueryInterface(iid); + } + + // nsIProgressEventSink + onProgress(request, progress, progressMax) {} + onStatus(request, status, statusArg) {} +} + +export var SearchUtils = { + BROWSER_SEARCH_PREF, + + /** + * This is the Remote Settings key that we use to get the ignore lists for + * engines. + */ + SETTINGS_IGNORELIST_KEY: "hijack-blocklists", + + /** + * This is the Remote Settings key that we use to get the allow lists for + * overriding the default engines. + */ + SETTINGS_ALLOWLIST_KEY: "search-default-override-allowlist", + + /** + * This is the Remote Settings key for getting the older search engine + * configuration. Tests may use `SETTINGS_KEY` if they want to get the key + * for the current configuration according to the preference. + */ + OLD_SETTINGS_KEY: "search-config", + + /** + * This is the Remote Settings key for getting the newer search engine + * configuration. Tests may use `SETTINGS_KEY` if they want to get the key + * for the current configuration according to the preference. + */ + NEW_SETTINGS_KEY: "search-config-v2", + + /** + * This is the Remote Settings key for getting the overrides for the + * older search engine configuration. Tests may use `SETTINGS_OVERRIDES_KEY` + * for the current configuration according to the preference. + */ + OLD_SETTINGS_OVERRIDES_KEY: "search-config-overrides", + + /** + * This is the Remote Settings key for getting the overrides for the + * newer search engine configuration. Tests may use `SETTINGS_OVERRIDES_KEY` + * for the current configuration according to the preference. + */ + NEW_SETTINGS_OVERRIDES_KEY: "search-config-overrides-v2", + + /** + * This is the Remote Settings key that we use to get the search engine + * configurations. + * + * @returns {string} + */ + get SETTINGS_KEY() { + return SearchUtils.newSearchConfigEnabled + ? SearchUtils.NEW_SETTINGS_KEY + : SearchUtils.OLD_SETTINGS_KEY; + }, + + /** + * This is the Remote Settings key that we use to get the search engine + * configuration overrides. + * + * @returns {string} + */ + get SETTINGS_OVERRIDES_KEY() { + return SearchUtils.newSearchConfigEnabled + ? SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + : SearchUtils.OLD_SETTINGS_OVERRIDES_KEY; + }, + + /** + * Topic used for events involving the service itself. + */ + TOPIC_SEARCH_SERVICE: "browser-search-service", + + // See documentation in nsISearchService.idl. + TOPIC_ENGINE_MODIFIED: "browser-search-engine-modified", + MODIFIED_TYPE: { + CHANGED: "engine-changed", + REMOVED: "engine-removed", + ADDED: "engine-added", + DEFAULT: "engine-default", + DEFAULT_PRIVATE: "engine-default-private", + }, + + URL_TYPE: { + SUGGEST_JSON: "application/x-suggestions+json", + SEARCH: "text/html", + OPENSEARCH: "application/opensearchdescription+xml", + TRENDING_JSON: "application/x-trending+json", + }, + + ENGINES_URLS: { + "prod-main": + "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records", + "prod-preview": + "https://firefox.settings.services.mozilla.com/v1/buckets/main-preview/collections/search-config/records", + "stage-main": + "https://firefox.settings.services.allizom.org/v1/buckets/main/collections/search-config/records", + "stage-preview": + "https://firefox.settings.services.allizom.org/v1/buckets/main-preview/collections/search-config/records", + }, + + // The following constants are left undocumented in nsISearchService.idl + // For the moment, they are meant for testing/debugging purposes only. + + // Set an arbitrary cap on the maximum icon size. Without this, large icons can + // cause big delays when loading them at startup. + MAX_ICON_SIZE: 20000, + + DEFAULT_QUERY_CHARSET: "UTF-8", + + // A tag to denote when we are using the "default_locale" of an engine. + DEFAULT_TAG: "default", + + MOZ_PARAM: { + LOCALE: "moz:locale", + }, + + // Query parameters can have the property "purpose", whose value + // indicates the context that initiated a search. This list contains + // defined search contexts. + PARAM_PURPOSES: { + CONTEXTMENU: "contextmenu", + HOMEPAGE: "homepage", + KEYWORD: "keyword", + NEWTAB: "newtab", + SEARCHBAR: "searchbar", + }, + + LoadListener, + + // This is a list of search engines that we currently consider to be "General" + // search, as opposed to a vertical search engine such as one used for + // shopping, book search, etc. + // + // Currently these are a list of hard-coded application provided ones. At some + // point in the future we expect to allow WebExtensions to specify by themselves, + // however this needs more definition on the "vertical" search terms, and the + // effects before we enable it. + GENERAL_SEARCH_ENGINE_IDS: new Set([ + "google@search.mozilla.org", + "ddg@search.mozilla.org", + "bing@search.mozilla.org", + "baidu@search.mozilla.org", + "ecosia@search.mozilla.org", + "qwant@search.mozilla.org", + "yahoo-jp@search.mozilla.org", + "yandex@search.mozilla.org", + ]), + + /** + * Notifies watchers of SEARCH_ENGINE_TOPIC about changes to an engine or to + * the state of the search service. + * + * @param {nsISearchEngine} engine + * The engine to which the change applies. + * @param {string} verb + * A verb describing the change. + * + * @see nsISearchService.idl + */ + notifyAction(engine, verb) { + if (Services.search.isInitialized) { + lazy.logConsole.debug("NOTIFY: Engine:", engine.name, "Verb:", verb); + Services.obs.notifyObservers(engine, this.TOPIC_ENGINE_MODIFIED, verb); + } + }, + + /** + * Wrapper function for nsIIOService::newURI. + * + * @param {string} urlSpec + * The URL string from which to create an nsIURI. + * @returns {nsIURI} an nsIURI object, or null if the creation of the URI failed. + */ + makeURI(urlSpec) { + try { + return Services.io.newURI(urlSpec); + } catch (ex) {} + + return null; + }, + + /** + * Wrapper function for nsIIOService::newChannel. + * + * @param {string|nsIURI} url + * The URL string from which to create an nsIChannel. + * @param {nsIContentPolicy} contentPolicyType + * The type of document being loaded. + * @returns {nsIChannel} + * an nsIChannel object, or null if the url is invalid. + */ + makeChannel(url, contentPolicyType) { + if (!contentPolicyType) { + throw new Error("makeChannel called with invalid content policy type"); + } + try { + let uri = typeof url == "string" ? Services.io.newURI(url) : url; + return Services.io.newChannelFromURI( + uri, + null /* loadingNode */, + Services.scriptSecurityManager.createNullPrincipal({}), + null /* triggeringPrincipal */, + Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + contentPolicyType + ); + } catch (ex) {} + + return null; + }, + + /** + * Tests whether this a partner distribution. + * + * @returns {boolean} + * Whether this is a partner distribution. + */ + isPartnerBuild() { + return SearchUtils.distroID && !SearchUtils.distroID.startsWith("mozilla"); + }, + + /** + * Current settings version. This should be incremented if the format of the + * settings file is modified. + * + * @returns {number} + * The current settings version. + */ + get SETTINGS_VERSION() { + return 9; + }, + + /** + * Indicates the channel that the build is on, with added hardening for ESR + * since some ESR builds may be self-built or not on the same channel. + * + * @returns {string} + * Returns the modified channel, with a focus on ESR if the application + * version is indicating ESR. + */ + get MODIFIED_APP_CHANNEL() { + return AppConstants.IS_ESR ? "esr" : AppConstants.MOZ_UPDATE_CHANNEL; + }, + + /** + * Sanitizes a name so that it can be used as an engine name. If it cannot be + * sanitized (e.g. no valid characters), then it returns a random name. + * + * @param {string} name + * The name to be sanitized. + * @returns {string} + * The sanitized name. + */ + sanitizeName(name) { + const maxLength = 60; + const minLength = 1; + var result = name.toLowerCase(); + result = result.replace(/\s+/g, "-"); + result = result.replace(/[^-a-z0-9]/g, ""); + + // Use a random name if our input had no valid characters. + if (result.length < minLength) { + result = Math.random().toString(36).replace(/^.*\./, ""); + } + + // Force max length. + return result.substring(0, maxLength); + }, + + getVerificationHash(name) { + let disclaimer = + "By modifying this file, I agree that I am doing so " + + "only within $appName itself, using official, user-driven search " + + "engine selection processes, and in a way which does not circumvent " + + "user consent. I acknowledge that any attempt to change this file " + + "from outside of $appName is a malicious act, and will be responded " + + "to accordingly."; + + let salt = + PathUtils.filename(PathUtils.profileDir) + + name + + disclaimer.replace(/\$appName/g, Services.appinfo.name); + + let data = new TextEncoder().encode(salt); + let hasher = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); + }, + + /** + * Tests whether the given URI is a secure OpenSearch submission URI or a + * secure OpenSearch update URI. + * + * Note: We don't want to count something served via localhost as insecure. + * We also don't want to count sites with .onion as their top-level domain + * as insecure because .onion URLs actually can't use https and are secured + * in other ways. + * + * @param {nsIURI} uri + * The URI to be tested. + * @returns {boolean} + * Whether the URI is secure for OpenSearch purposes. + */ + isSecureURIForOpenSearch(uri) { + const loopbackAddresses = ["127.0.0.1", "[::1]", "localhost"]; + + return ( + uri.schemeIs("https") || + loopbackAddresses.includes(uri.host) || + uri.host.toLowerCase().endsWith(".onion") + ); + }, +}; + +XPCOMUtils.defineLazyPreferenceGetter( + SearchUtils, + "loggingEnabled", + BROWSER_SEARCH_PREF + "log", + false +); + +XPCOMUtils.defineLazyPreferenceGetter( + SearchUtils, + "newSearchConfigEnabled", + "browser.search.newSearchConfig.enabled", + false +); + +// Can't use defineLazyPreferenceGetter because we want the value +// from the default branch +ChromeUtils.defineLazyGetter(SearchUtils, "distroID", () => { + return Services.prefs.getDefaultBranch("distribution.").getCharPref("id", ""); +}); diff --git a/toolkit/components/search/UserSearchEngine.sys.mjs b/toolkit/components/search/UserSearchEngine.sys.mjs new file mode 100644 index 0000000000..5b7c5c1e88 --- /dev/null +++ b/toolkit/components/search/UserSearchEngine.sys.mjs @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint no-shadow: error, mozilla/no-aArgs: error */ + +import { SearchEngine } from "resource://gre/modules/SearchEngine.sys.mjs"; + +/** + * UserSearchEngine represents a search engine defined by a user. + */ +export class UserSearchEngine extends SearchEngine { + /** + * Creates a UserSearchEngine. + * + * @param {object} options + * The options for this search engine. + * @param {object} [options.details] + * General information about the search engine. + * @param {string} [options.details.name] + * The search engine name. + * @param {string} [options.details.url] + * The search url for the engine. + * @param {string} [options.details.keyword] + * The keyword for the engine. + * @param {object} [options.json] + * An object that represents the saved JSON settings for the engine. + */ + constructor(options = {}) { + super({ + loadPath: "[user]", + }); + + if (options.details) { + this._initWithDetails({ + name: options.details.name, + search_url: encodeURI(options.details.url), + keyword: options.details.alias, + }); + } else { + this._initWithJSON(options.json); + } + } + + /** + * Returns the appropriate identifier to use for telemetry. + * + * @returns {string} + */ + get telemetryId() { + return `other-${this.name}`; + } +} diff --git a/toolkit/components/search/components.conf b/toolkit/components/search/components.conf new file mode 100644 index 0000000000..3e4a9b669b --- /dev/null +++ b/toolkit/components/search/components.conf @@ -0,0 +1,23 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'js_name': 'search', + 'cid': '{7319788a-fe93-4db3-9f39-818cf08f4256}', + 'contract_ids': ['@mozilla.org/browser/search-service;1'], + 'interfaces': ['nsISearchService'], + 'esModule': 'resource://gre/modules/SearchService.sys.mjs', + 'constructor': 'SearchService', + 'processes': ProcessSelector.MAIN_PROCESS_ONLY, + }, + { + 'cid': '{aa892eb4-ffbf-477d-9f9a-06c995ae9f27}', + 'contract_ids': ['@mozilla.org/autocomplete/search;1?name=search-autocomplete'], + 'esModule': 'resource://gre/modules/SearchSuggestions.sys.mjs', + 'constructor': 'SearchSuggestAutoComplete', + }, +] diff --git a/toolkit/components/search/docs/DefaultSearchEngines.rst b/toolkit/components/search/docs/DefaultSearchEngines.rst new file mode 100644 index 0000000000..3dfe68abb1 --- /dev/null +++ b/toolkit/components/search/docs/DefaultSearchEngines.rst @@ -0,0 +1,102 @@ +====================== +Default Search Engines +====================== + +Default Engine +============== + +The search service specifies default search engines via the `configuration +schema`_. + +Changing Defaults +================= + +The default engine may change when: + +* The user has the default engine set and the configuration for the locale/region + changes. +* The user has the default engine set and their locale/region changes to one + which has a different default. +* The user chooses to set a different engine via preferences. +* The user installs an add-on which sets its default as one of the application + provided engines. +* The user installs an add-on which supplies a different engine and the user allows + the different engine to be set as default. +* The user or Firefox (e.g. via blocklist) causes the default engine to be removed. + +Add-ons and Prompting for Default +--------------------------------- + +The prompt for selecting a search engine from an add-on as default is shown to +the user on installation of the add-on. It may also be shown if an add-on is +re-enabled, if the default engine was not changed in the meantime. + +The following diagram shows the full flow for search engines from add-ons: + +.. image:: ./search-add-on-prompts-flow.png + :align: center + :alt: Flowchart for prompting for default engine for Search Engines related to add-ons. + +When the Default Engine is Removed +================================== + +If the default engine is removed by the user, or by Firefox in the case of a +blocklist or for some other region, the new default engine is chosen by the +following process. + +* If the default engine specified by the configuration for the user's region and locale + is visible, then it will be selected as default. +* If there is another engine visible, fall back to the first engine identified + as a general search engine (see below). +* If there are no other visible engines, unhide the region/locale default engine + from the configuration and set it as default if it is not the one being removed. +* Otherwise, unhide the first general search engine, or the first visible engine. + +A general search engine is defined as one that returns general search results, +for example Google or DuckDuckGo. A non-general search engine returns results +for a specific area, e.g. shopping, books, dictionaries. + +Add-ons and App-provided Engines +================================ + +An add-on may set the name of the search provider in the manifest.json to be +the name of an app-provided engine. In this case: + +* If the add-on is a non-authorised partner, then we set the user's default + engine to be the name of the app-provided engine. +* If the add-on is from an authorised partner, then we set the users' default + engine to be the same as the app-provided engine, and we allow the + app-provided urls to be overridden with those of the add-on. + +If the specified engine is already default, then the add-on does +not override the app-provided engine, and it's settings are ignored and no +new engine is added. + +The list of authorised add-ons is stored in `remote settings`_ in the +`search-default-override-allowlist bucket`_. The list +includes records containing: + +* Third-party Add-on Id: The identifier of the third party add-on which will + override the app provided one. +* Add-on Id to Override: The identifier of the app-provided add-on to be + overridden. +* a list of the url / params that are authorised to be replaced. + +When an authorised add-on overrides the default, we record the add-on's id +with the app-provided engine in the ``overriddenBy`` field. This is used +when the engine is loaded on startup to known that it should load the parameters +from that add-on. + +The ``overriddenBy`` annotation may be removed when: + +* The associated authorised add-on is removed, disabled or can no longer be found. +* The user changes their default to another engine. + +If the ``overriddenBy`` annotation is present, but the add-on is not authorised, +then the annotation will be maintained in case the add-on is later re-authorised. +For example, a url is updated, but the update is performed before the allow list +is updated. + +.. _configuration schema: SearchConfigurationSchema.html +.. _remote settings: /services/settings/index.html +.. _search-default-override-allowlist bucket: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records diff --git a/toolkit/components/search/docs/Preferences.rst b/toolkit/components/search/docs/Preferences.rst new file mode 100644 index 0000000000..2fdf4699cb --- /dev/null +++ b/toolkit/components/search/docs/Preferences.rst @@ -0,0 +1,46 @@ +Preferences +=========== + +This document describes Preferences affecting the toolkit Search Service. +Preferences that are generated and updated by code won't be described here. + +User Exposed +------------ +These preferences are exposed through the Firefox UI. + +browser.search.suggest.enabled (default: true) + Whether search suggestions are enabled in the search bar. Turning this off + also currently prevents search suggestions in the address bar. + +Hidden +------ +These preferences are normally hidden, and should not be used unless you really +know what you are doing. + +browser.search.log (boolean, default: false) + Whether search service console logging is enabled or not. + +browser.search.suggest.timeout (number, default: 500) + When requesting suggestions from a search server, it has this long to respond. + +browser.search.update (boolean, default: true) + Whether updates are enabled for OpenSearch based engines. This only applies + to OpenSearch engines where their definition supplies an update url. + Note that this does not affect any of: application updates, add-on updates, + remote settings updates. + +Experimental +------------ +These preferences are experimental and not officially supported. They could be +removed at any time. + +browser.search.separatePrivateDefault.ui.enabled (boolean, default: false) + Whether the UI is enabled for having a separate default search engine in + private browsing mode. + +browser.search.separatePrivateDefault (boolean, default: false) + Whether the user has selected to have a separate default search engine in + private browsing mode. + +browser.search.suggest.enabled.private (boolean, default: false) + Whether search suggestions are enabled in a private browsing window. diff --git a/toolkit/components/search/docs/SearchConfigurationSchema.rst b/toolkit/components/search/docs/SearchConfigurationSchema.rst new file mode 100644 index 0000000000..b57f69fe13 --- /dev/null +++ b/toolkit/components/search/docs/SearchConfigurationSchema.rst @@ -0,0 +1,586 @@ +=========================== +Search Configuration Schema +=========================== + +This document outlines the details of the schema and how the various sub-parts +interact. For the full fields and descriptions, please see the `schema itself`_. + +.. note:: + In the examples, only relevant properties are displayed. + +Overview +======== + +The configuration is a JSON blob which is object with a `data` property which +is an array of engines: + +.. code-block:: js + + { + data: [ + { + // engine 1 details + }, + { + // engine 2 details + } + ] + } + +Engine Objects +============== + +An engine's details are located in the properties of the object associated with it. +An engine that is deployed globally could be listed simply as: + +.. code-block:: js + + { + "default": "no", + "telemetryId": "engine1-telem", + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + } + }] + } + +The ``appliesTo`` section is an array of objects. At least one object is required +to specify which regions/locales the engine is included within. If an +``appliesTo`` object lists additional attributes then these will override any +attributes at the top-level. + +For example, a more complex engine definition may be available only to users +located specific regions or with certain locales. For example: + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "region": "us" + }, + "webExtension": { + "id": "web-us@ext" + } + }, { + "included": { + "region": "gb" + }, + "webExtension": { + "id": "web-gb@ext" + } + }] + } + +In this case users identified as being in the US region would use the WebExtension +with identifier ``web-us@ext``. GB region users would get +``web-gb@ext``, and all other users would get ``web@ext``. + +To direct search engines to pull ``_locale`` data from a specific locale +directory, you can use ``webExtension.locales``. + +For example, in this code block: + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [ + { + "included": { + "locales": "en-US" + }, + "webExtension": { + "locales": [ + "us" + ] + } + }, { + "included": { + "locales": "en-GB" + }, + "webExtension": { + "locales": [ + "uk" + ] + } + } + ] + } + +There should exist a ``us`` and ``uk`` folder in the ``locales`` directory +of the extension, ``web``. + +If a locale is not provided, ``webExtension.locales`` is set to +``SearchUtils.DEFAULT_TAG``. + +`Search Extensions directory <https://searchfox.org/mozilla-central/source/browser/components/search/extensions>`__ + +`Example of a locales directory <https://searchfox.org/mozilla-central/source/browser/components/search/extensions/wikipedia/_locales>`__ + + +Special Attributes +================== + +$USER_LOCALE +------------ + +If a ``webExtension.locales`` property contains an element with the value +``"$USER_LOCALE"`` then the special value will be replaced in the +configuration object with the users locale. For example: + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "locales": { + "matches": [ + "en-US", + "en-GB" + ] + } + }, + "webExtension": { + "locales": ["$USER_LOCALE"] + } + }] + } + +Will report either ``[en-US]`` or ``[en-GB]`` as the ``webExtension.locales`` +property depending on the user's locale. + +Since the special string is replaced, custom folder names can be searched for +by adding the keyword in between a consistent prefix/suffix. + +For example, if ``webExtension.locales`` was ``["example-$USER_LOCALE"]``, +the locale generator will generate locale names in the form of ``example-en-US`` +and ``example-en-GB``. + +Note: Prior to Firefox 100.0, $USER_LOCALE used an exact match. +In Firefox 100.0 the replacement was updated to use a standard string replacement. + +From Firefox 98.0.1 and 97.7.1esr, ``"$USER_LOCALE"`` may also be used in the +``telemetryId`` field. + +$USER_REGION +------------ + +This can be used in the same situations as ``"$USER_LOCALE"``, instead +replacing ``webExtension.locale`` with a string that uses the users region. + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + }, + "webExtension": { + "locales": ["foo-$USER_REGION"] + } + }] + } + +In this example, if the user's region is ``fr``, the ``webExtension.locale`` +will be ``foo-fr``, and the code will look for the ``messages.json`` in +the ``foo-fr`` folder of the ``_locales`` folder for this extension. + +Note: ``"$USER_REGION"`` was added in Firefox 98.0.1 and 97.7.1esr and used an exact match. +In Firefox 100.0 the replacement was updated to use a standard string replacement. + +"default" +--------- + +You can specify ``"default"`` as a region in the configuration if +the engine is to be included when we do not know the user's region. + +"override" +---------- + +The ``"override"`` field can be set to true if you want a section to +only override otherwise included engines. ``"override"`` will only work for +sections which apply to distributions or experiments. The experiment case was +added in Firefox 81. + +Starting with Firefox 96, ``"override"`` sections may include ``included`` and +``excluded`` information which will be applied accordingly. If they are not +supplied, then the override section will be applied to everywhere. + +Example: + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + // Complicated and lengthy inclusion rules + }, { + "override": true, + "application": { "distributions": ["mydistrocode"]}, + "params": { + "searchUrlGetParams": [ + { "name": "custom", "value": "foobar" } + ] + } + }] + } + +Application Scoping +=================== + +An engine configuration may be scoped to a particular application. + +Name +---- + +One or more application names may be specified. Currently the only application +type supported is ``firefox``. If an application name is specified, then it +must be matched for the section to apply. If there are no application names +specified, then the section will match any consumer of the configuration. + +In the following example, ``web@ext`` would be included on any consumer +of the configuration, but ``web1@ext`` would only be included on Firefox desktop. + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "application": { + "name": [] + } + } + ]} + }, + { + "webExtension": { + "id": "web1@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "application": { + "name": ["firefox"] + } + } + ]} + } + +Channel +------- + +One or more channels may be specified in an array to restrict a configuration +to just those channels. The current known channels are: + + - default: Self-builds of Firefox, or possibly some self-distributed versions. + - nightly: Firefox Nightly builds. + - aurora: Firefox Developer Edition + - beta: Firefox Beta + - release: The main Firefox release channel. + - esr: The ESR Channel. This will also match versions of Firefox where the + displayed version number includes ``esr``. We do this to include Linux + distributions and other manual builds of ESR. + +In the following example, ``web@ext`` would be set as default on the default +channel only, whereas ``web1@ext`` would be set as default on release and esr +channels. + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "default": "yes", + "application": { + "channel": ["default"] + } + } + ]} + }, + { + "webExtension": { + "id": "web1@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "default": "yes", + "application": { + "channel": ["release", "esr"] + } + } + ]} + } + +Distributions +------------- + +Distributions may be specified to be included or excluded in an ``appliesTo`` +section. The ``distributions`` field in the ``application`` section is an array +of distribution identifiers. The identifiers match those supplied by the +``distribution.id`` preference. + +In the following, ``web@ext`` would be included in only the ``cake`` +distribution. ``web1@ext`` would be excluded from the ``apples`` distribution +but included in the main desktop application, and all other distributions. + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "application": { + "distributions": ["cake"] + } + } + ]} + }, + { + "webExtension": { + "id": "web1@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "application": { + "excludedDistributions": ["apples"] + } + } + ]} + } + +Version +------- + +Minimum and Maximum versions may be specified to restrict a configuration to +specific ranges. These may be open-ended. Version comparison is performed +using `the version comparator`_. + +Note: comparison against ``maxVersion`` is a less-than comparison. The +``maxVersion`` won't be matched directly. + +In the following example, ``web@ext`` would be included for any version after +72.0a1, whereas ``web1@ext`` would be included only between 68.0a1 and 71.x +version. + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "application": { + "minVersion": "72.0a1" + } + } + ]} + }, + { + "webExtension": { + "id": "web1@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + "default": "yes", + "application": { + "minVersion": "68.0a1" + "maxVersion": "72.0a1" + } + } + ]} + } + +Experiments +=========== + +We can run experiments by giving sections within ``appliesTo`` a +``experiment`` value, the Search Service can then optionally pass in a +matching ``experiment`` value to match those sections. + +Sections which have a ``experiment`` will not be used unless a matching +``experiment`` has been passed in, for example: + +.. code-block:: js + + { + "webExtension": { + "id": "web@ext" + }, + "appliesTo": [{ + "included": { + "everywhere": true + }, + "experiment": "nov-16", + "webExtension": { + "id": "web-experimental@ext" + } + }, { + "included": { + "everywhere": true + }, + "webExtension": { + "id": "web-gb@ext" + } + }] + } + +Engine Defaults +=============== + +An engine may be specified as the default for one of two purposes: + +#. normal browsing mode, +#. private browsing mode. + +If there is no engine specified for private browsing mode for a particular region/locale +pair, then the normal mode engine is used. + +If the instance of the application does not support a separate private browsing mode engine, +then it will only use the normal mode engine. + +An engine may or may not be default for particular regions/locales. The ``default`` +property is a tri-state value with states of ``yes``, ``yes-if-no-other`` and +``no``. Here's an example of how they apply: + +.. code-block:: js + + { + "webExtension": { + "id": "engine1@ext" + }, + "appliesTo": [{ + "included": { + "region": "us" + }, + "default": "yes" + }, { + "excluded": { + "region": "us" + }, + "default": "yes-if-no-other" + }] + }, + { + "webExtension": { + "id": "engine2@ext" + }, + "appliesTo": [{ + "included": { + "region": "gb" + }, + "default": "yes" + }] + }, + "webExtension": { + "id": "engine3@ext" + }, + "default": "no" + "appliesTo": [{ + "included": { + "everywhere": true + }, + }] + }, + { + "webExtension": { + "id": "engine4@ext" + }, + "defaultPrivate": "yes", + "appliesTo": [{ + "included": { + "region": "fr" + } + }] + } + +In this example, for normal mode: + + - engine1@ext is default in the US region, and all other regions except for GB + - engine2@ext is default in only the GB region + - engine3@ext and engine4 are never default anywhere + +In private browsing mode: + + - engine1@ext is default in the US region, and all other regions except for GB and FR + - engine2@ext is default in only the GB region + - engine3@ext is never default anywhere + - engine4@ext is default in the FR region. + +Engine Ordering +=============== + +The ``orderHint`` field indicates the suggested ordering of an engine relative to +other engines when displayed to the user, unless the user has customized their +ordering. + +The default ordering of engines is based on a combination of if the engine is +default, and the ``orderHint`` fields. The ordering is structured as follows: + +#. Default engine in normal mode +#. Default engine in private browsing mode (if different from the normal mode engine) +#. Other engines in order from the highest ``orderHint`` to the lowest. + +Example: + +.. code-block:: js + + { + "webExtension": { + "id": "engine1@ext" + }, + "orderHint": 2000, + "default": "no", + }, + { + "webExtension": { + "id": "engine2@ext" + }, + "orderHint": 1000, + "default": "yes" + }, + { + "webExtension": { + "id": "engine3@ext" + }, + "orderHint": 500, + "default": "no" + } + +This would result in the order: ``engine2@ext, engine1@ext, engine3@ext``. + +.. _schema itself: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ +.. _the version comparator: https://developer.mozilla.org/en-US/docs/Mozilla/Toolkit_version_format diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst new file mode 100644 index 0000000000..c782f9f7c3 --- /dev/null +++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst @@ -0,0 +1,72 @@ +=========================== +Search Engine Configuration +=========================== + +The search engine configuration is a mapping that is used to determine the +list of search engines for each user. The mapping is primarily based on the +user's region and locale. + +Configuration Management +======================== + +The application stores a dump of the configuration that is used for first +initialisation. Subsequent updates to the configuration are either updates to the +static dump, or they may be served via remote servers. + +The mechanism of delivering the settings dumps to the Search Service is +`the remote settings`_. + +Remote settings +--------------- + +The remote settings bucket for the search engine configuration list is +``search-config``. The version that is currently being delivered +to clients can be `viewed live`_. + +Configuration Schema +==================== + +The configuration format is defined via a `JSON schema`_. The search engine +configuration schema is `stored in mozilla-central`_ and is uploaded to the +Remote Settings server at convenient times after it changes. + +An outline of the schema may be found on the `Search Configuration Schema`_ page. + +Updating Search Engine WebExtensions +==================================== + +Updates for application provided search engine WebExtensions are provided via +`Normandy`_. + +It is likely that updates for search engine WebExtensions will be +received separately to configuration updates which may or may not be directly +related. As a result several situations may occur: + + - The updated WebExtension is for an app-provided engine already in-use by + the user. + + - In this case, the search service will apply the changes to the + app-provided engine's data. + + - A WebExtension addition/update is for an app-provided engine that is not + in-use by the user, or not in the configuration. + + - In this case, the search service will ignore the WebExtension. + - If the configuration (search or user) is updated later and the + new engine is added, then the Search Service will start to use the + new engine. + + - A configuration update is received that needs a WebExtension that is + not found locally. + + - In this case, the search service will ignore the missing engine and + continue without it. + - When the WebExtension is delivered, the search engine will then be + installed and added. + +.. _the remote settings: /services/settings/index.html +.. _JSON schema: https://json-schema.org/ +.. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ +.. _Search Configuration Schema: SearchConfigurationSchema.html +.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records +.. _Normandy: /toolkit/components/normandy/normandy/services.html diff --git a/toolkit/components/search/docs/SearchEngines.rst b/toolkit/components/search/docs/SearchEngines.rst new file mode 100644 index 0000000000..9993abf8c6 --- /dev/null +++ b/toolkit/components/search/docs/SearchEngines.rst @@ -0,0 +1,197 @@ +============== +Search Engines +============== +This document describes the three main ways Firefox serves search engines to the +user, enabling users to search the internet with different search providers. +The three main ways Firefox serves search engines to the users is through: + +- Add-on Search Engines +- OpenSearch Engines +- Enterprise Policy Engines + +An example of a search provider is Google, which is one of the Add-on Search +Engines described in the first section below. Another example of a search +provider is Bugzilla, which is an OpenSearch Engine described in the second +section below. Lastly, there are Enterprise Policy Search Engines, +which will be the third section described in this documentation. + +Add-on Search Engines +===================== +Add-ons are additional functionality that third-party developers provide for +users to install into Firefox. The add-on mechanism is also used by Firefox to +ship the search engines provided by the application. To define Add-on Search +Engines, developers use the `WebExtensions API`_. Since the WebExtensions API +technology is used, developers interchangeably used the term WebExtension Search +Engines when referring to Add-ons Search Engines. + +.. _WebExtensions API: + https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions + +The list of Add-on Search Engines provided by Firefox and their extension files +can be found in `mozilla-central/browser/components/search/extensions +<https://searchfox.org/mozilla-central/source/browser/components/search/extensions>`__. +Within each Add-on Search Engine folder, there is a manifest.json file. One of the +keys in that manifest.json file is `chrome_settings_overrides`, whose value is an object +that describes how to construct the url, images, strings, icon, etc. Here’s an example of +how the search provider is set within `chrome_settings_overrides`: + +.. code-block:: js + + "chrome_settings_overrides": { + "search_provider": { + "name": "Discogs", + "search_url": "https://www.discogs.com/search/?q={searchTerms}", + "keyword": "disc", + "favicon_url": "https://www.discogs.com/favicon.ico" + } + } + + +To see more details on the syntax and properties, visit the `chrome settings +overrides MDN documentation. +<https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/ +manifest.json/chrome_settings_overrides>`__ + +In Practice +----------- +All versions of Firefox support add-ons. Firefox switched over from OpenSearch +to Add-on Search Engines internally in Firefox version 78. Add-on Search engines +allows Firefox developers to have more flexibility and control in the +modification of formatting search engines as we support different search +providers. + +We maintain these Add-on Search Engines through a search configuration file that +is bundled and configured via Remote Settings. As of this writing, June 2022, we +use Remote Settings for managing search engines only for Firefox Desktop but not +outside of Firefox Desktop. + +OpenSearch Engines +=================== +OpenSearch is a plugin, software installed on Firefox to enhance capabilities +for searching. OpenSearch has a collection of formats that describe how to +construct the url, images, strings, icon, etc. These formats provided by +OpenSearch allow Firefox to make a search over the internet with a specific +search provider that is not an application provided search engine on Firefox. +The purpose of OpenSearch is to provide more convenient ways of searching and +different ways of searching. + +OpenSearch allows users to search with a vast variety of search providers which +do not come installed with Firefox out of the box. The main benefit of OpenSearch +is it allows site owners to easily provide users with a way to search a site. + +History +------- +Prior to OpenSearch, search plugins were first created by the `Mycroft Project +<https://mycroftproject.com/>`__ and based off of `Sherlock +<https://en.wikipedia.org/wiki/Sherlock_(software)>`__, a file and web search +tool created by Apple. + +The OpenSearch Protocol was created and launched by A9.com in 2005. OpenSearch +was added to Firefox version 2 in the year 2006. As of today in 2022, OpenSearch +is a collection of formats for sharing of search results. The code is stable but +unchanged for many years. + +See the `OpenSearch Documentation <https://github.com/dewitt/opensearch>`__ for +more information on the OpenSearch formats. + +Autodiscovery +------------- +Autodiscovery is a feature on Firefox which automatically notifies the user when +the webpage they visited has a search plugin. + +Here is an example of Autodiscovery from Bugzilla. You can visit +https://bugzilla.mozilla.org and Firefox will automatically detect that the +website has a provided search plugin. In the results dropdown, you can look at +the search engine shortcuts section at the bottom and it will show a green plus +sign over the Bugzilla search icon. The green plus sign indicates that the user +can add Bugzilla as an OpenSearch Engine. After the user adds Bugzilla as an +OpenSearch Engine, the green plus icon disappears. The user can now click the +Bugzilla icon to make a search directly on bugzilla.mozilla.org. + +.. figure:: assets/bugzilla-open-search1.png + :alt: Image of the address bar input showing a URL + :scale: 28% + :align: center + +.. figure:: assets/bugzilla-open-search2.png + :alt: Image of the address bar input showing a URL + :scale: 28% + :align: center + +See the `Autodiscovery MDN Documentation <https://developer.mozilla.org/en-US/ +docs/Web/OpenSearch#autodiscovery_of_search_plugins>`__ for more information on +Autodiscovery. + +Enterprise Policy Engines +========================= +Enterprise Policies are customizable configurations for the Firefox browser set +by enterprises or companies who want to distribute configuration for their +users. The idea of Enterprise Policies is to allow companies to customize Firefox +and how their users can or cannot change the usage of Firefox based on predefined +configuration that was set in place. + +Enterprise Policy Engines are search engines that a company has added as search +engines on Firefox for their users by setting the Enterprise Policy. In this +`Enterprise Policy Documentation +<https://mozilla.github.io/policy-templates/#searchengines +-this-policy-is-only-available-on-the-esr>`__, +it outlines the different options that are available for enterprises using +Firefox ESR (Extended Support Release) and what’s available in terms of adding, +updating, or removing search engines. The company can use the policy to define +which search engines are available on their Firefox ESR. + +See the `policy-templates +<https://mozilla.github.io/policy-templates/>`__ for more +information on Enterprise Policy templates and the different configuration rules +available. + +Configuration +------------- +In practice, there are different ways for a company to specify their policy, +depending on which operating system their machines are on. The admin can +configure the policy on a server and when the user logs in, those configurations +are automatically pushed to the user’s Firefox. + +For Windows, the `GPO (Group Policy Object) +<https://github.com/mozilla/policy-templates/tree/master/windows>`__ or `Intune +(Microsoft Endpoint Manager) <https://support.mozilla.org/en-US/kb/managing-firefox-intune>`__ is +used to set the policy. For macOS, `configuration profiles +<https://github.com/mozilla/policy-templates/tree/master/mac>`__ are created. +For the generic case, there is a JSON file to describe the policy. + +When these configurations are set, Firefox takes the configuration as inputs and +turns them into settings that Firefox can consume. + +A Hypothetical Use of Enterprise Policy +--------------------------------------- +A company that is in the banking industry and requires tighter security over +their users may not want their users to do something on Firefox without the +company's knowledge. It may make sense for the company to disable private +browsing for Firefox. + +Within a specific company, the employees of the finance department could use the +Firefox ESR version. In this situation, we think of the finance department as +the Firefox user rather than the individual employees as Firefox users. The +department makes choices for the individuals that use the Firefox browser +through the Enterprise Policy. + +Features On Enterprise Policy +----------------------------- +All Firefox versions have to honor the Enterprise Policy, but the Enterprise +Policy may not have effect on an individual who is not using Firefox ESR at a +company. There are features that are enterprise specific that are only available +in ESR. These features allow search engines to be configured, allowing for +unsigned extensions, installing search engines, and setting a default search +engine. + +How To Set Up and Use an Enterprise Policy for Firefox +------------------------------------------------------ +Install the ESR version of Firefox since Enterprise Policies are not supported on +rapid release. Then, create the JSON file that is located in the README.md within +https://github.com/mozilla/policy-templates. There are instructions there on how +to configure and use the policy. Once the JSON is created with the appropriate +settings, drop the JSON file in the directory outlined by the README.md and +Firefox will find it and Firefox will open and run with the policy. + +Common formatting mistakes are often made when creating the JSON file. The JSON +file can be validated using a JSON validator such as https://jsonlint.com/. diff --git a/toolkit/components/search/docs/SearchService.rst b/toolkit/components/search/docs/SearchService.rst new file mode 100644 index 0000000000..4f332ecedf --- /dev/null +++ b/toolkit/components/search/docs/SearchService.rst @@ -0,0 +1,6 @@ +SearchService Reference +======================= + +.. js:autoclass:: SearchService + :members: + :private-members: diff --git a/toolkit/components/search/docs/SearchServiceHighlevelOverview.rst b/toolkit/components/search/docs/SearchServiceHighlevelOverview.rst new file mode 100644 index 0000000000..1701f3a52b --- /dev/null +++ b/toolkit/components/search/docs/SearchServiceHighlevelOverview.rst @@ -0,0 +1,85 @@ +================================= +SearchService High-level Overview +================================= +``SearchService`` is the core component that manages Search Engines on the +Firefox browser. + +The following diagram is a high level systems context diagram of the +``SearchService``. The diagram illustrates which systems interface with the +``SearchService`` so that it can do its job of managing the Search Engines. + +The diagram and description is an over-simplification of the ``SearchService's`` +responsibilities. It specifically highlights how the ``SearchService`` serves +search engines to the browser on startup. However, the ``SearchService`` has +many other responsibilities that are not outlined in the diagram such as +maintaining and managing the Search Engines when various items change, e.g. the +user's region or locale, configuration changes received from remote settings, +updates received to search engine data. Nonetheless, the diagram gives a broad +overview of the main components the ``SearchService`` interacts with most often. + +Diagram +======= +.. figure:: assets/search-service-diagram.png + :scale: 85% + :align: center + +Description of the Diagram +========================== +These steps follow the same number on the diagram. Number 1 on the diagram is +described by number 1 below. + +1. When the user opens the Firefox Browser, the code starts to build the browser + UI components. During this startup phase, we have various systems making + calls to the ``SearchService``. E.g. `browser.js <https://searchfox.org/mozilla-central/rev/cb6f8d7b1f1782b9d4b2ee7312de1dcc284aaf06/browser/base/content/browser.js#3797>`_ + calls ``Services.search.getDefault`` to fetch the default Search Engine. + +2. The ``SearchService`` needs information from ``System Add-ons``, + ``SearchSettings``, and ``Remote Settings`` to build the correct engines in + the right order and to return the list of engines to its callers. + + a) First, the ``SearchService`` makes a request for the search configuration. + ``SearchService`` calls `SearchEngineSelector.fetchEngineConfiguration <https://searchfox.org/mozilla-central/rev/cb6f8d7b1f1782b9d4b2ee7312de1dcc284aaf06/toolkit/components/search/SearchService.sys.mjs#2247>`_ + which makes a call to `Remote Settings <https://searchfox.org/mozilla-central/rev/cb6f8d7b1f1782b9d4b2ee7312de1dcc284aaf06/toolkit/components/search/SearchEngineSelector.sys.mjs#129>`_ + for the search configuration. Remote Settings does not fetch the search + configuration from the remote database on startup. Instead Remote Settings + tries to load the :searchfox:`search configuration dump file <services/settings/dumps/main/search-config.json>` + from its local database and if that is empty, it will load the dump file into + its local database. Only after startup will Remote Settings connect to the + remote database when necessary. By connecting after startup, it avoids + a potential network request that could delay startup. + + b) Second, the ``SearchService`` `fetches a JSON file <https://searchfox.org/mozilla-central/rev/cb6f8d7b1f1782b9d4b2ee7312de1dcc284aaf06/toolkit/components/search/SearchService.sys.mjs#1296-1297>`_ + from the `SearchSettings <https://searchfox.org/mozilla-central/source/toolkit/components/search/SearchSettings.sys.mjs>`_. + This JSON file contains Search Engine metadata that is saved on the user's + computer. It's information that helps the ``SearchService`` remember the + user's custom settings for the Search Engines. + + c) Third, the `System Add-ons <https://searchfox.org/mozilla-central/rev/cb6f8d7b1f1782b9d4b2ee7312de1dcc284aaf06/browser/components/extensions/parent/ext-chrome-settings-overrides.js#536>`_ + passes the extension data to the ``SearchService``. At this point, the + ``SearchService`` only installs user installed search extensions. For the + Application Provided engines we create those when ``SearchService`` calls `_makeEngineFromConfig <https://searchfox.org/mozilla-central/rev/3002762e41363de8ee9ca80196d55e79651bcb6b/toolkit/components/search/SearchService.sys.mjs#3421-3440>`_. + Then ``_makeEngineFromConfig`` will create a new ``AddonSearchEngine``. + When the `AddonSearchEngine.init <https://searchfox.org/mozilla-central/rev/3002762e41363de8ee9ca80196d55e79651bcb6b/toolkit/components/search/AddonSearchEngine.sys.mjs#83-87,89>`_ + method is called, it combines both the extension and search configuration + data to create the correct engine for the user based on locale, region and + other information. + + After steps 2a, 2b, and 2c the ``SearchService`` has finished gathering + search engine data from ``System Add-ons``, ``SearchSettings``, and + ``Remote Settings``. Now the ``SearchService`` can build the different + types of Search Engines. + +3. The ``SearchService`` creates new instances of :searchfox:`SearchEngines <toolkit/components/search/SearchEngine.sys.mjs>` + by making an `Add-on, Open Search, or Enterprise Policy Search Engine <https://firefox-source-docs.mozilla.org/toolkit/search/SearchEngines.html>`_. + +4. The ``SearchService`` returns the engines to the caller that requested it. + E.g. the ``SearchService`` passes the default Search Engine back to + ``browser.js``, the system that initially requested it. + +Updates +======= +This page is up to date as of March 10, 2023. If the diagram or description +becomes out of date, please find access to the +``Search Service Diagram`` through the ``Firefox Search > Search Service +Documentation`` folder in the shared drive or through this link `here <https://drive.google.com/file/d/1vKRRK87kIGt6xamHJuclkC04EKrS69Qw/view?usp=sharing>`_. +Contributions are welcomed to keep this page up to date. diff --git a/toolkit/components/search/docs/Telemetry.rst b/toolkit/components/search/docs/Telemetry.rst new file mode 100644 index 0000000000..75f24b8d7e --- /dev/null +++ b/toolkit/components/search/docs/Telemetry.rst @@ -0,0 +1,94 @@ +Telemetry +========= + +This document describes search telemetry recorded by Toolkit such as search +service telemetry and telemetry related to fetching search suggestions. + +Other important search-related telemetry is recorded by Firefox and is +documented in :doc:`/browser/search/telemetry` in the Firefox documentation. + +Legacy Telemetry +---------------- + +Scalars +------- + +browser.searchinit.secure_opensearch_engine_count +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Records the number of secure (i.e., using https) OpenSearch search + engines a given user has installed. + +browser.searchinit.insecure_opensearch_engine_count +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Records the number of insecure (i.e., using http) OpenSearch search + engines a given user has installed. + +browser.searchinit.secure_opensearch_update_count +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Records the number of OpenSearch search engines with secure updates + enabled (i.e., using https) a given user has installed. + +browser.searchinit.insecure_opensearch_update_count +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Records the number of OpenSearch search engines with insecure updates + enabled (i.e., using http) a given user has installed. + +Keyed Scalars +------------- + +browser.searchinit.engine_invalid_webextension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Records the WebExtension ID of a search engine where the saved search engine + settings do not match the WebExtension. + + The keys are the WebExtension IDs. The values are integers: + + 1. Associated WebExtension is not installed. + 2. Associated WebExtension is disabled. + 3. The submission URL of the associated WebExtension is different to that of the saved settings. + +Histograms +---------- + +SEARCH_SUGGESTIONS_LATENCY_MS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This histogram records the latency in milliseconds of fetches to the + suggestions endpoints of search engines, or in other words, the time from + Firefox's request to a suggestions endpoint to the time Firefox receives a + response. It is a keyed exponential histogram with 50 buckets and values + between 0 and 30000 (0s and 30s). Keys in this histogram are search engine IDs + for built-in search engines and 'other' for non-built-in search engines. + +Default Search Engine +~~~~~~~~~~~~~~~~~~~~~ +Telemetry for the user's default search engine is currently reported via two +systems: + + 1. Legacy telemetry: + `Fields are reported within the legacy telemetry environment <https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/data/environment.html#defaultsearchengine>`__ + 2. Glean: + `Fields are documented in the Glean dictionary <https://dictionary.telemetry.mozilla.org/apps/firefox_desktop?search=search.engine>`__. + +Glean Telemetry +--------------- +`These search service fields are documented via Glean dictionary <https://dictionary.telemetry.mozilla.org/apps/firefox_desktop?page=1&search=search.service>`__. + +search.service.startup_time +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + The time duration it takes for the search service to start up. + +search.service.initializaitonStatus +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + A labeled counter for the type of initialization statuses that can occur on + start up. Labels include: ``failedSettings``, ``failedFetchEngines``, + ``failedLoadEngines``, ``success``. + + A counter for initialization successes on start up. diff --git a/toolkit/components/search/docs/assets/bugzilla-open-search1.png b/toolkit/components/search/docs/assets/bugzilla-open-search1.png Binary files differnew file mode 100644 index 0000000000..de265e5223 --- /dev/null +++ b/toolkit/components/search/docs/assets/bugzilla-open-search1.png diff --git a/toolkit/components/search/docs/assets/bugzilla-open-search2.png b/toolkit/components/search/docs/assets/bugzilla-open-search2.png Binary files differnew file mode 100644 index 0000000000..d30260d7f7 --- /dev/null +++ b/toolkit/components/search/docs/assets/bugzilla-open-search2.png diff --git a/toolkit/components/search/docs/assets/search-service-diagram.png b/toolkit/components/search/docs/assets/search-service-diagram.png Binary files differnew file mode 100644 index 0000000000..cdd244b8f4 --- /dev/null +++ b/toolkit/components/search/docs/assets/search-service-diagram.png diff --git a/toolkit/components/search/docs/index.rst b/toolkit/components/search/docs/index.rst new file mode 100644 index 0000000000..d8d4bc8d8f --- /dev/null +++ b/toolkit/components/search/docs/index.rst @@ -0,0 +1,40 @@ +============== +Search Service +============== + +This is documentation for the Search Service. + +Definitions +=========== + +* Application-provided engines (aka app-provided): These are engines provided + by the application to the user as part of the configurations for the user's + locale/region. +* Application default engine: The engine automatically selected by the + application as default, in the absence of user settings. +* Default engine: This is the engine that is the one used by default when + doing searches from the address bar, search bar and other places. This may be + the application default engine or an user selected engine. +* Default private engine: Same as for the default engine, but this is used by + default when in private browsing mode. + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + SearchServiceHighlevelOverview + SearchEngineConfiguration + SearchConfigurationSchema + SearchEngines + DefaultSearchEngines + Preferences + Telemetry + +API Reference +------------- + +.. toctree:: + + SearchService diff --git a/toolkit/components/search/docs/search-add-on-prompts-flow.png b/toolkit/components/search/docs/search-add-on-prompts-flow.png Binary files differnew file mode 100644 index 0000000000..ac2eeefadb --- /dev/null +++ b/toolkit/components/search/docs/search-add-on-prompts-flow.png diff --git a/toolkit/components/search/metrics.yaml b/toolkit/components/search/metrics.yaml new file mode 100644 index 0000000000..f2628dd39f --- /dev/null +++ b/toolkit/components/search/metrics.yaml @@ -0,0 +1,394 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 +$tags: + - 'Firefox :: Search' + +search.engine.default: + engine_id: + type: string + description: | + The telemetry id of the search engine. + For application provided engines, this is either supplied by the + configuration or from the first part of the associated WebExtension Id. + For other engines, this is `other-<extensionName>`. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1089670 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1138503 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1755549 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - interaction + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + - 'newtab' + + display_name: + type: string + description: | + The display name of the user's default engine. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159#c60 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - interaction + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + load_path: + type: string + description: | + A path relating to where the search engine was installed/loaded from. + For example: + `[addon]<extension id>` for a WebExtension based + engine. + `[https]developer.mozilla.org/mdn-web-docs.xml` for an OpenSearch based + engine. + Note: this metric is truncated at 100 characters. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159#c60 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + submission_url: + type: url + description: | + The submission URL of the default engine. This is only reported in the + cases where: + The engine is an application provided engine. + The engine has the same name as an application provided engine. + The engine matches one of a specific list of well known search engines. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1164159#c60 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + verified: + type: string + description: | + The verified status of the search engine. + For application provided engines, this will always be `default`. + For other engines this will either be `verified` or `unverified` depending + on if the loadPathHash is valid. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1259139 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1259139#c6 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - technical + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + changed: + type: event + description: | + Recorded when the default search engine is changed. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1634555 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1634555 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + extra_keys: + previous_engine_id: + type: string + description: The id of the previous default engine. + new_engine_id: + type: string + description: The id of the new default engine. + new_display_name: + type: string + description: The display name of the new default engine. + new_load_path: + type: string + description: | + The path relating to where the new default engine was installed/loaded + from. + new_submission_url: + type: string + description: | + The new submission URL of the new default engine. This is + limited to 100 characters. + change_source: + type: string + description: | + The source of the change of engine. For possible values, + see `REASON_CHANGE_MAP` in + https://searchfox.org/mozilla-central/source/toolkit/components/search/SearchService.sys.mjs + +search.engine.private: + engine_id: + type: string + description: | + The telemetry id of the search engine. + For application provided engines, this is either supplied by the + configuration or from the first part of the associated WebExtension Id. + For other engines, this is `other-<extensionName>`. + If this string is an empty string (`""`), this means that one or both of + the preferences `browser.search.separatePrivateDefault` and + `browser.search.separatePrivateDefault.ui.enabled` are set to false. + It is possible that the user selects the same private engine as for the + default engine, and hence both versions of these fields will be filled in. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016#c8 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - interaction + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + - 'newtab' + + display_name: + type: string + description: | + The display name of the user's default engine. + If this string is an empty string (`""`), this means that one or both of + the preferences `browser.search.separatePrivateDefault` and + `browser.search.separatePrivateDefault.ui.enabled` are set to false. + It is possible that the user selects the same private engine as for the + default engine, and hence both versions of these fields will be filled in. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016#c8 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - interaction + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + load_path: + type: string + description: | + A path relating to where the search engine was installed/loaded from. + For example: + `[addon]<extension id>` for a WebExtension based + engine. + `[https]developer.mozilla.org/mdn-web-docs.xml` for an OpenSearch based + engine. + Note: this metric is truncated at 100 characters. + If this string is an empty string (`""`), this means that one or both of + the preferences `browser.search.separatePrivateDefault` and + `browser.search.separatePrivateDefault.ui.enabled` are set to false. + It is possible that the user selects the same private engine as for the + default engine, and hence both versions of these fields will be filled in. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016#c8 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + submission_url: + type: url + description: | + The submission URL of the default engine. This is only reported in the + cases where: + The engine is an application provided engine. + The engine has the same name as an application provided engine. + The engine matches one of a specific list of well known search engines. + If this string is an empty string (`""`), this means that one or both of + the preferences `browser.search.separatePrivateDefault` and + `browser.search.separatePrivateDefault.ui.enabled` are set to false. + It is possible that the user selects the same private engine as for the + default engine, and hence both versions of these fields will be filled in. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016#c8 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + verified: + type: string + description: | + The verified status of the search engine. + For application provided engines, this will always be `default`. + For other engines this will either be `verified` or `unverified` depending + on if the loadPathHash is valid. + If this string is an empty string (`""`), this means that one or both of + the preferences `browser.search.separatePrivateDefault` and + `browser.search.separatePrivateDefault.ui.enabled` are set to false. + It is possible that the user selects the same private engine as for the + default engine, and hence both versions of these fields will be filled in. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1563016#c8 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1766999 + data_sensitivity: + - technical + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + lifetime: application + send_in_pings: + - 'metrics' + + changed: + type: event + description: | + Recorded when the default search engine is changed + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1634555 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1634555 + data_sensitivity: + - web_activity + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com + expires: never + extra_keys: + previous_engine_id: + type: string + description: The id of the previous default engine. + new_engine_id: + type: string + description: The id of the new default engine. + new_display_name: + type: string + description: The display name of the new default engine. + new_load_path: + type: string + description: | + The path relating to where the new default engine was installed/loaded + from. + new_submission_url: + type: string + description: | + The new submission URL of the new default engine. This is + limited to 100 characters. + change_source: + type: string + description: | + The source of the change of engine. For possible values, + see `REASON_CHANGE_MAP` in + https://searchfox.org/mozilla-central/source/toolkit/components/search/SearchService.sys.mjs + +search.service: + startup_time: + type: timing_distribution + time_unit: nanosecond + description: | + The time duration it takes for the search service to start up. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1832509 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1832509#c3 + notification_emails: + - fx-search-telemetry@mozilla.com + data_sensitivity: + - technical + expires: never + telemetry_mirror: SEARCH_SERVICE_INIT2_MS + + initialization_status: + type: labeled_counter + description: | + Counts the type of initialization statuses on start up. + failedSettings - failed loading and migrating the search settings file. + failedFetchEngines - failed to load or parse the configuration from remote + settings. + failedLoadEngines - failed to create all the search engines from the + configuration or user installed engines. + success - search service successfully initialized. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849013 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849013 + notification_emails: + - fx-search-telemetry@mozilla.com + data_sensitivity: + - technical + expires: never + labels: + - failedSettings + - failedFetchEngines + - failedLoadEngines + - success diff --git a/toolkit/components/search/moz.build b/toolkit/components/search/moz.build new file mode 100644 index 0000000000..81994b0345 --- /dev/null +++ b/toolkit/components/search/moz.build @@ -0,0 +1,47 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SPHINX_TREES["/toolkit/search"] = "docs" + +XPCSHELL_TESTS_MANIFESTS += [ + "tests/xpcshell/searchconfigs/xpcshell.toml", + "tests/xpcshell/xpcshell.toml", +] + +XPIDL_SOURCES += [ + "nsISearchService.idl", +] + +XPIDL_MODULE = "toolkit_search" + +EXTRA_JS_MODULES += [ + "AddonSearchEngine.sys.mjs", + "AppProvidedSearchEngine.sys.mjs", + "OpenSearchEngine.sys.mjs", + "OpenSearchLoader.sys.mjs", + "PolicySearchEngine.sys.mjs", + "SearchEngine.sys.mjs", + "SearchEngineSelector.sys.mjs", + "SearchEngineSelectorOld.sys.mjs", + "SearchService.sys.mjs", + "SearchSettings.sys.mjs", + "SearchStaticData.sys.mjs", + "SearchSuggestionController.sys.mjs", + "SearchSuggestions.sys.mjs", + "SearchUtils.sys.mjs", + "UserSearchEngine.sys.mjs", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] + +TESTING_JS_MODULES += [ + "tests/SearchTestUtils.sys.mjs", +] + +with Files("**"): + BUG_COMPONENT = ("Firefox", "Search") diff --git a/toolkit/components/search/nsISearchService.idl b/toolkit/components/search/nsISearchService.idl new file mode 100644 index 0000000000..769ba4eca7 --- /dev/null +++ b/toolkit/components/search/nsISearchService.idl @@ -0,0 +1,551 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; +interface nsIInputStream; + +[scriptable, uuid(5799251f-5b55-4df7-a9e7-0c27812c469a)] +interface nsISearchSubmission : nsISupports +{ + /** + * The POST data associated with a search submission, wrapped in a MIME + * input stream. May be null. + */ + readonly attribute nsIInputStream postData; + + /** + * The URI to submit a search to. + */ + readonly attribute nsIURI uri; +}; + +[scriptable, uuid(620bd920-0491-48c8-99a8-d6047e64802d)] +interface nsISearchEngine : nsISupports +{ + /** + * Gets a nsISearchSubmission object that contains information about what to + * send to the search engine, including the URI and postData, if applicable. + * + * @param searchTerms + * The search term(s) for the submission. + * + * @param responseType [optional] + * The MIME type that we'd like to receive in response + * to this submission. If null, will default to "text/html". + * + * @param purpose [optional] + * A string that indicates the context of the search request. This may then + * be used to provide different submission data depending on the context. + * + * @returns nsISearchSubmission + * The submission data. If no appropriate submission can be determined for + * the request type, this may be null. + */ + nsISearchSubmission getSubmission(in AString searchTerms, + [optional] in AString responseType, + [optional] in AString purpose); + /** + * Returns the search term of a possible search result URI if and only if: + * - The URI has the same scheme, host, and path as the engine. + * - All query parameters of the URI have a matching name and value in the engine. + * - An exception to the equality check is the engine's termsParameterName + * value, which contains a placeholder, i.e. {searchTerms}. + * - If an engine has query parameters with "null" values, they will be ignored. + * + * @param uri + * A URI that may or may not be from a search result matching the engine. + * + * @returns A string representing the termsParameterName value of the URI, + * or an empty string if the URI isn't matched to the engine. + */ + AString searchTermFromResult(in nsIURI uri); + + /** + * Returns the name of the parameter used for the search terms for a submission + * URL of type `SearchUtils.URL_TYPE.SEARCH`. + * + * @returns A string which is the name of the parameter, or empty string + * if no parameter cannot be found or is not supported (e.g. POST). + */ + readonly attribute AString searchUrlQueryParamName; + + /** + * Returns the public suffix for the submission URL of type + * `SearchUtils.URL_TYPE.SEARCH`. + * + * @returns A string which is a known public suffix, or empty string + * if one cannot be found. + */ + readonly attribute AString searchUrlPublicSuffix; + + /** + * Determines whether the engine can return responses in the given + * MIME type. Returns true if the engine spec has a URL with the + * given responseType, false otherwise. + * + * @param responseType + * The MIME type to check for + */ + boolean supportsResponseType(in AString responseType); + + /** + * Returns a string with the URL to an engine's icon matching both width and + * height. Returns undefined if icon with specified dimensions is not found. + * + * @param preferredWidth + * Width of the requested icon. If not specified, it is assumed that + * 16x16 is desired. + */ + jsval getIconURL([optional] in unsigned short preferredWidth); + + /** + * Opens a speculative connection to the engine's search URI + * (and suggest URI, if different) to reduce request latency + * + * @param options + * An object that must contain the following fields: + * {window} the content window for the window performing the search + * {originAttributes} the originAttributes for performing the search + * + * @throws NS_ERROR_INVALID_ARG if options is omitted or lacks required + * elemeents + */ + void speculativeConnect(in jsval options); + + /** + * A user defined alias for the engine. + * When not an empty string, this should be a unique identifier. + */ + attribute AString alias; + + /** + * An array of aliases including the user defined alias and + * ones specified by the webextensions keywords field. + */ + readonly attribute Array<AString> aliases; + + /** + * A text description describing the engine. + */ + readonly attribute AString description; + + /** + * Whether the engine should be hidden from the user. + */ + attribute boolean hidden; + + /** + * Whether the associated one off button should be hidden from the user. + */ + attribute boolean hideOneOffButton; + + /** + * The display name of the search engine. This is a unique identifier. + */ + readonly attribute AString name; + + /** + * The display of the search engine id. This is a unique identifier. + */ + readonly attribute AString id; + + /** + * The searchForm URL points to the engine's organic search page. This should + * not contain neither search term parameters nor partner codes, but may + * contain parameters which set the engine in the correct way. + * + * This URL is typically the prePath and filePath of the search submission URI, + * but may vary for different engines. For example, some engines may use a + * different domain, e.g. https://sub.example.com for the search URI but + * https://example.org/ for the organic search page. + */ + readonly attribute AString searchForm; + + /** + * The identifier to use for this engine when submitting to telemetry. + */ + readonly attribute AString telemetryId; + + /** + * An optional unique identifier for this search engine within the context of + * the distribution, as provided by the distributing entity. + */ + readonly attribute AString identifier; + + /** + * Whether or not this engine is provided by the application, e.g. it is + * in the list of configured search engines. + */ + readonly attribute boolean isAppProvided; + + /** + * Whether or not this engine is an in-memory only search engine. + * These engines are typically application provided or policy engines, + * where they are loaded every time on SearchService initialization + * using the policy JSON or the extension manifest. Minimal details of the + * in-memory engines are saved to disk, but they are never loaded + * from the user's saved settings file. + */ + readonly attribute boolean inMemory; + + /** + * Whether or not this engine is a "general" search engine, e.g. is it for + * generally searching the web, or does it have a specific purpose like + * shopping. + */ + readonly attribute boolean isGeneralPurposeEngine; + + /** + * The domain from which search results are returned for this engine. + * + * @return the domain of the the search URL. + */ + readonly attribute AString searchUrlDomain; + + /** + * The URL to report the search to. + * + * @return the reporting URL. + */ + readonly attribute AString clickUrl; +}; + +[scriptable, uuid(0dc93e51-a7bf-4a16-862d-4b3469ff6206)] +interface nsISearchParseSubmissionResult : nsISupports +{ + /** + * The search engine associated with the URL passed in to + * nsISearchEngine::parseSubmissionURL, or null if the URL does not represent + * a search submission. + */ + readonly attribute nsISearchEngine engine; + + /** + * String containing the sought terms. This can be an empty string in case no + * terms were specified or the URL does not represent a search submission. + */ + readonly attribute AString terms; + + /** + * The name of the query parameter used by `engine` for queries. E.g. "q". + */ + readonly attribute AString termsParameterName; +}; + +[scriptable, uuid(0301834b-2630-440e-8b98-db8dc55f34b9)] +interface nsISearchService : nsISupports +{ + const unsigned long ERROR_DOWNLOAD_FAILURE = 0x1; + const unsigned long ERROR_DUPLICATE_ENGINE = 0x2; + const unsigned long ERROR_ENGINE_CORRUPTED = 0x3; + + const unsigned short CHANGE_REASON_UNKNOWN = 0x0; + const unsigned short CHANGE_REASON_USER = 0x1; + const unsigned short CHANGE_REASON_USER_PRIVATE_SPLIT = 0x2; + const unsigned short CHANGE_REASON_USER_SEARCHBAR = 0x3; + const unsigned short CHANGE_REASON_USER_SEARCHBAR_CONTEXT = 0x4; + const unsigned short CHANGE_REASON_ADDON_INSTALL = 0x5; + const unsigned short CHANGE_REASON_ADDON_UNINSTALL = 0x6; + const unsigned short CHANGE_REASON_CONFIG = 0x7; + const unsigned short CHANGE_REASON_LOCALE = 0x8; + const unsigned short CHANGE_REASON_REGION = 0x9; + const unsigned short CHANGE_REASON_EXPERIMENT = 0xA; + const unsigned short CHANGE_REASON_ENTERPRISE = 0xB; + const unsigned short CHANGE_REASON_UITOUR = 0xC; + const unsigned short CHANGE_REASON_ENGINE_UPDATE = 0xD; + + /** + * Start asynchronous initialization. + * + * The promise is resolved once initialization is complete, which may be + * immediately, if initialization has already been completed by some previous + * call to this method. + * This method should only be called when you need or want to wait for the + * full initialization of the search service. + */ + Promise init(); + + /** + * A promise that is resolved when initialization has finished, but does not + * trigger initialization to begin. + * + * @returns {Promise} + * Resolved when initalization has successfully finished, and rejected if it + * has failed. + */ + readonly attribute Promise promiseInitialized; + + /** + * Determine whether initialization has been completed. + * + * Clients of the service can use this attribute to quickly determine whether + * initialization is complete, and decide to trigger some immediate treatment, + * to launch asynchronous initialization or to bailout. + * + * Note that this attribute does not indicate that initialization has succeeded. + * + * @return |true| if the search service is now initialized, |false| if + * initialization has not been triggered yet. + */ + readonly attribute bool isInitialized; + + /** + * Determine whether initialization has been completed successfully. + * + */ + readonly attribute bool hasSuccessfullyInitialized; + + + /** + * Runs background checks; Designed to be run on idle. + */ + Promise runBackgroundChecks(); + + /** + * Resets the default engine to its app default engine value. + */ + Promise resetToAppDefaultEngine(); + + /** + * Adds a new Open Search engine from the file at the supplied URI. + * + * @param engineURL + * The URL to the search engine's description file. + * + * @param iconURL + * A URL string to an icon file to be used as the search engine's + * icon. This value may be overridden by an icon specified in the + * engine description file. + * + * @throws NS_ERROR_FAILURE if the description file cannot be successfully + * loaded. + */ + Promise addOpenSearchEngine(in AString engineURL, in AString iconURL); + + /** + * Adds a new search engine defined by the user. + * + * @param name + * The name of the engine. + * @param url + * The url of the engine with %s denoting where to + * replace the search term. + * @param alias [optional] + * The alias to refer to the engine. + */ + Promise addUserEngine(in AString name, + in AString url, + [optional] in AString alias); + + /** + * Adds search providers to the search service. If the search + * service is configured to load multiple locales for the extension, + * it may load more than one search engine. If called directly + * ensure the extension has been initialised. + * + * @param extension + * The extension to load from. + * @returns Promise that resolves when finished. + */ + Promise addEnginesFromExtension(in jsval extension); + + /** + * Un-hides all engines in the set of engines returned by getAppProvidedEngines. + */ + void restoreDefaultEngines(); + + /** + * Returns an engine with the specified alias. + * + * @param alias + * The search engine's alias. + * @returns The corresponding nsISearchEngine object, or null if it doesn't + * exist. + */ + Promise getEngineByAlias(in AString alias); + + /** + * Returns an engine with the specified name. + * + * @param aEngineName + * The name of the engine. + * @returns The corresponding nsISearchEngine object, or null if it doesn't + * exist. + */ + nsISearchEngine getEngineByName(in AString aEngineName); + + + /** + * Returns an engine with the specified Id. + * + * @param aEngineId + * The Id of the engine. + * @returns The corresponding nsISearchEngine object, or null if it doesn't + * exist. + */ + nsISearchEngine getEngineById(in AString aEngineId); + + /** + * Returns an array of all installed search engines. + * The array is sorted either to the user requirements or the default order. + * + * @returns an array of nsISearchEngine objects. + */ + Promise getEngines(); + + /** + * Returns an array of all installed search engines whose hidden attribute is + * false. + * The array is sorted either to the user requirements or the default order. + * + * @returns an array of nsISearchEngine objects. + */ + Promise getVisibleEngines(); + + /** + * Returns an array of all default search engines. This includes all loaded + * engines that aren't in the user's profile directory. + * The array is sorted to the default order. + * + * @returns an array of nsISearchEngine objects. + */ + Promise getAppProvidedEngines(); + + /** + * Returns an array of search engines installed by a given extension. + * + * @returns an array of nsISearchEngine objects. + */ + Promise getEnginesByExtensionID(in AString extensionID); + + /** + * Moves a visible search engine. + * + * @param engine + * The engine to move. + * @param newIndex + * The engine's new index in the set of visible engines. + * + * @throws NS_ERROR_FAILURE if newIndex is out of bounds, or if engine is + * hidden. + */ + Promise moveEngine(in nsISearchEngine engine, in long newIndex); + + /** + * Removes the search engine. If the search engine is installed in a global + * location, this will just hide the engine. If the engine is in the user's + * profile directory, it will be removed from disk. + * + * @param engine + * The engine to remove. + */ + Promise removeEngine(in nsISearchEngine engine); + + /** + * Notify nsSearchService that an extension has been removed. Removes any + * engines that are associated with that extension. + * + * @param id + * The id of the extension. + */ + Promise removeWebExtensionEngine(in AString id); + + /** + * The Application Default Engine object that is the default for this region, + * ignoring changes the user may have subsequently made. + */ + readonly attribute nsISearchEngine appDefaultEngine; + + /** + * The Application Default Engine object that is the default for this region when in + * private browsing mode, ignoring changes the user may have subsequently made. + */ + readonly attribute nsISearchEngine appPrivateDefaultEngine; + + /** + * The currently active search engine. + * Unless the application doesn't ship any search plugin, this should never + * be null. If the currently active engine is removed, this attribute will + * fallback first to the application default engine if it's not hidden, then to + * the first visible engine, and as a last resort it will unhide the app + * default engine. + */ + attribute nsISearchEngine defaultEngine; + + Promise getDefault(); + Promise setDefault(in nsISearchEngine engine, in unsigned short changeSource); + + /** + * The currently active search engine for private browsing mode. + * @see defaultEngine. + */ + attribute nsISearchEngine defaultPrivateEngine; + + Promise getDefaultPrivate(); + Promise setDefaultPrivate( + in nsISearchEngine engine, + in unsigned short changeSource); + + /** + * Whether to display the "Search in Private Window" result in the urlbar. + */ + readonly attribute boolean separatePrivateDefaultUrlbarResultEnabled; + + /** + * Allows the add-on manager to discover if a WebExtension based search engine + * may change the default to an application provided search engine. + * If that WebExtension is on the allow list, then it will override the + * built-in engine's urls and parameters. + * + * @param extension + * The extension to load from. + * @returns An object with two booleans: + * - canChangeToAppProvided: indicates if the WebExtension engine may + * set the named engine as default e.g. it is application provided. + * - canInstallEngine: indicates if the WebExtension engine may be + * installed, e.g. it is not an app-provided engine. + */ + Promise maybeSetAndOverrideDefault(in jsval extension); + + /** + * Gets a representation of the default engine in an anonymized JSON + * string suitable for recording in the Telemetry environment. + * + * @return {object} result + * contains anonymized info about the default engine(s). + * @return {string} result.defaultSearchEngine + * contains the telemetry id of the default engine. + * @return {object} result.defaultSearchEngineData + * contains information about the default engine: + * name, loadPath, original submissionURL + * @return {string} [result.defaultPrivateSearchEngine] + * only returned if the preference for having a separate engine in private + * mode is turned on. + * contains the telemetry id of the default engine for private browsing mode. + * @return {object} [result.defaultPrivateSearchEngineData] + * only returned if the preference for having a separate engine in private + * mode is turned on. + * contains information about the default engine for private browsing mode: + * name, loadPath, original submissionURL + */ + jsval getDefaultEngineInfo(); + + /** + * Determines if the provided URL represents results from a search engine, and + * provides details about the match. + * + * The lookup mechanism checks whether the domain name and path of the + * provided HTTP or HTTPS URL matches one of the known values for the visible + * search engines. The match does not depend on which of the schemes is used. + * The expected URI parameter for the search terms must exist in the query + * string, but other parameters are ignored. + * + * @param url + * String containing the URL to parse, for example + * "https://www.google.com/search?q=terms". + */ + nsISearchParseSubmissionResult parseSubmissionURL(in AString url); +}; diff --git a/toolkit/components/search/schema/Readme.txt b/toolkit/components/search/schema/Readme.txt new file mode 100644 index 0000000000..14fffb5c10 --- /dev/null +++ b/toolkit/components/search/schema/Readme.txt @@ -0,0 +1,7 @@ +The schemas in this directory are the primary source for the schemas they represent. + +They are uploaded to the RemoteSettings server to validate new configurations. + +Any changes should be validated by the Search team. + +See the documentation for more information: https://firefox-source-docs.mozilla.org/ diff --git a/toolkit/components/search/schema/search-config-icons-schema.json b/toolkit/components/search/schema/search-config-icons-schema.json new file mode 100644 index 0000000000..eda23080dd --- /dev/null +++ b/toolkit/components/search/schema/search-config-icons-schema.json @@ -0,0 +1,28 @@ +{ + "title": "Search Engine Icons Schema", + "description": "This schema contains the structure of data for search engine icons which built from search-config-v2. The associated remote settings collection is search-config-icons.", + "type": "object", + "required": ["engineIdentifiers", "imageSize"], + "properties": { + "engineIdentifiers": { + "title": "Engine Identifiers", + "description": "An array of search engine identifiers that the attached icon is for. If an individual entry is suffixed with a star, matching is applied on a \"starts with\" basis.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*\\*?$" + } + }, + "imageSize": { + "title": "Image Size", + "description": "The image size in pixels. May be used to choose the most appropriate image for the display.", + "type": "number" + }, + "filter_expression": { + "type": "string", + "title": "Filter Expression", + "description": "A JEXL to filter records", + "pattern": "^[a-zA-Z0-9-_=!\"{} .]*$" + } + } +} diff --git a/toolkit/components/search/schema/search-config-icons-ui-schema.json b/toolkit/components/search/schema/search-config-icons-ui-schema.json new file mode 100644 index 0000000000..7757d699a0 --- /dev/null +++ b/toolkit/components/search/schema/search-config-icons-ui-schema.json @@ -0,0 +1,3 @@ +{ + "ui:order": ["engineIdentifiers", "imageSize", "filter_expression"] +} diff --git a/toolkit/components/search/schema/search-config-overrides-schema.json b/toolkit/components/search/schema/search-config-overrides-schema.json new file mode 100644 index 0000000000..784c059a81 --- /dev/null +++ b/toolkit/components/search/schema/search-config-overrides-schema.json @@ -0,0 +1,64 @@ +{ + "title": "Search Engine Overrides Schema", + "description": "This schema contains the details for overriding application provided search engines defined in search-config. The associated remote settings collection is search-config-overrides.", + "type": "object", + "required": ["telemetryId"], + "properties": { + "telemetryId": { + "type": "string", + "title": "Telemetry Id", + "description": "The telemetry Id used to match the engine that this record will override.", + "pattern": "^[a-zA-Z0-9-$_]{0,100}$" + }, + "params": { + "$ref": "#/definitions/params" + }, + "clickUrl": { + "type": "string", + "format": "uri", + "description": "The url used to for reporting clicks." + }, + "telemetrySuffix": { + "type": "string", + "title": "Telemetry Suffix", + "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`.", + "pattern": "^[a-zA-Z0-9-]*$" + } + }, + "definitions": { + "searchUrlCodes": { + "type": "array", + "title": "Codes", + "description": "A array of objects - map of parameter name to the parameter value.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "Name", + "pattern": "^[a-zA-Z0-9.-]{0,100}$", + "description": "Name of the parameter that will be used in the query" + }, + "value": { + "type": "string", + "title": "Value", + "pattern": "^[a-zA-Z0-9_{}:/.-]{0,100}$", + "description": "The value of parameter (pref or purpose)" + } + } + } + }, + "params": { + "type": "object", + "title": "Parameters", + "description": "Various parameters for the search engines", + "properties": { + "searchUrlGetParams": { + "title": "Search URL GET Parameters", + "description": "Extra parameters for search URLs (e.g. 'pc=foo').", + "$ref": "#/definitions/searchUrlCodes" + } + } + } + } +} diff --git a/toolkit/components/search/schema/search-config-overrides-v2-schema.json b/toolkit/components/search/schema/search-config-overrides-v2-schema.json new file mode 100644 index 0000000000..cac02d8f2a --- /dev/null +++ b/toolkit/components/search/schema/search-config-overrides-v2-schema.json @@ -0,0 +1,84 @@ +{ + "title": "Search Engine Overrides Schema", + "description": "This schema contains the details for overriding application provided search engines defined in search-config-v2. The associated remote settings collection is search-config-overrides-v2.", + "definitions": { + "partnerCode": { + "title": "Partner Code", + "description": "The partner code for the engine or variant. This will be inserted into parameters which include '{partnerCode}'", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "urls": { + "title": "URLs", + "description": "The URLs associated with the search engine.", + "type": "object", + "properties": { + "search": { + "title": "Search URL", + "description": "The URL to use for searches", + "$ref": "#/definitions/url" + } + } + }, + "url": { + "type": "object", + "properties": { + "base": { + "title": "Base", + "description": "The PrePath and FilePath of the URL. May include variables for engines which have a variable FilePath, e.g. {searchTerm} for when a search term is within the path of the url.", + "type": "string" + }, + "params": { + "title": "Parameters", + "description": "The parameters for this URL.", + "type": "array", + "items": { + "type": "object", + "title": "Parameter", + "properties": { + "name": { + "title": "Name", + "description": "The parameter name", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "value": { + "title": "Value", + "description": "The parameter value, this may be a static value, or additionally contain a parameter replacement, e.g. {inputEncoding}. For the partner code parameter, this field should be {pc}.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_{}]*$" + } + }, + "required": ["name", "value"] + } + } + } + } + }, + "type": "object", + "properties": { + "identifier": { + "title": "Identifier", + "description": "This is the identifier of the search engine in search-config-v2 that this record will override. It may be extended by telemetrySuffix.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "partnerCode": { + "$ref": "#/definitions/partnerCode" + }, + "telemetrySuffix": { + "title": "Telemetry Suffix", + "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`. There should always be a suffix supplied if the partner code is different.", + "type": "string", + "pattern": "^[a-zA-Z0-9-]*$" + }, + "clickUrl": { + "type": "string", + "format": "uri", + "description": "The url used to for reporting clicks." + }, + "urls": { + "$ref": "#/definitions/urls" + } + } +} diff --git a/toolkit/components/search/schema/search-config-schema.json b/toolkit/components/search/schema/search-config-schema.json new file mode 100644 index 0000000000..c902044b85 --- /dev/null +++ b/toolkit/components/search/schema/search-config-schema.json @@ -0,0 +1,392 @@ +{ + "type": "object", + "required": ["webExtension"], + "properties": { + "default": { + "$ref": "#/definitions/default" + }, + "defaultPrivate": { + "$ref": "#/definitions/defaultPrivate" + }, + "orderHint": { + "$ref": "#/definitions/orderHint" + }, + "appliesTo": { + "type": "array", + "title": "Applies To", + "description": "This section defines the region/locales/application information for where a search engine is available, and any specifics for that region/locale/application. If there are no entries in the list, it is considered to be included everywhere", + "items": { + "$ref": "#/definitions/appliesToSection" + } + }, + "telemetryId": { + "type": "string", + "title": "Telemetry Id", + "description": "The telemetry Id as used for search telemetry." + }, + "webExtension": { + "$ref": "#/definitions/webExtension" + }, + "urls": { + "$ref": "#/definitions/urls" + }, + "params": { + "$ref": "#/definitions/params" + }, + "extraParams": { + "$ref": "#/definitions/extraParams" + }, + "suggestExtraParams": { + "$ref": "#/definitions/extraParams" + } + }, + "definitions": { + "application": { + "type": "object", + "title": "Application Details", + "properties": { + "name": { + "type": "array", + "title": "Name", + "description": "The application(s) this applies to (default/not specified is everywhere)", + "items": { + "type": "string", + "pattern": "^[a-z]{0,100}$", + "enum": ["", "firefox"] + }, + "uniqueItems": true + }, + "channel": { + "type": "array", + "title": "Channel", + "description": "Which channel this belongs to (not set = everywhere). For ESR this is also keyed from the display version.", + "items": { + "type": "string", + "pattern": "^[a-z]{0,100}$", + "enum": ["default", "nightly", "aurora", "beta", "release", "esr"] + }, + "uniqueItems": true + }, + "distributions": { + "type": "array", + "title": "Distributions", + "description": "Which distributions this applies to.", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{0,100}$" + }, + "uniqueItems": true + }, + "excludedDistributions": { + "type": "array", + "title": "Excluded Distributions", + "description": "Which distributions this does not apply to.", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{0,100}$" + }, + "uniqueItems": true + }, + "minVersion": { + "type": "string", + "title": "Minimum Version", + "pattern": "^[0-9a-z.]{0,20}$", + "description": "The minimum version this applies to" + }, + "maxVersion": { + "type": "string", + "title": "Maximum Version", + "pattern": "^[0-9a-z.]{0,20}$", + "description": "The maximum version this applies to (less-than comparison)" + } + } + }, + "default": { + "type": "string", + "title": "Default Status", + "pattern": "^[a-z-]{0,20}$", + "description": "Whether or not this engine should be default.", + "enum": ["yes", "yes-if-no-other", "no"] + }, + "defaultPrivate": { + "type": "string", + "title": "Default Status (PBM)", + "pattern": "^[a-z-]{0,20}$", + "description": "Whether or not this engine should be default in private browsing mode.", + "enum": ["yes", "yes-if-no-other", "no"] + }, + "extraParams": { + "type": "array", + "title": "Extra Parameters", + "description": "Extra parameters for the search engine (aka MozParams)", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "Name", + "pattern": "^[a-z]{0,20}$", + "description": "Name of the parameter that will be used in the search query" + }, + "condition": { + "type": "string", + "title": "Condition", + "pattern": "^[a-z]{0,10}$", + "description": "The type of parameter (pref or purpose)", + "enum": ["pref", "purpose"] + }, + "purpose": { + "type": "string", + "title": "Purpose", + "pattern": "^[a-z{}]{0,100}$", + "description": "The search purpose that triggers this parameter being appended.", + "enum": [ + "searchbar", + "keyword", + "contextmenu", + "homepage", + "newtab" + ] + }, + "value": { + "type": "string", + "title": "Value", + "pattern": "^[a-zA-Z0-9_]{0,100}$", + "description": "If this is a purpose type, the value is used as the value of the parameter in the query" + }, + "pref": { + "type": "string", + "title": "Preference name", + "pattern": "^[a-z0-9_]{0,100}$", + "description": "The preference name to get the value from (i.e. browser.search.param.<preference name>)." + } + } + } + }, + "orderHint": { + "type": "number", + "title": "Order Hint", + "description": "A hint to the display order (higher is a higher rank)" + }, + "searchUrlCodes": { + "type": "array", + "title": "Codes", + "description": "A array of objects - map of parameter name to the parameter value.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "Name", + "pattern": "^[a-zA-Z0-9.-]{0,100}$", + "description": "Name of the parameter that will be used in the query" + }, + "value": { + "type": "string", + "title": "Value", + "pattern": "^[a-zA-Z0-9_{}:/.-]{0,100}$", + "description": "The value of parameter (pref or purpose)" + } + } + } + }, + "params": { + "type": "object", + "title": "Parameters", + "description": "Various parameters for the search engines", + "properties": { + "searchUrlGetParams": { + "title": "Search URL GET Parameters", + "description": "Extra parameters for search URLs (e.g. 'pc=foo').", + "$ref": "#/definitions/searchUrlCodes" + }, + "suggestUrlGetParams": { + "title": "Suggestion URL GET Parameters", + "description": "Extra parameters for search suggestion URLs (e.g. 'pc=foo').", + "$ref": "#/definitions/searchUrlCodes" + }, + "searchUrlPostParams": { + "title": "Search URL POST Parameters", + "description": "Extra parameters for search URLs (e.g. 'pc=foo').", + "$ref": "#/definitions/searchUrlCodes" + }, + "suggestUrlPostParams": { + "title": "Suggestion URL POST Parameters", + "description": "Extra parameters for search suggestion URLs (e.g. 'pc=foo').", + "$ref": "#/definitions/searchUrlCodes" + } + } + }, + "telemetryId": { + "type": "string", + "title": "Telemetry Id", + "pattern": "^[a-zA-Z0-9-$_]{0,100}$", + "description": "The telemetry Id as used for search telemetry." + }, + "webExtension": { + "type": "object", + "title": "WebExtension", + "properties": { + "id": { + "type": "string", + "title": "WebExtension Id", + "description": "The identifier (local part) of the associated WebExtension should be of the format example@search.mozilla.org", + "pattern": "^[a-zA-Z0-9-._]*@search.mozilla.org$" + }, + "locales": { + "type": "array", + "title": "WebExtension Locales", + "description": "Overrides the WebExtension locales and specifies to use a particular one. Ideally this should only be used when really necessary, otherwise considered deprecated.", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9-$_]{0,100}$" + } + } + } + }, + "urls": { + "type": "object", + "description": "Urls pertaining to this engine", + "properties": { + "trending": { + "type": "object", + "description": "Details of the url used to fetch trending suggestions", + "properties": { + "fullPath": { + "type": "string", + "title": "fullPath", + "format": "uri", + "description": "The url used to retrieve trending suggestions" + }, + "query": { + "type": "string", + "title": "query", + "description": "The query parameters to send in a trending suggestions request", + "pattern": "^[a-zA-Z0-9.={}&-]{0,100}$" + } + } + } + } + }, + "regionDetails": { + "type": "array", + "title": "Regions", + "description": "Two-letter region codes.", + "items": { + "type": "string", + "pattern": "^([a-z][a-z]|default)$", + "minLength": 2, + "maxLength": 7 + } + }, + "localeDetails": { + "type": "object", + "title": "Locales", + "description": "Locale codes.", + "properties": { + "matches": { + "type": "array", + "title": "Matches exactly the codes", + "items": { + "type": "string", + "pattern": "^([a-z]{2,3}(-[a-zA-Z]{2,})?(-macos)?|default)$", + "minLength": 2 + } + }, + "startsWith": { + "type": "array", + "title": "Matches any code starting with", + "items": { + "type": "string", + "pattern": "^[a-z]{2,3}$", + "minLength": 2, + "maxLength": 3 + } + } + } + }, + "included": { + "type": "object", + "title": "Included Locations", + "description": "The locations to which this section applies. Note: Regions and Locales are 'and'ed together.", + "properties": { + "everywhere": { + "type": "boolean", + "title": "Everywhere", + "description": "Set to true to signify that this is included everywhere." + }, + "regions": { + "$ref": "#/definitions/regionDetails" + }, + "locales": { + "$ref": "#/definitions/localeDetails" + } + } + }, + "excluded": { + "type": "object", + "title": "Excluded Locations", + "description": "The locations to which this section applies. Note: Regions and Locales are 'and'ed together.", + "properties": { + "regions": { + "$ref": "#/definitions/regionDetails" + }, + "locales": { + "$ref": "#/definitions/localeDetails" + } + } + }, + "appliesToSection": { + "type": "object", + "properties": { + "default": { + "$ref": "#/definitions/default" + }, + "defaultPrivate": { + "$ref": "#/definitions/defaultPrivate" + }, + "orderHint": { + "$ref": "#/definitions/orderHint" + }, + "included": { + "$ref": "#/definitions/included" + }, + "excluded": { + "$ref": "#/definitions/excluded" + }, + "application": { + "$ref": "#/definitions/application" + }, + "webExtension": { + "$ref": "#/definitions/webExtension" + }, + "urls": { + "$ref": "#/definitions/urls" + }, + "telemetryId": { + "$ref": "#/definitions/telemetryId" + }, + "params": { + "$ref": "#/definitions/params" + }, + "extraParams": { + "$ref": "#/definitions/extraParams" + }, + "experiment": { + "type": "string", + "title": "Experiment", + "pattern": "^[a-zA-Z0-9-]{0,100}$", + "description": "The experiment this section is associated with, if blank it is associated with any configuration." + }, + "override": { + "type": "boolean", + "title": "Override", + "description": "This section will override previous appliesTo sections, but not add new locations where this engine is deployed to." + }, + "suggestExtraParams": { + "$ref": "#/definitions/extraParams" + } + } + } + } +} diff --git a/toolkit/components/search/schema/search-config-ui-schema.json b/toolkit/components/search/schema/search-config-ui-schema.json new file mode 100644 index 0000000000..5b36e5e6ee --- /dev/null +++ b/toolkit/components/search/schema/search-config-ui-schema.json @@ -0,0 +1,14 @@ +{ + "ui:order": [ + "webExtension", + "default", + "defaultPrivate", + "orderHint", + "telemetryId", + "urls", + "params", + "extraParams", + "suggestExtraParams", + "appliesTo" + ] +} diff --git a/toolkit/components/search/schema/search-config-v2-schema.json b/toolkit/components/search/schema/search-config-v2-schema.json new file mode 100644 index 0000000000..cfd0124fa3 --- /dev/null +++ b/toolkit/components/search/schema/search-config-v2-schema.json @@ -0,0 +1,462 @@ +{ + "title": "Application Provided Search Engines Schema", + "description": "This schema contains the details to build application provided search engines shipped in some Mozilla products. The associated remote settings collection is search-config-v2. Note: the top-level properties section is only required to satisfy remoteSettings. The allOf section provides the full details of the different record types.", + "definitions": { + "recordType": { + "title": "Record Type", + "description": "The type of details that this record contains.", + "enum": ["engine", "defaultEngines", "engineOrders"] + }, + "environment": { + "title": "Environment", + "description": "Specifies details of possible user environments that the engine or variant applies to", + "type": "object", + "properties": { + "allRegionsAndLocales": { + "title": "All Regions and Locales", + "description": "Indicates that this section applies to all regions and locales. May be modified by excludedRegions/excludedLocales.", + "type": "boolean" + }, + "regions": { + "title": "Regions", + "description": "An array of regions that this section applies to. 'default' will apply to situations where we have not been able to detect the user's region.", + "type": "array", + "items": { + "type": "string", + "pattern": "^([a-z][a-z]|default)$", + "minLength": 2, + "maxLength": 7 + }, + "uniqueItems": true + }, + "locales": { + "title": "Locales", + "description": "An array of locales that this section applies to. 'default' will apply to situations where we have not been able to detect the user's locale.", + "type": "array", + "items": { + "type": "string", + "pattern": "^([a-z]{2,3}(-[a-zA-Z]{2,})?(-macos)?|default)$", + "minLength": 2 + }, + "uniqueItems": true + }, + "excludedRegions": { + "title": "Excluded Regions", + "description": "An array of regions that this section should be excluded from. 'default' will apply to situations where we have not been able to detect the user's region.", + "type": "array", + "items": { + "type": "string", + "pattern": "^([a-z]{2,3}|default)$", + "minLength": 2, + "maxLength": 7 + }, + "uniqueItems": true + }, + "excludedLocales": { + "title": "Excluded Locales", + "description": "An array of locales that this section should be excluded from. 'default' will apply to situations where we have not been able to detect the user's locale.", + "type": "array", + "items": { + "type": "string", + "pattern": "^([a-z]{2,3}(-[a-zA-Z]{2,})?(-macos)?|default)$", + "minLength": 2 + }, + "uniqueItems": true + }, + "distributions": { + "title": "Distributions", + "description": "An array of distribution identifiers that this applies to.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{0,100}$" + }, + "uniqueItems": true + }, + "excludedDistributions": { + "title": "Excluded Distributions", + "description": "An array of distribution identifiers that this does not apply to.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{0,100}$" + }, + "uniqueItems": true + }, + "channels": { + "title": "Release Channels", + "description": "An array of release channels that this section applies to (not set = everywhere).", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{0,100}$", + "enum": ["default", "nightly", "aurora", "beta", "release", "esr"] + }, + "uniqueItems": true + }, + "experiment": { + "title": "Experiment", + "description": "The experiment that this section applies to.", + "type": "string", + "pattern": "^[a-zA-Z0-9-]{0,100}$" + }, + "applications": { + "title": "Application Identifiers", + "description": "The application(s) this section applies to (default/not specified is everywhere).", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z-]{0,100}$", + "enum": [ + "", + "firefox", + "firefox-android", + "firefox-ios", + "focus-android", + "focus-ios" + ] + }, + "uniqueItems": true + }, + "minVersion": { + "title": "Minimum Version", + "description": "The minimum application version this section applies to.", + "type": "string", + "pattern": "^[0-9a-z.]{0,20}$" + }, + "maxVersion": { + "title": "Maximum Version", + "description": "The maximum application version this section applies to (less-than comparison).", + "type": "string", + "pattern": "^[0-9a-z.]{0,20}$" + } + } + }, + "partnerCode": { + "title": "Partner Code", + "description": "The partner code for the engine or variant. This will be inserted into parameters which include '{pc}'", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "urls": { + "title": "URLs", + "description": "The URLs associated with the search engine.", + "type": "object", + "properties": { + "search": { + "title": "Search URL", + "description": "The URL to use for searches", + "$ref": "#/definitions/url" + }, + "suggestions": { + "title": "Suggestions URL", + "description": "The URL to use for suggestions", + "$ref": "#/definitions/url" + }, + "trending": { + "title": "Trending Suggestions URL", + "description": "The URL to use for tending suggestions", + "$ref": "#/definitions/url" + } + } + }, + "url": { + "type": "object", + "properties": { + "base": { + "title": "Base", + "description": "The PrePath and FilePath of the URL. May include variables for engines which have a variable FilePath, e.g. {searchTerm} for when a search term is within the path of the url.", + "type": "string" + }, + "method": { + "title": "Method", + "description": "The HTTP method to use to send the request. If not specified, defaults to GET.", + "type": "string", + "pattern": "^(GET|POST)$", + "enum": ["GET", "POST"] + }, + "params": { + "title": "Parameters", + "description": "The parameters for this URL.", + "type": "array", + "items": { + "type": "object", + "title": "Parameter", + "properties": { + "name": { + "title": "Name", + "description": "The parameter name", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "value": { + "title": "Value", + "description": "The parameter value, this may be a static value, or additionally contain a parameter replacement, e.g. {inputEncoding}. For the partner code parameter, this field should be {pc}.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_{}]*$" + }, + "experimentConfig": { + "title": "Experiment Configuration", + "description": "The value for the parameter will be derived from the equivalent experiment configuration value. If not experiment is present, this parameter will not be included in the final url.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "searchAccessPoint": { + "title": "Search Access Point", + "description": "A parameter whose value depends on the access point where the search was initiated.", + "type": "object", + "properties": { + "searchbar": { + "title": "Name", + "description": "The value for the parameter when searched from the search bar.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "addressbar": { + "title": "Name", + "description": "The value for the parameter when searched from the address bar.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "contextmenu": { + "title": "Name", + "description": "The value for the parameter when searched from the context menu.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "homepage": { + "title": "Name", + "description": "The value for the parameter when searched from the homepage.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "newtab": { + "title": "Name", + "description": "The value for the parameter when searched from the new tab page.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + } + } + } + }, + "oneOf": [ + { "required": ["name", "value"] }, + { "required": ["name", "experimentConfig"] }, + { "required": ["name", "searchAccessPoint"] } + ] + } + }, + "searchTermParamName": { + "title": "Search Term Parameter Name", + "description": "The name of the query parameter for the search term. Automatically appended to the end of the query. This may be skipped if `{searchTerm}` is included in the base.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + } + } + } + }, + "type": "object", + "properties": { + "recordType": { "$ref": "#/definitions/recordType" } + }, + "allOf": [ + { + "if": { + "properties": { + "recordType": { + "const": "engine" + } + } + }, + "then": { + "properties": { + "recordType": { "$ref": "#/definitions/recordType" }, + "identifier": { + "title": "Identifier", + "description": "The identifier of the search engine. This is used as an internal identifier, e.g. for saving the user's settings for the engine. It is also used to form the base telemetry id and may be extended by telemetrySuffix.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "notes": { + "title": "Notes", + "description": "A short notes section used to potentially aid identification of this section for humans. Not intended for to be read by the application.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_.() ]*$" + }, + "base": { + "title": "Base Details", + "description": "Base details for the engine.", + "type": "object", + "properties": { + "aliases": { + "title": "Aliases", + "description": "An array of aliases that the user can use to search with this engine. The aliases will be prefix by '@' on desktop and potentially other platforms.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z\\xa1-\\uffff]*$" + }, + "uniqueItems": true + }, + "charset": { + "title": "Character Set", + "description": "The character set this engine uses for queries. Defaults to 'UTF=8' if not set.", + "type": "string", + "pattern": "^[a-zA-Z0-9-]*$" + }, + "classification": { + "title": "Classification", + "description": "The classification of search engine according to the main search types (e.g. general, shopping, travel, dictionary). Currently, only marking as a general search engine is supported.", + "type": "string", + "enum": ["general", "unknown"] + }, + "name": { + "title": "Name", + "description": "The user visible name for the search engine.", + "type": "string", + "maxLength": 100 + }, + "partnerCode": { + "$ref": "#/definitions/partnerCode" + }, + "urls": { + "$ref": "#/definitions/urls" + } + }, + "required": ["name", "urls", "classification"] + }, + "variants": { + "title": "Variants", + "description": "This section describes variations of this search engine that may occur depending on the user's environment. If multiple sections match a user's environment, then all matching sections are applied cumulatively in the order in the array.", + "type": "array", + "items": { + "type": "object", + "properties": { + "environment": { + "$ref": "#/definitions/environment" + }, + "partnerCode": { + "$ref": "#/definitions/partnerCode" + }, + "optional": { + "title": "Optional", + "description": "This search engine is presented as an option that the user may enable. It is not included in the initial list of search engines. If not specified, defaults to false.", + "type": "boolean" + }, + "telemetrySuffix": { + "title": "Telemetry Suffix", + "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`. There should always be a suffix supplied if the partner code is different.", + "type": "string", + "pattern": "^[a-zA-Z0-9-]*$" + }, + "urls": { + "$ref": "#/definitions/urls" + } + }, + "required": ["environment"], + "dependencies": { + "partnerCode": ["telemetrySuffix"] + } + } + } + }, + "required": ["recordType", "identifier", "base", "variants"] + } + }, + { + "if": { + "properties": { + "recordType": { + "const": "defaultEngines" + } + } + }, + "then": { + "properties": { + "recordType": { "$ref": "#/definitions/recordType" }, + "globalDefault": { + "title": "Global Default", + "description": "The identifier of the engine that will be used as the application default if no other engines are specified as default.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "globalDefaultPrivate": { + "title": "Global Default - Private Mode", + "description": "The identifier of the engine that will be used as the application default in private mode if no other engines are specified as default.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*$" + }, + "specificDefaults": { + "title": "Filters", + "description": "The specific environment filters to set a different default engine. The array is ordered, when multiple entries match on environments, the later entry will override earlier entries.", + "type": "array", + "items": { + "type": "object", + "properties": { + "environment": { + "$ref": "#/definitions/environment" + }, + "default": { + "title": "Default", + "description": "The identifier of the engine that will be used as the application default for the associated environment. If the entry is suffixed with a star, matching is applied on a \"starts with\" basis.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*\\*?$" + }, + "defaultPrivate": { + "title": "Default - Private Mode", + "description": "The identifier of the engine that will be used as the application default in private mode for the associated environment. If the entry is suffixed with a star, matching is applied on a \"starts with\" basis.", + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*\\*?$" + } + }, + "oneOf": [ + { "required": ["environment", "default"] }, + { "required": ["environment", "defaultPrivate"] }, + { "required": ["environment", "default", "defaultPrivate"] } + ] + } + } + }, + "required": ["recordType", "globalDefault"] + } + }, + { + "if": { + "properties": { + "recordType": { + "const": "engineOrders" + } + } + }, + "then": { + "properties": { + "recordType": { "$ref": "#/definitions/recordType" }, + "orders": { + "title": "Orders", + "description": "When a user's instance matches the defined environments, the associated engine order will be applied. The array is ordered, when multiple entries match on environments, the later entry will override earlier entries.", + "type": "array", + "items": { + "type": "object", + "properties": { + "environment": { + "$ref": "#/definitions/environment" + }, + "order": { + "title": "Engine Order", + "type": "array", + "description": "The order of the engines for the associated environment. If engines are present for the user but not included in this list, they will follow after the ones in this list in alphabetical order. If an individual entry is suffixed with a star, matching is applied on a \"starts with\" basis.", + "items": { + "type": "string", + "pattern": "^[a-zA-Z0-9-_]*\\*?$" + } + } + }, + "required": ["environment"] + } + } + }, + "required": ["recordType", "orders"] + } + } + ] +} diff --git a/toolkit/components/search/schema/search-config-v2-ui-schema.json b/toolkit/components/search/schema/search-config-v2-ui-schema.json new file mode 100644 index 0000000000..c301fc6ceb --- /dev/null +++ b/toolkit/components/search/schema/search-config-v2-ui-schema.json @@ -0,0 +1,3 @@ +{ + "ui:order": ["recordType"] +} diff --git a/toolkit/components/search/schema/search-default-override-allowlist-schema.json b/toolkit/components/search/schema/search-default-override-allowlist-schema.json new file mode 100644 index 0000000000..6e48745832 --- /dev/null +++ b/toolkit/components/search/schema/search-default-override-allowlist-schema.json @@ -0,0 +1,54 @@ +{ + "type": "object", + "required": ["thirdPartyId", "overridesId", "urls"], + "properties": { + "thirdPartyId": { + "type": "string", + "title": "Third-party Add-on Id", + "description": "The identifier of the third party add-on which will override the app provided one. Should be of the format example@foo. If the third party engine is an OpenSearch engine, then this must be opensearch@search.mozilla.org", + "pattern": "^[a-zA-Z0-9-._]*@[a-zA-Z0-9-._]*$" + }, + "overridesId": { + "type": "string", + "title": "Add-on Id to Override", + "description": "The identifier of the app-provided add-on to be overridden. Should be of the format example@search.mozilla.org", + "pattern": "^[a-zA-Z0-9-._]*@search.mozilla.org$" + }, + "engineName": { + "type": "string", + "title": "Engine Name", + "description": "The name of the search engine to be overridden, only applies to OpenSearch engines" + }, + "urls": { + "type": "array", + "title": "URLs", + "description": "An array of URL sets which must be matched (with the add-on's manifest settings) to allow the override", + "items": { + "type": "object", + "required": ["search_url"], + "properties": { + "search_url": { + "type": "string", + "title": "search_url", + "description": "The main search url" + }, + "search_url_get_params": { + "type": "string", + "title": "search_url_get_params", + "description": "Any get parameters" + }, + "search_url_post_params": { + "type": "string", + "title": "search_url_post_params", + "description": "Any post parameters" + }, + "search_form": { + "type": "string", + "title": "search_form", + "description": "The search form url" + } + } + } + } + } +} diff --git a/toolkit/components/search/schema/search-default-override-allowlist-ui-schema.json b/toolkit/components/search/schema/search-default-override-allowlist-ui-schema.json new file mode 100644 index 0000000000..1b85489c13 --- /dev/null +++ b/toolkit/components/search/schema/search-default-override-allowlist-ui-schema.json @@ -0,0 +1,3 @@ +{ + "ui:order": ["thirdPartyId", "overridesId", "urls"] +} diff --git a/toolkit/components/search/tests/SearchTestUtils.sys.mjs b/toolkit/components/search/tests/SearchTestUtils.sys.mjs new file mode 100644 index 0000000000..8922070154 --- /dev/null +++ b/toolkit/components/search/tests/SearchTestUtils.sys.mjs @@ -0,0 +1,586 @@ +import { MockRegistrar } from "resource://testing-common/MockRegistrar.sys.mjs"; +import { NON_SPLIT_ENGINE_IDS } from "resource://gre/modules/SearchService.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AddonTestUtils: "resource://testing-common/AddonTestUtils.sys.mjs", + ExtensionTestUtils: + "resource://testing-common/ExtensionXPCShellUtils.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", +}); + +var gTestScope; + +export var SearchTestUtils = { + init(testScope) { + gTestScope = testScope; + this._isMochitest = !Services.env.exists("XPCSHELL_TEST_PROFILE_DIR"); + if (this._isMochitest) { + this._isMochitest = true; + lazy.AddonTestUtils.initMochitest(testScope); + } else { + this._isMochitest = false; + // This handles xpcshell-tests. + gTestScope.ExtensionTestUtils = lazy.ExtensionTestUtils; + this.initXPCShellAddonManager(testScope); + } + }, + + /** + * Adds an OpenSearch based engine to the search service. It will remove + * the engine at the end of the test. + * + * @param {object} options + * The options for the new search engine. + * @param {string} options.url + * The URL of the engine to add. + * @param {boolean} [options.setAsDefault] + * Whether or not to set the engine as default automatically. If this is + * true, the engine will be set as default, and the previous default engine + * will be restored when the test exits. + * @param {boolean} [options.setAsDefaultPrivate] + * Whether or not to set the engine as default automatically for private mode. + * If this is true, the engine will be set as default, and the previous default + * engine will be restored when the test exits. + * @param {boolean} [options.skipReset] + * Skips resetting the default engine at the end of the test. + * @returns {Promise} Returns a promise that is resolved with the new engine + * or rejected if it fails. + */ + async promiseNewSearchEngine({ + url, + setAsDefault = false, + setAsDefaultPrivate = false, + skipReset = false, + }) { + // OpenSearch engines can only be added via http protocols. + url = url.replace("chrome://mochitests/content", "https://example.com"); + let engine = await Services.search.addOpenSearchEngine(url, ""); + let previousEngine = Services.search.defaultEngine; + let previousPrivateEngine = Services.search.defaultPrivateEngine; + if (setAsDefault) { + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + if (setAsDefaultPrivate) { + await Services.search.setDefaultPrivate( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + gTestScope.registerCleanupFunction(async () => { + if (setAsDefault && !skipReset) { + await Services.search.setDefault( + previousEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + if (setAsDefaultPrivate && !skipReset) { + await Services.search.setDefaultPrivate( + previousPrivateEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + try { + await Services.search.removeEngine(engine); + } catch (ex) { + // Don't throw if the test has already removed it. + } + }); + return engine; + }, + + /** + * Returns a promise that is resolved when an observer notification from the + * search service fires with the specified data. + * + * @param {*} expectedData + * The value the observer notification sends that causes us to resolve + * the promise. + * @param {string} topic + * The notification topic to observe. Defaults to 'browser-search-service'. + * @returns {Promise} + * Returns a promise that is resolved with the subject of the + * topic once the topic with the data has been observed. + */ + promiseSearchNotification(expectedData, topic = "browser-search-service") { + return new Promise(resolve => { + Services.obs.addObserver(function observer(aSubject, aTopic, aData) { + if (aData != expectedData) { + return; + } + + Services.obs.removeObserver(observer, topic); + // Let the stack unwind. + Services.tm.dispatchToMainThread(() => resolve(aSubject)); + }, topic); + }); + }, + + /** + * For xpcshell tests, configures loading engines from test data located in + * particular folders. + * + * @param {string} [folder] + * The folder name to use. + * @param {string} [subFolder] + * The subfolder to use, if any. + * @param {Array} [configData] + * An array which contains the configuration to set. + * @returns {object} + * An object that is a sinon stub for the configuration getter. + */ + async useTestEngines(folder = "data", subFolder = null, configData = null) { + if (!lazy.SearchUtils.newSearchConfigEnabled) { + let url = `resource://test/${folder}/`; + if (subFolder) { + url += `${subFolder}/`; + } + let resProt = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + resProt.setSubstitution("search-extensions", Services.io.newURI(url)); + } + + const settings = await lazy.RemoteSettings(lazy.SearchUtils.SETTINGS_KEY); + if (configData) { + return lazy.sinon.stub(settings, "get").returns(configData); + } + + let workDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile); + let configFileName = + "file://" + + PathUtils.join( + workDir.path, + folder, + subFolder ?? "", + lazy.SearchUtils.newSearchConfigEnabled + ? "search-config-v2.json" + : "engines.json" + ); + + let response = await fetch(configFileName); + let json = await response.json(); + return lazy.sinon.stub(settings, "get").returns(json.data); + }, + + /** + * For mochitests, configures loading engines from test data located in + * particular folders. This will cleanup at the end of the test. + * + * This will be removed when the old configuration is removed + * (newSearchConfigEnabled = false). + * + * @param {nsIFile} testDir + * The test directory to use. + */ + async useMochitestEngines(testDir) { + // Replace the path we load search engines from with + // the path to our test data. + let resProt = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + let originalSubstitution = resProt.getSubstitution("search-extensions"); + resProt.setSubstitution( + "search-extensions", + Services.io.newURI("file://" + testDir.path) + ); + gTestScope.registerCleanupFunction(() => { + resProt.setSubstitution("search-extensions", originalSubstitution); + }); + }, + + /** + * Utility function for mochitests to configure a custom search engine + * directory and search configuration. + * + * @param {string} testDir + * The test directory to use. + * @param {Array} searchConfig + * The test search configuration to use. + */ + async setupTestEngines(testDir, searchConfig) { + let searchExtensions = gTestScope.getChromeDir( + gTestScope.getResolvedURI(gTestScope.gTestPath) + ); + searchExtensions.append(testDir); + await this.useMochitestEngines(searchExtensions); + + this.useMockIdleService(); + + await this.updateRemoteSettingsConfig(searchConfig); + + gTestScope.registerCleanupFunction(async () => { + let settingsWritten = SearchTestUtils.promiseSearchNotification( + "write-settings-to-disk-complete" + ); + await SearchTestUtils.updateRemoteSettingsConfig(); + await settingsWritten; + }); + }, + + /** + * Convert a list of engine configurations into engine objects. + * + * @param {Array} engineConfigurations + * An array of engine configurations. + */ + async searchConfigToEngines(engineConfigurations) { + let engines = []; + + for (let e of engineConfigurations) { + if (!e.webExtension) { + e.webExtension = {}; + } + e.webExtension.locale = + e.webExtension.locale ?? lazy.SearchUtils.DEFAULT_TAG; + + // TODO Bug 1875912 - Remove the webextension.id and webextension.locale when + // we're ready to remove old search-config and use search-config-v2 for all + // clients. The id in appProvidedSearchEngine should be changed to + // engine.identifier. + if (lazy.SearchUtils.newSearchConfigEnabled) { + let identifierComponents = NON_SPLIT_ENGINE_IDS.includes(e.identifier) + ? [e.identifier] + : e.identifier.split("-"); + + e.webExtension.locale = + identifierComponents.slice(1).join("-") || "default"; + e.webExtension.id = identifierComponents[0] + "@search.mozilla.org"; + } + } + + for (let config of engineConfigurations) { + let engine = await Services.search.wrappedJSObject._makeEngineFromConfig( + config + ); + engines.push(engine); + } + return engines; + }, + + /** + * Provides various setup for xpcshell-tests installing WebExtensions. Should + * be called from the global scope of the test. + * + * @param {object} scope + * The global scope of the test being run. + * @param {*} usePrivilegedSignatures + * How to sign created addons. + */ + initXPCShellAddonManager(scope, usePrivilegedSignatures = false) { + let scopes = + lazy.AddonManager.SCOPE_PROFILE | lazy.AddonManager.SCOPE_APPLICATION; + Services.prefs.setIntPref("extensions.enabledScopes", scopes); + // Only do this once. + try { + gTestScope.ExtensionTestUtils.init(scope); + } catch (ex) { + // This can happen if init is called twice. + if (ex.result != Cr.NS_ERROR_FILE_ALREADY_EXISTS) { + throw ex; + } + } + lazy.AddonTestUtils.usePrivilegedSignatures = usePrivilegedSignatures; + lazy.AddonTestUtils.overrideCertDB(); + }, + + /** + * Add a search engine as a WebExtension. + * + * Note: for tests, the extension must generally be unloaded before + * `registerCleanupFunction`s are triggered. See bug 1694409. + * + * This function automatically registers an unload for the extension, this + * may be skipped with the skipUnload argument. + * + * @param {object} [manifest] + * See {@link createEngineManifest} + * @param {object} [options] + * Options for how the engine is installed and uninstalled. + * @param {boolean} [options.setAsDefault] + * Whether or not to set the engine as default automatically. If this is + * true, the engine will be set as default, and the previous default engine + * will be restored when the test exits. + * @param {boolean} [options.setAsDefaultPrivate] + * Whether or not to set the engine as default automatically for private mode. + * If this is true, the engine will be set as default, and the previous default + * engine will be restored when the test exits. + * @param {boolean} [options.skipUnload] + * If true, this will skip the automatic unloading of the extension. + * @param {object} [files] + * A key value object where the keys are the filenames and their contents are + * the values. Used for simulating locales and other files in the WebExtension. + * @returns {object} + * The loaded extension. + */ + async installSearchExtension( + manifest = {}, + { + setAsDefault = false, + setAsDefaultPrivate = false, + skipUnload = false, + } = {}, + files = {} + ) { + await Services.search.init(); + + let extensionInfo = { + useAddonManager: "permanent", + files, + manifest: this.createEngineManifest(manifest), + }; + + let extension; + + let previousEngine = Services.search.defaultEngine; + let previousPrivateEngine = Services.search.defaultPrivateEngine; + + async function cleanup() { + if (setAsDefault) { + await Services.search.setDefault( + previousEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + if (setAsDefaultPrivate) { + await Services.search.setDefaultPrivate( + previousPrivateEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + await extension.unload(); + } + + // Cleanup must be registered before loading the extension to avoid + // failures for mochitests. + if (!skipUnload && this._isMochitest) { + gTestScope.registerCleanupFunction(cleanup); + } + + extension = gTestScope.ExtensionTestUtils.loadExtension(extensionInfo); + await extension.startup(); + await lazy.AddonTestUtils.waitForSearchProviderStartup(extension); + let engine = Services.search.getEngineByName(manifest.name); + + if (setAsDefault) { + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + if (setAsDefaultPrivate) { + await Services.search.setDefaultPrivate( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + } + + // For xpcshell-tests we must register the unload after adding the extension. + // See bug 1694409 for why this is. + if (!skipUnload && !this._isMochitest) { + gTestScope.registerCleanupFunction(cleanup); + } + + return extension; + }, + + /** + * Install a search engine as a system extension to simulate + * Normandy updates. For xpcshell-tests only. + * + * @param {object} [options] + * See {@link createEngineManifest} + */ + async installSystemSearchExtension(options = {}) { + options.id = (options.id ?? "example") + "@search.mozilla.org"; + let xpi = await lazy.AddonTestUtils.createTempWebExtensionFile({ + manifest: this.createEngineManifest(options), + background() { + // eslint-disable-next-line no-undef + browser.test.sendMessage("started"); + }, + }); + let wrapper = gTestScope.ExtensionTestUtils.expectExtension(options.id); + + const install = await lazy.AddonManager.getInstallForURL( + `file://${xpi.path}`, + { + useSystemLocation: true, + } + ); + + install.install(); + + await wrapper.awaitStartup(); + await wrapper.awaitMessage("started"); + + return wrapper; + }, + + /** + * Create a search engine extension manifest. + * + * @param {object} [options] + * The options for the manifest. + * @param {string} [options.id] + * The id to use for the WebExtension. + * @param {string} [options.name] + * The display name to use for the WebExtension. + * @param {string} [options.version] + * The version to use for the WebExtension. + * @param {string} [options.favicon_url] + * The favicon to use for the search engine in the WebExtension. + * @param {string} [options.keyword] + * The keyword to use for the search engine. + * @param {string} [options.encoding] + * The encoding to use for the search engine. + * @param {string} [options.search_url] + * The search URL to use for the search engine. + * @param {string} [options.search_url_get_params] + * The GET search URL parameters to use for the search engine + * @param {string} [options.search_url_post_params] + * The POST search URL parameters to use for the search engine + * @param {string} [options.suggest_url] + * The suggestion URL to use for the search engine. + * @param {string} [options.suggest_url_get_params] + * The suggestion URL parameters to use for the search engine. + * @param {string} [options.search_form] + * The search form to use for the search engine. + * @returns {object} + * The generated manifest. + */ + createEngineManifest(options = {}) { + options.name = options.name ?? "Example"; + options.id = options.id ?? options.name.toLowerCase().replaceAll(" ", ""); + if (!options.id.includes("@")) { + options.id += "@tests.mozilla.org"; + } + options.version = options.version ?? "1.0"; + let manifest = { + version: options.version, + browser_specific_settings: { + gecko: { + id: options.id, + }, + }, + chrome_settings_overrides: { + search_provider: { + name: options.name, + search_url: options.search_url ?? "https://example.com/", + }, + }, + }; + + if (options.default_locale) { + manifest.default_locale = options.default_locale; + } + + if (options.search_url_post_params) { + manifest.chrome_settings_overrides.search_provider.search_url_post_params = + options.search_url_post_params; + } else { + manifest.chrome_settings_overrides.search_provider.search_url_get_params = + options.search_url_get_params ?? "?q={searchTerms}"; + } + + if (options.favicon_url) { + manifest.chrome_settings_overrides.search_provider.favicon_url = + options.favicon_url; + } + if (options.encoding) { + manifest.chrome_settings_overrides.search_provider.encoding = + options.encoding; + } + if (options.keyword) { + manifest.chrome_settings_overrides.search_provider.keyword = + options.keyword; + } + if (options.suggest_url) { + manifest.chrome_settings_overrides.search_provider.suggest_url = + options.suggest_url; + } + if (options.suggest_url) { + manifest.chrome_settings_overrides.search_provider.suggest_url_get_params = + options.suggest_url_get_params; + } + if (options.search_form) { + manifest.chrome_settings_overrides.search_provider.search_form = + options.search_form; + } + if (options.favicon_url) { + manifest.chrome_settings_overrides.search_provider.favicon_url = + options.favicon_url; + } + return manifest; + }, + + /** + * A mock idleService that allows us to simulate RemoteSettings + * configuration updates. + */ + idleService: { + _observers: new Set(), + + _reset() { + this._observers.clear(); + }, + + _fireObservers(state) { + for (let observer of this._observers.values()) { + observer.observe(observer, state, null); + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsIUserIdleService"]), + idleTime: 19999, + + addIdleObserver(observer, time) { + this._observers.add(observer); + }, + + removeIdleObserver(observer, time) { + this._observers.delete(observer); + }, + }, + + /** + * Register the mock idleSerice. + */ + useMockIdleService() { + let fakeIdleService = MockRegistrar.register( + "@mozilla.org/widget/useridleservice;1", + SearchTestUtils.idleService + ); + gTestScope.registerCleanupFunction(() => { + MockRegistrar.unregister(fakeIdleService); + }); + }, + + /** + * Simulates an update to the RemoteSettings configuration. + * + * @param {object} [config] + * The new configuration. + */ + async updateRemoteSettingsConfig(config) { + if (!config) { + let settings = lazy.RemoteSettings(lazy.SearchUtils.SETTINGS_KEY); + config = await settings.get(); + } + const reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + await lazy.RemoteSettings(lazy.SearchUtils.SETTINGS_KEY).emit("sync", { + data: { current: config }, + }); + + this.idleService._fireObservers("idle"); + await reloadObserved; + }, +}; diff --git a/toolkit/components/search/tests/xpcshell/data/bigIcon.ico b/toolkit/components/search/tests/xpcshell/data/bigIcon.ico Binary files differnew file mode 100644 index 0000000000..f22522411d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/bigIcon.ico diff --git a/toolkit/components/search/tests/xpcshell/data/engine-app/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-app/manifest.json new file mode 100644 index 0000000000..14bc27bdd4 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-app/manifest.json @@ -0,0 +1,24 @@ +{ + "name": "TestEngineApp", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-app@search.mozilla.org" + } + }, + "hidden": true, + "description": "A test search engine installed in the application directory", + "chrome_settings_overrides": { + "search_provider": { + "name": "TestEngineApp", + "search_url": "https://localhost/", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-chromeicon/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-chromeicon/manifest.json new file mode 100644 index 0000000000..d2896b78f8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-chromeicon/manifest.json @@ -0,0 +1,23 @@ +{ + "name": "engine-chromeicon", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-chromeicon@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine-chromeicon", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/en/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/en/messages.json new file mode 100644 index 0000000000..07580b8ea5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/en/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-diff-name-en" + }, + "searchUrl": { + "message": "https://en.wikipedia.com/search" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/gd/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/gd/messages.json new file mode 100644 index 0000000000..de01d16bf0 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/_locales/gd/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-diff-name-gd" + }, + "searchUrl": { + "message": "https://gd.wikipedia.com/search" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-diff-name/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/manifest.json new file mode 100644 index 0000000000..3c80765f61 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-diff-name/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "engine-diff-name", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-diff-name@search.mozilla.org" + } + }, + "hidden": true, + "icons": { + "16": "favicon.png" + }, + "default_locale": "en", + "chrome_settings_overrides": { + "search_provider": { + "name": "__MSG_extensionName__", + "search_url": "__MSG_searchUrl__" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-fr.xml b/toolkit/components/search/tests/xpcshell/data/engine-fr.xml new file mode 100644 index 0000000000..4bb4426a12 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-fr.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="UTF-8"?>
+<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
+<ShortName>Test search engine (fr)</ShortName>
+<Description>A test search engine (based on Google search for a different locale)</Description>
+<InputEncoding>ISO-8859-1</InputEncoding>
+<Url type="text/html" method="GET" template="http://www.google.fr/search">
+ <Param name="q" value="{searchTerms}"/>
+ <Param name="ie" value="iso-8859-1"/>
+ <Param name="oe" value="iso-8859-1"/>
+</Url>
+<SearchForm>http://www.google.fr/</SearchForm>
+</SearchPlugin>
diff --git a/toolkit/components/search/tests/xpcshell/data/engine-fr/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-fr/manifest.json new file mode 100644 index 0000000000..cc895d26d9 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-fr/manifest.json @@ -0,0 +1,32 @@ +{ + "name": "Test search engine (fr)", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-fr@search.mozilla.org" + } + }, + "hidden": true, + "description": "A test search engine (based on Google search for a different locale)", + "chrome_settings_overrides": { + "search_provider": { + "name": "Test search engine (fr)", + "search_url": "https://www.google.fr/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "ie", + "value": "iso-8859-1" + }, + { + "name": "oe", + "value": "iso-8859-1" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-override/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-override/manifest.json new file mode 100644 index 0000000000..a894be0a41 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-override/manifest.json @@ -0,0 +1,24 @@ +{ + "name": "bug645970", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-override@search.mozilla.org" + } + }, + "hidden": true, + "description": "override", + "chrome_settings_overrides": { + "search_provider": { + "name": "bug645970", + "search_url": "https://searchtest.local", + "params": [ + { + "name": "search", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-pref/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-pref/manifest.json new file mode 100644 index 0000000000..d796b65b63 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-pref/manifest.json @@ -0,0 +1,33 @@ +{ + "name": "engine-pref", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-pref@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine-pref", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "code", + "condition": "pref", + "pref": "code" + }, + { + "name": "test", + "condition": "pref", + "pref": "test" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-purposes/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-purposes/manifest.json new file mode 100644 index 0000000000..fc709063e1 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-purposes/manifest.json @@ -0,0 +1,62 @@ +{ + "name": "Test Engine With Purposes", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-purposes@search.mozilla.org" + } + }, + "description": "A test search engine with purposes", + "chrome_settings_overrides": { + "search_provider": { + "name": "Test Engine With Purposes", + "search_url": "https://www.example.com/search", + "params": [ + { + "name": "form", + "condition": "purpose", + "purpose": "keyword", + "value": "MOZKEYWORD" + }, + { + "name": "form", + "condition": "purpose", + "purpose": "contextmenu", + "value": "MOZCONTEXT" + }, + { + "name": "form", + "condition": "purpose", + "purpose": "newtab", + "value": "MOZNEWTAB" + }, + { + "name": "form", + "condition": "purpose", + "purpose": "searchbar", + "value": "MOZSEARCHBAR" + }, + { + "name": "form", + "condition": "purpose", + "purpose": "homepage", + "value": "MOZHOMEPAGE" + }, + { + "name": "pc", + "value": "FIREFOX" + }, + { + "name": "channel", + "condition": "pref", + "pref": "testChannelEnabled" + }, + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-rel-searchform-purpose/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-rel-searchform-purpose/manifest.json new file mode 100644 index 0000000000..ed4a609e7c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-rel-searchform-purpose/manifest.json @@ -0,0 +1,41 @@ +{ + "name": "engine-rel-searchform-purpose", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-rel-searchform-purpose@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine-rel-searchform-purpose", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "searchbar", + "value": "sb" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-reordered/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-reordered/manifest.json new file mode 100644 index 0000000000..cc3fc95430 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-reordered/manifest.json @@ -0,0 +1,40 @@ +{ + "name": "Test search engine (Reordered)", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-reordered@search.mozilla.org" + } + }, + "hidden": true, + "description": "A test search engine (based on Google search)", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "Test search engine (Reordered)", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/en/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/en/messages.json new file mode 100644 index 0000000000..1cc3f68ee1 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/en/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-resourceicon" + }, + "searchUrl": { + "message": "https://www.google.com/search" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/gd/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/gd/messages.json new file mode 100644 index 0000000000..3c02e6a2af --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/_locales/gd/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-resourceicon-gd" + }, + "searchUrl": { + "message": "https://www.google.com/search" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/manifest.json new file mode 100644 index 0000000000..dc62336145 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-resourceicon/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "engine-resourceicon", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-resourceicon@search.mozilla.org" + } + }, + "hidden": true, + "icons": { + "16": "favicon.png" + }, + "default_locale": "en", + "chrome_settings_overrides": { + "search_provider": { + "name": "__MSG_extensionName__", + "search_url": "__MSG_searchUrl__" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/en/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/en/messages.json new file mode 100644 index 0000000000..ee808e7a62 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/en/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-same-name" + }, + "searchUrl": { + "message": "https://www.google.com/search?q={searchTerms}" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/gd/messages.json b/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/gd/messages.json new file mode 100644 index 0000000000..476a9e56cc --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-same-name/_locales/gd/messages.json @@ -0,0 +1,8 @@ +{ + "extensionName": { + "message": "engine-same-name" + }, + "searchUrl": { + "message": "https://www.example.com/search?q={searchTerms}" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-same-name/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-same-name/manifest.json new file mode 100644 index 0000000000..dc8a4c5d45 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-same-name/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "engine-same-name", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-same-name@search.mozilla.org" + } + }, + "hidden": true, + "icons": { + "16": "favicon.png" + }, + "default_locale": "en", + "chrome_settings_overrides": { + "search_provider": { + "name": "__MSG_extensionName__", + "search_url": "__MSG_searchUrl__" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine-system-purpose/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine-system-purpose/manifest.json new file mode 100644 index 0000000000..d268af8eab --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-system-purpose/manifest.json @@ -0,0 +1,35 @@ +{ + "name": "engine-system-purpose", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine-system-purpose@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine-system-purpose", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "searchbar", + "value": "sb" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "system", + "value": "sys" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine.xml b/toolkit/components/search/tests/xpcshell/data/engine.xml new file mode 100644 index 0000000000..a665e46b0b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8"?> +<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/"> +<ShortName>Test search engine</ShortName> +<Description>A test search engine (based on Google search)</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA</Image> +<Url type="application/x-suggestions+json" method="GET" template="https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}"/> +<Url type="text/html" method="GET" template="https://www.google.com/search"> + <Param name="q" value="{searchTerms}"/> + <!-- Dynamic parameters --> + <MozParam name="channel" condition="purpose" purpose="contextmenu" value="rcs"/> + <MozParam name="channel" condition="purpose" purpose="keyword" value="fflb"/> +</Url> +<SearchForm>http://www.google.com/</SearchForm> +</SearchPlugin> diff --git a/toolkit/components/search/tests/xpcshell/data/engine/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine/manifest.json new file mode 100644 index 0000000000..30d221f388 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine/manifest.json @@ -0,0 +1,40 @@ +{ + "name": "Test search engine", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine@search.mozilla.org" + } + }, + "hidden": true, + "description": "A test search engine (based on Google search)", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "Test search engine", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engine2.xml b/toolkit/components/search/tests/xpcshell/data/engine2.xml new file mode 100644 index 0000000000..9957bfdf48 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine2.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"> + <ShortName>A second test engine</ShortName> + <Description>A second test search engine (based on DuckDuckGo)</Description> + <InputEncoding>UTF-8</InputEncoding> + <LongName>A second test search engine (based on DuckDuckGo)</LongName> + <Image width="16" height="16">data:image/vnd.microsoft.icon;base64,AAABAAMAEBAAAAEACABoBQAANgAAACAgAAABAAgAqAgAAJ4FAAAwMAAAAQAIAKgOAABGDgAAKAAAABAAAAAgAAAAAQAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUvgAAFMAAABfDAAAYwQAAGscAAB7KAAAgzAAAIcwAACHNAAAizgAAI9AAACTQAAAnzAAAJtMAACnXABktyAAALtgAAC/bAAAx3QAAM+AAADbjAAA34wAAOOQAADzYAAA65wAWPdcAADvoAABG1QC6cg0AAFXdAERS0AAAVd8AAF3cAABp5wAAcOUATGziAAB36AAAeugATHLqAF165ABlg+0A0qRkAACS7wAAlO8AeI7nAIGX6gCVndwAAKX1AACr9gAAr/YAALX4AAC3+QCdrvIAALz6AAC9+QAAv/kAAMD7AADH+wAAyvwAAND9ALHB9QAA0vwA5c68AADT/gAA1P0AANT+AADU/wAA1/8AANj/AADZ/wAA3P8AAN3/AGrV+wAA3/8AAOH/AMrS9ADs3tYA39zmAOHj8AB+6v8A5ur6AKDw/wCq8f8A9fDtAPby7wD38vAA6/D8APfz8QD69O4A8PP8APL0/AD69/IA+/fzAPr39QDd+f8A+/n1APb4/QD7+fgA+/r5AOz8/gD1/P8A+vz/AP7+/QD//v8A////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABpLGhoaGhQHgEAAQEBAQFpGUtoaGhoSw8CAwICAgICAiNQaGhoaC0EBAwgIBsEBAQnWmhoaGgnIjY5PUBHOyUFLWhoaGheR0MdBgYGByQrCEtoaGhoT0I3CQkJCQkJCQlQaGhoaFI/QTIzNSoXCgsLWmhoaGhoUUVEQ0RKRDEfDWhoXFxoaGhjZWRILzpJRjhoXBwcWGhoaFtbYg4QIS8waFwcKVhoaF8cHF8REREREWhnXFxoaGhfHClOEhISEhJVU2hoaGhoaF9fNBMTExMTVz5MZmhoaFdMTSYUFBQUFDxhVGhoaGhdPi4VFhYWFhZpVmBoaGhoWSgYGhoaGhppgAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAACgAAAAgAAAAQAAAAAEACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFL8AABTAAAEWvAABF74AABbCAAAXwwABGMAAABjDAAAYxAAAGsUAABrGAAAbxgAAG8cAAh3BAAEcxQAAHMgAAR3HAAgewgAAHcoAAB/LAAAgzAAFIckAiEYiAAAhzgAAI88AACPQAAAk0AAAJc8AACXSAAUmzQAAJ9IAACfTABAqxwAAKNQAjE0rAAAp1QAAKdYAACrXAAAr2AAGLdMAAC3ZAAAu2wAAL9wAADPVAAAx3QAAMt8AADPgAAA04QAAONQAADnXAAA24wAGN94ACjfdAAA34wAAN+QACznbAAA45QAJOeAAFTzUABY81AAAOeYAFj3UAAo74AAAP9YAADrnAAs74AAAPtoABT/pAABF2wAARd0AAEbaAABI1gAASdoAEETmAABJ2wAAS98AAFHfAAFV3ABTc2EAWYA+AAFa4gAAXd0AAF3hADyeAAABZd4AAWTjADZf5wABZuIAQKETAD2jDwBGZ9gARKEYADhk6wA8pBMAPKUVAK+DbABKbN8ATGziAAF15ABTpioAaKE1AEGtJAABfOkAOrIiAH6hRQABgOkAAYLmAAGC5wACgucAXXrkAAGE5wBEr0AAAYToAAGG6gA9s0AAObkuAD20QQA5ui8AaIHhAEe3OABxgeAAAo3sAFasbABshukAOcA5AGmI7gABleoAbovsAAKZ7QCFtmEAAp7vAD7CagCBl+oAPsNuAAKo8QACqvIAA6zyAAOs9AACr/QAkKfxAGLJlQBfypUAZMqWAAO49QDSuawAc8qaAMPCngADwfgAA8H5AAPE+QAEyPsABMr7AATO/AAE0PwABND9AATR/QAE0v0ACNP9ABvW/QDj1MwAMtr9ADvc/QDK0vQA59rTAOjb1ADp3tcA6+DaAG/l/gB15v4A29/0AO3k3wDu5eAA4eT3AJDr/gDy6+cA5uj3APLs6ADj6PsA8+zpAObq+gDz7uoA8+7rANHt+gD07usA6e35APbx7wDy9PwA5fr/APv7+wD2+/8A7fz/APz8+gDw/P8A/fz7APv8/gD9/f4A+P7/AP/+/gD+/v8A////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAyMgAAAAAPYyDb4HHx8fHw05dUwAAAAAAAAAAAAAAyMjIAQEBAQFhkYNyXmV3Y2h6dF1TAgEBAQEBAQEBAQEByAQEBAQEBG2Og3JedXxnWYV0XVMDBAQEBAQEBAQEBAQEBQUFBQURhIyDcl5zfGdbhXRdUwYFBQUFBQUFBQUFBQUICAgICDuijYNyXmV3ZE96dF1TBwgICAgICAgICAgICAoKCgoKYbOMg5K/x8esCg5OWFMJCgoKCgoKCgoKCgoKDAwMDAxtusfHx8fHx3gMDAwLDQwMDAwMDAwMDAwMDAwPDw8PEITHx8fHx8fHFQ8PDw8PDw8PDw8PDw8PDw8PDxISEhIgosfHx8fHx8YSEhISEhISEhISEhISEhISEhISExMTEzuzx8fHx8fHrxMTExMTExMTExMTExMTExMTExMUFBQUYbrHx8fHx8epFBQUboiXnJqGVEcUFBQUFBQUFBcXFxdtx8fHx8fHx7YbbJyYj2tigpObnIAXFxcXFxcXGBgYHYTHx8fHx8e7npybcE0wGBgYP1F+ahgYGBgYGBgaGho6osfHx8fHx6icnIcZGhoaGhoaGhoaGhoaGhoaGhwcHGGzx8fHx8fHp5ycnEgeRjErHBwcHBwcHBwcHBwcISEhbbrHx8fHx8e+oZycnJycnJyViVdEISEhISEhISEhIR+Ex8fHx8fHx8fEraCdnJycnJycnJZxSiEhISEhISMjJ6LHx8fHx8fHx8fHx8fAvcBCVXmUnJyZUiMjIyMjJSU6s8fHx6Wlx8fHx8fHx8fHxyUkJUxpmZuKJSUlJSUmJmG6x8WjIhaqx8fHx8fHtLTHJiYmJkVQZksmJiYmJigobcXHx6QiX6rHx8fHx7AiFrUoKCgoKCgoKCgoKCgoKSmEx8fHx6urx8fHx8fHsiJfeykpKSkpKSkpKSkpKSkqKoTHt8fHx8fHx8fHx8fHubk0KioqKioqKioqKioqKiwsbcemx8fHx8fHx8fHwcfHuCwsLCwsLCwsLCwsLCwsLi43s8eQkJ/Hx8fHx8eukJA5Li4uLi4uLi4uLi4uLi4uLi2Ex8fHx8fHx8fHx8fHfy4uLi4uLi4uLi4uLi4uLi8vLzOzx8fHx8fHx8fHx4svLy8vLy8vLy8vLy8vLy8vMjIyMkGEvMfHx8fHx7FWMjIyMjIyMjIyMjIyMjIyMjI2NjY2MjI+drPHx8d9NTY2NjY2NjY2NjY2NjY2NjY2Njg4NlrHx8fHx8fGSTg4ODg4ODg4ODg4ODg4ODg4ODg4yEBAPGDHx8fCXEBAQEBAQEBAQEBAQEBAQEBAQEBAQMjIyENDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0PIyMAAAAOAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAHAAAADKAAAADAAAABgAAAAAQAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARwQAAE8IAABXEAAAXxQAAGcYAARvHAAAfwAAAHsoAFSucAIdGHgAAIs0Ai0kjAAAlzwAHJc8AID59AAAo0gCPTSgAACnTACQ8lwAYMsgAADDaAAI01AA+WkgAADTeABU4zgAEM+YAl1o3AAg52QAANukAADfqAAA65AA5Y0cAAEDYABc+3QAARtYANEPPAJ1lQgA6eSUAGUbeAAFK2wAVRucAMknWAAJQ2gA8hBkANoQfAB9M5QAXTeYANo0FAABT3gAmUeEAVoMvAEaYAAAAXeIASJIcAABg3gA9WeEAPZ4AADefBACneGMAA2TiAAFn3gAwX+oAU4ZkADqiDwAAauIArH9hAKx9aABWZtYAW4pfAFJp2gAAcOIAX2raAEOkJABJoi4AXKErAEtt4ABGpykANa0bAElu6gA6qiYAaXLbAAZ66ABFricAPq4tAFZ24wBseNoAeaVCALiPdwBtedsANrgeAF955gBBsjIAY6dMAFx86gBYfu0AAIjrAD61RwBCuzMAZILpAAKM6QCipWMAYrZCAL+bhwA3wjQAaIbuADy6UABrifEAAJXtAHCL6wBZtmgATLduAG2O8AACnO8APMFkAIuW4wCltXMAq7J+AIKX6AB9mOsAyamaAImb5AA5xXMAi6DjAACq9ACsuYgAT8iDAAaw8wBBzIIAjqbyAACz9wCVqu0AmK3xAJ+u8AClru4AC8P2AKa48wAAx/wAAMv5ALvQtgAJz/4AAND/ALPB9QDDzscAANP8AMDF7QAQ1PwAwsz3ACna/AC9zfoANtv8AObYzACr4sYA6NrOAMvS8wCu5M8ATuD/AGDi/wCy59MAaOP/ANPn1ABp5vwA5eHjAOzk2wDU3vgAwurYAIHq/ADr6t0Akev/AObm8wDh5vYA9e3kAOTq+gDz8OsAsPH/AOjt/QC38v8A6u//AN327QDx8f4A+PT2AM73/gDz9foA+/jzAPX2/AD8+fQA3fr8AP369QD4+v8A+v37AP/++AD8//4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC/vwAAAAAAAAAAAAAAAAF2obO9vr6+vr6+vr28RwAAAAAAAAAAAAAAAAAAAAAAv7+/AAYAAAAAAAAAAAABAgGZfW18vr6+vr6+vr6pAAAAAAAAAAAAAAAAAAAAAQAGAL8BAQEBQ5Cyvru+vb1QAxiff3FgSWS+vr6+vr5HAggWDgETvb67vr2+vrWQQwEGAQEBAQFyvlUTAAEBAQEAAUukfXFgTzhMYFJKc44+ST9IKwEBAQEBAQEBAQFVvnIBAQEBAUO+IwEBAQEAAQEBAVSdf3FgT0hnZ1lNSHlpWz85LwEBAQEBAQEBAQEBI7tDAQECApBYAgICAgICAgICAnWaf3FgUjhnZ2FPP31pUzhILwICAgICAgICAgICAliQAgICArUCAgICAgICAgICApmaf3FgTzlnZ1lNSHlpUzg5LwICAgICAgICAgICAgK1AgICAr0CAgICAgICAgICS6mXf3FgTzNWZVtPMm5pWz9ILAICAgICAgICAgICAgK7AgICAr4CAgICAgICAgICVLWXf3FcdL6+vr6FBAJEUzg4JQICAgICAgICAgICAgK9AgICArsCAgICAgICAgICdb6xiqa+vr6+vrMpCgICEjU4HwICAgICAgICAgICAgK7AgIDA74DAwMDAwMDAwMDmb6+vr6+vr6+vrAKAwMDAwMSAwMDAwMDAwMDAwMDAwO+AwMDA7sDAwMDAwMDAwMDqb6+vr6+vr6+voUDAwMDAwMDAwMDAwMDAwMDAwMDAwO9AwMFBb4FBQUFBQUFBQVLtb6+vr6+vr6+vkUFBQUFBQUFBQUFBQUFBQUFBQUFBQW+BQUFBbsFBQUFBQUFBQVavr6+vr6+vr6+vikFBQUFBQUFBQUFBQUFBQUFBQUFBQW9BQUHB7sHBwcHBwcHBwd1vr6+vr6+vr6+vgcHBwcHBwcHBwcHBwcHBwcHBwcHBwe+BwcHB70HBwcHBwcHBweZvr6+vr6+vr6+vgcHBwciUXuIj4+PhmsqBwcHBwcHBwe+BwcKCr4MCgoKCgoKCgqrvr6+vr6+vr6+vgoKY4uRj5GRkYyPi4uRiDYKCgoKCgq7CgoKCr4MCgoKCgoKCku1vr6+vr6+vr6+r4aPi4twOycgIjBAcIaLj488CgoKCgq9CgoKCr4KCgoKCgoKClS+vr6+vr6+vr60kYyMiyoKCgoKCgoKCgoVKkYVCgoKCgq9CgoMDLsMDAwMDAwMDHW+vr6+vr6+vrubjI+RYw0KCgoKCgoMCgwMDAwMDAwMCgy+CgwMDL4MDAwMDAwMDJm+vr6+vr6+vr2Tj4+PiBUMDAwMDAwMDAwMDAwMDAwMDAy9DwwMDL4MDAwMDAwMIam+vr6+vr6+vrugi4+Mj4uIi4yIflEgDAwMDAwMDAwMDAy8DAwMDL4MDAwMDAwMS7W+vr6+vr6+vr6+npGLj4yPj4+Pj4+PiF8gDAwMDAwMDAy+DAwPDL0PDwwPDA8PWr6+vr6+vr6+vr6+vq2Tj4+Lj4yPj5GLj4+PgTsPDA8MDwy9DA8PD74PDw8PDw8Pdb6+vr6+vr6+vr6+vr67ua2npZyVjI+LkY+Rj4+INA8PDw++Dw8PD70PDw8PDw8Pmb6+vr6+vr6+vr6+vr69vr6+vr69VCpffoyLjI+Jj4EPDw++Dw8PD70PDw8PDw8Pq76+vqEaGqq+vr6+vr6+vr68vr6+NxEPDw82e5GMiVEPDw++Dw8REbwRERERERFLtb6+vgsQCzq+vr6+vr6+vpYaGqy7ERERERERERERERERERG9EREUEb4RFBEUERRdvr6+vhAQoma+vr6+vr6+vgkaCTq+FBEUERQRFBEUERQRFBG+ERQUFL4UFBQUFBR1vr6+vqpBQra+vr6+vr6+uxAQoWa+FBQUFBQUFBQUFBQUFBS9FBQUFL4UFBQUFBR2vr6+vr6+vr6+vr6+vr6+vqpCQrayFBQUFBQUFBQUFBQUFBS+FBQUFL4UFBQUFBRivr6qvr6+vr6+vr6+vr6+vr6+vr6CFBQUFBQUFBQUFBQUFBS9FBQXFL4UFxQXFBdLvrNXvr6+vr6+vr6+vr6+vr6+vr4UFxQXFBcUFxQXFBcUFxS+FBcUF74XFBcUFxQXvr62JFd3s72+vr6+vr67rLO7mKgXFBcUFxQXFBcUFxQXFBe+FxQXF74XFxcXFxcXrr6+vrq9vr6+vr6+vr6+uCRXljEXFxcXFxcXFxcXFxcXFxe+FxcXF74XFxcXFxcXhL2+vr6+vr6+vr6+vr6+vb69gxcXFxcXFxcXFxcXFxcXFxe+FxcXF74XFxcXFxcXG667vr6+vr6+vr6+vr6+vrupFxcXFxcXFxcXFxcXFxcXFxe+FxcXHr4eFx4XHhceFx68vb6+vr6+vr6+vr69vrUXHhceFx4XHhceFx4XHhceFx6+HhceF74XHhceFx4XHhceeLu9vb6+vr6+vb6zhxceFx4XHhceFx4XHhceFx4XHhe+Fx4eHr4eHh4eHh4eHh4eHhctXZm+vr6+sGIeHh4eHh4eHh4eHh4eHh4eHh4eHh69Hh4eHr0eHh4eHh4eHmJsdoKNo7u+vr61LR4eHh4eHh4eHh4eHh4eHh4eHh4eHh69Hh4eHLUcHhweHB4cG6u8vr6+vr6+voMeHB4cHhweHB4cHhweHB4cHhweHB4cHhy1HB4cHpJqHB4cHhweHCaZvr6+vr2DLhwcHhweHB4cHhweHB4cHhweHB4cHhweHGqSHhwcHF6+PRwcHBwcHlR6hINoTigZGRwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcPb5eHBwcHByAvW8cHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBxvvYAcHBwcHBwcXpS3vr6+vr6+vr6+vr6+vr6+vr6+vr6+vr6+vr6+vr6+vr6+vreUXhwcHBy/HBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHL+/vx0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dv7/AAAAAAAMAAIAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAQAAwAAAAAADAAA=</Image> + <Url type="text/html" method="get" template="https://duckduckgo.com/?q={searchTerms}"/> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/data/engine2/manifest.json b/toolkit/components/search/tests/xpcshell/data/engine2/manifest.json new file mode 100644 index 0000000000..7dd4b15931 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine2/manifest.json @@ -0,0 +1,18 @@ +{ + "name": "A second test engine", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine2@search.mozilla.org" + } + }, + "hidden": true, + "description": "A second test search engine (based on DuckDuckGo)", + "chrome_settings_overrides": { + "search_provider": { + "name": "A second test engine", + "search_url": "https://duckduckgo.com/?q={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engineImages.xml b/toolkit/components/search/tests/xpcshell/data/engineImages.xml new file mode 100644 index 0000000000..65b550b31b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engineImages.xml @@ -0,0 +1,22 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/"> + <ShortName>IconsTest</ShortName> + <Description>IconsTest. Search by Test.</Description> + <InputEncoding>UTF-8</InputEncoding> + <Image width="16" height="16">data:image/x-icon;base64,ico16</Image> + <Image width="32" height="32">data:image/x-icon;base64,ico32</Image> + <Image width="74" height="74">data:image/png;base64,ico74</Image> + <Url type="application/x-suggestions+json" template="http://api.bing.com/osjson.aspx"> + <Param name="query" value="{searchTerms}"/> + <Param name="form" value="MOZW"/> + </Url> + <Url type="text/html" method="GET" template="http://www.bing.com/search"> + <Param name="q" value="{searchTerms}"/> + <MozParam name="pc" condition="pref" pref="ms-pc"/> + <Param name="form" value="MOZW"/> + </Url> + <SearchForm>http://www.bing.com/search</SearchForm> +</SearchPlugin> diff --git a/toolkit/components/search/tests/xpcshell/data/engineImages/manifest.json b/toolkit/components/search/tests/xpcshell/data/engineImages/manifest.json new file mode 100644 index 0000000000..b5366a6eb6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engineImages/manifest.json @@ -0,0 +1,37 @@ +{ + "name": "IconsTest", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engineImages@search.mozilla.org" + } + }, + "hidden": true, + "description": "IconsTest. Search by Test.", + "icons": { + "16": "data:image/x-icon;base64,ico16" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "IconsTest", + "search_url": "https://www.bing.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "form", + "value": "MOZW" + }, + { + "name": "pc", + "condition": "pref", + "pref": "ms-pc" + } + ], + "suggest_url": "https://api.bing.com/osjson.aspxquery={searchTerms}&form=MOZW" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/engineMaker.sjs b/toolkit/components/search/tests/xpcshell/data/engineMaker.sjs new file mode 100644 index 0000000000..4f97bd7635 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engineMaker.sjs @@ -0,0 +1,79 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Dynamically create an OpenSearch search engine offering search suggestions + * via searchSuggestions.sjs. + * + * The engine is constructed by passing a JSON object with engine details as the query string. + */ + +function handleRequest(request, response) { + let engineData = JSON.parse(unescape(request.queryString).replace("+", " ")); + + if (!engineData.baseURL) { + response.setStatusLine(request.httpVersion, 500, "baseURL required"); + return; + } + + engineData.name = engineData.name || "Generated test engine"; + engineData.description = + engineData.description || "Generated test engine description"; + engineData.method = engineData.method || "GET"; + + response.setStatusLine(request.httpVersion, 200, "OK"); + createOpenSearchEngine(response, engineData); +} + +/** + * Create an OpenSearch engine for the given base URL. + * + * @param {Response} response + * The response object to write the engine to. + * @param {object} engineData + * Information about the search engine to write to the response. + */ +function createOpenSearchEngine(response, engineData) { + let params = ""; + let queryString = ""; + if (engineData.queryString) { + queryString = engineData.queryString.replace("&", "&"); + } else if (engineData.method == "POST") { + params = "<Param name='q' value='{searchTerms}'/>"; + } else { + queryString = "?q={searchTerms}"; + } + let type = "type='application/x-suggestions+json'"; + if (engineData.alternativeJSONType) { + type = "type='application/json' rel='suggestions'"; + } + let image = ""; + if (engineData.image) { + image = `<Image width="16" height="16">${engineData.baseURL}${engineData.image}</Image>`; + } + let updateFile = ""; + if (engineData.updateFile) { + updateFile = `<Url type="application/opensearchdescription+xml" + rel="self" + template="${engineData.baseURL}${engineData.updateFile}" /> + `; + } + + let result = `<?xml version='1.0' encoding='utf-8'?> +<OpenSearchDescription xmlns='http://a9.com/-/spec/opensearch/1.1/'> + <ShortName>${engineData.name}</ShortName> + <Description>${engineData.description}</Description> + <InputEncoding>UTF-8</InputEncoding> + <LongName>${engineData.name}</LongName> + ${image} + <Url ${type} method='${engineData.method}' + template='${engineData.baseURL}searchSuggestions.sjs${queryString}'> + ${params} + </Url> + <Url type='text/html' method='${engineData.method}' + template='${engineData.baseURL}${queryString}'/> + ${updateFile} +</OpenSearchDescription> +`; + response.write(result); +} diff --git a/toolkit/components/search/tests/xpcshell/data/engines-no-order-hint.json b/toolkit/components/search/tests/xpcshell/data/engines-no-order-hint.json new file mode 100644 index 0000000000..3c8befc726 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engines-no-order-hint.json @@ -0,0 +1,189 @@ +{ + "data": [ + { + "webExtension": { + "id": "engine@search.mozilla.org", + "name": "Test search engine", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + }, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes" + } + ] + }, + { + "webExtension": { + "id": "engine-rel-searchform-purpose@search.mozilla.org", + "name": "engine-rel-searchform-purpose", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "searchbar", + "value": "sb" + } + ] + }, + "orderHint": 1000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + } + ] + }, + { + "webExtension": { + "id": "engine-chromeicon@search.mozilla.org", + "name": "engine-chromeicon", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 1000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + }, + { + "included": { "regions": ["ru"] }, + "default": "no" + } + ] + }, + { + "webExtension": { + "id": "engine-resourceicon@search.mozilla.org", + "name": "engine-resourceicon", + "search_url": "https://www.google.com/search", + "search_provider": { + "en": { + "name": "engine-resourceicon", + "search_url": "https://www.google.com/search" + }, + "gd": { + "name": "engine-resourceicon-gd", + "search_url": "https://www.google.com/search" + } + } + }, + "appliesTo": [ + { + "included": { "locales": { "matches": ["en-US", "fr"] } }, + "excluded": { + "regions": ["ru"] + }, + "default": "no" + } + ] + }, + { + "webExtension": { + "id": "engine-reordered@search.mozilla.org", + "name": "Test search engine (Reordered)", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + }, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + } + ] + }, + { + "webExtension": { + "id": "engine-pref@search.mozilla.org", + "name": "engine-pref", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "code", + "condition": "pref", + "pref": "code" + }, + { + "name": "test", + "condition": "pref", + "pref": "test" + } + ] + }, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de"] } }, + "default": "no" + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/engines.json b/toolkit/components/search/tests/xpcshell/data/engines.json new file mode 100644 index 0000000000..f27a4844df --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engines.json @@ -0,0 +1,203 @@ +{ + "data": [ + { + "webExtension": { + "id": "engine@search.mozilla.org", + "name": "Test search engine", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + }, + "orderHint": 10000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["gd"] } }, + "default": "yes" + } + ] + }, + { + "webExtension": { + "id": "engine-pref@search.mozilla.org", + "name": "engine-pref", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "code", + "condition": "pref", + "pref": "code" + }, + { + "name": "test", + "condition": "pref", + "pref": "test" + } + ] + }, + "orderHint": 7000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de"] } }, + "default": "no", + "defaultPrivate": "yes" + } + ] + }, + { + "webExtension": { + "id": "engine-rel-searchform-purpose@search.mozilla.org", + "name": "engine-rel-searchform-purpose", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "searchbar", + "value": "sb" + } + ] + }, + "orderHint": 6000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + }, + { + "included": { "locales": { "matches": ["gd"] } }, + "orderHint": 9000 + } + ] + }, + { + "webExtension": { + "id": "engine-chromeicon@search.mozilla.org", + "name": "engine-chromeicon", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 8000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + }, + { + "included": { "regions": ["ru"] }, + "default": "no" + } + ] + }, + { + "webExtension": { + "id": "engine-resourceicon@search.mozilla.org", + "default_locale": "en", + "searchProvider": { + "en": { + "name": "engine-resourceicon", + "search_url": "https://www.google.com/search" + }, + "gd": { + "name": "engine-resourceicon-gd", + "search_url": "https://www.google.com/search" + } + } + }, + "orderHint": 9000, + "appliesTo": [ + { + "included": { "locales": { "matches": ["en-US", "fr"] } }, + "excluded": { "regions": ["ru"] }, + "default": "no" + }, + { + "included": { "locales": { "matches": ["gd"] } }, + "default": "yes", + "webExtension": { + "locales": ["gd"] + } + } + ] + }, + { + "webExtension": { + "id": "engine-reordered@search.mozilla.org", + "name": "Test search engine (Reordered)", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "contextmenu", + "value": "rcs" + }, + { + "name": "channel", + "condition": "purpose", + "purpose": "keyword", + "value": "fflb" + } + ], + "suggest_url": "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}" + }, + "orderHint": 5000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "excluded": { "locales": { "matches": ["de", "fr"] } }, + "default": "no" + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/af/messages.json b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/af/messages.json new file mode 100644 index 0000000000..29ddd24df5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/af/messages.json @@ -0,0 +1,17 @@ +{ + "extensionName": { + "message": "Multilocale" + }, + "extensionDescription": { + "message": "Wikipedia, die vrye ensiklopedie" + }, + "url_lang": { + "message": "af" + }, + "searchUrl": { + "message": "https://af.wikipedia.org/wiki/Spesiaal:Soek" + }, + "suggestUrl": { + "message": "https://af.wikipedia.org/w/api.php" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/an/messages.json b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/an/messages.json new file mode 100644 index 0000000000..d21d910463 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/_locales/an/messages.json @@ -0,0 +1,17 @@ +{ + "extensionName": { + "message": "Multilocale" + }, + "extensionDescription": { + "message": "A enciclopedia Libre" + }, + "url_lang": { + "message": "an" + }, + "searchUrl": { + "message": "https://an.wikipedia.org/wiki/Especial:Mirar" + }, + "suggestUrl": { + "message": "https://an.wikipedia.org/w/api.php" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/favicon.ico b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/favicon.ico Binary files differnew file mode 100644 index 0000000000..4314071e24 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/favicon.ico diff --git a/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/manifest.json b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/manifest.json new file mode 100644 index 0000000000..0fd835ca40 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/geolookup-extensions/multilocale/manifest.json @@ -0,0 +1,23 @@ +{ + "name": "__MSG_extensionName__", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "multilocale@search.mozilla.org" + } + }, + "hidden": true, + "description": "__MSG_extensionDescription__", + "icons": { + "16": "favicon.ico" + }, + "default_locale": "af", + "chrome_settings_overrides": { + "search_provider": { + "name": "__MSG_extensionName__", + "search_url": "__MSG_searchUrl__", + "suggest_url": "__MSG_searchUrl__" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/iconsRedirect.sjs b/toolkit/components/search/tests/xpcshell/data/iconsRedirect.sjs new file mode 100644 index 0000000000..98f9aed4d0 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/iconsRedirect.sjs @@ -0,0 +1,18 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Redirect a request for an icon to a different place, using a different + * content-type. + */ + +function handleRequest(request, response) { + response.setStatusLine("1.1", 302, "Moved"); + if (request.queryString == "type=invalid") { + response.setHeader("Content-Type", "image/png", false); + response.setHeader("Location", "/head_search.js", false); + } else { + response.setHeader("Content-Type", "text/html", false); + response.setHeader("Location", "remoteIcon.ico", false); + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/remoteIcon.ico b/toolkit/components/search/tests/xpcshell/data/remoteIcon.ico Binary files differnew file mode 100644 index 0000000000..442ab4dc80 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/remoteIcon.ico diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy-correct-default-engine-hashes.json b/toolkit/components/search/tests/xpcshell/data/search-legacy-correct-default-engine-hashes.json new file mode 100644 index 0000000000..e6091b7230 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy-correct-default-engine-hashes.json @@ -0,0 +1,112 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": { + "current": "engine2", + "private": "engine2" + }, + "engines": [ + { + "_name": "engine1", + "_shortName": "engine1", + "_loadPath": "[other]addEngineWithDetails:engine1@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico" + }, + "_metaData": { + "alias": "testAlias" + }, + "_urls": [ + { + "template": "https://1.example.com/search", + "rels": [], + "resultDomain": "1.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine1@search.mozilla.org" + }, + { + "_name": "engine2", + "_shortName": "engine2", + "_loadPath": "[other]addEngineWithDetails:engine2@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico" + }, + "_metaData": { + "alias": null, + "hidden": false + }, + "_urls": [ + { + "template": "https://2.example.com/search", + "rels": [], + "resultDomain": "2.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine2@search.mozilla.org" + }, + { + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "resultDomain": "suggestqueries.google.com", + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "keyword" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8", + "extensionID": "test-addon-id@mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy-no-ids.json b/toolkit/components/search/tests/xpcshell/data/search-legacy-no-ids.json new file mode 100644 index 0000000000..733c323876 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy-no-ids.json @@ -0,0 +1,87 @@ +{ + "version": 6, + "engines": [ + { "_name": "Google", "_isAppProvided": true, "_metaData": { "order": 1 } }, + { + "_name": "Wikipedia (en)", + "_isAppProvided": true, + "_metaData": { "order": 7 } + }, + { "_name": "Bing", "_isAppProvided": true, "_metaData": { "order": 3 } }, + { + "_name": "Amazon.co.uk", + "_isAppProvided": true, + "_metaData": { "order": 2 } + }, + { + "_name": "DuckDuckGo", + "_isAppProvided": true, + "_metaData": { "order": 4 } + }, + { "_name": "eBay", "_isAppProvided": true, "_metaData": { "order": 5 } }, + { + "_name": "Policy", + "_loadPath": "[other]addEngineWithDetails:set-via-policy", + "_metaData": { "alias": "PolicyAlias", "order": 6 } + }, + { + "_name": "Bugzilla@Mozilla", + "_loadPath": "[https]bugzilla.mozilla.org/bugzillamozilla.xml", + "description": "Bugzilla@Mozilla Quick Search", + "_metaData": { + "loadPathHash": "Bxz6jVe3IIBxLLaafUus536LMyLKoGZm7xsBv/yiTw8=", + "order": 8, + "alias": "bugzillaAlias" + }, + "_urls": [ + { + "params": [], + "rels": [], + "template": "https://bugzilla.mozilla.org/buglist.cgi?quicksearch={searchTerms}" + } + ], + "_orderHint": null, + "_telemetryId": null, + "_updateInterval": null, + "_updateURL": null, + "_iconUpdateURL": null, + "_extensionID": null, + "_locale": null, + "_definedAliases": [] + }, + { + "_name": "User", + "_loadPath": "[other]addEngineWithDetails:set-via-user", + "_metaData": { + "order": 9, + "alias": "UserAlias" + }, + "_urls": [ + { + "params": [], + "rels": [], + "template": "https://example.com/test?q={searchTerms}" + } + ], + "_telemetryId": null, + "_updateInterval": null, + "_updateURL": null, + "_iconUpdateURL": null, + "_extensionID": null, + "_locale": null, + "_hasPreferredIcon": null + }, + { "_name": "Amazon.com", "_isAppProvided": true, "_metaData": {} } + ], + "metaData": { + "useSavedOrder": true, + "locale": "en-US", + "region": "GB", + "channel": "default", + "experiment": "", + "distroID": "", + "appDefaultEngine": "Google", + "current": "Bing", + "hash": "5Of6s1D+BDjPRti1wqtFyBTH1PnOf9n6cRwWlEXZhd0=" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy-old-loadPaths.json b/toolkit/components/search/tests/xpcshell/data/search-legacy-old-loadPaths.json new file mode 100644 index 0000000000..f716fb3e4a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy-old-loadPaths.json @@ -0,0 +1,135 @@ +{ + "version": 7, + "engines": [ + { + "id": "google@search.mozilla.orgdefault", + "_name": "Google", + "_isAppProvided": true, + "_metaData": { "order": 1 } + }, + { + "id": "amazon@search.mozilla.orgen-GB", + "_name": "Amazon.co.uk", + "_isAppProvided": true, + "_metaData": { "order": 2 } + }, + { + "id": "bing@search.mozilla.orgdefault", + "_name": "Bing", + "_isAppProvided": true, + "_metaData": { "order": 3 } + }, + { + "id": "ddg@search.mozilla.orgdefault", + "_name": "DuckDuckGo", + "_isAppProvided": true, + "_metaData": { "order": 4 } + }, + { + "id": "ebay@search.mozilla.orguk", + "_name": "eBay", + "_isAppProvided": true, + "_metaData": { "order": 5 } + }, + { + "id": "wikipedia@search.mozilla.orgdefault", + "_name": "Wikipedia (en)", + "_isAppProvided": true, + "_metaData": { "order": 6 } + }, + { + "id": "policy-Policy", + "_name": "Policy", + "_loadPath": "[other]addEngineWithDetails:set-via-policy", + "_metaData": { "alias": "PolicyAlias", "order": 6 } + }, + { + "id": "bbc163e7-7b1a-47aa-a32c-c59062de2753", + "_name": "Bugzilla@Mozilla", + "_loadPath": "[https]bugzilla.mozilla.org/bugzillamozilla.xml", + "description": "Bugzilla@Mozilla Quick Search", + "_metaData": { + "loadPathHash": "Bxz6jVe3IIBxLLaafUus536LMyLKoGZm7xsBv/yiTw8=", + "order": 8, + "alias": "bugzillaAlias" + }, + "_urls": [ + { + "params": [], + "rels": [], + "template": "https://bugzilla.mozilla.org/buglist.cgi?quicksearch={searchTerms}" + } + ], + "_orderHint": null, + "_telemetryId": null, + "_updateInterval": null, + "_updateURL": null, + "_iconUpdateURL": null, + "_extensionID": null, + "_locale": null, + "_definedAliases": [] + }, + { + "id": "bbc163e7-7b1a-47aa-a32c-c59062de2754", + "_name": "User", + "_loadPath": "[other]addEngineWithDetails:set-via-user", + "_metaData": { + "order": 9, + "alias": "UserAlias" + }, + "_urls": [ + { + "params": [], + "rels": [], + "template": "https://example.com/test?q={searchTerms}" + } + ], + "_telemetryId": null, + "_updateInterval": null, + "_updateURL": null, + "_iconUpdateURL": null, + "_extensionID": null, + "_locale": null, + "_hasPreferredIcon": null + }, + { + "id": "example@tests.mozilla.orgdefault", + "_name": "Example", + "_loadPath": "[other]addEngineWithDetails:example@tests.mozilla.org", + "description": null, + "_iconURL": "", + "_metaData": {}, + "_urls": [ + { + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ], + "rels": [], + "template": "https://example.com/" + } + ], + "_telemetryId": null, + "_updateInterval": null, + "_updateURL": null, + "_iconUpdateURL": null, + "_extensionID": "example@tests.mozilla.org", + "_locale": "default", + "_definedAliases": [], + "_hasPreferredIcon": null + } + ], + "metaData": { + "useSavedOrder": true, + "locale": "en-US", + "region": "GB", + "channel": "default", + "experiment": "", + "distroID": "", + "appDefaultEngine": "Google", + "current": "Bing", + "hash": "5Of6s1D+BDjPRti1wqtFyBTH1PnOf9n6cRwWlEXZhd0=" + } +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-default-engine-hashes.json b/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-default-engine-hashes.json new file mode 100644 index 0000000000..ca7081f565 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-default-engine-hashes.json @@ -0,0 +1,114 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": { + "current": "engine2", + "private": "engine2", + "hash": "wrong-hash-o/HzjHlVpb97AFGH3pY1GZ6CoTQkQslUKRd38/qasto=", + "privateHash": "wrong-hash-o/HzjHlVpb97AFGH3pY1GZ6CoTQkQslUKRd38/qasto=" + }, + "engines": [ + { + "_name": "engine1", + "_shortName": "engine1", + "_loadPath": "[other]addEngineWithDetails:engine1@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico" + }, + "_metaData": { + "alias": "testAlias" + }, + "_urls": [ + { + "template": "https://1.example.com/search", + "rels": [], + "resultDomain": "1.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine1@search.mozilla.org" + }, + { + "_name": "engine2", + "_shortName": "engine2", + "_loadPath": "[other]addEngineWithDetails:engine2@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico" + }, + "_metaData": { + "alias": null, + "hidden": false + }, + "_urls": [ + { + "template": "https://2.example.com/search", + "rels": [], + "resultDomain": "2.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine2@search.mozilla.org" + }, + { + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "resultDomain": "suggestqueries.google.com", + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "keyword" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8", + "extensionID": "test-addon-id@mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-third-party-engine-hashes.json b/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-third-party-engine-hashes.json new file mode 100644 index 0000000000..25329e083f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy-wrong-third-party-engine-hashes.json @@ -0,0 +1,114 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": { + "current": "Test search engine", + "private": "Test search engine", + "hash": "wrong-hash-o/HzjHlVpb97AFGH3pY1GZ6CoTQkQslUKRd38/qasto=", + "privateHash": "wrong-hash-o/HzjHlVpb97AFGH3pY1GZ6CoTQkQslUKRd38/qasto=" + }, + "engines": [ + { + "_name": "engine1", + "_shortName": "engine1", + "_loadPath": "[other]addEngineWithDetails:engine1@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico" + }, + "_metaData": { + "alias": "testAlias" + }, + "_urls": [ + { + "template": "https://1.example.com/search", + "rels": [], + "resultDomain": "1.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine1@search.mozilla.org" + }, + { + "_name": "engine2", + "_shortName": "engine2", + "_loadPath": "[other]addEngineWithDetails:engine2@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico" + }, + "_metaData": { + "alias": null, + "hidden": false + }, + "_urls": [ + { + "template": "https://2.example.com/search", + "rels": [], + "resultDomain": "2.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine2@search.mozilla.org" + }, + { + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "resultDomain": "suggestqueries.google.com", + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "keyword" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8", + "extensionID": "test-addon-id@mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-legacy.json b/toolkit/components/search/tests/xpcshell/data/search-legacy.json new file mode 100644 index 0000000000..c8416f3813 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-legacy.json @@ -0,0 +1,109 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "engine1", + "_shortName": "engine1", + "_loadPath": "[other]addEngineWithDetails:engine1@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://9c38b851-bede-2244-a086-9be8128dd64d/favicon.ico" + }, + "_metaData": { + "alias": "testAlias" + }, + "_urls": [ + { + "template": "https://1.example.com/search", + "rels": [], + "resultDomain": "1.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine1@search.mozilla.org" + }, + { + "_name": "engine2", + "_shortName": "engine2", + "_loadPath": "[other]addEngineWithDetails:engine2@search.mozilla.org", + "description": "A small test engine", + "__searchForm": null, + "_iconURL": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico", + "_iconMapObj": { + "{}": "moz-extension://0ea1d9b5-a14c-0e42-afaf-f25e8261c135/favicon.ico" + }, + "_metaData": { + "alias": null, + "hidden": true + }, + "_urls": [ + { + "template": "https://2.example.com/search", + "rels": [], + "resultDomain": "2.example.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "_isBuiltin": true, + "queryCharset": "UTF-8", + "extensionID": "engine2@search.mozilla.org" + }, + { + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "resultDomain": "suggestqueries.google.com", + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "keyword" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8", + "extensionID": "test-addon-id@mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-migration.json b/toolkit/components/search/tests/xpcshell/data/search-migration.json new file mode 100644 index 0000000000..520149a370 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-migration.json @@ -0,0 +1,68 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "engine1", + "_metaData": { + "alias": "testAlias" + }, + "_isAppProvided": true + }, + { + "_name": "engine2", + "_metaData": { + "alias": null, + "hidden": true + }, + "_isAppProvided": true + }, + { + "_name": "simple", + "_loadPath": "jar:[profile]/extensions/simple@tests.mozilla.org.xpi!/simple.xml", + "_shortName": "simple", + "description": "A migration test engine", + "__searchForm": "http://www.example.com/", + "_metaData": {}, + "_urls": [ + { + "template": "http://www.example.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "queryCharset": "UTF-8" + }, + { + "_name": "simple search", + "_loadPath": "[other]addEngineWithDetails:simple@tests.mozilla.org", + "_shortName": "simple search", + "description": "A migration test engine", + "__searchForm": "http://www.example.com/", + "_metaData": {}, + "_urls": [ + { + "template": "http://www.example.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + ], + "queryCharset": "UTF-8", + "_extensionID": "simple@tests.mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-obsolete-app.json b/toolkit/components/search/tests/xpcshell/data/search-obsolete-app.json new file mode 100644 index 0000000000..151359ff73 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-obsolete-app.json @@ -0,0 +1,41 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "engine1", + "_metaData": { + "alias": "testAlias" + }, + "_isAppProvided": true + }, + { + "_name": "engine2", + "_metaData": { + "alias": null, + "hidden": true + }, + "_isAppProvided": true + }, + { + "_name": "App", + "_shortName": "app", + "_loadPath": "jar:[app]/omni.ja!distribution.xml", + "description": "App Search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example.com/search", + "rels": ["searchform"], + "resultDomain": "example.com", + "params": [] + } + ], + "queryCharset": "UTF-8", + "_readOnly": false + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-obsolete-distribution.json b/toolkit/components/search/tests/xpcshell/data/search-obsolete-distribution.json new file mode 100644 index 0000000000..efc609a5af --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-obsolete-distribution.json @@ -0,0 +1,41 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "engine1", + "_metaData": { + "alias": "testAlias" + }, + "_isAppProvided": true + }, + { + "_name": "engine2", + "_metaData": { + "alias": null, + "hidden": true + }, + "_isAppProvided": true + }, + { + "_name": "Distribution", + "_shortName": "distribution", + "_loadPath": "[distribution]/searchplugins/common/distribution.xml", + "description": "Distribution Search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example.com/search", + "rels": ["searchform"], + "resultDomain": "example.com", + "params": [] + } + ], + "queryCharset": "UTF-8", + "_readOnly": false + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search-obsolete-langpack.json b/toolkit/components/search/tests/xpcshell/data/search-obsolete-langpack.json new file mode 100644 index 0000000000..8c45b4d61f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search-obsolete-langpack.json @@ -0,0 +1,91 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "engine1", + "_metaData": { + "alias": "testAlias" + }, + "_isAppProvided": true + }, + { + "_name": "engine2", + "_metaData": { + "alias": null, + "hidden": true + }, + "_isAppProvided": true + }, + { + "_name": "Langpack", + "_shortName": "langpack-ru", + "_loadPath": "jar:[app]/extensions/langpack-ru@firefox.mozilla.org.xpi!browser/langpack.xml", + "description": "Langpack search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example.com/search", + "rels": ["searchform"], + "resultDomain": "example.com", + "params": [] + } + ], + "queryCharset": "UTF-8" + }, + { + "_name": "Langpack1", + "_shortName": "langpack1-ru", + "_loadPath": "[app]/extensions/langpack-ru@firefox.mozilla.org.xpi!browser/langpack1.xml", + "description": "Langpack1 search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example1.com/search", + "rels": ["searchform"], + "resultDomain": "example1.com", + "params": [] + } + ], + "queryCharset": "UTF-8" + }, + { + "_name": "Langpack2", + "_shortName": "langpack2-ru", + "_loadPath": "jar:[profile]/extensions/langpack-ru@firefox.mozilla.org.xpi!browser/langpack2.xml", + "description": "Langpack2 search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example2.com/search", + "rels": ["searchform"], + "resultDomain": "example2.com", + "params": [] + } + ], + "queryCharset": "UTF-8" + }, + { + "_name": "Langpack3", + "_shortName": "langpack3-ru", + "_loadPath": "jar:[other]/langpack-ru@firefox.mozilla.org.xpi!browser/langpack3.xml", + "description": "Langpack3 search", + "__searchForm": null, + "_metaData": {}, + "_urls": [ + { + "template": "https://example3.com/search", + "rels": ["searchform"], + "resultDomain": "example3.com", + "params": [] + } + ], + "queryCharset": "UTF-8" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/search.json b/toolkit/components/search/tests/xpcshell/data/search.json new file mode 100644 index 0000000000..f757af2ab3 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search.json @@ -0,0 +1,66 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "id": "engine1@search.mozilla.orgdefault", + "_name": "engine1", + "_metaData": { + "alias": "testAlias" + }, + "_isAppProvided": true + }, + { + "id": "engine2@search.mozilla.orgdefault", + "_name": "engine2", + "_metaData": { + "alias": null, + "hidden": true + }, + "_isAppProvided": true + }, + { + "id": "test-addon-id@mozilla.orgdefault", + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "resultDomain": "suggestqueries.google.com", + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "rels": [], + "resultDomain": "google.com", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "searchbar" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8", + "_extensionID": "test-addon-id@mozilla.org" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/searchSuggestions.sjs b/toolkit/components/search/tests/xpcshell/data/searchSuggestions.sjs new file mode 100644 index 0000000000..2eaa2684c4 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/searchSuggestions.sjs @@ -0,0 +1,187 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +let { setTimeout } = ChromeUtils.importESModule( + "resource://gre/modules/Timer.sys.mjs" +); +let { NetUtil } = ChromeUtils.importESModule( + "resource://gre/modules/NetUtil.sys.mjs" +); + +/** + * Provide search suggestions in the OpenSearch JSON format. + */ + +function handleRequest(request, response) { + // Get the query parameters from the query string. + let query = parseQueryString(request.queryString); + + function convertToUtf8(str) { + return String.fromCharCode(...new TextEncoder().encode(str)); + } + + function writeSuggestions(q, completions = []) { + let jsonString = JSON.stringify([q, completions]); + + // This script must be evaluated as UTF-8 for this to write out the bytes of + // the string in UTF-8. If it's evaluated as Latin-1, the written bytes + // will be the result of UTF-8-encoding the result-string *twice*, which + // will break the "I ❤️" case further down. + let stringOfUtf8Bytes = convertToUtf8(jsonString); + + response.write(stringOfUtf8Bytes); + } + + /** + * Sends `data` as suggestions directly. This is useful when testing rich + * suggestions, which do not conform to the object shape sent by + * writeSuggestions. + * + * @param {Array} data The data to send as suggestions. + */ + function writeSuggestionsDirectly(data) { + let jsonString = JSON.stringify(data); + let stringOfUtf8Bytes = convertToUtf8(jsonString); + response.setHeader("Content-Type", "application/json", false); + response.write(stringOfUtf8Bytes); + } + + response.setStatusLine(request.httpVersion, 200, "OK"); + + let q = request.method == "GET" ? query.q : undefined; + if (q == "cookie") { + response.setHeader("Set-Cookie", "cookie=1"); + writeSuggestions(q); + } else if (q == "no remote" || q == "no results") { + writeSuggestions(q); + } else if (q == "Query Mismatch") { + writeSuggestions("This is an incorrect query string", ["some result"]); + } else if (q == "Query Case Mismatch") { + writeSuggestions(q.toUpperCase(), [q]); + } else if (q == "") { + writeSuggestions("", ["The server should never be sent an empty query"]); + } else if (q?.startsWith("mo")) { + writeSuggestions(q, ["Mozilla", "modern", "mom"]); + } else if (q?.startsWith("I ❤️")) { + writeSuggestions(q, ["I ❤️ Mozilla"]); + } else if (q?.startsWith("stü")) { + writeSuggestions("st\\u00FC", ["stühle", "stüssy"]); + } else if (q?.startsWith("tailjunk ")) { + writeSuggestionsDirectly([ + q, + [q + " normal", q + " tail 1", q + " tail 2"], + [], + { + "google:irrelevantparameter": [], + "google:badformat": { + "google:suggestdetail": [ + {}, + { mp: "… ", t: "tail 1" }, + { mp: "… ", t: "tail 2" }, + ], + }, + }, + ]); + } else if (q?.startsWith("tailjunk few ")) { + writeSuggestionsDirectly([ + q, + [q + " normal", q + " tail 1", q + " tail 2"], + [], + { + "google:irrelevantparameter": [], + "google:badformat": { + "google:suggestdetail": [{ mp: "… ", t: "tail 1" }], + }, + }, + ]); + } else if (q?.startsWith("tailalt ")) { + writeSuggestionsDirectly([ + q, + [q + " normal", q + " tail 1", q + " tail 2"], + { + "google:suggestdetail": [ + {}, + { mp: "… ", t: "tail 1" }, + { mp: "… ", t: "tail 2" }, + ], + }, + ]); + } else if (q?.startsWith("tail ")) { + writeSuggestionsDirectly([ + q, + [q + " normal", q + " tail 1", q + " tail 2"], + [], + { + "google:irrelevantparameter": [], + "google:suggestdetail": [ + {}, + { mp: "… ", t: "tail 1" }, + { mp: "… ", t: "tail 2" }, + ], + }, + ]); + } else if (q?.startsWith("richempty ")) { + writeSuggestionsDirectly([ + q, + [q + " normal", q + " tail 1", q + " tail 2"], + [], + { + "google:irrelevantparameter": [], + "google:suggestdetail": [], + }, + ]); + } else if (q?.startsWith("letter ")) { + let letters = []; + for ( + let charCode = "A".charCodeAt(); + charCode <= "Z".charCodeAt(); + charCode++ + ) { + letters.push("letter " + String.fromCharCode(charCode)); + } + writeSuggestions(q, letters); + } else if (q?.startsWith("HTTP ")) { + response.setStatusLine(request.httpVersion, q.replace("HTTP ", ""), q); + writeSuggestions(q, [q]); + } else if (q == "invalidJSON") { + response.setHeader("Content-Type", "application/json", false); + response.write('["invalid"]'); + } else if (q == "invalidContentType") { + response.setHeader("Content-Type", "text/xml", false); + writeSuggestions(q, ["invalidContentType response"]); + } else if (q?.startsWith("delay")) { + // Delay the response by delayMs milliseconds. 200ms is the default, less + // than the timeout but hopefully enough to abort before completion. + let match = /^delay([0-9]+)/.exec(q); + let delayMs = match ? parseInt(match[1]) : 200; + response.processAsync(); + writeSuggestions(q, [q]); + setTimeout(() => response.finish(), delayMs); + } else if (q?.startsWith("slow ")) { + // Delay the response by 10 seconds so the client timeout is reached. + response.processAsync(); + writeSuggestions(q, [q]); + setTimeout(() => response.finish(), 10000); + } else if (request.method == "POST") { + // This includes headers, not just the body + let requestText = NetUtil.readInputStreamToString( + request.bodyInputStream, + request.bodyInputStream.available() + ); + // Only use the last line which contains the encoded params + let requestLines = requestText.split("\n"); + let postParams = parseQueryString(requestLines[requestLines.length - 1]); + writeSuggestions(postParams.q, ["Mozilla", "modern", "mom"]); + } else { + response.setStatusLine(request.httpVersion, 404, "Not Found"); + } +} + +function parseQueryString(queryString) { + let query = {}; + queryString.split("&").forEach(function (val) { + let [name, value] = val.split("="); + query[name] = decodeURIComponent(value).replace(/[+]/g, " "); + }); + return query; +} diff --git a/toolkit/components/search/tests/xpcshell/data/search_ignorelist.json b/toolkit/components/search/tests/xpcshell/data/search_ignorelist.json new file mode 100644 index 0000000000..35240893ec --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/search_ignorelist.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "buildID": "20121106", + "locale": "en-US", + "metaData": {}, + "engines": [ + { + "_name": "Test search engine", + "_shortName": "test-search-engine", + "description": "A test search engine (based on Google search)", + "extensionID": "test-addon-id@mozilla.org", + "__searchForm": "http://www.google.com/", + "_iconURL": "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + "_metaData": {}, + "_urls": [ + { + "template": "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + "rels": [], + "type": "application/x-suggestions+json", + "params": [] + }, + { + "template": "http://www.google.com/search", + "resultDomain": "google.com", + "rels": [], + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "ignore", + "value": "true" + }, + { + "name": "channel", + "value": "fflb", + "purpose": "keyword" + }, + { + "name": "channel", + "value": "rcs", + "purpose": "contextmenu" + } + ] + } + ], + "queryCharset": "UTF-8" + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data/svgIcon.svg b/toolkit/components/search/tests/xpcshell/data/svgIcon.svg new file mode 100644 index 0000000000..e2550f8d5d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/svgIcon.svg @@ -0,0 +1,4 @@ +<svg xmlns="http://www.w3.org/2000/svg" + width="16" height="16" viewBox="0 0 16 16"> + <rect x="4" y="4" width="8px" height="8px" style="fill: blue" /> +</svg> diff --git a/toolkit/components/search/tests/xpcshell/data1/engine1/manifest.json b/toolkit/components/search/tests/xpcshell/data1/engine1/manifest.json new file mode 100644 index 0000000000..5fa44ea692 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/engine1/manifest.json @@ -0,0 +1,27 @@ +{ + "name": "engine1", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine1@search.mozilla.org" + } + }, + "hidden": true, + "description": "A small test engine", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine1", + "search_url": "https://1.example.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data1/engine2/manifest.json b/toolkit/components/search/tests/xpcshell/data1/engine2/manifest.json new file mode 100644 index 0000000000..7ab094198b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/engine2/manifest.json @@ -0,0 +1,27 @@ +{ + "name": "engine2", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "engine2@search.mozilla.org" + } + }, + "hidden": true, + "description": "A small test engine", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "engine2", + "search_url": "https://2.example.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data1/engines.json b/toolkit/components/search/tests/xpcshell/data1/engines.json new file mode 100644 index 0000000000..92ffbd48a0 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/engines.json @@ -0,0 +1,92 @@ +{ + "data": [ + { + "webExtension": { + "id": "engine1@search.mozilla.org", + "name": "engine1", + "search_url": "https://1.example.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 10000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes-if-no-other", + "defaultPrivate": "yes-if-no-other" + } + ] + }, + { + "webExtension": { + "id": "engine2@search.mozilla.org", + "name": "engine2", + "search_url": "https://2.example.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 7000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "no" + }, + { + "included": { "everywhere": true }, + "default": "yes", + "experiment": "exp1" + } + ] + }, + { + "webExtension": { + "id": "exp2@search.mozilla.org", + "name": "exp2", + "search_url": "https://2.example.com/searchexp", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 5000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "defaultPrivate": "yes", + "experiment": "exp2" + } + ] + }, + { + "webExtension": { + "id": "exp3@search.mozilla.org", + "name": "exp3", + "search_url": "https://3.example.com/searchexp", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + }, + "orderHint": 20000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes", + "experiment": "exp3" + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/data1/exp2/manifest.json b/toolkit/components/search/tests/xpcshell/data1/exp2/manifest.json new file mode 100644 index 0000000000..0cd0e080b9 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/exp2/manifest.json @@ -0,0 +1,27 @@ +{ + "name": "exp2", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "exp2@search.mozilla.org" + } + }, + "hidden": true, + "description": "A small test engine", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "exp2", + "search_url": "https://2.example.com/searchexp", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data1/exp3/manifest.json b/toolkit/components/search/tests/xpcshell/data1/exp3/manifest.json new file mode 100644 index 0000000000..4e023e0fef --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/exp3/manifest.json @@ -0,0 +1,27 @@ +{ + "name": "exp3", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "exp3@search.mozilla.org" + } + }, + "hidden": true, + "description": "A small test engine", + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "exp3", + "search_url": "https://3.example.com/searchexp", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + } + ] + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json b/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json new file mode 100644 index 0000000000..98bdfa26ff --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json @@ -0,0 +1,101 @@ +{ + "data": [ + { + "base": { + "name": "engine1", + "urls": { + "search": { + "base": "https://1.example.com/search", + "searchTermParamName": "q" + } + }, + "classification": "unknown" + }, + "variants": [{ "environment": { "allRegionsAndLocales": true } }], + "identifier": "engine1", + "recordType": "engine" + }, + { + "base": { + "name": "engine2", + "urls": { + "search": { + "base": "https://2.example.com/search", + "searchTermParamName": "q" + } + }, + "classification": "unknown" + }, + "variants": [{ "environment": { "allRegionsAndLocales": true } }], + "identifier": "engine2", + "recordType": "engine" + }, + { + "base": { + "name": "exp2", + "urls": { + "search": { + "base": "https://2.example.com/searchexp", + "searchTermParamName": "q" + } + }, + "classification": "unknown" + }, + "variants": [ + { + "environment": { "allRegionsAndLocales": true, "experiment": "exp2" } + } + ], + "identifier": "exp2", + "recordType": "engine" + }, + { + "base": { + "name": "exp3", + "urls": { + "search": { + "base": "https://3.example.com/searchexp", + "searchTermParamName": "q" + } + }, + "classification": "unknown" + }, + "variants": [ + { + "environment": { "allRegionsAndLocales": true, "experiment": "exp3" } + } + ], + "identifier": "exp3", + "recordType": "engine" + }, + { + "recordType": "defaultEngines", + "globalDefault": "engine1", + "globalDefaultPrivate": "engine1", + "specificDefaults": [ + { + "environment": { "experiment": "exp1" }, + "default": "engine2" + }, + { + "environment": { "experiment": "exp2" }, + "defaultPrivate": "exp2" + }, + { + "environment": { "experiment": "exp3" }, + "default": "exp3" + } + ] + }, + { + "recordType": "engineOrders", + "orders": [ + { + "environment": { "allRegionsAndLocales": true }, + "order": ["exp3", "engine1", "engine2", "exp2"] + } + ] + } + ], + "timestamp": 1704229342821 +} diff --git a/toolkit/components/search/tests/xpcshell/head_search.js b/toolkit/components/search/tests/xpcshell/head_search.js new file mode 100644 index 0000000000..1c0504e277 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/head_search.js @@ -0,0 +1,508 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et: */ + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +ChromeUtils.defineESModuleGetters(this, { + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + RemoteSettingsClient: + "resource://services-settings/RemoteSettingsClient.sys.mjs", + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", + SearchService: "resource://gre/modules/SearchService.sys.mjs", + SearchSettings: "resource://gre/modules/SearchSettings.sys.mjs", + SearchTestUtils: "resource://testing-common/SearchTestUtils.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", + TestUtils: "resource://testing-common/TestUtils.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", +}); + +var { HttpServer } = ChromeUtils.importESModule( + "resource://testing-common/httpd.sys.mjs" +); +var { AddonTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/AddonTestUtils.sys.mjs" +); +const { ExtensionTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/ExtensionXPCShellUtils.sys.mjs" +); + +SearchTestUtils.init(this); + +const SETTINGS_FILENAME = "search.json.mozlz4"; + +// nsSearchService.js uses Services.appinfo.name to build a salt for a hash. +// eslint-disable-next-line mozilla/use-services +var XULRuntime = Cc["@mozilla.org/xre/runtime;1"].getService(Ci.nsIXULRuntime); + +// Expand the amount of information available in error logs +Services.prefs.setBoolPref("browser.search.log", true); +Services.prefs.setBoolPref("browser.region.log", true); + +AddonTestUtils.init(this, false); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "42", + "42" +); + +// Allow telemetry probes which may otherwise be disabled for some applications (e.g. Thunderbird) +Services.prefs.setBoolPref( + "toolkit.telemetry.testing.overrideProductsCheck", + true +); + +// For tests, allow the settings to write sooner than it would do normally so that +// the tests that need to wait for it can run a bit faster. +SearchSettings.SETTNGS_INVALIDATION_DELAY = 250; + +async function promiseSettingsData() { + let path = PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME); + return IOUtils.readJSON(path, { decompress: true }); +} + +function promiseSaveSettingsData(data) { + return IOUtils.writeJSON( + PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME), + data, + { compress: true } + ); +} + +async function promiseEngineMetadata() { + let settings = await promiseSettingsData(); + let data = {}; + for (let engine of settings.engines) { + data[engine._name] = engine._metaData; + } + return data; +} + +async function promiseGlobalMetadata() { + return (await promiseSettingsData()).metaData; +} + +async function promiseSaveGlobalMetadata(globalData) { + let data = await promiseSettingsData(); + data.metaData = globalData; + await promiseSaveSettingsData(data); +} + +function promiseDefaultNotification(type = "normal") { + return SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE[ + type == "private" ? "DEFAULT_PRIVATE" : "DEFAULT" + ], + SearchUtils.TOPIC_ENGINE_MODIFIED + ); +} + +/** + * Clean the profile of any settings file left from a previous run. + * + * @returns {boolean} + * Indicates if the settings file existed. + */ +function removeSettingsFile() { + let file = do_get_profile().clone(); + file.append(SETTINGS_FILENAME); + if (file.exists()) { + file.remove(false); + return true; + } + return false; +} + +/** + * isUSTimezone taken from nsSearchService.js + * + * @returns {boolean} + */ +function isUSTimezone() { + // Timezone assumptions! We assume that if the system clock's timezone is + // between Newfoundland and Hawaii, that the user is in North America. + + // This includes all of South America as well, but we have relatively few + // en-US users there, so that's OK. + + // 150 minutes = 2.5 hours (UTC-2.5), which is + // Newfoundland Daylight Time (http://www.timeanddate.com/time/zones/ndt) + + // 600 minutes = 10 hours (UTC-10), which is + // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast) + + let UTCOffset = new Date().getTimezoneOffset(); + return UTCOffset >= 150 && UTCOffset <= 600; +} + +const kTestEngineName = "Test search engine"; + +/** + * Waits for the settings file to be saved. + * + * @returns {Promise} Resolved when the settings file is saved. + */ +function promiseAfterSettings() { + return SearchTestUtils.promiseSearchNotification( + "write-settings-to-disk-complete" + ); +} + +/** + * Sets the home region, and waits for the search service to reload the engines. + * + * @param {string} region + * The region to set. + */ +async function promiseSetHomeRegion(region) { + let promise = SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Region._setHomeRegion(region); + await promise; +} + +/** + * Sets the requested/available locales and waits for the search service to + * reload the engines. + * + * @param {string} locale + * The locale to set. + */ +async function promiseSetLocale(locale) { + if (!Services.locale.availableLocales.includes(locale)) { + throw new Error( + `"${locale}" needs to be included in Services.locales.availableLocales at the start of the test.` + ); + } + + let promise = SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Services.locale.requestedLocales = [locale]; + await promise; +} + +/** + * Read a JSON file and return the JS object + * + * @param {nsIFile} file + * The file to read. + * @returns {object} + * Returns the JSON object if the file was successfully read, + * false otherwise. + */ +async function readJSONFile(file) { + return JSON.parse(await IOUtils.readUTF8(file.path)); +} + +/** + * Recursively compare two objects and check that every property of expectedObj has the same value + * on actualObj. + * + * @param {object} expectedObj + * The source object that we expect to match + * @param {object} actualObj + * The object to check against the source + * @param {Function} skipProp + * A function that is called with the property name and its value, to see if + * testing that property should be skipped or not. + */ +function isSubObjectOf(expectedObj, actualObj, skipProp) { + for (let prop in expectedObj) { + if (skipProp && skipProp(prop, expectedObj[prop])) { + continue; + } + if (expectedObj[prop] instanceof Object) { + Assert.equal( + actualObj[prop]?.length, + expectedObj[prop].length, + `Should have the correct length for property ${prop}` + ); + isSubObjectOf(expectedObj[prop], actualObj[prop], skipProp); + } else { + Assert.equal( + actualObj[prop], + expectedObj[prop], + `Should have the correct value for property ${prop}` + ); + } + } +} + +/** + * After useHttpServer() is called, this string contains the URL of the "data" + * directory, including the final slash. + */ +var gDataUrl; + +/** + * Initializes the HTTP server and ensures that it is terminated when tests end. + * + * @param {string} dir + * The test sub-directory to use for the engines. + * @returns {HttpServer} + * The HttpServer object in case further customization is needed. + */ +function useHttpServer(dir = "data") { + let httpServer = new HttpServer(); + httpServer.start(-1); + httpServer.registerDirectory("/", do_get_cwd()); + gDataUrl = `http://localhost:${httpServer.identity.primaryPort}/${dir}/`; + registerCleanupFunction(async function cleanup_httpServer() { + await new Promise(resolve => { + httpServer.stop(resolve); + }); + }); + return httpServer; +} + +// This "enum" from nsSearchService.js +const TELEMETRY_RESULT_ENUM = { + SUCCESS: 0, + SUCCESS_WITHOUT_DATA: 1, + TIMEOUT: 2, + ERROR: 3, +}; + +/** + * Checks the value of the SEARCH_SERVICE_COUNTRY_FETCH_RESULT probe. + * + * @param {string|null} aExpectedValue + * If a value from TELEMETRY_RESULT_ENUM, we expect to see this value + * recorded exactly once in the probe. If |null|, we expect to see + * nothing recorded in the probe at all. + */ +function checkCountryResultTelemetry(aExpectedValue) { + let histogram = Services.telemetry.getHistogramById( + "SEARCH_SERVICE_COUNTRY_FETCH_RESULT" + ); + let snapshot = histogram.snapshot(); + if (aExpectedValue != null) { + equal(snapshot.values[aExpectedValue], 1); + } else { + deepEqual(snapshot.values, {}); + } +} + +/** + * Provides a basic set of remote settings for use in tests. + */ +async function setupRemoteSettings() { + const settings = await RemoteSettings("hijack-blocklists"); + sinon.stub(settings, "get").returns([ + { + id: "load-paths", + matches: ["[addon]searchignore@mozilla.com"], + _status: "synced", + }, + { + id: "submission-urls", + matches: ["ignore=true"], + _status: "synced", + }, + ]); +} + +/** + * Helper function that sets up a server and respnds to region + * fetch requests. + * + * @param {string} region + * The region that the server will respond with. + * @param {Promise|null} waitToRespond + * A promise that the server will await on to delay responding + * to the request. + */ +function useCustomGeoServer(region, waitToRespond = Promise.resolve()) { + let srv = useHttpServer(); + srv.registerPathHandler("/fetch_region", async (req, res) => { + res.processAsync(); + await waitToRespond; + res.setStatusLine("1.1", 200, "OK"); + res.write(JSON.stringify({ country_code: region })); + res.finish(); + }); + + Services.prefs.setCharPref( + "browser.region.network.url", + `http://localhost:${srv.identity.primaryPort}/fetch_region` + ); +} + +/** + * @typedef {object} TelemetryDetails + * @property {string} engineId + * The telemetry ID for the search engine. + * @property {string} [displayName] + * The search engine's display name. + * @property {string} [loadPath] + * The load path for the search engine. + * @property {string} [submissionUrl] + * The submission URL for the search engine. + * @property {string} [verified] + * Whether the search engine is verified. + */ + +/** + * Asserts that default search engine telemetry has been correctly reported + * to Glean. + * + * @param {object} expected + * An object containing telemetry details for normal and private engines. + * @param {TelemetryDetails} expected.normal + * An object with the expected details for the normal search engine. + * @param {TelemetryDetails} [expected.private] + * An object with the expected details for the private search engine. + */ +async function assertGleanDefaultEngine(expected) { + await TestUtils.waitForCondition( + () => + Glean.searchEngineDefault.engineId.testGetValue() == + (expected.normal.engineId ?? ""), + "Should have set the correct telemetry id for the normal engine" + ); + + await TestUtils.waitForCondition( + () => + Glean.searchEnginePrivate.engineId.testGetValue() == + (expected.private?.engineId ?? ""), + "Should have set the correct telemetry id for the private engine" + ); + + for (let property of [ + "displayName", + "loadPath", + "submissionUrl", + "verified", + ]) { + if (property in expected.normal) { + Assert.equal( + Glean.searchEngineDefault[property].testGetValue(), + expected.normal[property] ?? "", + `Should have set ${property} correctly` + ); + } + if (expected.private && property in expected.private) { + Assert.equal( + Glean.searchEnginePrivate[property].testGetValue(), + expected.private[property] ?? "", + `Should have set ${property} correctly` + ); + } + } +} + +/** + * A simple observer to ensure we get only the expected notifications. + */ +class SearchObserver { + constructor(expectedNotifications, returnEngineForNotification = false) { + this.observer = this.observer.bind(this); + this.deferred = Promise.withResolvers(); + this.expectedNotifications = expectedNotifications; + this.returnEngineForNotification = returnEngineForNotification; + + Services.obs.addObserver(this.observer, SearchUtils.TOPIC_ENGINE_MODIFIED); + + this.timeout = setTimeout(this.handleTimeout.bind(this), 5000); + } + + get promise() { + return this.deferred.promise; + } + + handleTimeout() { + this.deferred.reject( + new Error( + "Waiting for Notifications timed out, only received: " + + this.expectedNotifications.join(",") + ) + ); + } + + observer(subject, topic, data) { + Assert.greater( + this.expectedNotifications.length, + 0, + "Should be expecting a notification" + ); + Assert.equal( + data, + this.expectedNotifications[0], + "Should have received the next expected notification" + ); + + if ( + this.returnEngineForNotification && + data == this.returnEngineForNotification + ) { + this.engineToReturn = subject.QueryInterface(Ci.nsISearchEngine); + } + + this.expectedNotifications.shift(); + + if (!this.expectedNotifications.length) { + clearTimeout(this.timeout); + delete this.timeout; + this.deferred.resolve(this.engineToReturn); + Services.obs.removeObserver( + this.observer, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + } + } +} + +/** + * Some tests might trigger initialisation which will trigger the search settings + * update. We need to make sure we wait for that to finish before we exit, otherwise + * it may cause shutdown issues. + */ +let updatePromise = SearchTestUtils.promiseSearchNotification( + "settings-update-complete" +); + +registerCleanupFunction(async () => { + if (Services.search.isInitialized) { + await updatePromise; + } +}); + +let consoleAllowList = [ + // Harness issues. + 'property "localProfileDir" is non-configurable and can\'t be deleted', + 'property "profileDir" is non-configurable and can\'t be deleted', +]; + +let endConsoleListening = TestUtils.listenForConsoleMessages(); + +registerCleanupFunction(async () => { + let msgs = await endConsoleListening(); + for (let msg of msgs) { + msg = msg.wrappedJSObject; + if (msg.level != "error") { + continue; + } + + if (!msg.arguments?.length) { + Assert.ok( + false, + "Unexpected console message received during test: " + msg + ); + } else { + let firstArg = msg.arguments[0]; + // Use the appropriate message depending on the object supplied to + // the first argument. + let message = firstArg.messageContents ?? firstArg.message ?? firstArg; + if (!consoleAllowList.some(e => message.includes(e))) { + Assert.ok( + false, + "Unexpected console message received during test: " + message + ); + } + } + } +}); diff --git a/toolkit/components/search/tests/xpcshell/method-extensions/engines.json b/toolkit/components/search/tests/xpcshell/method-extensions/engines.json new file mode 100644 index 0000000000..ddd4e45acc --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/method-extensions/engines.json @@ -0,0 +1,55 @@ +{ + "data": [ + { + "webExtension": { + "id": "get@search.mozilla.org", + "name": "Get Engine", + "search_url": "https://example.com", + "search_url_get_params": "webExtension=1&search={searchTerms}", + "suggest_url": "https://example.com", + "suggest_url_get_params": "webExtension=1&suggest={searchTerms}" + }, + "params": { + "searchUrlGetParams": [ + { "name": "config", "value": "1" }, + { "name": "search", "value": "{searchTerms}" } + ], + "suggestUrlGetParams": [ + { "name": "config", "value": "1" }, + { "name": "suggest", "value": "{searchTerms}" } + ] + }, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes" + } + ] + }, + { + "webExtension": { + "id": "post@search.mozilla.org", + "name": "Post Engine", + "search_url": "https://example.com", + "search_url_post_params": "webExtension=1&search={searchTerms}", + "suggest_url": "https://example.com", + "suggest_url_post_params": "webExtension=1&suggest={searchTerms}" + }, + "params": { + "searchUrlPostParams": [ + { "name": "config", "value": "1" }, + { "name": "search", "value": "{searchTerms}" } + ], + "suggestUrlPostParams": [ + { "name": "config", "value": "1" }, + { "name": "suggest", "value": "{searchTerms}" } + ] + }, + "appliesTo": [ + { + "included": { "everywhere": true } + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/method-extensions/get/manifest.json b/toolkit/components/search/tests/xpcshell/method-extensions/get/manifest.json new file mode 100644 index 0000000000..a85cdaaa0f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/method-extensions/get/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "Get Engine", + "manifest_version": 2, + "version": "1.0", + "description": "Get engine to test get params", + "browser_specific_settings": { + "gecko": { + "id": "get@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "Get Engine", + "search_url": "https://example.com", + "search_url_get_params": "webExtension=1&search={searchTerms}", + "suggest_url": "https://example.com", + "suggest_url_get_params": "webExtension=1&suggest={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/method-extensions/post/manifest.json b/toolkit/components/search/tests/xpcshell/method-extensions/post/manifest.json new file mode 100644 index 0000000000..dce9bfb512 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/method-extensions/post/manifest.json @@ -0,0 +1,21 @@ +{ + "name": "Post Engine", + "manifest_version": 2, + "version": "1.0", + "description": "Get engine to test ost params", + "browser_specific_settings": { + "gecko": { + "id": "post@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "Post Engine", + "search_url": "https://example.com", + "search_url_post_params": "webExtension=1&search={searchTerms}", + "suggest_url": "https://example.com", + "suggest_url_post_params": "webExtension=1&suggest={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/opensearch/chromeicon.xml b/toolkit/components/search/tests/xpcshell/opensearch/chromeicon.xml new file mode 100644 index 0000000000..856732c6d6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/chromeicon.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/"> +<ShortName>engine-chromeicon</ShortName> +<Image width="16" height="16">chrome://branding/content/icon16.png</Image> +<Image width="32" height="32">chrome://branding/content/icon32.png</Image> +<Url type="text/html" method="GET" template="http://www.google.com/search"> + <Param name="q" value="{searchTerms}"/> +</Url> +</SearchPlugin> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated1.xml b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated1.xml new file mode 100644 index 0000000000..3131c25f37 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated1.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ii1</ShortName> +<Description>Insecure and insecurely updated 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://example.com/ii1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="http://example.com/ii1.xml" /> +<SearchForm>http://example.com/ii1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated2.xml b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated2.xml new file mode 100644 index 0000000000..a3c850d4d9 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-insecurely-updated2.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ii2</ShortName> +<Description>Insecure and insecurely updated 2</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://example.com/ii2"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="http://example.com/ii2.xml" /> +<SearchForm>http://example.com/ii2</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-no-update-url1.xml b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-no-update-url1.xml new file mode 100644 index 0000000000..75a5da8e7f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-no-update-url1.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>inu1</ShortName> +<Description>Insecure and no update URL 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://example.com/inu1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>http://example.com/inu1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-securely-updated1.xml b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-securely-updated1.xml new file mode 100644 index 0000000000..9427747722 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/insecure-and-securely-updated1.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>is1</ShortName> +<Description>Insecure and securely updated 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://example.com/is1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="https://example.com/is1.xml" /> +<SearchForm>http://example.com/is1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/invalid.xml b/toolkit/components/search/tests/xpcshell/opensearch/invalid.xml new file mode 100644 index 0000000000..e8efce6726 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/invalid.xml @@ -0,0 +1 @@ +# An invalid xml engine file. diff --git a/toolkit/components/search/tests/xpcshell/opensearch/mozilla-ns.xml b/toolkit/components/search/tests/xpcshell/opensearch/mozilla-ns.xml new file mode 100644 index 0000000000..f185f94868 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/mozilla-ns.xml @@ -0,0 +1,12 @@ +<?xml version="1.0" encoding="UTF-8"?> +<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/"> +<ShortName>mozilla-ns</ShortName> +<Description>An engine using mozilla namespace</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/search"> + <Param name="q" value="{searchTerms}"/> + <MozParam name="channel" condition="purpose" purpose="searchbar" value="test"/> +</Url> +<SearchForm>https://example.com/</SearchForm> +</SearchPlugin> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/post.xml b/toolkit/components/search/tests/xpcshell/opensearch/post.xml new file mode 100644 index 0000000000..621e49c872 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/post.xml @@ -0,0 +1,8 @@ +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"> + <ShortName>Post</ShortName> + <Url type="text/html" method="POST" template="https://example.com/post"> + <Param name="searchterms" value="{searchTerms}"/> + </Url> + <Url type="text/html" method="POST" template="http://engine-rel-searchform-post.xml/POST" rel="searchform"/> + <SearchForm>http://engine-rel-searchform-post.xml/?search</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/resourceicon.xml b/toolkit/components/search/tests/xpcshell/opensearch/resourceicon.xml new file mode 100644 index 0000000000..32861c34ea --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/resourceicon.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/"> +<ShortName>engine-resourceicon</ShortName> +<Image width="16" height="16">resource://search-extensions/icon16.png</Image> +<Image width="32" height="32">resource://search-extensions/icon32.png</Image> +<Url type="text/html" method="GET" template="http://www.google.com/search"> + <Param name="q" value="{searchTerms}"/> +</Url> +</SearchPlugin> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/searchform-invalid.xml b/toolkit/components/search/tests/xpcshell/opensearch/searchform-invalid.xml new file mode 100644 index 0000000000..4cc059b59e --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/searchform-invalid.xml @@ -0,0 +1,10 @@ +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" + xmlns:moz="http://www.mozilla.org/2006/browser/search/"> + <ShortName>searchform-invalid</ShortName> + <Description>Bug 483086 Test 1</Description> + <Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABGklEQVQoz2NgGB6AnZ1dUlJSXl4eSDIyMhLW4Ovr%2B%2Fr168uXL69Zs4YoG%2BLi4i5dusTExMTGxsbNzd3f37937976%2BnpmZmagbHR09J49e5YvX66kpATVEBYW9ubNm2nTphkbG7e2tp44cQLIuHfvXm5urpaWFlDKysqqu7v73LlzECMYIiIiHj58mJCQoKKicvXq1bS0NKBgW1vbjh074uPjgeqAXE1NzSdPnvDz84M0AEUvXLgAsW379u1z5swBen3jxo2zZ892cHB4%2BvQp0KlAfwI1cHJyghQFBwfv2rULokFXV%2FfixYu7d%2B8GGqGgoMDKyrpu3br9%2B%2FcDuXl5eVA%2FAEWBfoWHAdAYoNuAYQ0XAeoUERFhGDYAAPoUaT2dfWJuAAAAAElFTkSuQmCC</Image> + <Url type="text/html" method="GET" template="http://mochi.test:8888/browser/browser/components/search/test/browser/?search"> + <Param name="test" value="{searchTerms}"/> + </Url> + <moz:SearchForm>foo://example.com</moz:SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated1.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated1.xml new file mode 100644 index 0000000000..d8a62d0e18 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated1.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>si1</ShortName> +<Description>Secure and insecurely updated 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/si1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="http://example.com/si1.xml" /> +<SearchForm>https://example.com/si1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated2.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated2.xml new file mode 100644 index 0000000000..f707e5eb3d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-insecurely-updated2.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>si2</ShortName> +<Description>Secure and insecurely updated 2</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/si2"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="http://example.com/si2.xml" /> +<SearchForm>https://example.com/si2</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-no-update-url1.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-no-update-url1.xml new file mode 100644 index 0000000000..6dcbbb126c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-no-update-url1.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>snu1</ShortName> +<Description>Secure and no update URL 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/snu1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>https://example.com/snu1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated-insecure-form.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated-insecure-form.xml new file mode 100644 index 0000000000..15a0b6a517 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated-insecure-form.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ssif</ShortName> +<Description>Secure and securely updated insecure form</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/ssif"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="https://example.com/ssif.xml" /> +<SearchForm>http://example.com/ssif</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated1.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated1.xml new file mode 100644 index 0000000000..593c8bec8c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated1.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ss1</ShortName> +<Description>Secure and securely updated 1</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/ss1"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="https://example.com/ss1.xml" /> +<SearchForm>https://example.com/ss1</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated2.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated2.xml new file mode 100644 index 0000000000..30a20b754a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated2.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ss2</ShortName> +<Description>Secure and securely updated 2</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/ss2"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="https://example.com/ss2.xml" /> +<SearchForm>https://example.com/ss2</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated3.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated3.xml new file mode 100644 index 0000000000..8b86a82199 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-and-securely-updated3.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>ss3</ShortName> +<Description>Secure and securely updated 3</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/ss3"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/opensearchdescription+xml" + rel="self" + template="https://example.com/ss3.xml" /> +<SearchForm>https://example.com/ss3</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-localhost.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-localhost.xml new file mode 100644 index 0000000000..89d96f2c43 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-localhost.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>sl</ShortName> +<Description>Secure localhost</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://localhost:8080"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>http://localhost:8080/sl</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv2.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv2.xml new file mode 100644 index 0000000000..8da3995a71 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv2.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>sov2</ShortName> +<Description>Secure onion v2</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://s3zkf3ortukqklec.onion"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>http://s3zkf3ortukqklec.onion/sov2</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv3.xml b/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv3.xml new file mode 100644 index 0000000000..c8256ca28a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/secure-onionv3.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>sov3</ShortName> +<Description>Secure onion v3</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="http://ydemw5wg5cseltau22u4fjfrmfshopaldpoznsirb3rgo2gv6uh4s2y5.onion"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>http://ydemw5wg5cseltau22u4fjfrmfshopaldpoznsirb3rgo2gv6uh4s2y5.onion/sov3</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/simple.xml b/toolkit/components/search/tests/xpcshell/opensearch/simple.xml new file mode 100644 index 0000000000..ee38e51bca --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/simple.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.0/"> +<ShortName>simple</ShortName> +<Description>A small test engine</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/search"> + <Param name="q" value="{searchTerms}"/> +</Url> +<SearchForm>https://example.com/</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/suggestion-alternate.xml b/toolkit/components/search/tests/xpcshell/opensearch/suggestion-alternate.xml new file mode 100644 index 0000000000..7a961520b9 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/suggestion-alternate.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearchdescription/1.1/"> +<ShortName>suggestion-alternate</ShortName> +<Description>A small engine with suggestions</Description> +<InputEncoding>UTF-8</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/search"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/json" rel="suggestions" method="GET" + template="https://example.com/suggest"> + <Param name="suggestion" value="{searchTerms}"/> +</Url> + +<SearchForm>https://example.com/</SearchForm> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/opensearch/suggestion.xml b/toolkit/components/search/tests/xpcshell/opensearch/suggestion.xml new file mode 100644 index 0000000000..8d2f701a36 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/opensearch/suggestion.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8"?> +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearchdescription/1.0/"> +<ShortName>suggestion</ShortName> +<Description>A small engine with suggestions</Description> +<InputEncoding>windows-1252</InputEncoding> +<Image width="16" height="16">data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4//8/AAX+Av7czFnnAAAAAElFTkSuQmCC</Image> +<Url type="text/html" method="GET" template="https://example.com/search"> + <Param name="q" value="{searchTerms}"/> +</Url> +<Url type="application/x-suggestions+json" method="GET" + template="https://example.com/suggest"> + <Param name="suggestion" value="{searchTerms}"/> +</Url> +<Url type="text/html" method="GET" template="http://engine-rel-searchform.xml/?search" rel="searchform"/> +</OpenSearchDescription> diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js b/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js new file mode 100644 index 0000000000..72c4d4f04f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js @@ -0,0 +1,604 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +ChromeUtils.defineESModuleGetters(this, { + // Only needed when SearchUtils.newSearchConfigEnabled is false. + AddonTestUtils: "resource://testing-common/AddonTestUtils.sys.mjs", + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + SearchEngine: "resource://gre/modules/SearchEngine.sys.mjs", + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", + SearchTestUtils: "resource://testing-common/SearchTestUtils.sys.mjs", + SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", + sinon: "resource://testing-common/Sinon.sys.mjs", + updateAppInfo: "resource://testing-common/AppInfo.sys.mjs", +}); + +const GLOBAL_SCOPE = this; +const TEST_DEBUG = Services.env.get("TEST_DEBUG"); + +const URLTYPE_SUGGEST_JSON = "application/x-suggestions+json"; +const URLTYPE_SEARCH_HTML = "text/html"; +const SUBMISSION_PURPOSES = [ + "searchbar", + "keyword", + "contextmenu", + "homepage", + "newtab", +]; + +let engineSelector; + +/** + * This function is used to override the remote settings configuration + * if the SEARCH_CONFIG environment variable is set. This allows testing + * against a remote server. + */ +async function maybeSetupConfig() { + const SEARCH_CONFIG = Services.env.get("SEARCH_CONFIG"); + if (SEARCH_CONFIG) { + if (!(SEARCH_CONFIG in SearchUtils.ENGINES_URLS)) { + throw new Error(`Invalid value for SEARCH_CONFIG`); + } + const url = SearchUtils.ENGINES_URLS[SEARCH_CONFIG]; + const response = await fetch(url); + const config = await response.json(); + const settings = await RemoteSettings(SearchUtils.SETTINGS_KEY); + sinon.stub(settings, "get").returns(config.data); + } +} + +/** + * This class implements the test harness for search configuration tests. + * These tests are designed to ensure that the correct search engines are + * loaded for the various region/locale configurations. + * + * The configuration for each test is represented by an object having the + * following properties: + * + * - identifier (string) + * The identifier for the search engine under test. + * - default (object) + * An inclusion/exclusion configuration (see below) to detail when this engine + * should be listed as default. + * + * The inclusion/exclusion configuration is represented as an object having the + * following properties: + * + * - included (array) + * An optional array of region/locale pairs. + * - excluded (array) + * An optional array of region/locale pairs. + * + * If the object is empty, the engine is assumed not to be part of any locale/region + * pair. + * If the object has `excluded` but not `included`, then the engine is assumed to + * be part of every locale/region pair except for where it matches the exclusions. + * + * The region/locale pairs are represented as an object having the following + * properties: + * + * - region (array) + * An array of two-letter region codes. + * - locale (object) + * A locale object which may consist of: + * - matches (array) + * An array of locale strings which should exactly match the locale. + * - startsWith (array) + * An array of locale strings which the locale should start with. + */ +class SearchConfigTest { + /** + * @param {object} config + * The initial configuration for this test, see above. + */ + constructor(config = {}) { + this._config = config; + } + + /** + * Sets up the test. + * + * @param {string} [version] + * The version to simulate for running the tests. + */ + async setup(version = "42.0") { + if (SearchUtils.newSearchConfigEnabled) { + updateAppInfo({ + name: "XPCShell", + ID: "xpcshell@tests.mozilla.org", + version, + platformVersion: version, + }); + } else { + AddonTestUtils.init(GLOBAL_SCOPE); + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + version, + version + ); + } + + await maybeSetupConfig(); + + // Disable region checks. + Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", false); + + // Enable separatePrivateDefault testing. We test with this on, as we have + // separate tests for ensuring the normal = private when this is off. + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + if (!SearchUtils.newSearchConfigEnabled) { + await AddonTestUtils.promiseStartupManager(); + } + await Services.search.init(); + + // We must use the engine selector that the search service has created (if + // it has), as remote settings can only easily deal with us loading the + // configuration once - after that, it tries to access the network. + engineSelector = + Services.search.wrappedJSObject._engineSelector || + SearchUtils.newSearchConfigEnabled + ? new SearchEngineSelector() + : new SearchEngineSelectorOld(); + + // Note: we don't use the helper function here, so that we have at least + // one message output per process. + Assert.ok( + Services.search.isInitialized, + "Should have correctly initialized the search service" + ); + } + + /** + * Runs the test. + */ + async run() { + const locales = await this.getLocales(); + const regions = this._regions; + + // We loop on region and then locale, so that we always cause a re-init + // when updating the requested/available locales. + for (let region of regions) { + for (let locale of locales) { + const engines = await this._getEngines(region, locale); + this._assertEngineRules([engines[0]], region, locale, "default"); + const isPresent = this._assertAvailableEngines(region, locale, engines); + if (isPresent) { + this._assertEngineDetails(region, locale, engines); + } + } + } + } + + async _getEngines(region, locale) { + let configs = await engineSelector.fetchEngineConfiguration({ + locale, + region: region || "default", + channel: SearchUtils.MODIFIED_APP_CHANNEL, + }); + + return SearchTestUtils.searchConfigToEngines(configs.engines); + } + + /** + * @returns {Set} the list of regions for the tests to run with. + */ + get _regions() { + // TODO: The legacy configuration worked with null as an unknown region, + // for the search engine selector, we expect "default" but apply the + // fallback in _getEngines. Once we remove the legacy configuration, we can + // simplify this. + if (TEST_DEBUG) { + return new Set(["by", "cn", "kz", "us", "ru", "tr", null]); + } + return [...Services.intl.getAvailableLocaleDisplayNames("region"), null]; + } + + /** + * @returns {Array} the list of locales for the tests to run with. + */ + async getLocales() { + if (TEST_DEBUG) { + return ["be", "en-US", "kk", "tr", "ru", "zh-CN", "ach", "unknown"]; + } + const data = await IOUtils.readUTF8(do_get_file("all-locales").path); + // "en-US" is not in all-locales as it is the default locale + // add it manually to ensure it is tested. + let locales = [...data.split("\n").filter(e => e != ""), "en-US"]; + // BCP47 requires all variants are 5-8 characters long. Our + // build sytem uses the short `mac` variant, this is invalid, and inside + // the app we turn it into `ja-JP-macos` + locales = locales.map(l => (l == "ja-JP-mac" ? "ja-JP-macos" : l)); + // The locale sometimes can be unknown or a strange name, e.g. if the updater + // is disabled, it may be "und", add one here so we know what happens if we + // hit it. + locales.push("unknown"); + return locales; + } + + /** + * Determines if a locale/region pair match a section of the configuration. + * + * @param {object} section + * The configuration section to match against. + * @param {string} region + * The two-letter region code. + * @param {string} locale + * The two-letter locale code. + * @returns {boolean} + * True if the locale/region pair matches the section. + */ + _localeRegionInSection(section, region, locale) { + for (const { regions, locales } of section) { + // If we only specify a regions or locales section then + // it is always considered included in the other section. + const inRegions = !regions || regions.includes(region); + const inLocales = !locales || locales.includes(locale); + if (inRegions && inLocales) { + return true; + } + } + return false; + } + + /** + * Helper function to find an engine from within a list. + * + * @param {Array} engines + * The list of engines to check. + * @param {string} identifier + * The identifier to look for in the list. + * @param {boolean} exactMatch + * Whether to use an exactMatch for the identifier. + * @returns {Engine} + * Returns the engine if found, null otherwise. + */ + _findEngine(engines, identifier, exactMatch) { + return engines.find(engine => + exactMatch + ? engine.identifier == identifier + : engine.identifier.startsWith(identifier) + ); + } + + /** + * Asserts whether the engines rules defined in the configuration are met. + * + * @param {Array} engines + * The list of engines to check. + * @param {string} region + * The two-letter region code. + * @param {string} locale + * The two-letter locale code. + * @param {string} section + * The section of the configuration to check. + * @returns {boolean} + * Returns true if the engine is expected to be present, false otherwise. + */ + _assertEngineRules(engines, region, locale, section) { + const infoString = `region: "${region}" locale: "${locale}"`; + const config = this._config[section]; + const hasIncluded = "included" in config; + const hasExcluded = "excluded" in config; + const identifierIncluded = !!this._findEngine( + engines, + this._config.identifier, + this._config.identifierExactMatch ?? false + ); + + // If there's not included/excluded, then this shouldn't be the default anywhere. + if (section == "default" && !hasIncluded && !hasExcluded) { + this.assertOk( + !identifierIncluded, + `Should not be ${section} for any locale/region, + currently set for ${infoString}` + ); + return false; + } + + // If there's no included section, we assume the engine is default everywhere + // and we should apply the exclusions instead. + let included = + hasIncluded && + this._localeRegionInSection(config.included, region, locale); + + let excluded = + hasExcluded && + this._localeRegionInSection(config.excluded, region, locale); + if ( + (included && (!hasExcluded || !excluded)) || + (!hasIncluded && hasExcluded && !excluded) + ) { + this.assertOk( + identifierIncluded, + `Should be ${section} for ${infoString}` + ); + return true; + } + this.assertOk( + !identifierIncluded, + `Should not be ${section} for ${infoString}` + ); + return false; + } + + /** + * Asserts whether the engine is correctly set as default or not. + * + * @param {string} region + * The two-letter region code. + * @param {string} locale + * The two-letter locale code. + */ + _assertDefaultEngines(region, locale) { + this._assertEngineRules( + [Services.search.appDefaultEngine], + region, + locale, + "default" + ); + // At the moment, this uses the same section as the normal default, as + // we don't set this differently for any region/locale. + this._assertEngineRules( + [Services.search.appPrivateDefaultEngine], + region, + locale, + "default" + ); + } + + /** + * Asserts whether the engine is correctly available or not. + * + * @param {string} region + * The two-letter region code. + * @param {string} locale + * The two-letter locale code. + * @param {Array} engines + * The current visible engines. + * @returns {boolean} + * Returns true if the engine is expected to be present, false otherwise. + */ + _assertAvailableEngines(region, locale, engines) { + return this._assertEngineRules(engines, region, locale, "available"); + } + + /** + * Asserts the engine follows various rules. + * + * @param {string} region + * The two-letter region code. + * @param {string} locale + * The two-letter locale code. + * @param {Array} engines + * The current visible engines. + */ + _assertEngineDetails(region, locale, engines) { + const details = this._config.details.filter(value => { + const included = this._localeRegionInSection( + value.included, + region, + locale + ); + const excluded = + value.excluded && + this._localeRegionInSection(value.excluded, region, locale); + return included && !excluded; + }); + this.assertEqual( + details.length, + 1, + `Should have just one details section for region: ${region} locale: ${locale}` + ); + + const engine = this._findEngine( + engines, + this._config.identifier, + this._config.identifierExactMatch ?? false + ); + this.assertOk(engine, "Should have an engine present"); + + if (this._config.aliases) { + this.assertDeepEqual( + engine.aliases, + this._config.aliases, + "Should have the correct aliases for the engine" + ); + } + + const location = `in region:${region}, locale:${locale}`; + + for (const rule of details) { + this._assertCorrectDomains(location, engine, rule); + if (rule.codes) { + this._assertCorrectCodes(location, engine, rule); + } + if (rule.searchUrlCode || rule.suggestUrlCode) { + this._assertCorrectUrlCode(location, engine, rule); + } + if (rule.aliases) { + this.assertDeepEqual( + engine.aliases, + rule.aliases, + "Should have the correct aliases for the engine" + ); + } + if (rule.telemetryId) { + this.assertEqual( + engine.telemetryId, + rule.telemetryId, + `Should have the correct telemetryId ${location}.` + ); + } + } + } + + /** + * Asserts whether the engine is using the correct domains or not. + * + * @param {string} location + * Debug string with locale + region information. + * @param {object} engine + * The engine being tested. + * @param {object} rules + * Rules to test. + */ + _assertCorrectDomains(location, engine, rules) { + this.assertOk( + rules.domain, + `Should have an expectedDomain for the engine ${location}` + ); + + const searchForm = new URL(engine.searchForm); + this.assertOk( + searchForm.host.endsWith(rules.domain), + `Should have the correct search form domain ${location}. + Got "${searchForm.host}", expected to end with "${rules.domain}".` + ); + + let submission = engine.getSubmission("test", URLTYPE_SEARCH_HTML); + + this.assertOk( + submission.uri.host.endsWith(rules.domain), + `Should have the correct domain for type: ${URLTYPE_SEARCH_HTML} ${location}. + Got "${submission.uri.host}", expected to end with "${rules.domain}".` + ); + + submission = engine.getSubmission("test", URLTYPE_SUGGEST_JSON); + if (this._config.noSuggestionsURL || rules.noSuggestionsURL) { + this.assertOk(!submission, "Should not have a submission url"); + } else if (this._config.suggestionUrlBase) { + this.assertEqual( + submission.uri.prePath + submission.uri.filePath, + this._config.suggestionUrlBase, + `Should have the correct domain for type: ${URLTYPE_SUGGEST_JSON} ${location}.` + ); + this.assertOk( + submission.uri.query.includes(rules.suggestUrlCode), + `Should have the code in the uri` + ); + } + } + + /** + * Asserts whether the engine is using the correct codes or not. + * + * @param {string} location + * Debug string with locale + region information. + * @param {object} engine + * The engine being tested. + * @param {object} rules + * Rules to test. + */ + _assertCorrectCodes(location, engine, rules) { + for (const purpose of SUBMISSION_PURPOSES) { + // Don't need to repeat the code if we use it for all purposes. + const code = + typeof rules.codes === "string" ? rules.codes : rules.codes[purpose]; + const submission = engine.getSubmission("test", "text/html", purpose); + const submissionQueryParams = submission.uri.query.split("&"); + this.assertOk( + submissionQueryParams.includes(code), + `Expected "${code}" in url "${submission.uri.spec}" from purpose "${purpose}" ${location}` + ); + + const paramName = code.split("=")[0]; + this.assertOk( + submissionQueryParams.filter(param => param.startsWith(paramName)) + .length == 1, + `Expected only one "${paramName}" parameter in "${submission.uri.spec}" from purpose "${purpose}" ${location}` + ); + } + } + + /** + * Asserts whether the engine is using the correct URL codes or not. + * + * @param {string} location + * Debug string with locale + region information. + * @param {object} engine + * The engine being tested. + * @param {object} rule + * Rules to test. + */ + _assertCorrectUrlCode(location, engine, rule) { + if (rule.searchUrlCode) { + const submission = engine.getSubmission("test", URLTYPE_SEARCH_HTML); + this.assertOk( + submission.uri.query.split("&").includes(rule.searchUrlCode), + `Expected "${rule.searchUrlCode}" in search url "${submission.uri.spec}"` + ); + let uri = engine.searchForm; + this.assertOk( + !uri.includes(rule.searchUrlCode), + `"${rule.searchUrlCode}" should not be in the search form URL.` + ); + } + if (rule.searchUrlCodeNotInQuery) { + const submission = engine.getSubmission("test", URLTYPE_SEARCH_HTML); + this.assertOk( + submission.uri.includes(rule.searchUrlCodeNotInQuery), + `Expected "${rule.searchUrlCodeNotInQuery}" in search url "${submission.uri.spec}"` + ); + } + if (rule.suggestUrlCode) { + const submission = engine.getSubmission("test", URLTYPE_SUGGEST_JSON); + this.assertOk( + submission.uri.query.split("&").includes(rule.suggestUrlCode), + `Expected "${rule.suggestUrlCode}" in suggestion url "${submission.uri.spec}"` + ); + } + } + + /** + * Helper functions which avoid outputting test results when there are no + * failures. These help the tests to run faster, and avoid clogging up the + * python test runner process. + */ + + assertOk(value, message) { + if (!value || TEST_DEBUG) { + Assert.ok(value, message); + } + } + + assertEqual(actual, expected, message) { + if (actual != expected || TEST_DEBUG) { + Assert.equal(actual, expected, message); + } + } + + assertDeepEqual(actual, expected, message) { + if (!ObjectUtils.deepEqual(actual, expected)) { + Assert.deepEqual(actual, expected, message); + } + } +} + +async function checkUISchemaValid(configSchema, uiSchema) { + for (let key of Object.keys(configSchema.properties)) { + Assert.ok( + uiSchema["ui:order"].includes(key), + `Should have ${key} listed at the top-level of the ui schema` + ); + } +} diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_amazon.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_amazon.js new file mode 100644 index 0000000000..bfe05e1596 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_amazon.js @@ -0,0 +1,75 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "amazon", + default: { + // Not default anywhere. + }, + available: { + included: [ + { + // The main regions we ship Amazon to. Below this are special cases. + regions: ["us", "jp"], + }, + ], + }, + details: [ + { + domain: "amazon.co.jp", + telemetryId: "amazon-jp", + aliases: ["@amazon"], + included: [ + { + regions: ["jp"], + }, + ], + searchUrlCode: "tag=mozillajapan-fx-22", + noSuggestionsURL: true, + }, + { + domain: "amazon.com", + telemetryId: "amazondotcom-us-adm", + aliases: ["@amazon"], + included: [ + { + regions: ["us"], + }, + ], + noSuggestionsURL: true, + searchUrlCode: "tag=admarketus-20", + }, + ], +}); + +add_setup(async function () { + // We only need to do setup on one of the tests. + await test.setup("89.0"); +}); + +add_task(async function test_searchConfig_amazon() { + await test.run(); +}); + +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_searchConfig_amazon_pre89() { + const version = "88.0"; + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + version, + version + ); + // For pre-89, Amazon has a slightly different config. + let details = test._config.details.find( + d => d.telemetryId == "amazondotcom-us-adm" + ); + details.telemetryId = "amazondotcom"; + delete details.searchUrlCode; + + await test.run(); + } +); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_baidu.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_baidu.js new file mode 100644 index 0000000000..3c66708bdc --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_baidu.js @@ -0,0 +1,39 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "baidu", + aliases: ["@\u767E\u5EA6", "@baidu"], + default: { + included: [ + { + regions: ["cn"], + locales: ["zh-CN"], + }, + ], + }, + available: { + included: [ + { + locales: ["zh-CN"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "baidu.com", + telemetryId: "baidu", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_baidu() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_bing.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_bing.js new file mode 100644 index 0000000000..6dbddd8a08 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_bing.js @@ -0,0 +1,134 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "bing", + aliases: ["@bing"], + default: { + // Not included anywhere. + }, + available: { + included: [ + { + // regions: [ + // These arent currently enforced. + // "au", "at", "be", "br", "ca", "fi", "fr", "de", + // "in", "ie", "it", "jp", "my", "mx", "nl", "nz", + // "no", "sg", "es", "se", "ch", "gb", "us", + // ], + locales: [ + "ach", + "af", + "an", + "ar", + "ast", + "az", + "bn", + "bs", + "ca", + "ca-valencia", + "cak", + "cs", + "cy", + "da", + "de", + "dsb", + "el", + "en-CA", + "en-GB", + "en-US", + "eo", + "es-CL", + "es-ES", + "es-MX", + "eu", + "fa", + "ff", + "fi", + "fr", + "fur", + "fy-NL", + "gd", + "gl", + "gn", + "gu-IN", + "he", + "hi-IN", + "hr", + "hsb", + "hy-AM", + "ia", + "id", + "is", + "it", + "ja-JP-macos", + "ja", + "ka", + "kab", + "km", + "kn", + "lij", + "lo", + "lt", + "meh", + "mk", + "ms", + "my", + "nb-NO", + "ne-NP", + "nl", + "nn-NO", + "oc", + "pa-IN", + "pt-BR", + "rm", + "ro", + "sc", + "sco", + "son", + "sq", + "sr", + "sv-SE", + "te", + "th", + "tl", + "tr", + "trs", + "uk", + "ur", + "uz", + "wo", + "xh", + "zh-CN", + ], + }, + ], + }, + details: [ + { + included: [{}], + domain: "bing.com", + telemetryId: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "bing-esr" : "bing", + codes: { + searchbar: "form=MOZSBR", + keyword: "form=MOZLBR", + contextmenu: "form=MOZCON", + homepage: "form=MOZSPG", + newtab: "form=MOZTSB", + }, + searchUrlCode: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "pc=MOZR" : "pc=MOZI", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_bing() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_distributions.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_distributions.js new file mode 100644 index 0000000000..513fdaafef --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_distributions.js @@ -0,0 +1,348 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchService: "resource://gre/modules/SearchService.sys.mjs", +}); + +const tests = []; + +for (let canonicalId of ["canonical", "canonical-001"]) { + tests.push({ + locale: "en-US", + region: "US", + distribution: canonicalId, + test: engines => + hasParams(engines, "Google", "searchbar", "client=ubuntu") && + hasParams(engines, "Google", "searchbar", "channel=fs") && + hasTelemetryId(engines, "Google", "google-canonical"), + }); + + tests.push({ + locale: "en-US", + region: "GB", + distribution: canonicalId, + test: engines => + hasParams(engines, "Google", "searchbar", "client=ubuntu") && + hasParams(engines, "Google", "searchbar", "channel=fs") && + hasTelemetryId(engines, "Google", "google-canonical"), + }); +} + +tests.push({ + locale: "en-US", + region: "US", + distribution: "canonical-002", + test: engines => + hasParams(engines, "Google", "searchbar", "client=ubuntu-sn") && + hasParams(engines, "Google", "searchbar", "channel=fs") && + hasTelemetryId(engines, "Google", "google-ubuntu-sn"), +}); + +tests.push({ + locale: "en-US", + region: "GB", + distribution: "canonical-002", + test: engines => + hasParams(engines, "Google", "searchbar", "client=ubuntu-sn") && + hasParams(engines, "Google", "searchbar", "channel=fs") && + hasTelemetryId(engines, "Google", "google-ubuntu-sn"), +}); + +tests.push({ + locale: "zh-CN", + region: "CN", + distribution: "MozillaOnline", + test: engines => + hasEnginesFirst(engines, ["百度", "Bing", "Google", "维基百科"]), +}); + +tests.push({ + locale: "fr", + distribution: "qwant-001", + test: engines => + hasParams(engines, "Qwant", "searchbar", "client=firefoxqwant") && + hasDefault(engines, "Qwant") && + hasEnginesFirst(engines, ["Qwant", "Qwant Junior"]), +}); + +tests.push({ + locale: "fr", + distribution: "qwant-001", + test: engines => + hasParams(engines, "Qwant Junior", "searchbar", "client=firefoxqwant"), +}); + +tests.push({ + locale: "fr", + distribution: "qwant-002", + test: engines => + hasParams(engines, "Qwant", "searchbar", "client=firefoxqwant") && + hasDefault(engines, "Qwant") && + hasEnginesFirst(engines, ["Qwant", "Qwant Junior"]), +}); + +tests.push({ + locale: "fr", + distribution: "qwant-002", + test: engines => + hasParams(engines, "Qwant Junior", "searchbar", "client=firefoxqwant"), +}); + +for (const locale of ["en-US", "de"]) { + tests.push({ + locale, + distribution: "1und1", + test: engines => + hasParams(engines, "1&1 Suche", "searchbar", "enc=UTF-8") && + hasDefault(engines, "1&1 Suche") && + hasEnginesFirst(engines, ["1&1 Suche"]), + }); + + tests.push({ + locale, + distribution: "gmx", + test: engines => + hasParams(engines, "GMX Suche", "searchbar", "enc=UTF-8") && + hasDefault(engines, "GMX Suche") && + hasEnginesFirst(engines, ["GMX Suche"]), + }); + + tests.push({ + locale, + distribution: "gmx", + test: engines => + hasParams(engines, "GMX Shopping", "searchbar", "origin=br_osd"), + }); + + tests.push({ + locale, + distribution: "mail.com", + test: engines => + hasParams(engines, "mail.com search", "searchbar", "enc=UTF-8") && + hasDefault(engines, "mail.com search") && + hasEnginesFirst(engines, ["mail.com search"]), + }); + + tests.push({ + locale, + distribution: "webde", + test: engines => + hasParams(engines, "WEB.DE Suche", "searchbar", "enc=UTF-8") && + hasDefault(engines, "WEB.DE Suche") && + hasEnginesFirst(engines, ["WEB.DE Suche"]), + }); +} + +tests.push({ + locale: "ru", + region: "RU", + distribution: "gmx", + test: engines => hasDefault(engines, "GMX Suche"), +}); + +tests.push({ + locale: "en-GB", + distribution: "gmxcouk", + test: engines => + hasURLs( + engines, + "GMX Search", + "https://go.gmx.co.uk/br/moz_search_web/?enc=UTF-8&q=test", + SearchUtils.newSearchConfigEnabled + ? "https://suggestplugin.gmx.co.uk/s?brand=gmxcouk&origin=moz_splugin_ff&enc=UTF-8&q=test" + : "https://suggestplugin.gmx.co.uk/s?q=test&brand=gmxcouk&origin=moz_splugin_ff&enc=UTF-8" + ) && + hasDefault(engines, "GMX Search") && + hasEnginesFirst(engines, ["GMX Search"]), +}); + +tests.push({ + locale: "ru", + region: "RU", + distribution: "gmxcouk", + test: engines => hasDefault(engines, "GMX Search"), +}); + +tests.push({ + locale: "es", + distribution: "gmxes", + test: engines => + hasURLs( + engines, + "GMX - Búsqueda web", + "https://go.gmx.es/br/moz_search_web/?enc=UTF-8&q=test", + SearchUtils.newSearchConfigEnabled + ? "https://suggestplugin.gmx.es/s?brand=gmxes&origin=moz_splugin_ff&enc=UTF-8&q=test" + : "https://suggestplugin.gmx.es/s?q=test&brand=gmxes&origin=moz_splugin_ff&enc=UTF-8" + ) && + hasDefault(engines, "GMX Search") && + hasEnginesFirst(engines, ["GMX Search"]), +}); + +tests.push({ + locale: "ru", + region: "RU", + distribution: "gmxes", + test: engines => hasDefault(engines, "GMX - Búsqueda web"), +}); + +tests.push({ + locale: "fr", + distribution: "gmxfr", + test: engines => + hasURLs( + engines, + "GMX - Recherche web", + "https://go.gmx.fr/br/moz_search_web/?enc=UTF-8&q=test", + SearchUtils.newSearchConfigEnabled + ? "https://suggestplugin.gmx.fr/s?brand=gmxfr&origin=moz_splugin_ff&enc=UTF-8&q=test" + : "https://suggestplugin.gmx.fr/s?q=test&brand=gmxfr&origin=moz_splugin_ff&enc=UTF-8" + ) && + hasDefault(engines, "GMX Search") && + hasEnginesFirst(engines, ["GMX Search"]), +}); + +tests.push({ + locale: "ru", + region: "RU", + distribution: "gmxfr", + test: engines => hasDefault(engines, "GMX - Recherche web"), +}); + +tests.push({ + locale: "en-US", + region: "US", + distribution: "mint-001", + test: engines => + hasParams(engines, "DuckDuckGo", "searchbar", "t=lm") && + hasParams(engines, "Google", "searchbar", "client=firefox-b-1-lm") && + hasDefault(engines, "Google") && + hasEnginesFirst(engines, ["Google"]) && + hasTelemetryId(engines, "Google", "google-b-1-lm"), +}); + +tests.push({ + locale: "en-GB", + region: "GB", + distribution: "mint-001", + test: engines => + hasParams(engines, "DuckDuckGo", "searchbar", "t=lm") && + hasParams(engines, "Google", "searchbar", "client=firefox-b-lm") && + hasDefault(engines, "Google") && + hasEnginesFirst(engines, ["Google"]) && + hasTelemetryId(engines, "Google", "google-b-lm"), +}); + +function hasURLs(engines, engineName, url, suggestURL) { + let engine = engines.find(e => e._name === engineName); + Assert.ok(engine, `Should be able to find ${engineName}`); + + let submission = engine.getSubmission("test", "text/html"); + Assert.equal( + submission.uri.spec, + url, + `Should have the correct submission url for ${engineName}` + ); + + submission = engine.getSubmission("test", "application/x-suggestions+json"); + Assert.equal( + submission.uri.spec, + suggestURL, + `Should have the correct suggestion url for ${engineName}` + ); +} + +function hasParams(engines, engineName, purpose, param) { + let engine = engines.find(e => e._name === engineName); + Assert.ok(engine, `Should be able to find ${engineName}`); + + let submission = engine.getSubmission("test", "text/html", purpose); + let queries = submission.uri.query.split("&"); + + let paramNames = new Set(); + for (let query of queries) { + let queryParam = query.split("=")[0]; + Assert.ok( + !paramNames.has(queryParam), + `Should not have a duplicate ${queryParam} param` + ); + paramNames.add(queryParam); + } + + let result = queries.includes(param); + Assert.ok(result, `expect ${submission.uri.query} to include ${param}`); + return true; +} + +function hasTelemetryId(engines, engineName, telemetryId) { + let engine = engines.find(e => e._name === engineName); + Assert.ok(engine, `Should be able to find ${engineName}`); + + Assert.equal( + engine.telemetryId, + telemetryId, + "Should have the correct telemetryId" + ); + return true; +} + +function hasDefault(engines, expectedDefaultName) { + Assert.equal( + engines[0].name, + expectedDefaultName, + "Should have the expected engine set as default" + ); + return true; +} + +function hasEnginesFirst(engines, expectedEngines) { + for (let [i, expectedEngine] of expectedEngines.entries()) { + Assert.equal( + engines[i].name, + expectedEngine, + `Should have the expected engine in position ${i}` + ); + } +} + +engineSelector = SearchUtils.newSearchConfigEnabled + ? new SearchEngineSelector() + : new SearchEngineSelectorOld(); + +add_setup(async function () { + if (!SearchUtils.newSearchConfigEnabled) { + AddonTestUtils.init(GLOBAL_SCOPE); + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "42", + "42" + ); + await AddonTestUtils.promiseStartupManager(); + } + + await maybeSetupConfig(); +}); + +add_task(async function test_expected_distribution_engines() { + let searchService = new SearchService(); + for (const { distribution, locale = "en-US", region = "US", test } of tests) { + let config = await engineSelector.fetchEngineConfiguration({ + locale, + region, + distroID: distribution, + }); + let engines = await SearchTestUtils.searchConfigToEngines(config.engines); + searchService._engines = engines; + searchService._searchDefault = { + id: config.engines[0].webExtension.id, + locale: + config.engines[0]?.webExtension?.locale ?? SearchUtils.DEFAULT_TAG, + }; + engines = searchService._sortEnginesByDefaults(engines); + test(engines); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_duckduckgo.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_duckduckgo.js new file mode 100644 index 0000000000..ffbd3fb1ce --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_duckduckgo.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "ddg", + aliases: ["@duckduckgo", "@ddg"], + default: { + // Not included anywhere. + }, + available: { + excluded: [ + // Should be available everywhere. + ], + }, + details: [ + { + included: [{}], + domain: "duckduckgo.com", + telemetryId: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "ddg-esr" : "ddg", + searchUrlCode: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "t=ftsa" : "t=ffab", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_duckduckgo() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_ebay.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_ebay.js new file mode 100644 index 0000000000..ed8e5147ee --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_ebay.js @@ -0,0 +1,276 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const DOMAIN_LOCALES = { + "ebay-ca": ["en-CA"], + "ebay-ch": ["rm"], + "ebay-de": ["de", "dsb", "hsb"], + "ebay-es": ["an", "ast", "ca", "ca-valencia", "es-ES", "eu", "gl"], + "ebay-ie": ["ga-IE", "ie"], + "ebay-it": ["fur", "it", "lij", "sc"], + "ebay-nl": ["fy-NL", "nl"], + "ebay-uk": ["cy", "en-GB", "gd"], +}; + +const test = new SearchConfigTest({ + identifier: "ebay", + aliases: ["@ebay"], + default: { + // Not included anywhere. + }, + available: { + included: [ + { + // We don't currently enforce by region, but do locale instead. + // regions: [ + // "us", "gb", "ca", "ie", "fr", "it", "de", "at", "es", "nl", "ch", "au" + // ], + locales: [ + "an", + "ast", + "br", + "ca", + "ca-valencia", + "cy", + "de", + "dsb", + "en-CA", + "en-GB", + "es-ES", + "eu", + "fur", + "fr", + "fy-NL", + "ga-IE", + "gd", + "gl", + "hsb", + "it", + "lij", + "nl", + "rm", + "sc", + "wo", + ], + }, + { + regions: ["au", "be", "ca", "ch", "gb", "ie", "nl", "us"], + locales: ["en-US"], + }, + { + regions: ["gb"], + locales: ["sco"], + }, + ], + }, + suggestionUrlBase: "https://autosug.ebay.com/autosug", + details: [ + { + // Note: These should be based on region, but we don't currently enforce that. + // Note: the order here is important. A region/locale match higher up in the + // list will override a region/locale match lower down. + domain: "www.befr.ebay.be", + telemetryId: "ebay-be", + included: [ + { + regions: ["be"], + locales: ["br", "unknown", "en-US", "fr", "fy-NL", "nl", "wo"], + }, + ], + searchUrlCode: "mkrid=1553-53471-19255-0", + suggestUrlCode: "sId=23", + }, + { + domain: "www.ebay.at", + telemetryId: "ebay-at", + included: [ + { + regions: ["at"], + locales: ["de", "dsb", "hsb"], + }, + ], + searchUrlCode: "mkrid=5221-53469-19255-0", + suggestUrlCode: "sId=16", + }, + { + domain: "www.ebay.ca", + telemetryId: "ebay-ca", + included: [ + { + locales: DOMAIN_LOCALES["ebay-ca"], + }, + { + regions: ["ca"], + }, + ], + excluded: [ + { + locales: [ + ...DOMAIN_LOCALES["ebay-ch"], + ...DOMAIN_LOCALES["ebay-de"], + ...DOMAIN_LOCALES["ebay-es"], + ...DOMAIN_LOCALES["ebay-ie"], + ...DOMAIN_LOCALES["ebay-it"], + ...DOMAIN_LOCALES["ebay-nl"], + ...DOMAIN_LOCALES["ebay-uk"], + ], + }, + ], + searchUrlCode: "mkrid=706-53473-19255-0", + suggestUrlCode: "sId=2", + }, + { + domain: "www.ebay.ch", + telemetryId: "ebay-ch", + included: [ + { + locales: DOMAIN_LOCALES["ebay-ch"], + }, + { + regions: ["ch"], + }, + ], + excluded: [ + { + locales: [ + ...DOMAIN_LOCALES["ebay-ca"], + ...DOMAIN_LOCALES["ebay-es"], + ...DOMAIN_LOCALES["ebay-ie"], + ...DOMAIN_LOCALES["ebay-it"], + ...DOMAIN_LOCALES["ebay-nl"], + ...DOMAIN_LOCALES["ebay-uk"], + ], + }, + ], + searchUrlCode: "mkrid=5222-53480-19255-0", + suggestUrlCode: "sId=193", + }, + { + domain: "www.ebay.com", + telemetryId: "ebay", + included: [ + { + locales: ["unknown", "en-US"], + }, + ], + excluded: [{ regions: ["au", "be", "ca", "ch", "gb", "ie", "nl"] }], + searchUrlCode: "mkrid=711-53200-19255-0", + suggestUrlCode: "sId=0", + }, + { + domain: "www.ebay.com.au", + telemetryId: "ebay-au", + included: [ + { + regions: ["au"], + locales: ["cy", "unknown", "en-GB", "en-US", "gd"], + }, + ], + searchUrlCode: "mkrid=705-53470-19255-0", + suggestUrlCode: "sId=15", + }, + { + domain: "www.ebay.ie", + telemetryId: "ebay-ie", + included: [ + { + locales: DOMAIN_LOCALES["ebay-ie"], + }, + { + regions: ["ie"], + locales: ["cy", "unknown", "en-GB", "en-US", "gd"], + }, + ], + searchUrlCode: "mkrid=5282-53468-19255-0", + suggestUrlCode: "sId=205", + }, + { + domain: "www.ebay.co.uk", + telemetryId: "ebay-uk", + included: [ + { + locales: DOMAIN_LOCALES["ebay-uk"], + }, + { + locales: ["unknown", "en-US", "sco"], + regions: ["gb"], + }, + ], + excluded: [{ regions: ["au", "ie"] }], + searchUrlCode: "mkrid=710-53481-19255-0", + suggestUrlCode: "sId=3", + }, + { + domain: "www.ebay.de", + telemetryId: "ebay-de", + included: [ + { + locales: DOMAIN_LOCALES["ebay-de"], + }, + ], + excluded: [{ regions: ["at", "ch"] }], + searchUrlCode: "mkrid=707-53477-19255-0", + suggestUrlCode: "sId=77", + }, + { + domain: "www.ebay.es", + telemetryId: "ebay-es", + included: [ + { + locales: DOMAIN_LOCALES["ebay-es"], + }, + ], + searchUrlCode: "mkrid=1185-53479-19255-0", + suggestUrlCode: "sId=186", + }, + { + domain: "www.ebay.fr", + telemetryId: "ebay-fr", + included: [ + { + locales: ["br", "fr", "wo"], + }, + ], + excluded: [{ regions: ["be", "ca", "ch"] }], + searchUrlCode: "mkrid=709-53476-19255-0", + suggestUrlCode: "sId=71", + }, + { + domain: "www.ebay.it", + telemetryId: "ebay-it", + included: [ + { + locales: DOMAIN_LOCALES["ebay-it"], + }, + ], + searchUrlCode: "mkrid=724-53478-19255-0", + suggestUrlCode: "sId=101", + }, + { + domain: "www.ebay.nl", + telemetryId: "ebay-nl", + included: [ + { + locales: DOMAIN_LOCALES["ebay-nl"], + }, + { + locales: ["unknown", "en-US"], + regions: ["nl"], + }, + ], + excluded: [{ regions: ["be"] }], + searchUrlCode: "mkrid=1346-53482-19255-0", + suggestUrlCode: "sId=146", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_ebay() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_ecosia.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_ecosia.js new file mode 100644 index 0000000000..e9fc7241b1 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_ecosia.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "ecosia", + aliases: [], + default: { + // Not default anywhere. + }, + available: { + included: [ + { + locales: ["de"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "www.ecosia.org", + telemetryId: "ecosia", + searchUrlCode: "tt=mzl", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_ecosia() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_google.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_google.js new file mode 100644 index 0000000000..c6b9d6a991 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_google.js @@ -0,0 +1,171 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", +}); + +const test = new SearchConfigTest({ + identifier: "google", + aliases: ["@google"], + default: { + // Included everywhere apart from the exclusions below. These are basically + // just excluding what Yandex and Baidu include. + excluded: [ + { + regions: ["cn"], + locales: ["zh-CN"], + }, + ], + }, + available: { + excluded: [ + // Should be available everywhere. + ], + }, + details: [ + { + included: [{ regions: ["us"] }], + domain: "google.com", + telemetryId: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" + ? "google-b-1-e" + : "google-b-1-d", + codes: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" + ? "client=firefox-b-1-e" + : "client=firefox-b-1-d", + }, + { + excluded: [{ regions: ["us", "by", "kz", "ru", "tr"] }], + included: [{}], + domain: "google.com", + telemetryId: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "google-b-e" : "google-b-d", + codes: + SearchUtils.MODIFIED_APP_CHANNEL == "esr" + ? "client=firefox-b-e" + : "client=firefox-b-d", + }, + { + included: [{ regions: ["by", "kz", "ru", "tr"] }], + domain: "google.com", + telemetryId: "google-com-nocodes", + }, + ], +}); + +add_setup(async function () { + sinon.spy(NimbusFeatures.search, "onUpdate"); + sinon.stub(NimbusFeatures.search, "ready").resolves(); + await test.setup(); +}); + +add_task(async function test_searchConfig_google() { + await test.run(); +}); + +add_task(async function test_searchConfig_google_with_mozparam() { + // Test a couple of configurations with a MozParam set up. + const TEST_DATA = [ + { + locale: "en-US", + region: "US", + pref: "google_channel_us", + expected: "us_param", + }, + { + locale: "en-US", + region: "GB", + pref: "google_channel_row", + expected: "row_param", + }, + ]; + + const defaultBranch = Services.prefs.getDefaultBranch( + SearchUtils.BROWSER_SEARCH_PREF + ); + for (const testData of TEST_DATA) { + defaultBranch.setCharPref("param." + testData.pref, testData.expected); + } + + for (const testData of TEST_DATA) { + info(`Checking region ${testData.region}, locale ${testData.locale}`); + const engines = await test._getEngines(testData.region, testData.locale); + + Assert.ok( + engines[0].identifier.startsWith("google"), + "Should have the correct engine" + ); + console.log(engines[0]); + + const submission = engines[0].getSubmission("test", URLTYPE_SEARCH_HTML); + Assert.ok( + submission.uri.query.split("&").includes("channel=" + testData.expected), + "Should be including the correct MozParam parameter for the engine" + ); + } + + // Reset the pref values for next tests + for (const testData of TEST_DATA) { + defaultBranch.setCharPref("param." + testData.pref, ""); + } +}); + +add_task(async function test_searchConfig_google_with_nimbus() { + let sandbox = sinon.createSandbox(); + // Test a couple of configurations with a MozParam set up. + const TEST_DATA = [ + { + locale: "en-US", + region: "US", + expected: "nimbus_us_param", + }, + { + locale: "en-US", + region: "GB", + expected: "nimbus_row_param", + }, + ]; + + Assert.ok( + NimbusFeatures.search.onUpdate.called, + "Should register an update listener for Nimbus experiments" + ); + // Stub getVariable to populate the cache with our expected data + sandbox.stub(NimbusFeatures.search, "getVariable").returns([ + { key: "google_channel_us", value: "nimbus_us_param" }, + { key: "google_channel_row", value: "nimbus_row_param" }, + ]); + // Set the pref cache with Nimbus values + NimbusFeatures.search.onUpdate.firstCall.args[0](); + + for (const testData of TEST_DATA) { + info(`Checking region ${testData.region}, locale ${testData.locale}`); + const engines = await test._getEngines(testData.region, testData.locale); + + Assert.ok( + engines[0].identifier.startsWith("google"), + "Should have the correct engine" + ); + console.log(engines[0]); + + const submission = engines[0].getSubmission("test", URLTYPE_SEARCH_HTML); + Assert.ok( + NimbusFeatures.search.ready.called, + "Should wait for Nimbus to get ready" + ); + Assert.ok( + NimbusFeatures.search.getVariable, + "Should call NimbusFeatures.search.getVariable to populate the cache" + ); + Assert.ok( + submission.uri.query.split("&").includes("channel=" + testData.expected), + "Should be including the correct MozParam parameter for the engine" + ); + } + + sandbox.restore(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_mailru.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_mailru.js new file mode 100644 index 0000000000..e1a9c66b12 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_mailru.js @@ -0,0 +1,36 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "mailru", + aliases: [], + default: { + // Not default anywhere. + }, + available: { + included: [ + { + locales: ["ru"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "go.mail.ru", + telemetryId: "mailru", + codes: "gp=900200", + searchUrlCode: "frc=900200", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_mailru() { + await test.run(); +}).skip(); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js new file mode 100644 index 0000000000..4024385729 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js @@ -0,0 +1,36 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "qwant", + aliases: ["@qwant"], + default: { + // Not default anywhere. + }, + available: { + included: [ + { + locales: ["fr"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "www.qwant.com", + telemetryId: "qwant", + searchUrlCode: "client=brz-moz", + suggestUrlCode: "client=opensearch", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_qwant() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_rakuten.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_rakuten.js new file mode 100644 index 0000000000..2c0010e2b3 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_rakuten.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "rakuten", + aliases: [], + default: { + // Not default anywhere. + }, + available: { + included: [ + { + locales: ["ja", "ja-JP-macos"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "rakuten.co.jp", + telemetryId: "rakuten", + searchUrlCodeNotInQuery: "013ca98b.cd7c5f0c", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_rakuten() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js new file mode 100644 index 0000000000..51e71ff573 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js @@ -0,0 +1,209 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + JsonSchema: "resource://gre/modules/JsonSchema.sys.mjs", + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +/** + * Checks to see if a value is an object or not. + * + * @param {*} value + * The value to check. + * @returns {boolean} + */ +function isObject(value) { + return value != null && typeof value == "object" && !Array.isArray(value); +} + +/** + * This function modifies the schema to prevent allowing additional properties + * on objects. This is used to enforce that the schema contains everything that + * we deliver via the search configuration. + * + * These checks are not enabled in-product, as we want to allow older versions + * to keep working if we add new properties for whatever reason. + * + * @param {object} section + * The section to check to see if an additionalProperties flag should be added. + */ +function disallowAdditionalProperties(section) { + // It is generally acceptable for new properties to be added to the + // configuration as older builds will ignore them. + // + // As a result, we only check for new properties on nightly builds, and this + // avoids us having to uplift schema changes. This also helps preserve the + // schemas as documentation of "what was supported in this version". + if (!AppConstants.NIGHTLY_BUILD) { + return; + } + + // If the section is a `oneOf` section, avoid the additionalProperties check. + // Otherwise, the validator expects all properties of any `oneOf` item to be + // present. + if (isObject(section)) { + if (section.properties && !("recordType" in section.properties)) { + section.additionalProperties = false; + } + if ("then" in section) { + section.then.additionalProperties = false; + } + } + + for (let value of Object.values(section)) { + if (isObject(value)) { + disallowAdditionalProperties(value); + } else if (Array.isArray(value)) { + for (let item of value) { + disallowAdditionalProperties(item); + } + } + } +} + +let searchConfigSchemaV1; +let searchConfigSchema; + +add_setup(async function () { + searchConfigSchemaV1 = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-schema.json") + ); + searchConfigSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-v2-schema.json") + ); +}); + +async function checkSearchConfigValidates(schema, searchConfig) { + disallowAdditionalProperties(schema); + let validator = new JsonSchema.Validator(schema); + + for (let entry of searchConfig) { + // Records in Remote Settings contain additional properties independent of + // the schema. Hence, we don't want to validate their presence. + delete entry.schema; + delete entry.id; + delete entry.last_modified; + + let result = validator.validate(entry); + // entry.webExtension.id supports search-config v1. + let message = `Should validate ${ + entry.identifier ?? entry.recordType ?? entry.webExtension.id + }`; + if (!result.valid) { + message += `:\n${JSON.stringify(result.errors, null, 2)}`; + } + Assert.ok(result.valid, message); + + // All engine objects should have the base URL defined for each entry in + // entry.base.urls. + // Unfortunately this is difficult to enforce in the schema as it would + // need a `required` field that works across multiple levels. + if (entry.recordType == "engine") { + for (let urlEntry of Object.values(entry.base.urls)) { + Assert.ok( + urlEntry.base, + "Should have a base url for every URL defined on the top-level base object." + ); + } + } + } +} + +async function checkSearchConfigOverrideValidates( + schema, + searchConfigOverride +) { + let validator = new JsonSchema.Validator(schema); + + for (let entry of searchConfigOverride) { + // Records in Remote Settings contain additional properties independent of + // the schema. Hence, we don't want to validate their presence. + delete entry.schema; + delete entry.id; + delete entry.last_modified; + + let result = validator.validate(entry); + + let message = `Should validate ${entry.identifier ?? entry.telemetryId}`; + if (!result.valid) { + message += `:\n${JSON.stringify(result.errors, null, 2)}`; + } + Assert.ok(result.valid, message); + } +} + +add_task(async function test_search_config_validates_to_schema_v1() { + let selector = new SearchEngineSelectorOld(() => {}); + let searchConfig = await selector.getEngineConfiguration(); + + await checkSearchConfigValidates(searchConfigSchemaV1, searchConfig); +}); + +add_task(async function test_ui_schema_valid_v1() { + let uiSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-ui-schema.json") + ); + + await checkUISchemaValid(searchConfigSchemaV1, uiSchema); +}); + +add_task(async function test_search_config_override_validates_to_schema_v1() { + let selector = new SearchEngineSelectorOld(() => {}); + let searchConfigOverrides = await selector.getEngineConfigurationOverrides(); + let overrideSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-overrides-schema.json") + ); + + await checkSearchConfigOverrideValidates( + overrideSchema, + searchConfigOverrides + ); +}); + +add_task( + { skip_if: () => !SearchUtils.newSearchConfigEnabled }, + async function test_search_config_validates_to_schema() { + delete SearchUtils.newSearchConfigEnabled; + SearchUtils.newSearchConfigEnabled = true; + + let selector = new SearchEngineSelector(() => {}); + let searchConfig = await selector.getEngineConfiguration(); + + await checkSearchConfigValidates(searchConfigSchema, searchConfig); + } +); + +add_task( + { skip_if: () => !SearchUtils.newSearchConfigEnabled }, + async function test_ui_schema_valid() { + let uiSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-v2-ui-schema.json") + ); + + await checkUISchemaValid(searchConfigSchema, uiSchema); + } +); + +add_task( + { skip_if: () => !SearchUtils.newSearchConfigEnabled }, + async function test_search_config_override_validates_to_schema() { + let selector = new SearchEngineSelector(() => {}); + let searchConfigOverrides = + await selector.getEngineConfigurationOverrides(); + let overrideSchema = await IOUtils.readJSON( + PathUtils.join( + do_get_cwd().path, + "search-config-overrides-v2-schema.json" + ) + ); + + await checkSearchConfigOverrideValidates( + overrideSchema, + searchConfigOverrides + ); + } +); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchicons_validates.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchicons_validates.js new file mode 100644 index 0000000000..c830bb7ade --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchicons_validates.js @@ -0,0 +1,20 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +let searchIconsSchema; + +add_setup(async function () { + searchIconsSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-icons-schema.json") + ); +}); + +add_task(async function test_ui_schema_valid() { + let uiSchema = await IOUtils.readJSON( + PathUtils.join(do_get_cwd().path, "search-config-icons-ui-schema.json") + ); + + await checkUISchemaValid(searchIconsSchema, uiSchema); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js new file mode 100644 index 0000000000..9bd032c3b8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js @@ -0,0 +1,93 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + RemoteSettingsWorker: + "resource://services-settings/RemoteSettingsWorker.sys.mjs", +}); + +do_get_profile(); + +add_task(async function test_selector_db_out_of_date() { + let searchConfig = RemoteSettings(SearchUtils.SETTINGS_KEY); + + // Do an initial get to pre-seed the database. + await searchConfig.get(); + + // Now clear the database and re-fill it. + let db = searchConfig.db; + await db.clear(); + let databaseEntries = await db.list(); + Assert.equal(databaseEntries.length, 0, "Should have cleared the database."); + + // Add a dummy record with an out-of-date last modified. + if (SearchUtils.newSearchConfigEnabled) { + await RemoteSettingsWorker._execute("_test_only_import", [ + "main", + SearchUtils.SETTINGS_KEY, + [ + { + id: "b70edfdd-1c3f-4b7b-ab55-38cb048636c0", + identifier: "outofdate", + recordType: "engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + last_modified: 1606227264000, + }, + ], + 1606227264000, + ]); + } else { + await RemoteSettingsWorker._execute("_test_only_import", [ + "main", + SearchUtils.SETTINGS_KEY, + [ + { + id: "b70edfdd-1c3f-4b7b-ab55-38cb048636c0", + default: "yes", + webExtension: { id: "outofdate@search.mozilla.org" }, + appliesTo: [{ included: { everywhere: true } }], + last_modified: 1606227264000, + }, + ], + 1606227264000, + ]); + } + + // Now load the configuration and check we get what we expect. + let engineSelector = SearchUtils.newSearchConfigEnabled + ? new SearchEngineSelector() + : new SearchEngineSelectorOld(); + + let result = await engineSelector.fetchEngineConfiguration({ + // Use the fallback default locale/regions to get a simple list. + locale: "default", + region: "default", + }); + + if (SearchUtils.newSearchConfigEnabled) { + Assert.deepEqual( + result.engines.map(e => e.identifier), + ["google", "ddg", "wikipedia"], + "Should have returned the correct data." + ); + } else { + Assert.deepEqual( + result.engines.map(e => e.webExtension.id), + [ + "google@search.mozilla.org", + "wikipedia@search.mozilla.org", + "ddg@search.mozilla.org", + ], + "Should have returned the correct data." + ); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_wikipedia.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_wikipedia.js new file mode 100644 index 0000000000..54cf764830 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_wikipedia.js @@ -0,0 +1,171 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const testConfiguration = { + identifier: "wikipedia", + default: { + // Not default anywhere. + }, + available: { + excluded: [ + // Should be available everywhere. + ], + }, + details: [ + // Details generated below. + ], +}; + +/** + * Generates the expected details for the given locales and inserts + * them into the testConfiguration. + * + * @param {string[]} locales + * The locales for this details entry - which locales this variant of + * Wikipedia is expected to be deployed to. + * @param {string} [subDomainName] + * The expected sub domain name for this variant of Wikipedia. If not + * specified, defaults to the first item in the locales array. + * @param {string} [telemetrySuffix] + * The expected suffix used when this variant is reported via telemetry. If + * not specified, defaults to the first item in the array. If this is the + * empty string, then it "wikipedia" (i.e. no suffix) will be the expected + * value. + */ +function generateExpectedDetails(locales, subDomainName, telemetrySuffix) { + if (!subDomainName) { + subDomainName = locales[0]; + } + if (telemetrySuffix == undefined) { + telemetrySuffix = locales[0]; + } + testConfiguration.details.push({ + domain: `${subDomainName}.wikipedia.org`, + telemetryId: telemetrySuffix ? `wikipedia-${telemetrySuffix}` : "wikipedia", + aliases: ["@wikipedia"], + included: [{ locales }], + }); +} + +// This is an array of an array of arguments to be passed to generateExpectedDetails(). +// These are the locale, sub domain name and telemetry id suffix expectations for +// the test to check. +// Note that the expectations for en.wikipedia.com are generated in add_setup. +const LOCALES_INFO = [ + [["af"]], + [["an"]], + [["ar"]], + [["ast"]], + [["az"]], + [["be"]], + [["bg"]], + [["bn"]], + [["br"]], + [["bs"]], + [["ca", "ca-valencia"], "ca", "ca"], + [["cs"], "cs", "cz"], + [["cy"]], + [["da"]], + [["de"]], + [["dsb"]], + [["el"]], + [["eo"]], + [["cak", "es-AR", "es-CL", "es-ES", "es-MX", "trs"], "es", "es"], + [["et"]], + [["eu"]], + [["fa"]], + [["fi"]], + [["fr", "ff", "son"], "fr", "fr"], + [["fy-NL"], "fy", "fy-NL"], + [["ga-IE"], "ga", "ga-IE"], + [["gd"]], + [["gl"]], + [["gn"]], + [["gu-IN"], "gu", "gu"], + [["hi-IN"], "hi", "hi"], + [["he"]], + [["hr"]], + [["hsb"]], + [["hu"]], + [["hy-AM"], "hy", "hy"], + [["ia"]], + [["id"]], + [["is"]], + [["ja", "ja-JP-macos"], "ja", "ja"], + [["ka"]], + [["kab"]], + [["kk"]], + [["km"]], + [["kn"]], + [["ko"], "ko", "kr"], + [["it", "fur", "sc"], "it", "it"], + [["lij"]], + [["lo"]], + [["lt"]], + [["ltg"]], + [["lv"]], + [["mk"]], + [["mr"]], + [["ms"]], + [["my"]], + [["nb-NO"], "no", "NO"], + [["ne-NP"], "ne", "ne"], + [["nl"]], + [["nn-NO"], "nn", "NN"], + [["oc"]], + [["pa-IN"], "pa", "pa"], + [["pl", "szl"], "pl", "pl"], + [["pt-BR", "pt-PT"], "pt", "pt"], + [["rm"]], + [["ro"]], + [["ru"]], + [["si"]], + [["sk"]], + [["sl"]], + [["sq"]], + [["sr"]], + [["sv-SE"], "sv", "sv-SE"], + [["ta"]], + [["te"]], + [["th"]], + [["tl"]], + [["tr"]], + [["uk"]], + [["ur"]], + [["uz"]], + [["vi"]], + [["wo"]], + [["zh-CN"], "zh", "zh-CN"], + [["zh-TW"], "zh", "zh-TW"], +]; + +const test = new SearchConfigTest(testConfiguration); + +add_setup(async function () { + const allLocales = await test.getLocales(); + + // For the "en" version of Wikipedia, we ship it to all locales where other + // Wikipedias are not shipped. We form the list based on all-locales to avoid + // needing to update the test whenever all-locales is updated. + let enLocales = []; + for (let locale of allLocales) { + if (!LOCALES_INFO.find(d => d[0].includes(locale))) { + enLocales.push(locale); + } + } + + console.log("en.wikipedia.org expected locales are:", enLocales); + generateExpectedDetails(enLocales, "en", ""); + + for (let details of LOCALES_INFO) { + generateExpectedDetails(...details); + } + + await test.setup(); +}); + +add_task(async function test_searchConfig_wikipedia() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_yahoojp.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_yahoojp.js new file mode 100644 index 0000000000..2b80dffd17 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_yahoojp.js @@ -0,0 +1,36 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "yahoo-jp", + identifierExactMatch: true, + aliases: [], + default: { + // Not default anywhere. + }, + available: { + included: [ + { + locales: ["ja", "ja-JP-macos"], + }, + ], + }, + details: [ + { + included: [{}], + domain: "search.yahoo.co.jp", + telemetryId: "yahoo-jp", + searchUrlCode: "fr=mozff", + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_yahoojp() { + await test.run(); +}); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_yandex.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_yandex.js new file mode 100644 index 0000000000..6e2575b3c3 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_yandex.js @@ -0,0 +1,110 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const test = new SearchConfigTest({ + identifier: "yandex", + aliases: ["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"], + default: { + included: [ + { + regions: ["ru", "tr", "by", "kz"], + locales: ["ru", "tr", "be", "kk", "en-CA", "en-GB", "en-US"], + }, + ], + }, + available: { + included: [ + { + locales: ["az", "ru", "be", "kk", "tr"], + }, + { + regions: ["ru", "tr", "by", "kz"], + locales: ["en-CA", "en-GB", "en-US"], + }, + ], + }, + details: [ + { + included: [{ locales: ["az"] }], + domain: "yandex.az", + telemetryId: "yandex-az", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + { + included: [{ locales: { startsWith: ["en"] } }], + domain: "yandex.com", + telemetryId: "yandex-en", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + { + included: [{ locales: ["ru"] }], + domain: "yandex.ru", + telemetryId: "yandex-ru", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + { + included: [{ locales: ["be"] }], + domain: "yandex.by", + telemetryId: "yandex-by", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + { + included: [{ locales: ["kk"] }], + domain: "yandex.kz", + telemetryId: "yandex-kk", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + { + included: [{ locales: ["tr"] }], + domain: "yandex.com.tr", + telemetryId: "yandex-tr", + codes: { + searchbar: "clid=2186618", + keyword: "clid=2186621", + contextmenu: "clid=2186623", + homepage: "clid=2186617", + newtab: "clid=2186620", + }, + }, + ], +}); + +add_setup(async function () { + await test.setup(); +}); + +add_task(async function test_searchConfig_yandex() { + await test.run(); +}).skip(); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.toml b/toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.toml new file mode 100644 index 0000000000..8baff2a38d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.toml @@ -0,0 +1,65 @@ +[DEFAULT] +firefox-appdir = "browser" +head = "head_searchconfig.js" +dupe-manifest = "" +support-files = ["../../../../../../browser/locales/all-locales"] +tags = "searchconfig remote-settings" +# These are extensive tests, we don't need to run them on asan/tsan. +# They are also skipped for mobile and Thunderbird as these are specifically +# testing the Firefox config at the moment. +skip-if = [ + "os == 'android'", + "appname == 'thunderbird'", + "asan", + "tsan", + "debug", + "os == 'win' && ccov", +] +# These tests do take a little longer on Linux ccov, so allow that here. +requesttimeoutfactor = 2 + +["test_amazon.js"] + +["test_baidu.js"] + +["test_bing.js"] + +["test_distributions.js"] + +["test_duckduckgo.js"] + +["test_ebay.js"] + +["test_ecosia.js"] + +["test_google.js"] + +["test_mailru.js"] + +["test_qwant.js"] + +["test_rakuten.js"] + +["test_searchconfig_validates.js"] +support-files = [ + "../../../schema/search-config-overrides-schema.json", + "../../../schema/search-config-overrides-v2-schema.json", + "../../../schema/search-config-schema.json", + "../../../schema/search-config-ui-schema.json", + "../../../schema/search-config-v2-schema.json", + "../../../schema/search-config-v2-ui-schema.json", +] + +["test_searchicons_validates.js"] +support-files = [ + "../../../schema/search-config-icons-schema.json", + "../../../schema/search-config-icons-ui-schema.json", +] + +["test_selector_db_out_of_date.js"] + +["test_wikipedia.js"] + +["test_yahoojp.js"] + +["test_yandex.js"] diff --git a/toolkit/components/search/tests/xpcshell/simple-engines/basic/manifest.json b/toolkit/components/search/tests/xpcshell/simple-engines/basic/manifest.json new file mode 100644 index 0000000000..b62eb9bb2b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/simple-engines/basic/manifest.json @@ -0,0 +1,29 @@ +{ + "name": "basic", + "manifest_version": 2, + "version": "1.0", + "description": "basic", + "browser_specific_settings": { + "gecko": { + "id": "basic@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "basic", + "search_url": "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + "params": [ + { + "name": "search", + "value": "{searchTerms}" + }, + { + "name": "sourceId", + "value": "Mozilla-search" + } + ], + "suggest_url": "https://ar.wikipedia.org/w/api.php?action=opensearch&search={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/simple-engines/engines.json b/toolkit/components/search/tests/xpcshell/simple-engines/engines.json new file mode 100644 index 0000000000..170ed8c4fe --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/simple-engines/engines.json @@ -0,0 +1,53 @@ +{ + "data": [ + { + "webExtension": { + "id": "basic@search.mozilla.org", + "name": "basic", + "search_url": "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + "params": [ + { + "name": "search", + "value": "{searchTerms}" + }, + { + "name": "sourceId", + "value": "Mozilla-search" + } + ], + "suggest_url": "https://ar.wikipedia.org/w/api.php?action=opensearch&search={searchTerms}" + }, + "telemetryId": "basic-telemetry", + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes" + } + ] + }, + { + "webExtension": { + "id": "simple@search.mozilla.org", + "name": "Simple Engine", + "search_url": "https://example.com", + "params": [ + { + "name": "sourceId", + "value": "Mozilla-search" + }, + { + "name": "search", + "value": "{searchTerms}" + } + ], + "suggest_url": "https://example.com?search={searchTerms}" + }, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "no" + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/simple-engines/search-config-v2.json b/toolkit/components/search/tests/xpcshell/simple-engines/search-config-v2.json new file mode 100644 index 0000000000..6b0541ab2c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/simple-engines/search-config-v2.json @@ -0,0 +1,66 @@ +{ + "data": [ + { + "identifier": "basic", + "recordType": "engine", + "base": { + "name": "basic", + "urls": { + "search": { + "base": "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + "params": [ + { + "name": "sourceId", + "value": "Mozilla-search" + } + ], + "searchTermParamName": "search" + }, + "suggestions": { + "base": "https://ar.wikipedia.org/w/api.php?action=opensearch&search={searchTerms}" + } + } + }, + "variants": [ + { + "environment": { "allRegionsAndLocales": true }, + "telemetrySuffix": "telemetry" + } + ] + }, + { + "identifier": "simple", + "recordType": "engine", + "base": { + "name": "Simple Engine", + "urls": { + "search": { + "base": "https://example.com", + "params": [ + { + "name": "sourceId", + "value": "Mozilla-search" + } + ], + "searchTermParamName": "search" + }, + "suggestions": { "base": "https://example.com?search={searchTerms}" } + } + }, + "variants": [ + { + "environment": { "allRegionsAndLocales": true } + } + ] + }, + { + "recordType": "defaultEngines", + "globalDefault": "basic", + "specificDefaults": [] + }, + { + "recordType": "engineOrders", + "orders": [] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/simple-engines/simple/manifest.json b/toolkit/components/search/tests/xpcshell/simple-engines/simple/manifest.json new file mode 100644 index 0000000000..67d2974753 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/simple-engines/simple/manifest.json @@ -0,0 +1,29 @@ +{ + "name": "Simple Engine", + "manifest_version": 2, + "version": "1.0", + "description": "Simple engine with a different name from the WebExtension id prefix", + "browser_specific_settings": { + "gecko": { + "id": "simple@search.mozilla.org" + } + }, + "hidden": true, + "chrome_settings_overrides": { + "search_provider": { + "name": "Simple Engine", + "search_url": "https://example.com", + "params": [ + { + "name": "sourceId", + "value": "Mozilla-search" + }, + { + "name": "search", + "value": "{searchTerms}" + } + ], + "suggest_url": "https://example.com?search={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/engines.json b/toolkit/components/search/tests/xpcshell/test-extensions/engines.json new file mode 100644 index 0000000000..0bedb236ab --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/engines.json @@ -0,0 +1,128 @@ +{ + "data": [ + { + "webExtension": { + "id": "plainengine@search.mozilla.org", + "name": "Plain", + "search_url": "https://duckduckgo.com/", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "contextmenu", + "value": "ffcm" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "keyword", + "value": "ffab" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "searchbar", + "value": "ffsb" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "homepage", + "value": "ffhp" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "newtab", + "value": "ffnt" + } + ], + "suggest_url": "https://ac.duckduckgo.com/ac/q={searchTerms}&type=list" + }, + "orderHint": 10000, + "appliesTo": [ + { + "included": { "everywhere": true }, + "default": "yes-if-no-other" + } + ] + }, + { + "webExtension": { + "id": "special-engine@search.mozilla.org", + "name": "Special", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "client", + "condition": "purpose", + "purpose": "keyword", + "value": "firefox-b-1-ab" + }, + { + "name": "client", + "condition": "purpose", + "purpose": "searchbar", + "value": "firefox-b-1" + } + ], + "suggest_url": "https://www.google.com/complete/search?client=firefox&q={searchTerms}" + }, + "orderHint": 7000, + "appliesTo": [ + { + "included": { "regions": ["tr"] }, + "default": "yes" + }, + { + "included": { "everywhere": true } + } + ] + }, + { + "webExtension": { + "id": "multilocale@search.mozilla.org", + "default_locale": "af", + "searchProvider": { + "an": { + "name": "Multilocale AN", + "description": "A enciclopedia Libre", + "search_url": "https://an.wikipedia.org/wiki/Especial:Mirar", + "suggest_url": "https://an.wikipedia.org/w/api.php" + }, + "af": { + "name": "Multilocale AF", + "description": "Wikipedia, die vrye ensiklopedie", + "search_url": "https://af.wikipedia.org/wiki/Spesiaal:Soek", + "suggest_url": "https://af.wikipedia.org/w/api.php" + } + } + }, + "orderHint": 6000, + "appliesTo": [ + { + "included": { "regions": ["an"] }, + "webExtension": { + "locales": ["an"] + }, + "default": "yes" + }, + { + "included": { "regions": ["af"] }, + "webExtension": { + "locales": ["af", "an"] + }, + "orderHint": 6500 + } + ] + } + ] +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/af/messages.json b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/af/messages.json new file mode 100644 index 0000000000..95e49f9bc5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/af/messages.json @@ -0,0 +1,20 @@ +{ + "extensionName": { + "message": "Multilocale AF" + }, + "extensionDescription": { + "message": "Wikipedia, die vrye ensiklopedie" + }, + "url_lang": { + "message": "af" + }, + "searchUrl": { + "message": "https://af.wikipedia.org/wiki/Spesiaal:Soek" + }, + "suggestUrl": { + "message": "https://af.wikipedia.org/w/api.php" + }, + "extensionIcon": { + "message": "favicon-af.ico" + } +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/an/messages.json b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/an/messages.json new file mode 100644 index 0000000000..6222338596 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/_locales/an/messages.json @@ -0,0 +1,20 @@ +{ + "extensionName": { + "message": "Multilocale AN" + }, + "extensionDescription": { + "message": "A enciclopedia Libre" + }, + "url_lang": { + "message": "an" + }, + "searchUrl": { + "message": "https://an.wikipedia.org/wiki/Especial:Mirar" + }, + "suggestUrl": { + "message": "https://an.wikipedia.org/w/api.php" + }, + "extensionIcon": { + "message": "favicon-an.ico" + } +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-af.ico b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-af.ico Binary files differnew file mode 100644 index 0000000000..4314071e24 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-af.ico diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-an.ico b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-an.ico Binary files differnew file mode 100644 index 0000000000..dda80dfd88 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/favicon-an.ico diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/manifest.json b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/manifest.json new file mode 100644 index 0000000000..a117ffb0db --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/multilocale/manifest.json @@ -0,0 +1,23 @@ +{ + "name": "__MSG_extensionName__", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "multilocale@search.mozilla.org" + } + }, + "hidden": true, + "description": "__MSG_extensionDescription__", + "icons": { + "16": "__MSG_extensionIcon__" + }, + "default_locale": "af", + "chrome_settings_overrides": { + "search_provider": { + "name": "__MSG_extensionName__", + "search_url": "__MSG_searchUrl__", + "suggest_url": "__MSG_searchUrl__" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/favicon.ico b/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/favicon.ico Binary files differnew file mode 100644 index 0000000000..dda80dfd88 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/favicon.ico diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/manifest.json b/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/manifest.json new file mode 100644 index 0000000000..cabb4c9f9a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/plainengine/manifest.json @@ -0,0 +1,58 @@ +{ + "name": "Plain", + "description": "Plain Engine", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "plainengine@search.mozilla.org" + } + }, + "hidden": true, + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "Plain", + "search_url": "https://duckduckgo.com/", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "contextmenu", + "value": "ffcm" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "keyword", + "value": "ffab" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "searchbar", + "value": "ffsb" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "homepage", + "value": "ffhp" + }, + { + "name": "t", + "condition": "purpose", + "purpose": "newtab", + "value": "ffnt" + } + ], + "suggest_url": "https://ac.duckduckgo.com/ac/q={searchTerms}&type=list" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/favicon.ico b/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/favicon.ico Binary files differnew file mode 100644 index 0000000000..82339b3b1d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/favicon.ico diff --git a/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/manifest.json b/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/manifest.json new file mode 100644 index 0000000000..1568c6ed55 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test-extensions/special-engine/manifest.json @@ -0,0 +1,40 @@ +{ + "name": "Special", + "description": "Special Engine", + "manifest_version": 2, + "version": "1.0", + "browser_specific_settings": { + "gecko": { + "id": "special-engine@search.mozilla.org" + } + }, + "hidden": true, + "icons": { + "16": "favicon.ico" + }, + "chrome_settings_overrides": { + "search_provider": { + "name": "Special", + "search_url": "https://www.google.com/search", + "params": [ + { + "name": "q", + "value": "{searchTerms}" + }, + { + "name": "client", + "condition": "purpose", + "purpose": "keyword", + "value": "firefox-b-1-ab" + }, + { + "name": "client", + "condition": "purpose", + "purpose": "searchbar", + "value": "firefox-b-1" + } + ], + "suggest_url": "https://www.google.com/complete/search?client=firefox&q={searchTerms}" + } + } +} diff --git a/toolkit/components/search/tests/xpcshell/test_SearchStaticData.js b/toolkit/components/search/tests/xpcshell/test_SearchStaticData.js new file mode 100644 index 0000000000..741953a1cf --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_SearchStaticData.js @@ -0,0 +1,40 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests the SearchStaticData module. + */ + +"use strict"; + +const { SearchStaticData } = ChromeUtils.importESModule( + "resource://gre/modules/SearchStaticData.sys.mjs" +); + +function run_test() { + Assert.ok( + SearchStaticData.getAlternateDomains("www.google.com").includes( + "www.google.fr" + ) + ); + Assert.ok( + SearchStaticData.getAlternateDomains("www.google.fr").includes( + "www.google.com" + ) + ); + Assert.ok( + SearchStaticData.getAlternateDomains("www.google.com").every(d => + d.startsWith("www.google.") + ) + ); + Assert.ok(!SearchStaticData.getAlternateDomains("google.com").length); + + // Test that methods from SearchStaticData module can be overwritten, + // needed for hotfixing. + let backup = SearchStaticData.getAlternateDomains; + SearchStaticData.getAlternateDomains = () => ["www.bing.fr"]; + Assert.deepEqual(SearchStaticData.getAlternateDomains("www.bing.com"), [ + "www.bing.fr", + ]); + SearchStaticData.getAlternateDomains = backup; +} diff --git a/toolkit/components/search/tests/xpcshell/test_appDefaultEngine.js b/toolkit/components/search/tests/xpcshell/test_appDefaultEngine.js new file mode 100644 index 0000000000..243c0cd967 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_appDefaultEngine.js @@ -0,0 +1,41 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that appDefaultEngine property is set and switches correctly. + */ + +"use strict"; + +add_setup(async function () { + Region._setHomeRegion("an", false); + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines("test-extensions"); +}); + +add_task(async function test_appDefaultEngine() { + await Promise.all([Services.search.init(), promiseAfterSettings()]); + Assert.equal( + Services.search.appDefaultEngine.name, + "Multilocale AN", + "Should have returned the correct app default engine" + ); +}); + +add_task(async function test_changeRegion() { + // Now change the region, and check we get the correct default according to + // the config file. + + // Note: the test could be done with changing regions or locales. The important + // part is that the default engine is changing across the switch, and that + // the engine is not the first one in the new sorted engines list. + await promiseSetHomeRegion("tr"); + + Assert.equal( + Services.search.appDefaultEngine.name, + // Very important this default is not the first one in the list (which is + // the next fallback if the config one can't be found). + "Special", + "Should have returned the correct engine for the new locale" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_async.js b/toolkit/components/search/tests/xpcshell/test_async.js new file mode 100644 index 0000000000..d66d402743 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_async.js @@ -0,0 +1,36 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines("simple-engines"); + Services.fog.initializeFOG(); +}); + +add_task(async function test_async() { + Assert.ok(!Services.search.isInitialized); + + let aStatus = await Services.search.init(); + Assert.ok(Components.isSuccessCode(aStatus)); + Assert.ok(Services.search.isInitialized); + + // test engines from dir are not loaded. + let engines = await Services.search.getEngines(); + Assert.equal(engines.length, 2); + + // test jar engine is loaded ok. + let engine = Services.search.getEngineByName("basic"); + Assert.notEqual(engine, null); + Assert.ok(engine.isAppProvided, "Should be shown as an app-provided engine"); + + engine = Services.search.getEngineByName("Simple Engine"); + Assert.notEqual(engine, null); + Assert.ok(engine.isAppProvided, "Should be shown as an app-provided engine"); + + // Check if there is a value for startup_time + Assert.notEqual( + await Glean.searchService.startupTime.testGetValue(), + undefined, + "Should have a value stored in startup_time" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_config_engine_params.js b/toolkit/components/search/tests/xpcshell/test_config_engine_params.js new file mode 100644 index 0000000000..190ccbba2c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_config_engine_params.js @@ -0,0 +1,68 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_setup(async function () { + await SearchTestUtils.useTestEngines("method-extensions"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_get_extension() { + let engine = Services.search.getEngineByName("Get Engine"); + Assert.notEqual(engine, null, "Should have found an engine"); + + let url = engine.wrappedJSObject._getURLOfType(SearchUtils.URL_TYPE.SEARCH); + Assert.equal(url.method, "GET", "Search URLs method is GET"); + + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://example.com/?config=1&search=foo", + "Search URLs should match" + ); + + let submissionSuggest = engine.getSubmission( + "bar", + SearchUtils.URL_TYPE.SUGGEST_JSON + ); + Assert.equal( + submissionSuggest.uri.spec, + "https://example.com/?config=1&suggest=bar", + "Suggest URLs should match" + ); +}); + +add_task(async function test_post_extension() { + let engine = Services.search.getEngineByName("Post Engine"); + Assert.ok(!!engine, "Should have found an engine"); + + let url = engine.wrappedJSObject._getURLOfType(SearchUtils.URL_TYPE.SEARCH); + Assert.equal(url.method, "POST", "Search URLs method is POST"); + + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://example.com/", + "Search URLs should match" + ); + Assert.equal( + submission.postData.data.data, + "config=1&search=foo", + "Search postData should match" + ); + + let submissionSuggest = engine.getSubmission( + "bar", + SearchUtils.URL_TYPE.SUGGEST_JSON + ); + Assert.equal( + submissionSuggest.uri.spec, + "https://example.com/", + "Suggest URLs should match" + ); + Assert.equal( + submissionSuggest.postData.data.data, + "config=1&suggest=bar", + "Suggest postData should match" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_defaultEngine.js b/toolkit/components/search/tests/xpcshell/test_defaultEngine.js new file mode 100644 index 0000000000..9bab6b5423 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_defaultEngine.js @@ -0,0 +1,120 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that defaultEngine property can be set and yields the proper events and\ + * behavior (search results) + */ + +"use strict"; + +let engine1; +let engine2; + +add_setup(async () => { + do_get_profile(); + Services.fog.initializeFOG(); + + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); + + await Services.search.init(); + + engine1 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + engine2 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine2.xml`, + }); +}); + +function promiseDefaultNotification() { + return SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); +} + +add_task(async function test_defaultEngine() { + let promise = promiseDefaultNotification(); + Services.search.defaultEngine = engine1; + Assert.equal((await promise).wrappedJSObject, engine1); + Assert.equal(Services.search.defaultEngine.wrappedJSObject, engine1); + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-Test search engine", + displayName: "Test search engine", + loadPath: "[http]localhost/test-search-engine.xml", + submissionUrl: "https://www.google.com/search?q=", + verified: "verified", + }, + }); + + promise = promiseDefaultNotification(); + Services.search.defaultEngine = engine2; + Assert.equal((await promise).wrappedJSObject, engine2); + Assert.equal(Services.search.defaultEngine.wrappedJSObject, engine2); + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-A second test engine", + displayName: "A second test engine", + loadPath: "[http]localhost/a-second-test-engine.xml", + submissionUrl: "https://duckduckgo.com/?q=", + verified: "verified", + }, + }); + + promise = promiseDefaultNotification(); + Services.search.defaultEngine = engine1; + Assert.equal((await promise).wrappedJSObject, engine1); + Assert.equal(Services.search.defaultEngine.wrappedJSObject, engine1); + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-Test search engine", + displayName: "Test search engine", + loadPath: "[http]localhost/test-search-engine.xml", + submissionUrl: "https://www.google.com/search?q=", + verified: "verified", + }, + }); +}); + +add_task(async function test_telemetry_empty_submission_url() { + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}../opensearch/simple.xml`, + setAsDefaultPrivate: true, + }); + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-simple", + displayName: "simple", + loadPath: "[http]localhost/simple.xml", + submissionUrl: "blank:", + verified: "verified", + }, + private: { + engineId: "", + displayName: "", + loadPath: "", + submissionUrl: "blank:", + verified: "", + }, + }); +}); + +add_task(async function test_switch_with_invalid_overriddenBy() { + engine1.wrappedJSObject.setAttr("overriddenBy", "random@id"); + + consoleAllowList.push( + "Test search engine had overriddenBy set, but no _overriddenData" + ); + + let promise = promiseDefaultNotification(); + Services.search.defaultEngine = engine2; + Assert.equal((await promise).wrappedJSObject, engine2); + Assert.equal(Services.search.defaultEngine.wrappedJSObject, engine2); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_defaultEngine_experiments.js b/toolkit/components/search/tests/xpcshell/test_defaultEngine_experiments.js new file mode 100644 index 0000000000..96fdf03d58 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_defaultEngine_experiments.js @@ -0,0 +1,480 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that defaultEngine property can be set and yields the proper events and\ + * behavior (search results) + */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", +}); + +let getVariableStub; + +let defaultGetVariable = name => { + if (name == "seperatePrivateDefaultUIEnabled") { + return true; + } + if (name == "seperatePrivateDefaultUrlbarResultEnabled") { + return false; + } + return undefined; +}; + +add_setup(async () => { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + sinon.spy(NimbusFeatures.searchConfiguration, "onUpdate"); + sinon.stub(NimbusFeatures.searchConfiguration, "ready").resolves(); + getVariableStub = sinon.stub( + NimbusFeatures.searchConfiguration, + "getVariable" + ); + getVariableStub.callsFake(defaultGetVariable); + + do_get_profile(); + Services.fog.initializeFOG(); + + await SearchTestUtils.useTestEngines("data1"); + + await AddonTestUtils.promiseStartupManager(); + + let promiseSaved = promiseSaveSettingsData(); + await Services.search.init(); + await promiseSaved; +}); + +async function switchExperiment(newExperiment) { + let promiseReloaded = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + let promiseSaved = promiseSaveSettingsData(); + + // Stub getVariable to populate the cache with our expected data + getVariableStub.callsFake(name => { + if (name == "experiment") { + return newExperiment; + } + return defaultGetVariable(name); + }); + for (let call of NimbusFeatures.searchConfiguration.onUpdate.getCalls()) { + call.args[0](); + } + + await promiseReloaded; + await promiseSaved; +} + +function getSettingsAttribute(setting, isAppProvided) { + return Services.search.wrappedJSObject._settings.getVerifiedMetaDataAttribute( + setting, + isAppProvided + ); +} + +add_task(async function test_experiment_setting() { + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the application default engine as default" + ); + + // Start the experiment. + await switchExperiment("exp1"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have set the experiment engine as default" + ); + + // End the experiment. + await switchExperiment(""); + + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have reset the default engine to the application default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should have kept the saved attribute as empty" + ); +}); + +add_task(async function test_experiment_setting_to_same_as_user() { + Services.search.defaultEngine = Services.search.getEngineByName("engine2"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have the user selected engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault" + ); + + // Start the experiment, ensure user default is maintained. + await switchExperiment("exp1"); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault" + ); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine2", + "Should have set the experiment engine as default" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have set the experiment engine as default" + ); + + // End the experiment. + await switchExperiment(""); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine1", + "Should have set the app default engine correctly" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have kept the engine the same " + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault", + "Should have kept the saved attribute as the user's preference" + ); +}); + +add_task(async function test_experiment_setting_user_changed_back_during() { + Services.search.defaultEngine = Services.search.getEngineByName("engine1"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the application default engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should have an empty settings attribute" + ); + + // Start the experiment. + await switchExperiment("exp1"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have set the experiment engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should still have an empty settings attribute" + ); + + // User resets to the original default engine. + Services.search.defaultEngine = Services.search.getEngineByName("engine1"); + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the user selected engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine1@search.mozilla.orgdefault" + ); + + // Ending the experiment should keep the original default and reset the + // saved attribute. + await switchExperiment(""); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine1", + "Should have set the app default engine correctly" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have kept the engine the same" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should have reset the saved attribute to empty after the experiment ended" + ); +}); + +add_task(async function test_experiment_setting_user_changed_back_private() { + Services.search.defaultPrivateEngine = + Services.search.getEngineByName("engine1"); + + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine1", + "Should have the user selected engine as default" + ); + Assert.equal( + getSettingsAttribute( + "privateDefaultEngineId", + Services.search.defaultPrivateEngine.isAppProvided + ), + "", + "Should have an empty settings attribute" + ); + + // Start the experiment. + await switchExperiment("exp2"); + + Assert.equal( + Services.search.defaultPrivateEngine.name, + "exp2", + "Should have set the experiment engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should still have an empty settings attribute" + ); + + // User resets to the original default engine. + Services.search.defaultPrivateEngine = + Services.search.getEngineByName("engine1"); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine1", + "Should have the user selected engine as default" + ); + Assert.equal( + getSettingsAttribute( + "privateDefaultEngineId", + Services.search.defaultPrivateEngine.isAppProvided + ), + "engine1@search.mozilla.orgdefault" + ); + + // Ending the experiment should keep the original default and reset the + // saved attribute. + await switchExperiment(""); + + Assert.equal(Services.search.appPrivateDefaultEngine.name, "engine1"); + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have kept the engine the same " + ); + Assert.equal( + getSettingsAttribute( + "privateDefaultEngineId", + Services.search.defaultPrivateEngine.isAppProvided + ), + "", + "Should have reset the saved attribute to empty after the experiment ended" + ); +}); + +add_task(async function test_experiment_setting_user_changed_to_other_during() { + Services.search.defaultEngine = Services.search.getEngineByName("engine1"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the application default engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should have an empty settings attribute" + ); + + // Start the experiment. + await switchExperiment("exp3"); + + Assert.equal( + Services.search.defaultEngine.name, + "exp3", + "Should have set the experiment engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should still have an empty settings attribute" + ); + + // User changes to a different default engine + Services.search.defaultEngine = Services.search.getEngineByName("engine2"); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have the user selected engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault", + "Should have correctly set the user's default in settings" + ); + + // Ending the experiment should keep the original default and reset the + // saved attribute. + await switchExperiment(""); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine1", + "Should have set the app default engine correctly" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have kept the user's choice of engine" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault", + "Should have kept the user's choice in settings" + ); +}); + +add_task(async function test_experiment_setting_user_hid_app_default_during() { + // Add all the test engines to be general search engines. This is important + // for the test, as the removed experiment engine needs to be a general search + // engine, and the first in the list (aided by the orderHint in + // data1/engines.json). + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.add("engine1@search.mozilla.org"); + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.add("engine2@search.mozilla.org"); + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.add("exp2@search.mozilla.org"); + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.add("exp3@search.mozilla.org"); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + Services.search.defaultEngine = Services.search.getEngineByName("engine1"); + + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the application default engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should have an empty settings attribute" + ); + + // Start the experiment. + await switchExperiment("exp3"); + + Assert.equal( + Services.search.defaultEngine.name, + "exp3", + "Should have set the experiment engine as default" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "", + "Should still have an empty settings attribute" + ); + + // User hides the original application engine + await Services.search.removeEngine( + Services.search.getEngineByName("engine1") + ); + Assert.equal( + Services.search.getEngineByName("engine1").hidden, + true, + "Should have hid the selected engine" + ); + + // Ending the experiment should keep the original default and reset the + // saved attribute. + await switchExperiment(""); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine1", + "Should have set the app default engine correctly" + ); + Assert.equal( + Services.search.defaultEngine.hidden, + false, + "Should not have set default engine to an engine that is hidden" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have reset the user's engine to the next available engine" + ); + Assert.equal( + getSettingsAttribute( + "defaultEngineId", + Services.search.defaultEngine.isAppProvided + ), + "engine2@search.mozilla.orgdefault", + "Should have saved the choice in settings" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js b/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js new file mode 100644 index 0000000000..12cb6568e7 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js @@ -0,0 +1,406 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * This test is checking the fallbacks when an engine that is default is + * removed or hidden. + * + * The fallback procedure is: + * + * - Region/Locale default (if visible) + * - First visible engine + * - If no other visible engines, unhide the region/locale default and use it. + */ + +let appDefault; +let appPrivateDefault; + +add_setup(async function () { + useHttpServer(); + await SearchTestUtils.useTestEngines(); + + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + SearchUtils.GENERAL_SEARCH_ENGINE_IDS = new Set([ + "engine-resourceicon@search.mozilla.org", + "engine-reordered@search.mozilla.org", + ]); + + await AddonTestUtils.promiseStartupManager(); + + appDefault = await Services.search.getDefault(); + appPrivateDefault = await Services.search.getDefaultPrivate(); +}); + +function getDefault(privateMode) { + return privateMode + ? Services.search.getDefaultPrivate() + : Services.search.getDefault(); +} + +function setDefault(privateMode, engine) { + return privateMode + ? Services.search.setDefaultPrivate( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ) + : Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); +} + +async function checkFallbackDefaultRegion(checkPrivate) { + let defaultEngine = checkPrivate ? appPrivateDefault : appDefault; + let expectedDefaultNotification = checkPrivate + ? SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + : SearchUtils.MODIFIED_TYPE.DEFAULT; + Services.search.restoreDefaultEngines(); + + let otherEngine = Services.search.getEngineByName("engine-chromeicon"); + await setDefault(checkPrivate, otherEngine); + + Assert.notEqual( + otherEngine, + defaultEngine, + "Sanity check engines are different" + ); + + const observer = new SearchObserver( + [ + expectedDefaultNotification, + // For hiding (removing) the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ], + expectedDefaultNotification + ); + + await Services.search.removeEngine(otherEngine); + + let notified = await observer.promise; + + Assert.ok(otherEngine.hidden, "Should have hidden the removed engine"); + Assert.equal( + (await getDefault(checkPrivate)).name, + defaultEngine.name, + "Should have reverted the defaultEngine to the region default" + ); + Assert.equal( + notified.name, + defaultEngine.name, + "Should have notified the correct default engine" + ); +} + +add_task(async function test_default_fallback_to_region_default() { + await checkFallbackDefaultRegion(false); +}); + +add_task(async function test_default_private_fallback_to_region_default() { + await checkFallbackDefaultRegion(true); +}); + +async function checkFallbackFirstVisible(checkPrivate) { + let defaultEngine = checkPrivate ? appPrivateDefault : appDefault; + let expectedDefaultNotification = checkPrivate + ? SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + : SearchUtils.MODIFIED_TYPE.DEFAULT; + Services.search.restoreDefaultEngines(); + + let otherEngine = Services.search.getEngineByName("engine-chromeicon"); + await setDefault(checkPrivate, otherEngine); + await Services.search.removeEngine(defaultEngine); + + Assert.notEqual( + otherEngine, + defaultEngine, + "Sanity check engines are different" + ); + + const observer = new SearchObserver( + checkPrivate + ? [ + expectedDefaultNotification, + // For hiding (removing) the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ] + : [ + expectedDefaultNotification, + // For hiding (removing) the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ], + expectedDefaultNotification + ); + + await Services.search.removeEngine(otherEngine); + + let notified = await observer.promise; + + Assert.equal( + (await getDefault(checkPrivate)).name, + "engine-resourceicon", + "Should have set the default engine to the first visible general engine" + ); + Assert.equal( + notified.name, + "engine-resourceicon", + "Should have notified the correct default general engine" + ); +} + +add_task(async function test_default_fallback_to_first_gen_visible() { + await checkFallbackFirstVisible(false); +}); + +add_task(async function test_default_private_fallback_to_first_gen_visible() { + await checkFallbackFirstVisible(true); +}); + +// Removing all visible engines affects both the default and private default +// engines. +add_task(async function test_default_fallback_when_no_others_visible() { + // Remove all but one of the visible engines. + let visibleEngines = await Services.search.getVisibleEngines(); + for (let i = 0; i < visibleEngines.length - 1; i++) { + await Services.search.removeEngine(visibleEngines[i]); + } + Assert.equal( + (await Services.search.getVisibleEngines()).length, + 1, + "Should only have one visible engine" + ); + + const observer = new SearchObserver( + [ + // Unhiding of the default engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + // Change of the default. + SearchUtils.MODIFIED_TYPE.DEFAULT, + // Unhiding of the default private. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE, + // Hiding the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ], + SearchUtils.MODIFIED_TYPE.DEFAULT + ); + + // Now remove the last engine, which should set the new default. + await Services.search.removeEngine(visibleEngines[visibleEngines.length - 1]); + + let notified = await observer.promise; + + Assert.equal( + (await getDefault(false)).name, + appDefault.name, + "Should fallback to the app default engine after removing all engines" + ); + Assert.equal( + (await getDefault(true)).name, + appPrivateDefault.name, + "Should fallback to the app default private engine after removing all engines" + ); + Assert.equal( + notified.name, + appDefault.name, + "Should have notified the correct default engine" + ); + Assert.ok( + !appPrivateDefault.hidden, + "Should have unhidden the app default private engine" + ); + Assert.equal( + (await Services.search.getVisibleEngines()).length, + 2, + "Should now have two engines visible" + ); +}); + +add_task(async function test_default_fallback_remove_default_no_visible() { + // Remove all but the default engine. + Services.search.defaultPrivateEngine = Services.search.defaultEngine; + let visibleEngines = await Services.search.getVisibleEngines(); + for (let engine of visibleEngines) { + if (engine.name != appDefault.name) { + await Services.search.removeEngine(engine); + } + } + Assert.equal( + (await Services.search.getVisibleEngines()).length, + 1, + "Should only have one visible engine" + ); + + const observer = new SearchObserver( + [ + // Unhiding of the default engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + // Change of the default. + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE, + // Hiding the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ], + SearchUtils.MODIFIED_TYPE.DEFAULT + ); + + // Now remove the last engine, which should set the new default. + await Services.search.removeEngine(appDefault); + + let notified = await observer.promise; + + Assert.equal( + (await getDefault(false)).name, + "engine-resourceicon", + "Should fallback the default engine to the first general search engine" + ); + Assert.equal( + (await getDefault(true)).name, + "engine-resourceicon", + "Should fallback the default private engine to the first general search engine" + ); + Assert.equal( + notified.name, + "engine-resourceicon", + "Should have notified the correct default engine" + ); + Assert.ok( + !Services.search.getEngineByName("engine-resourceicon").hidden, + "Should have unhidden the new engine" + ); + Assert.equal( + (await Services.search.getVisibleEngines()).length, + 1, + "Should now have one engines visible" + ); +}); + +add_task( + async function test_default_fallback_remove_default_no_visible_or_general() { + // Reset. + Services.search.restoreDefaultEngines(); + Services.search.defaultEngine = Services.search.defaultPrivateEngine = + appPrivateDefault; + + // Remove all but the default engine. + let visibleEngines = await Services.search.getVisibleEngines(); + for (let engine of visibleEngines) { + if (engine.name != appPrivateDefault.name) { + await Services.search.removeEngine(engine); + } + } + Assert.deepEqual( + (await Services.search.getVisibleEngines()).map(e => e.name), + appPrivateDefault.name, + "Should only have one visible engine" + ); + + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.clear(); + + const observer = new SearchObserver( + [ + // Unhiding of the default engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + // Change of the default. + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE, + // Hiding the engine. + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.MODIFIED_TYPE.REMOVED, + ], + SearchUtils.MODIFIED_TYPE.DEFAULT + ); + + // Now remove the last engine, which should set the new default. + await Services.search.removeEngine(appPrivateDefault); + + let notified = await observer.promise; + + Assert.equal( + (await getDefault(false)).name, + "Test search engine", + "Should fallback to the first engine that isn't a general search engine" + ); + Assert.equal( + (await getDefault(true)).name, + "Test search engine", + "Should fallback the private engine to the first engine that isn't a general search engine" + ); + Assert.equal( + notified.name, + "Test search engine", + "Should have notified the correct default engine" + ); + Assert.ok( + !Services.search.getEngineByName("Test search engine").hidden, + "Should have unhidden the new engine" + ); + Assert.equal( + (await Services.search.getVisibleEngines()).length, + 1, + "Should now have one engines visible" + ); + } +); + +// Test the other remove engine route - for removing non-application provided +// engines. + +async function checkNonBuiltinFallback(checkPrivate) { + let defaultEngine = checkPrivate ? appPrivateDefault : appDefault; + let expectedDefaultNotification = checkPrivate + ? SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + : SearchUtils.MODIFIED_TYPE.DEFAULT; + Services.search.restoreDefaultEngines(); + + let addedEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine2.xml`, + }); + + await setDefault(checkPrivate, addedEngine); + + const observer = new SearchObserver( + [expectedDefaultNotification, SearchUtils.MODIFIED_TYPE.REMOVED], + expectedDefaultNotification + ); + + // Remove the current engine... + await Services.search.removeEngine(addedEngine); + + // ... and verify we've reverted to the normal default engine. + Assert.equal( + (await getDefault(checkPrivate)).name, + defaultEngine.name, + "Should revert to the app default engine" + ); + + let notified = await observer.promise; + Assert.equal( + notified.name, + defaultEngine.name, + "Should have notified the correct default engine" + ); +} + +add_task(async function test_default_fallback_non_builtin() { + await checkNonBuiltinFallback(false); +}); + +add_task(async function test_default_fallback_non_builtin_private() { + await checkNonBuiltinFallback(true); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_defaultPrivateEngine.js b/toolkit/components/search/tests/xpcshell/test_defaultPrivateEngine.js new file mode 100644 index 0000000000..053d91fe48 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_defaultPrivateEngine.js @@ -0,0 +1,593 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that defaultEngine property can be set and yields the proper events and\ + * behavior (search results) + */ + +"use strict"; + +let engine1; +let engine2; +let appDefault; +let appPrivateDefault; + +add_setup(async () => { + do_get_profile(); + Services.fog.initializeFOG(); + + await SearchTestUtils.useTestEngines(); + + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + useHttpServer("opensearch"); + await AddonTestUtils.promiseStartupManager(); + + await Services.search.init(); + + appDefault = Services.search.appDefaultEngine; + appPrivateDefault = Services.search.appPrivateDefaultEngine; + engine1 = Services.search.getEngineByName("engine-rel-searchform-purpose"); + engine2 = Services.search.getEngineByName("engine-chromeicon"); +}); + +add_task(async function test_defaultPrivateEngine() { + Assert.equal( + Services.search.defaultPrivateEngine, + appPrivateDefault, + "Should have the app private default as the default private engine" + ); + Assert.equal( + Services.search.defaultEngine, + appDefault, + "Should have the app default as the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + displayName: "Test search engine", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine@search.mozilla.org" + : "[addon]engine@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + private: { + engineId: "engine-pref", + displayName: "engine-pref", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-pref@search.mozilla.org" + : "[addon]engine-pref@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + }); + + let promise = promiseDefaultNotification("private"); + Services.search.defaultPrivateEngine = engine1; + Assert.equal( + await promise, + engine1, + "Should have notified setting the private engine to the new one" + ); + + Assert.equal( + Services.search.defaultPrivateEngine, + engine1, + "Should have set the private engine to the new one" + ); + Assert.equal( + Services.search.defaultEngine, + appDefault, + "Should not have changed the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + displayName: "Test search engine", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine@search.mozilla.org" + : "[addon]engine@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + private: { + engineId: "engine-rel-searchform-purpose", + displayName: "engine-rel-searchform-purpose", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-rel-searchform-purpose@search.mozilla.org" + : "[addon]engine-rel-searchform-purpose@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=&channel=sb", + verified: "default", + }, + }); + + promise = promiseDefaultNotification("private"); + await Services.search.setDefaultPrivate( + engine2, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal( + await promise, + engine2, + "Should have notified setting the private engine to the new one using async api" + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should have set the private engine to the new one using the async api" + ); + + // We use the names here as for some reason the getDefaultPrivate promise + // returns something which is an nsISearchEngine but doesn't compare + // exactly to what engine2 is. + Assert.equal( + (await Services.search.getDefaultPrivate()).name, + engine2.name, + "Should have got the correct private engine with the async api" + ); + Assert.equal( + Services.search.defaultEngine, + appDefault, + "Should not have changed the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + displayName: "Test search engine", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine@search.mozilla.org" + : "[addon]engine@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + private: { + engineId: "engine-chromeicon", + displayName: "engine-chromeicon", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-chromeicon@search.mozilla.org" + : "[addon]engine-chromeicon@search.mozilla.org", + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + }); + + promise = promiseDefaultNotification("private"); + await Services.search.setDefaultPrivate( + engine1, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal( + await promise, + engine1, + "Should have notified reverting the private engine to the selected one using async api" + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine1, + "Should have reverted the private engine to the selected one using the async api" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + }, + private: { + engineId: "engine-rel-searchform-purpose", + }, + }); + + engine1.hidden = true; + Assert.equal( + Services.search.defaultPrivateEngine, + appPrivateDefault, + "Should reset to the app default private engine when hiding the default" + ); + Assert.equal( + Services.search.defaultEngine, + appDefault, + "Should not have changed the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + }, + private: { + engineId: "engine-pref", + }, + }); + + engine1.hidden = false; + Services.search.defaultEngine = engine1; + Assert.equal( + Services.search.defaultPrivateEngine, + appPrivateDefault, + "Setting the default engine should not affect the private default" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-rel-searchform-purpose", + }, + private: { + engineId: "engine-pref", + }, + }); + + Services.search.defaultEngine = appDefault; +}); + +add_task(async function test_telemetry_private_empty_submission_url() { + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}simple.xml`, + setAsDefaultPrivate: true, + }); + + await assertGleanDefaultEngine({ + normal: { + engineId: appDefault.telemetryId, + }, + private: { + engineId: "other-simple", + displayName: "simple", + loadPath: "[http]localhost/simple.xml", + submissionUrl: "blank:", + verified: "verified", + }, + }); + + Services.search.defaultEngine = appDefault; +}); + +add_task(async function test_defaultPrivateEngine_turned_off() { + Services.search.defaultEngine = appDefault; + Services.search.defaultPrivateEngine = engine1; + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + }, + private: { + engineId: "engine-rel-searchform-purpose", + }, + }); + + let promise = promiseDefaultNotification("private"); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + Assert.equal( + await promise, + appDefault, + "Should have notified setting the first engine correctly." + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + }, + private: { + engineId: "", + }, + }); + + promise = promiseDefaultNotification("normal"); + let privatePromise = promiseDefaultNotification("private"); + Services.search.defaultEngine = engine1; + Assert.equal( + await promise, + engine1, + "Should have notified setting the first engine correctly." + ); + Assert.equal( + await privatePromise, + engine1, + "Should have notified setting of the private engine as well." + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine1, + "Should be set to the first engine correctly" + ); + Assert.equal( + Services.search.defaultEngine, + engine1, + "Should keep the default engine in sync with the pref off" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-rel-searchform-purpose", + }, + private: { + engineId: "", + }, + }); + + promise = promiseDefaultNotification("private"); + Services.search.defaultPrivateEngine = engine2; + Assert.equal( + await promise, + engine2, + "Should have notified setting the second engine correctly." + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should be set to the second engine correctly" + ); + Assert.equal( + Services.search.defaultEngine, + engine1, + "Should not change the normal mode default engine" + ); + Assert.equal( + Services.prefs.getBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ), + true, + "Should have set the separate private default pref to true" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-rel-searchform-purpose", + }, + private: { + engineId: "engine-chromeicon", + }, + }); + + promise = promiseDefaultNotification("private"); + Services.search.defaultPrivateEngine = engine1; + Assert.equal( + await promise, + engine1, + "Should have notified resetting to the first engine again" + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine1, + "Should be reset to the first engine again" + ); + Assert.equal( + Services.search.defaultEngine, + engine1, + "Should keep the default engine in sync with the pref off" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-rel-searchform-purpose", + }, + private: { + engineId: "engine-rel-searchform-purpose", + }, + }); +}); + +add_task(async function test_defaultPrivateEngine_ui_turned_off() { + engine1.hidden = false; + engine2.hidden = false; + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + Services.search.defaultEngine = engine2; + Services.search.defaultPrivateEngine = engine1; + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "engine-rel-searchform-purpose", + }, + }); + + let promise = promiseDefaultNotification("private"); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + false + ); + Assert.equal( + await promise, + engine2, + "Should have notified for resetting of the private pref." + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "", + }, + }); + + promise = promiseDefaultNotification("normal"); + Services.search.defaultPrivateEngine = engine1; + Assert.equal( + await promise, + engine1, + "Should have notified setting the first engine correctly." + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine1, + "Should be set to the first engine correctly" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-rel-searchform-purpose", + }, + private: { + engineId: "", + }, + }); +}); + +add_task(async function test_defaultPrivateEngine_same_engine_toggle_pref() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + // Set the normal and private engines to be the same + Services.search.defaultEngine = engine2; + Services.search.defaultPrivateEngine = engine2; + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "engine-chromeicon", + }, + }); + + // Disable pref + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should not change the default private engine" + ); + Assert.equal( + Services.search.defaultEngine, + engine2, + "Should not change the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "", + }, + }); + + // Re-enable pref + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should not change the default private engine" + ); + Assert.equal( + Services.search.defaultEngine, + engine2, + "Should not change the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "engine-chromeicon", + }, + }); +}); + +add_task(async function test_defaultPrivateEngine_same_engine_toggle_ui_pref() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + // Set the normal and private engines to be the same + Services.search.defaultEngine = engine2; + Services.search.defaultPrivateEngine = engine2; + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "engine-chromeicon", + }, + }); + + // Disable UI pref + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + false + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should not change the default private engine" + ); + Assert.equal( + Services.search.defaultEngine, + engine2, + "Should not change the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "", + }, + }); + + // Re-enable UI pref + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Assert.equal( + Services.search.defaultPrivateEngine, + engine2, + "Should not change the default private engine" + ); + Assert.equal( + Services.search.defaultEngine, + engine2, + "Should not change the default engine" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine-chromeicon", + }, + private: { + engineId: "engine-chromeicon", + }, + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_alias.js b/toolkit/components/search/tests/xpcshell/test_engine_alias.js new file mode 100644 index 0000000000..2b0aa41850 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_alias.js @@ -0,0 +1,31 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const NAME = "Test Alias Engine"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + let settingsFileWritten = promiseAfterSettings(); + await Services.search.init(); + await settingsFileWritten; +}); + +add_task(async function upgrade_with_configuration_change_test() { + let settingsFileWritten = promiseAfterSettings(); + await SearchTestUtils.installSearchExtension({ + name: NAME, + keyword: "testalias", + }); + await settingsFileWritten; + + let engine = await Services.search.getEngineByAlias("testalias"); + Assert.equal(engine?.name, NAME, "Engine can be fetched by alias"); + + // Restart the search service but not the AddonManager, we will + // load the engines from settings. + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + engine = await Services.search.getEngineByAlias("testalias"); + Assert.equal(engine?.name, NAME, "Engine can be fetched by alias"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_ids.js b/toolkit/components/search/tests/xpcshell/test_engine_ids.js new file mode 100644 index 0000000000..cef6a17c92 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_ids.js @@ -0,0 +1,158 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests Search Engine IDs are created correctly. + */ + +"use strict"; + +const { EnterprisePolicyTesting } = ChromeUtils.importESModule( + "resource://testing-common/EnterprisePolicyTesting.sys.mjs" +); + +const CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, +]; + +add_setup(async function () { + useHttpServer("opensearch"); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_add_on_engine_id() { + let addOnEngine = Services.search.defaultEngine; + + Assert.equal( + addOnEngine.name, + "Test search engine", + "Should have installed the Test search engine as default." + ); + Assert.ok(addOnEngine.id, "The Addon Search Engine should have an id."); + Assert.equal( + addOnEngine.id, + "engine@search.mozilla.orgdefault", + "The Addon Search Engine id should be the webextension id + the locale." + ); +}); + +add_task(async function test_user_engine_id() { + let promiseEngineAdded = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.ADDED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + await Services.search.addUserEngine( + "user", + "https://example.com/user?q={searchTerms}", + "u" + ); + + await promiseEngineAdded; + let userEngine = Services.search.getEngineByName("user"); + + Assert.ok(userEngine, "Should have installed the User Search Engine."); + Assert.ok(userEngine.id, "The User Search Engine should have an id."); + Assert.equal( + userEngine.id.length, + 36, + "The User Search Engine id should be a 36 character uuid." + ); +}); + +add_task(async function test_open_search_engine_id() { + let openSearchEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: gDataUrl + "simple.xml", + }); + + Assert.ok(openSearchEngine, "Should have installed the Open Search Engine."); + Assert.ok(openSearchEngine.id, "The Open Search Engine should have an id."); + Assert.equal( + openSearchEngine.id.length, + 36, + "The Open Search Engine id should be a 36 character uuid." + ); +}); + +add_task(async function test_enterprise_policy_engine_id() { + await setupPolicyEngineWithJson({ + policies: { + SearchEngines: { + Add: [ + { + Name: "policy", + Description: "Test policy engine", + IconURL: "data:image/gif;base64,R0lGODl", + Alias: "p", + URLTemplate: "https://example.com?q={searchTerms}", + SuggestURLTemplate: "https://example.com/suggest/?q={searchTerms}", + }, + ], + }, + }, + }); + + let policyEngine = Services.search.getEngineByName("policy"); + + Assert.ok(policyEngine, "Should have installed the Policy Engine."); + Assert.ok(policyEngine.id, "The Policy Engine should have an id."); + Assert.equal( + policyEngine.id, + "policy-policy", + "The Policy Engine id should be 'policy-' + 'the name of the policy engine'." + ); +}); + +/** + * Loads a new enterprise policy, and re-initialise the search service + * with the new policy. Also waits for the search service to write the settings + * file to disk. + * + * @param {object} policy + * The enterprise policy to use. + */ +async function setupPolicyEngineWithJson(policy) { + Services.search.wrappedJSObject.reset(); + + await EnterprisePolicyTesting.setupPolicyEngineWithJson(policy); + + let settingsWritten = SearchTestUtils.promiseSearchNotification( + "write-settings-to-disk-complete" + ); + await Services.search.init(); + await settingsWritten; +} diff --git a/toolkit/components/search/tests/xpcshell/test_engine_multiple_alias.js b/toolkit/components/search/tests/xpcshell/test_engine_multiple_alias.js new file mode 100644 index 0000000000..5b3138ced1 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_multiple_alias.js @@ -0,0 +1,23 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const NAME = "Test Alias Engine"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function upgrade_with_configuration_change_test() { + let settingsFileWritten = promiseAfterSettings(); + await SearchTestUtils.installSearchExtension({ + name: NAME, + keyword: [" test", "alias "], + }); + await settingsFileWritten; + + let engine = await Services.search.getEngineByAlias("test"); + Assert.equal(engine?.name, NAME, "Can be fetched by either alias"); + engine = await Services.search.getEngineByAlias("alias"); + Assert.equal(engine?.name, NAME, "Can be fetched by either alias"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector.js new file mode 100644 index 0000000000..12344021d2 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector.js @@ -0,0 +1,242 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const TEST_CONFIG = [ + { + engineName: "aol", + orderHint: 500, + webExtension: { + locales: ["default"], + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { regions: ["us"] }, + webExtension: { + locales: ["baz-$USER_LOCALE"], + }, + telemetryId: "foo-$USER_LOCALE", + }, + { + included: { regions: ["fr"] }, + webExtension: { + locales: ["region-$USER_REGION"], + }, + telemetryId: "bar-$USER_REGION", + }, + { + included: { regions: ["be"] }, + webExtension: { + locales: ["$USER_LOCALE"], + }, + telemetryId: "$USER_LOCALE", + }, + { + included: { regions: ["au"] }, + webExtension: { + locales: ["$USER_REGION"], + }, + telemetryId: "$USER_REGION", + }, + ], + }, + { + engineName: "lycos", + orderHint: 1000, + default: "yes", + appliesTo: [ + { + included: { everywhere: true }, + excluded: { locales: { matches: ["zh-CN"] } }, + }, + ], + }, + { + engineName: "altavista", + orderHint: 2000, + defaultPrivate: "yes", + appliesTo: [ + { + included: { locales: { matches: ["en-US"] } }, + }, + { + included: { regions: ["default"] }, + }, + ], + }, + { + engineName: "excite", + default: "yes-if-no-other", + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["us"] }, + }, + { + included: { everywhere: true }, + experiment: "acohortid", + }, + ], + }, + { + engineName: "askjeeves", + }, +]; + +const engineSelector = new SearchEngineSelectorOld(); + +add_setup(async function () { + const settings = await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY); + sinon.stub(settings, "get").returns(TEST_CONFIG); +}); + +add_task(async function test_engine_selector() { + let { engines, privateDefault } = + await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + Assert.equal( + privateDefault.engineName, + "altavista", + "Should set altavista as privateDefault" + ); + let names = engines.map(obj => obj.engineName); + Assert.deepEqual(names, ["lycos", "altavista", "aol"], "Correct order"); + Assert.equal( + engines[2].webExtension.locale, + "baz-en-US", + "Subsequent matches in applies to can override default" + ); + + ({ engines, privateDefault } = await engineSelector.fetchEngineConfiguration({ + locale: "zh-CN", + region: "kz", + })); + Assert.equal(engines.length, 2, "Correct engines are returns"); + Assert.equal(privateDefault, null, "There should be no privateDefault"); + names = engines.map(obj => obj.engineName); + Assert.deepEqual( + names, + ["excite", "aol"], + "The engines should be in the correct order" + ); + + ({ engines, privateDefault } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + experiment: "acohortid", + })); + Assert.deepEqual( + engines.map(obj => obj.engineName), + ["lycos", "altavista", "aol", "excite"], + "Engines are in the correct order and include the experiment engine" + ); + + ({ engines, privateDefault } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + experiment: "acohortid", + })); + Assert.deepEqual( + engines.map(obj => obj.engineName), + ["lycos", "altavista", "aol", "excite"], + "The engines should be in the correct order" + ); + Assert.equal( + privateDefault.engineName, + "altavista", + "Should set altavista as privateDefault" + ); +}); + +add_task(async function test_locale_region_replacement() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + let engine = engines.find(e => e.engineName == "aol"); + Assert.equal( + engine.webExtension.locale, + "baz-en-US", + "The locale is correctly inserted into the locale field" + ); + Assert.equal( + engine.telemetryId, + "foo-en-US", + "The locale is correctly inserted into the telemetryId" + ); + + ({ engines } = await engineSelector.fetchEngineConfiguration({ + locale: "it", + region: "us", + })); + engine = engines.find(e => e.engineName == "aol"); + + Assert.equal( + engines.find(e => e.engineName == "aol").webExtension.locale, + "baz-it", + "The locale is correctly inserted into the locale field" + ); + Assert.equal( + engine.telemetryId, + "foo-it", + "The locale is correctly inserted into the telemetryId" + ); + + ({ engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-CA", + region: "fr", + })); + engine = engines.find(e => e.engineName == "aol"); + Assert.equal( + engine.webExtension.locale, + "region-fr", + "The region is correctly inserted into the locale field" + ); + Assert.equal( + engine.telemetryId, + "bar-fr", + "The region is correctly inserted into the telemetryId" + ); + + ({ engines } = await engineSelector.fetchEngineConfiguration({ + locale: "fy-NL", + region: "be", + })); + engine = engines.find(e => e.engineName == "aol"); + Assert.equal( + engine.webExtension.locale, + "fy-NL", + "The locale is correctly inserted into the locale field" + ); + Assert.equal( + engine.telemetryId, + "fy-NL", + "The locale is correctly inserted into the telemetryId" + ); + ({ engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "au", + })); + engine = engines.find(e => e.engineName == "aol"); + Assert.equal( + engine.webExtension.locale, + "au", + "The region is correctly inserted into the locale field" + ); + Assert.equal( + engine.telemetryId, + "au", + "The region is correctly inserted into the telemetryId" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application.js new file mode 100644 index 0000000000..3270bdb55e --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application.js @@ -0,0 +1,116 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const TEST_CONFIG = [ + { + webExtension: { + id: "aol@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + default: "yes-if-no-other", + }, + { + webExtension: { + id: "lycos@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + channel: ["nightly"], + }, + }, + ], + default: "yes", + }, + { + webExtension: { + id: "altavista@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + channel: ["nightly", "esr"], + }, + }, + ], + }, + { + webExtension: { + id: "excite@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { everywhere: true }, + application: { + channel: ["release"], + }, + default: "yes", + }, + ], + }, +]; + +const expectedEnginesPerChannel = { + default: ["aol@example.com", "excite@example.com"], + nightly: [ + "lycos@example.com", + "aol@example.com", + "altavista@example.com", + "excite@example.com", + ], + beta: ["aol@example.com", "excite@example.com"], + release: ["excite@example.com", "aol@example.com"], + esr: ["aol@example.com", "altavista@example.com", "excite@example.com"], +}; + +const expectedDefaultEngine = { + default: "aol@example.com", + nightly: "lycos@example.com", + beta: "aol@example.com", + release: "excite@example.com", + esr: "aol@example.com", +}; + +const engineSelector = new SearchEngineSelectorOld(); + +add_task(async function test_engine_selector_channels() { + const settings = await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY); + sinon.stub(settings, "get").returns(TEST_CONFIG); + + for (let [channel, expected] of Object.entries(expectedEnginesPerChannel)) { + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + channel, + }); + + const engineIds = engines.map(obj => obj.webExtension.id); + Assert.deepEqual( + engineIds, + expected, + `Should have the expected engines for channel "${channel}"` + ); + + Assert.equal( + engineIds[0], + expectedDefaultEngine[channel], + `Should have the correct default for channel "${channel}"` + ); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_distribution.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_distribution.js new file mode 100644 index 0000000000..01252740a5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_distribution.js @@ -0,0 +1,122 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const CONFIG = [ + { + webExtension: { + id: "aol@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + default: "yes-if-no-other", + }, + { + webExtension: { + id: "excite@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + // Test with a application/distributions section present but an + // empty list. + application: { + distributions: [], + }, + }, + ], + }, + { + webExtension: { + id: "lycos@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + distributions: ["cake"], + }, + }, + ], + default: "yes", + }, + { + webExtension: { + id: "altavista@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + excludedDistributions: ["apples"], + }, + }, + ], + }, +]; + +const engineSelector = new SearchEngineSelectorOld(); +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.newSearchConfig.enabled", false); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_no_distribution_preference() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "default", + region: "default", + channel: "", + distroID: "", + }); + const engineIds = engines.map(obj => obj.webExtension.id); + Assert.deepEqual( + engineIds, + ["aol@example.com", "excite@example.com", "altavista@example.com"], + `Should have the expected engines for a normal build.` + ); +}); + +add_task(async function test_distribution_included() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "default", + region: "default", + channel: "", + distroID: "cake", + }); + const engineIds = engines.map(obj => obj.webExtension.id); + Assert.deepEqual( + engineIds, + [ + "lycos@example.com", + "aol@example.com", + "excite@example.com", + "altavista@example.com", + ], + `Should have the expected engines for the "cake" distribution.` + ); +}); + +add_task(async function test_distribution_excluded() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "default", + region: "default", + channel: "", + distroID: "apples", + }); + const engineIds = engines.map(obj => obj.webExtension.id); + Assert.deepEqual( + engineIds, + ["aol@example.com", "excite@example.com"], + `Should have the expected engines for the "apples" distribution.` + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_name.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_name.js new file mode 100644 index 0000000000..f38198aca6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_application_name.js @@ -0,0 +1,128 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const CONFIG = [ + { + webExtension: { + id: "aol@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + default: "yes-if-no-other", + }, + { + webExtension: { + id: "lycos@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + name: ["firefox"], + }, + }, + ], + default: "yes", + }, + { + webExtension: { + id: "altavista@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + name: ["fenix"], + }, + }, + ], + }, + { + webExtension: { + id: "excite@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + application: { + name: ["firefox"], + minVersion: "10", + maxVersion: "30", + }, + default: "yes", + }, + ], + }, +]; + +const engineSelector = new SearchEngineSelectorOld(); + +const tests = [ + { + name: "Firefox", + version: "1", + expected: ["lycos@example.com", "aol@example.com"], + }, + { + name: "Firefox", + version: "20", + expected: ["lycos@example.com", "aol@example.com", "excite@example.com"], + }, + { + name: "Fenix", + version: "20", + expected: ["aol@example.com", "altavista@example.com"], + }, + { + name: "Firefox", + version: "31", + expected: ["lycos@example.com", "aol@example.com"], + }, + { + name: "Firefox", + version: "30", + expected: ["lycos@example.com", "aol@example.com", "excite@example.com"], + }, + { + name: "Firefox", + version: "10", + expected: ["lycos@example.com", "aol@example.com", "excite@example.com"], + }, +]; + +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.newSearchConfig.enabled", false); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); + + let confUrl = `data:application/json,${JSON.stringify(CONFIG)}`; + Services.prefs.setStringPref("search.config.url", confUrl); +}); + +add_task(async function test_application_name() { + for (const { name, version, expected } of tests) { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "default", + region: "default", + name, + version, + }); + const engineIds = engines.map(obj => obj.webExtension.id); + Assert.deepEqual( + engineIds, + expected, + `Should have the expected engines for app: "${name}" + and version: "${version}"` + ); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_order.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_order.js new file mode 100644 index 0000000000..6a7b63f9a2 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_order.js @@ -0,0 +1,137 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +/** + * This constant defines the tests for the order. The input is an array of + * engines that will be constructed. The engine definitions are arrays with + * fields in order: + * name, orderHint, default, defaultPrivate + * + * The expected is an array of engine names. + */ +const TESTS = [ + { + // Basic tests to ensure correct order for default engine. + input: [ + ["A", 750, "no", "no"], + ["B", 3000, "no", "no"], + ["C", 2000, "yes", "no"], + ["D", 1000, "yes-if-no-other", "no"], + ["E", 500, "no", "no"], + ], + expected: ["C", "B", "D", "A", "E"], + expectedPrivate: undefined, + }, + { + input: [ + ["A", 750, "no", "no"], + ["B", 3000, "no", "no"], + ["C", 2000, "yes-if-no-other", "no"], + ["D", 1000, "yes", "no"], + ["E", 500, "no", "no"], + ], + expected: ["D", "B", "C", "A", "E"], + expectedPrivate: undefined, + }, + // Check that yes-if-no-other works correctly. + { + input: [ + ["A", 750, "no", "no"], + ["B", 3000, "no", "no"], + ["C", 2000, "no", "no"], + ["D", 1000, "yes-if-no-other", "no"], + ["E", 500, "no", "no"], + ], + expected: ["D", "B", "C", "A", "E"], + expectedPrivate: undefined, + }, + // Basic tests to ensure correct order with private engine. + { + input: [ + ["A", 750, "no", "no"], + ["B", 3000, "yes-if-no-other", "no"], + ["C", 2000, "no", "yes"], + ["D", 1000, "yes", "yes-if-no-other"], + ["E", 500, "no", "no"], + ], + expected: ["D", "C", "B", "A", "E"], + expectedPrivate: "C", + }, + { + input: [ + ["A", 750, "no", "yes-if-no-other"], + ["B", 3000, "yes-if-no-other", "no"], + ["C", 2000, "no", "yes"], + ["D", 1000, "yes", "no"], + ["E", 500, "no", "no"], + ], + expected: ["D", "C", "B", "A", "E"], + expectedPrivate: "C", + }, + // Private engine test for yes-if-no-other. + { + input: [ + ["A", 750, "no", "yes-if-no-other"], + ["B", 3000, "yes-if-no-other", "no"], + ["C", 2000, "no", "no"], + ["D", 1000, "yes", "no"], + ["E", 500, "no", "no"], + ], + expected: ["D", "A", "B", "C", "E"], + expectedPrivate: "A", + }, +]; + +function getConfigData(testInput) { + return testInput.map(info => ({ + engineName: info[0], + orderHint: info[1], + default: info[2], + defaultPrivate: info[3], + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + })); +} + +const engineSelector = new SearchEngineSelectorOld(); + +add_task(async function () { + const settings = await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY); + const getStub = sinon.stub(settings, "get"); + + let i = 0; + for (const test of TESTS) { + // Remove the existing configuration and update the stub to return the data + // for this test. + delete engineSelector._configuration; + getStub.returns(getConfigData(test.input)); + + const { engines, privateDefault } = + await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + + let names = engines.map(obj => obj.engineName); + Assert.deepEqual( + names, + test.expected, + `Should have the correct order for the engines: test ${i}` + ); + Assert.equal( + privateDefault && privateDefault.engineName, + test.expectedPrivate, + `Should have the correct selection for the private engine: test ${i++}` + ); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_override.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_override.js new file mode 100644 index 0000000000..b4575be9ce --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_override.js @@ -0,0 +1,196 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const TEST_CONFIG = [ + { + webExtension: { + id: "aol@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + override: true, + application: { + distributions: ["distro1"], + }, + params: { + searchUrlGetParams: [ + { + name: "field-keywords", + value: "{searchTerms}", + }, + ], + }, + }, + ], + default: "yes-if-no-other", + }, + { + webExtension: { + id: "lycos@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + override: true, + experiment: "experiment1", + params: { + searchUrlGetParams: [ + { + name: "experiment-params", + value: "{searchTerms}", + }, + ], + }, + }, + ], + default: "yes", + }, + { + webExtension: { + id: "altavista@example.com", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + override: true, + application: { + distributions: ["distro2"], + }, + included: { regions: ["gb"] }, + params: { + searchUrlGetParams: [ + { + name: "field-keywords2", + value: "{searchTerms}", + }, + ], + }, + }, + ], + default: "yes", + }, +]; + +const engineSelector = new SearchEngineSelectorOld(); + +add_setup(async function () { + const settings = await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY); + sinon.stub(settings, "get").returns(TEST_CONFIG); +}); + +add_task(async function test_engine_selector_defaults() { + // Check that with no override sections matching, we have no overrides active. + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + + let engine = engines.find(e => e.webExtension.id == "aol@example.com"); + + Assert.ok( + !("params" in engine), + "Should not have overriden the parameters of the aol engine." + ); + + engine = engines.find(e => e.webExtension.id == "lycos@example.com"); + + Assert.ok( + !("params" in engine), + "Should not have overriden the parameters of the lycos engine." + ); +}); + +add_task(async function test_engine_selector_override_distributions() { + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + distroID: "distro1", + }); + + let engine = engines.find(e => e.webExtension.id == "aol@example.com"); + + Assert.deepEqual( + engine.params, + { + searchUrlGetParams: [ + { + name: "field-keywords", + value: "{searchTerms}", + }, + ], + }, + "Should have overriden the parameters of the engine." + ); +}); + +add_task(async function test_engine_selector_override_experiments() { + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + experiment: "experiment1", + }); + + let engine = engines.find(e => e.webExtension.id == "lycos@example.com"); + + Assert.deepEqual( + engine.params, + { + searchUrlGetParams: [ + { + name: "experiment-params", + value: "{searchTerms}", + }, + ], + }, + "Should have overriden the parameters of the engine." + ); +}); + +add_task(async function test_engine_selector_override_with_included() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + distroID: "distro2", + }); + + let engine = engines.find(e => e.webExtension.id == "altavista@example.com"); + Assert.ok( + !("params" in engine), + "Should not have overriden the parameters of the engine." + ); + + let result = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "gb", + distroID: "distro2", + }); + engine = result.engines.find( + e => e.webExtension.id == "altavista@example.com" + ); + Assert.deepEqual( + engine.params, + { + searchUrlGetParams: [ + { + name: "field-keywords2", + value: "{searchTerms}", + }, + ], + }, + "Should have overriden the parameters of the engine." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_override.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_override.js new file mode 100644 index 0000000000..9535adfb28 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_override.js @@ -0,0 +1,135 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", +}); + +const TEST_CONFIG_OLD = [ + { + engineName: "aol", + telemetryId: "aol", + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + params: { + searchUrlGetParams: [ + { + name: "original_param", + value: "original_value", + }, + ], + }, + }, +]; + +const TEST_CONFIG_OVERRIDE_OLD = [ + { + telemetryId: "aol", + telemetrySuffix: "tsfx", + params: { + searchUrlGetParams: [ + { + name: "new_param", + value: "new_value", + }, + ], + }, + }, +]; + +const TEST_CONFIG = [ + { + base: { + urls: { + search: { + base: "https://www.bing.com/search", + params: [ + { + name: "old_param", + value: "old_value", + }, + ], + searchTermParamName: "q", + }, + }, + }, + variants: [ + { + environment: { + locales: ["en-US"], + }, + }, + ], + identifier: "aol", + recordType: "engine", + }, + { + recordType: "defaultEngines", + globalDefault: "aol", + specificDefaults: [], + }, + { + orders: [], + recordType: "engineOrders", + }, +]; + +const TEST_CONFIG_OVERRIDE = [ + { + identifier: "aol", + urls: { + search: { + params: [{ name: "new_param", value: "new_value" }], + }, + }, + telemetrySuffix: "tsfx", + clickUrl: "https://aol.url", + }, +]; + +const engineSelectorOld = new SearchEngineSelectorOld(); +const engineSelector = new SearchEngineSelector(); + +add_setup(async function () { + const settingsOld = await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY); + sinon.stub(settingsOld, "get").returns(TEST_CONFIG_OLD); + const overridesOld = await RemoteSettings( + SearchUtils.OLD_SETTINGS_OVERRIDES_KEY + ); + sinon.stub(overridesOld, "get").returns(TEST_CONFIG_OVERRIDE_OLD); + + const settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + sinon.stub(settings, "get").returns(TEST_CONFIG); + const overrides = await RemoteSettings( + SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + ); + sinon.stub(overrides, "get").returns(TEST_CONFIG_OVERRIDE); +}); + +add_task(async function test_engine_selector_old() { + let { engines } = await engineSelectorOld.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + Assert.equal(engines[0].telemetryId, "aol-tsfx"); + Assert.equal(engines[0].params.searchUrlGetParams[0].name, "new_param"); + Assert.equal(engines[0].params.searchUrlGetParams[0].value, "new_value"); +}); + +add_task(async function test_engine_selector() { + let { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "us", + }); + Assert.equal(engines[0].telemetrySuffix, "tsfx"); + Assert.equal(engines[0].clickUrl, "https://aol.url"); + Assert.equal(engines[0].urls.search.params[0].name, "new_param"); + Assert.equal(engines[0].urls.search.params[0].value, "new_value"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_settings.js b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_settings.js new file mode 100644 index 0000000000..622db4a291 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_old_selector_remote_settings.js @@ -0,0 +1,345 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelectorOld: + "resource://gre/modules/SearchEngineSelectorOld.sys.mjs", +}); + +const TEST_CONFIG = [ + { + engineName: "aol", + orderHint: 500, + webExtension: { + locales: ["default"], + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { regions: ["us"] }, + webExtension: { + locales: ["$USER_LOCALE"], + }, + }, + ], + }, + { + engineName: "lycos", + orderHint: 1000, + default: "yes", + appliesTo: [ + { + included: { everywhere: true }, + excluded: { locales: { matches: ["zh-CN"] } }, + }, + ], + }, + { + engineName: "altavista", + orderHint: 2000, + defaultPrivate: "yes", + appliesTo: [ + { + included: { locales: { matches: ["en-US"] } }, + }, + { + included: { regions: ["default"] }, + }, + ], + }, + { + engineName: "excite", + default: "yes-if-no-other", + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["us"] }, + }, + { + included: { everywhere: true }, + cohort: "acohortid", + }, + ], + }, + { + engineName: "askjeeves", + }, +]; + +let getStub; + +add_setup(async function () { + const searchConfigSettings = await RemoteSettings( + SearchUtils.OLD_SETTINGS_KEY + ); + getStub = sinon.stub(searchConfigSettings, "get"); + + // We expect this error from remove settings as we're invalidating the + // signature. + consoleAllowList.push("Invalid content signature (abc)"); + // We also test returning an empty configuration. + consoleAllowList.push("Received empty search configuration"); +}); + +add_task(async function test_selector_basic_get() { + const listenerSpy = sinon.spy(); + const engineSelector = new SearchEngineSelectorOld(listenerSpy); + getStub.onFirstCall().returns(TEST_CONFIG); + + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }); + + Assert.deepEqual( + engines.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have obtained the correct data from the database." + ); + Assert.ok(listenerSpy.notCalled, "Should not have called the listener"); +}); + +add_task(async function test_selector_get_reentry() { + const listenerSpy = sinon.spy(); + const engineSelector = new SearchEngineSelectorOld(listenerSpy); + let promise = Promise.withResolvers(); + getStub.resetHistory(); + getStub.onFirstCall().returns(promise.promise); + delete engineSelector._configuration; + + let firstResult; + let secondResult; + + const firstCallPromise = engineSelector + .fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }) + .then(result => (firstResult = result.engines)); + + const secondCallPromise = engineSelector + .fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }) + .then(result => (secondResult = result.engines)); + + Assert.strictEqual( + firstResult, + undefined, + "Should not have returned the first result yet." + ); + + Assert.strictEqual( + secondResult, + undefined, + "Should not have returned the second result yet." + ); + + promise.resolve(TEST_CONFIG); + + await Promise.all([firstCallPromise, secondCallPromise]); + Assert.deepEqual( + firstResult.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have returned the correct data to the first call" + ); + + Assert.deepEqual( + secondResult.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have returned the correct data to the second call" + ); + Assert.ok(listenerSpy.notCalled, "Should not have called the listener"); +}); + +add_task(async function test_selector_config_update() { + const listenerSpy = sinon.spy(); + const engineSelector = new SearchEngineSelectorOld(listenerSpy); + getStub.resetHistory(); + getStub.onFirstCall().returns(TEST_CONFIG); + + const { engines } = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }); + + Assert.deepEqual( + engines.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have got the correct configuration" + ); + + Assert.ok(listenerSpy.notCalled, "Should not have called the listener yet"); + + const NEW_DATA = [ + { + default: "yes", + engineName: "askjeeves", + appliesTo: [{ included: { everywhere: true } }], + schema: 1553857697843, + last_modified: 1553859483588, + }, + ]; + + getStub.resetHistory(); + getStub.onFirstCall().returns(NEW_DATA); + await RemoteSettings(SearchUtils.OLD_SETTINGS_KEY).emit("sync", { + data: { + current: NEW_DATA, + }, + }); + + Assert.ok(listenerSpy.called, "Should have called the listener"); + + const result = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }); + + Assert.deepEqual( + result.engines.map(e => e.engineName), + ["askjeeves"], + "Should have updated the configuration with the new data" + ); +}); + +add_task(async function test_selector_db_modification() { + const engineSelector = new SearchEngineSelectorOld(); + // Fill the database with some values that we can use to test that it is cleared. + const db = RemoteSettings(SearchUtils.OLD_SETTINGS_KEY).db; + await db.importChanges( + {}, + Date.now(), + [ + { + id: "85e1f268-9ca5-4b52-a4ac-922df5c07264", + default: "yes", + engineName: "askjeeves", + appliesTo: [{ included: { everywhere: true } }], + }, + ], + { clear: true } + ); + + // Stub the get() so that the first call simulates a signature error, and + // the second simulates success reading from the dump. + getStub.resetHistory(); + getStub + .onFirstCall() + .rejects(new RemoteSettingsClient.InvalidSignatureError("abc")); + getStub.onSecondCall().returns(TEST_CONFIG); + + let result = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }); + + Assert.ok( + getStub.calledTwice, + "Should have called the get() function twice." + ); + + const databaseEntries = await db.list(); + Assert.equal(databaseEntries.length, 0, "Should have cleared the database."); + + Assert.deepEqual( + result.engines.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have returned the correct data." + ); +}); + +add_task(async function test_selector_db_modification_never_succeeds() { + const engineSelector = new SearchEngineSelectorOld(); + // Fill the database with some values that we can use to test that it is cleared. + const db = RemoteSettings(SearchUtils.OLD_SETTINGS_KEY).db; + await db.importChanges( + {}, + Date.now(), + [ + { + id: "b70edfdd-1c3f-4b7b-ab55-38cb048636c0", + default: "yes", + engineName: "askjeeves", + appliesTo: [{ included: { everywhere: true } }], + }, + ], + { + clear: true, + } + ); + + // Now simulate the condition where for some reason we never get a + // valid result. + getStub.reset(); + getStub.rejects(new RemoteSettingsClient.InvalidSignatureError("abc")); + + await Assert.rejects( + engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }), + ex => ex.result == Cr.NS_ERROR_UNEXPECTED, + "Should have rejected loading the engine configuration" + ); + + Assert.ok( + getStub.calledTwice, + "Should have called the get() function twice." + ); + + const databaseEntries = await db.list(); + Assert.equal(databaseEntries.length, 0, "Should have cleared the database."); +}); + +add_task(async function test_empty_results() { + // Check that returning an empty result re-tries. + const engineSelector = new SearchEngineSelectorOld(); + // Fill the database with some values that we can use to test that it is cleared. + const db = RemoteSettings(SearchUtils.OLD_SETTINGS_KEY).db; + await db.importChanges( + {}, + Date.now(), + [ + { + id: "df5655ca-e045-4f8c-a7ee-047eeb654722", + default: "yes", + engineName: "askjeeves", + appliesTo: [{ included: { everywhere: true } }], + }, + ], + { + clear: true, + } + ); + + // Stub the get() so that the first call simulates an empty database, and + // the second simulates success reading from the dump. + getStub.resetHistory(); + getStub.onFirstCall().returns([]); + getStub.onSecondCall().returns(TEST_CONFIG); + + let result = await engineSelector.fetchEngineConfiguration({ + locale: "en-US", + region: "default", + }); + + Assert.ok( + getStub.calledTwice, + "Should have called the get() function twice." + ); + + const databaseEntries = await db.list(); + Assert.equal(databaseEntries.length, 0, "Should have cleared the database."); + + Assert.deepEqual( + result.engines.map(e => e.engineName), + ["lycos", "altavista", "aol", "excite"], + "Should have returned the correct data." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_defaults.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_defaults.js new file mode 100644 index 0000000000..bb904fcc86 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_defaults.js @@ -0,0 +1,349 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This tests the SearchEngineSelector in finding the correct default engine + * and private default engine based on the user's environment. + */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", +}); + +const CONFIG = [ + { + recordType: "engine", + identifier: "global-default", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "global-default-private", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-specific-location", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-specific-distro", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-starts-with-ca", + base: {}, + variants: [ + { + environment: { locales: ["en-US"], regions: ["CA"] }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-starts-with-us", + base: {}, + variants: [ + { + environment: { locales: ["en-US"], regions: ["US"] }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "global-default", + globalDefaultPrivate: "global-default-private", + specificDefaults: [ + { + environment: { locales: ["zh-CN"], regions: ["cn"] }, + default: "default-specific-location", + defaultPrivate: "default-specific-location", + }, + { + environment: { distributions: ["specific-distro"] }, + default: "default-specific-distro", + defaultPrivate: "default-specific-distro", + }, + { + environment: { locales: ["en-US"], regions: ["CA"] }, + default: "default-starts-with*", + defaultPrivate: "default-starts-with*", + }, + { + environment: { locales: ["en-US"], regions: ["US"] }, + default: "default-starts-with*", + defaultPrivate: "default-starts-with*", + }, + ], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_DEFAULTS_OVERRIDE = [ + { + recordType: "engine", + identifier: "engine-global", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-locale-de", + base: {}, + variants: [ + { + environment: { + locales: ["de"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-distro", + base: {}, + variants: [ + { + environment: { + distributions: ["distro"], + regions: ["FR"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "engine-global", + specificDefaults: [ + { + environment: { locales: ["de"] }, + default: "engine-locale-de", + }, + { + environment: { distributions: ["distro"] }, + default: "engine-distro", + }, + ], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const engineSelector = new SearchEngineSelector(); +let settings; +let settingOverrides; +let configStub; +let overrideStub; + +/** + * This function asserts if the actual engine identifiers returned equals + * the expected engines. + * + * @param {object} config + * A mock search config contain engines. + * @param {object} userEnv + * A fake user's environment including locale and region, experiment, etc. + * @param {string} expectedDefault + * The identifer of the expected default engine. + * @param {string} expectedDefaultPrivate + * The identifer of the expected default private engine. + * @param {string} message + * The description of the test. + */ +async function assertActualEnginesEqualsExpected( + config, + userEnv, + expectedDefault, + expectedDefaultPrivate, + message +) { + engineSelector._configuration = null; + configStub.returns(config); + + let { engines, privateDefault } = + await engineSelector.fetchEngineConfiguration(userEnv); + let actualEngines = engines.map(engine => engine.identifier); + + info(`${message}`); + Assert.equal( + actualEngines[0], + expectedDefault, + `Should match the default engine ${expectedDefault}.` + ); + + Assert.equal( + privateDefault ? privateDefault.identifier : undefined, + expectedDefaultPrivate, + `Should match default private engine ${expectedDefaultPrivate}.` + ); +} + +add_setup(async function () { + settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + configStub = sinon.stub(settings, "get"); + settingOverrides = await RemoteSettings( + SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + ); + overrideStub = sinon.stub(settingOverrides, "get"); + overrideStub.returns([]); +}); + +add_task(async function test_default_engines() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-CA", + region: "ca", + }, + "global-default", + "global-default-private", + "Should use the global default engine and global default private when no specific defaults are matched." + ); + + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "zh-CN", + region: "cn", + }, + "default-specific-location", + "default-specific-location", + "Should use the matched locale and region default engine." + ); + + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "fi", + region: "FI", + distroID: "specific-distro", + }, + "default-specific-distro", + "default-specific-distro", + "Should use the matched distribution default engine." + ); + + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-US", + region: "CA", + }, + "default-starts-with-ca", + "default-starts-with-ca", + "Should use the matched default engine with specific suffix." + ); + + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-US", + region: "US", + }, + "default-starts-with-us", + "default-starts-with-us", + "Should use the matched default engine with specific suffix." + ); +}); + +add_task(async function test_default_engines_override() { + await assertActualEnginesEqualsExpected( + CONFIG_DEFAULTS_OVERRIDE, + { + locale: "en-US", + region: "US", + }, + "engine-global", + undefined, + "Should use the globalDefault for default when no specific defaults are matched. Private default should be undefined when no default private." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DEFAULTS_OVERRIDE, + { + locale: "de", + region: "US", + }, + "engine-locale-de", + undefined, + "Should use the matched locale default engine." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DEFAULTS_OVERRIDE, + { + locale: "de", + region: "FR", + }, + "engine-locale-de", + undefined, + "Should use the matched locale default engine." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DEFAULTS_OVERRIDE, + { + locale: "en-US", + region: "FR", + distroID: "distro", + }, + "engine-distro", + undefined, + "Should use the matched distro default engine." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DEFAULTS_OVERRIDE, + { + locale: "de", + region: "FR", + distroID: "distro", + }, + "engine-distro", + undefined, + "Should use the last matched default engine when multiple default engines are matched." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_engine_orders.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_engine_orders.js new file mode 100644 index 0000000000..e7725ceb21 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_engine_orders.js @@ -0,0 +1,348 @@ +/* Any copyright is dedicated to the Public Domain. +http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This tests the SearchEngineSelector in ordering the engines correctly based + * on the user's environment. + */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", +}); + +const ENGINE_ORDERS_CONFIG = [ + { + recordType: "engine", + identifier: "b-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "a-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "c-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [ + { + environment: { distributions: ["distro"] }, + order: ["a-engine", "b-engine", "c-engine"], + }, + { + environment: { + distributions: ["distro"], + locales: ["en-CA"], + regions: ["CA"], + }, + order: ["c-engine", "b-engine", "a-engine"], + }, + { + environment: { + distributions: ["distro-2"], + }, + order: ["a-engine", "b-engine"], + }, + ], + }, +]; + +const STARTS_WITH_WIKI_CONFIG = [ + { + recordType: "engine", + identifier: "wiki-ca", + base: {}, + variants: [ + { + environment: { + locales: ["en-CA"], + regions: ["CA"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "wiki-uk", + base: {}, + variants: [ + { + environment: { + locales: ["en-GB"], + regions: ["GB"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-1", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-2", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [ + { + environment: { + locales: ["en-CA"], + regions: ["CA"], + }, + order: ["wiki*", "engine-1", "engine-2"], + }, + { + environment: { + locales: ["en-GB"], + regions: ["GB"], + }, + order: ["wiki*", "engine-1", "engine-2"], + }, + ], + }, +]; + +const DEFAULTS_CONFIG = [ + { + recordType: "engine", + identifier: "b-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "a-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "default-private-engine", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "default-engine", + globalDefaultPrivate: "default-private-engine", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [ + { + environment: { + locales: ["en-CA"], + regions: ["CA"], + }, + order: ["a-engine", "b-engine"], + }, + ], + }, +]; + +const engineSelector = new SearchEngineSelector(); +let settings; +let settingOverrides; +let configStub; +let overrideStub; + +/** + * This function asserts if the actual engine identifiers returned equals + * the expected engines. + * + * @param {object} config + * A mock search config contain engines. + * @param {object} userEnv + * A fake user's environment including locale and region, experiment, etc. + * @param {Array} expectedEngineOrders + * The array of engine identifers in the expected order. + * @param {string} message + * The description of the test. + */ +async function assertActualEnginesEqualsExpected( + config, + userEnv, + expectedEngineOrders, + message +) { + engineSelector._configuration = null; + configStub.returns(config); + + let { engines } = await engineSelector.fetchEngineConfiguration(userEnv); + let actualEngineOrders = engines.map(engine => engine.identifier); + + info(`${message}`); + Assert.deepEqual(actualEngineOrders, expectedEngineOrders, message); +} + +add_setup(async function () { + settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + configStub = sinon.stub(settings, "get"); + settingOverrides = await RemoteSettings( + SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + ); + overrideStub = sinon.stub(settingOverrides, "get"); + overrideStub.returns([]); +}); + +add_task(async function test_selector_match_engine_orders() { + await assertActualEnginesEqualsExpected( + ENGINE_ORDERS_CONFIG, + { + locale: "fr", + region: "FR", + distroID: "distro", + }, + ["a-engine", "b-engine", "c-engine"], + "Should match engine orders with the distro distribution." + ); + + await assertActualEnginesEqualsExpected( + ENGINE_ORDERS_CONFIG, + { + locale: "en-CA", + region: "CA", + distroID: "distro", + }, + ["c-engine", "b-engine", "a-engine"], + "Should match engine orders with the distro distribution, en-CA locale and CA region." + ); + + await assertActualEnginesEqualsExpected( + ENGINE_ORDERS_CONFIG, + { + locale: "en-CA", + region: "CA", + distroID: "distro-2", + }, + ["a-engine", "b-engine", "c-engine"], + "Should order the first two engines correctly for distro-2 distribution" + ); + + await assertActualEnginesEqualsExpected( + ENGINE_ORDERS_CONFIG, + { + locale: "en-CA", + region: "CA", + }, + ["b-engine", "a-engine", "c-engine"], + "Should be in the same engine order as the config when there's no engine order environments matched." + ); +}); + +add_task(async function test_selector_match_engine_orders_starts_with() { + await assertActualEnginesEqualsExpected( + STARTS_WITH_WIKI_CONFIG, + { + locale: "en-CA", + region: "CA", + }, + ["wiki-ca", "engine-1", "engine-2"], + "Should list the wiki-ca engine and other engines in correct orders with the en-CA and CA locale region environment." + ); + + await assertActualEnginesEqualsExpected( + STARTS_WITH_WIKI_CONFIG, + { + locale: "en-GB", + region: "GB", + }, + ["wiki-uk", "engine-1", "engine-2"], + "Should list the wiki-ca engine and other engines in correct orders with the en-CA and CA locale region environment." + ); +}); + +add_task(async function test_selector_match_engine_orders_with_defaults() { + await assertActualEnginesEqualsExpected( + DEFAULTS_CONFIG, + { + locale: "en-CA", + region: "CA", + }, + ["default-engine", "default-private-engine", "a-engine", "b-engine"], + "Should order the default engine first, default private engine second, and the rest of the engines in the correct order." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js new file mode 100644 index 0000000000..bf56984cde --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js @@ -0,0 +1,795 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This tests the SearchEngineSelector's functionality in correctly filtering the + * engines from the config based on the user's environment. + */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchEngineSelector: "resource://gre/modules/SearchEngineSelector.sys.mjs", +}); + +const CONFIG_EVERYWHERE = [ + { + recordType: "engine", + identifier: "engine-everywhere", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-everywhere-except-en-US", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + excludedLocales: ["en-US"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-everywhere-except-FI", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + excludedRegions: ["FI"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-everywhere-except-en-CA-and-CA", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + excludedRegions: ["CA"], + excludedLocales: ["en-CA"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_EXPERIMENT = [ + { + recordType: "engine", + identifier: "engine-experiment", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + experiment: "experiment", + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_LOCALES_AND_REGIONS = [ + { + recordType: "engine", + identifier: "engine-canada", + base: {}, + variants: [ + { + environment: { + locales: ["en-CA"], + regions: ["CA"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-exclude-regions", + base: {}, + variants: [ + { + environment: { + locales: ["en-GB"], + excludedRegions: ["US"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-specific-locale-in-all-regions", + base: {}, + variants: [ + { + environment: { + locales: ["en-US"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-exclude-locale", + base: {}, + variants: [ + { + environment: { + excludedLocales: ["fr"], + regions: ["BE"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-specific-region-with-any-locales", + base: {}, + variants: [ + { + environment: { + regions: ["FI"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_DISTRIBUTION = [ + { + recordType: "engine", + identifier: "engine-distribution-1", + base: {}, + variants: [ + { + environment: { + distributions: ["distribution-1"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-multiple-distributions", + base: {}, + variants: [ + { + environment: { + distributions: ["distribution-2", "distribution-3"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-distribution-region-locales", + base: {}, + variants: [ + { + environment: { + distributions: ["distribution-4"], + locales: ["fi"], + regions: ["FI"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-distribution-experiment", + base: {}, + variants: [ + { + environment: { + distributions: ["distribution-5"], + experiment: "experiment", + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-distribution-excluded", + base: {}, + variants: [ + { + environment: { + distributions: ["distribution-include"], + excludedDistributions: ["distribution-exclude"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_CHANNEL_APPLICATION = [ + { + recordType: "engine", + identifier: "engine-channel", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + channels: ["esr"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-application", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + applications: ["firefox"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_OPTIONAL = [ + { + recordType: "engine", + identifier: "engine-optional-true", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + applications: ["firefox"], + }, + optional: true, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-optional-false", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + applications: ["firefox"], + }, + optional: false, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-optional-undefined", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + applications: ["firefox"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_VERSIONS = [ + { + recordType: "engine", + identifier: "engine-min-1", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + minVersion: "1", + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-max-20", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + maxVersion: "20", + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-min-max-1-10", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + minVersion: "1", + maxVersion: "10", + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const engineSelector = new SearchEngineSelector(); +let settings; +let settingOverrides; +let configStub; +let overrideStub; + +/** + * This function asserts if the actual engine identifiers returned equals + * the expected engines. + * + * @param {object} config + * A mock search config contain engines. + * @param {object} userEnv + * A fake user's environment including locale and region, experiment, etc. + * @param {Array} expectedEngines + * The array of expected engine identifiers to be returned from the config. + * @param {string} message + * The assertion message. + */ +async function assertActualEnginesEqualsExpected( + config, + userEnv, + expectedEngines, + message +) { + engineSelector._configuration = null; + configStub.returns(config); + + let { engines } = await engineSelector.fetchEngineConfiguration(userEnv); + let actualEngines = engines.map(engine => engine.identifier); + Assert.deepEqual(actualEngines, expectedEngines, message); +} + +add_setup(async function () { + settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + configStub = sinon.stub(settings, "get"); + settingOverrides = await RemoteSettings( + SearchUtils.NEW_SETTINGS_OVERRIDES_KEY + ); + overrideStub = sinon.stub(settingOverrides, "get"); + overrideStub.returns([]); +}); + +add_task(async function test_selector_match_experiment() { + await assertActualEnginesEqualsExpected( + CONFIG_EXPERIMENT, + { + locale: "en-CA", + region: "ca", + experiment: "experiment", + }, + ["engine-experiment"], + "Should match engine with the same experiment." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_EXPERIMENT, + { + locale: "en-CA", + region: "ca", + experiment: "no-match-experiment", + }, + [], + "Should not match any engines without experiment." + ); +}); + +add_task(async function test_everywhere_and_excluded_locale() { + await assertActualEnginesEqualsExpected( + CONFIG_EVERYWHERE, + { + locale: "en-GB", + region: "GB", + }, + [ + "engine-everywhere", + "engine-everywhere-except-en-US", + "engine-everywhere-except-FI", + "engine-everywhere-except-en-CA-and-CA", + ], + "Should match the engines for all locales and regions." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_EVERYWHERE, + { + locale: "en-US", + region: "US", + }, + [ + "engine-everywhere", + "engine-everywhere-except-FI", + "engine-everywhere-except-en-CA-and-CA", + ], + "Should match engines that do not exclude user's locale." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_EVERYWHERE, + { + locale: "fi", + region: "FI", + }, + [ + "engine-everywhere", + "engine-everywhere-except-en-US", + "engine-everywhere-except-en-CA-and-CA", + ], + "Should match engines that do not exclude user's region." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_EVERYWHERE, + { + locale: "en-CA", + region: "CA", + }, + [ + "engine-everywhere", + "engine-everywhere-except-en-US", + "engine-everywhere-except-FI", + ], + "Should match engine that do not exclude user's region and locale." + ); +}); + +add_task(async function test_selector_locales_and_regions() { + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "en-CA", + region: "CA", + }, + ["engine-canada"], + "Should match engine with same locale and region." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "en-GB", + region: "US", + }, + [], + "Should not match any engines when region is excluded." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "en-US", + region: "AU", + }, + ["engine-specific-locale-in-all-regions"], + "Should match engine with specified locale in any region." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "en-US", + region: "NL", + }, + ["engine-specific-locale-in-all-regions"], + "Should match engine with specified locale in any region." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "fr", + region: "BE", + }, + [], + "Should not match any engines when locale is excluded." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "fi", + region: "FI", + }, + ["engine-specific-region-with-any-locales"], + "Should match engine with specified region with any locale." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_LOCALES_AND_REGIONS, + { + locale: "tlh", + region: "FI", + }, + ["engine-specific-region-with-any-locales"], + "Should match engine with specified region with any locale." + ); +}); + +add_task(async function test_selector_match_distribution() { + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-1", + }, + ["engine-distribution-1"], + "Should match engine with the same distribution." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-2", + }, + ["engine-multiple-distributions"], + "Should match engine with multiple distributions." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-3", + }, + ["engine-multiple-distributions"], + "Should match engine with multiple distributions." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "fi", + region: "FI", + distroID: "distribution-4", + }, + ["engine-distribution-region-locales"], + "Should match engine with distribution, specific region and locale." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-4", + }, + [], + "Should not match any engines with no matching distribution, region and locale." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-5", + experiment: "experiment", + }, + ["engine-distribution-experiment"], + "Should match engine with distribution and experiment." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-5", + experiment: "no-match-experiment", + }, + [], + "Should not match any engines with no matching distribution and experiment." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-include", + }, + ["engine-distribution-excluded"], + "Should match engines with included distributions." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_DISTRIBUTION, + { + locale: "en-CA", + region: "CA", + distroID: "distribution-exclude", + }, + [], + "Should not match any engines with excluded distribution." + ); +}); + +add_task(async function test_engine_selector_match_applications() { + await assertActualEnginesEqualsExpected( + CONFIG_CHANNEL_APPLICATION, + { + locale: "en-CA", + region: "CA", + channel: "esr", + }, + ["engine-channel"], + "Should match engine for esr channel." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_CHANNEL_APPLICATION, + { + locale: "en-CA", + region: "CA", + appName: "firefox", + }, + ["engine-application"], + "Should match engine for application." + ); +}); + +add_task(async function test_engine_selector_match_version() { + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "1", + }, + ["engine-min-1", "engine-max-20", "engine-min-max-1-10"], + "Should match engines with that support versions equal or above the minimum." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "30", + }, + ["engine-min-1"], + "Should match engines with that support versions above the minimum." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "20", + }, + ["engine-min-1", "engine-max-20"], + "Should match engines with that support versions equal or below the maximum." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "5", + }, + ["engine-min-1", "engine-max-20", "engine-min-max-1-10"], + "Should match engines with that support the versions above the minimum and below the maximum." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "15", + }, + ["engine-min-1", "engine-max-20"], + "Should match engines with that support the versions above the minimum and below the maximum." + ); + + await assertActualEnginesEqualsExpected( + CONFIG_VERSIONS, + { + locale: "en-CA", + region: "CA", + version: "", + }, + [], + "Should match no engines with no matching versions." + ); +}); + +add_task(async function test_engine_selector_does_not_match_optional_engines() { + await assertActualEnginesEqualsExpected( + CONFIG_OPTIONAL, + { + locale: "en-CA", + region: "CA", + appName: "firefox", + }, + ["engine-optional-false", "engine-optional-undefined"], + "Should match engines where optional flag is false or undefined" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js new file mode 100644 index 0000000000..0fd57f2094 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js @@ -0,0 +1,203 @@ +/* Any copyright is dedicated to the Public Domain. +http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This tests the SearchEngineSelector matching the correct variant engine. + * If multiple variants match, it should add the matching variants to the base + * cumulatively. + */ + +"use strict"; + +const CONFIG = [ + { + recordType: "engine", + identifier: "engine-1", + base: {}, + urls: { + search: { + params: [ + { + name: "partner-code", + value: "code", + }, + ], + }, + }, + variants: [ + { + environment: { regions: ["CA", "GB", "IT"] }, + urls: { + search: { + params: [], + }, + }, + }, + { + environment: { regions: ["CA", "US"] }, + urls: { + search: { + params: [ + { + name: "partner-code", + value: "bar", + }, + ], + }, + }, + telemetrySuffix: "telemetry", + }, + { + environment: { regions: ["CA", "GB"] }, + urls: { + search: { + params: [ + { + name: "partner-code", + value: "foo", + }, + ], + }, + }, + searchTermParamName: "search-param", + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_CLONE = structuredClone(CONFIG); + +const engineSelector = new SearchEngineSelector(); +let settings; +let configStub; + +/** + * This function asserts if the actual engines returned equals the expected + * engines. + * + * @param {object} config + * A fake search config containing engines. + * @param {object} userEnv + * A fake user's environment including locale and region, experiment, etc. + * @param {Array} expectedEngines + * The array of expected engines to be returned from the fake config. + * @param {string} message + * The assertion message. + */ +async function assertActualEnginesEqualsExpected( + config, + userEnv, + expectedEngines, + message +) { + engineSelector._configuration = null; + configStub.returns(config); + let { engines } = await engineSelector.fetchEngineConfiguration(userEnv); + + Assert.deepEqual(engines, expectedEngines, message); +} + +add_setup(async function () { + settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + configStub = sinon.stub(settings, "get"); +}); + +add_task(async function test_no_variants_match() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "fi", + region: "FI", + }, + [], + "Should match no variants." + ); +}); + +add_task(async function test_match_and_apply_all_variants() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-US", + region: "CA", + }, + [ + { + identifier: "engine-1", + urls: { search: { params: [{ name: "partner-code", value: "foo" }] } }, + telemetrySuffix: "telemetry", + searchTermParamName: "search-param", + }, + ], + "Should match all variants and apply each variant property cumulatively." + ); +}); + +add_task(async function test_match_middle_variant() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-US", + region: "US", + }, + [ + { + identifier: "engine-1", + urls: { search: { params: [{ name: "partner-code", value: "bar" }] } }, + telemetrySuffix: "telemetry", + }, + ], + "Should match first and second variants." + ); +}); + +add_task(async function test_match_first_and_last_variant() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-GB", + region: "GB", + }, + [ + { + identifier: "engine-1", + urls: { search: { params: [{ name: "partner-code", value: "foo" }] } }, + searchTermParamName: "search-param", + }, + ], + "Should match first and last variant." + ); +}); + +add_task(async function test_match_variant_with_empty_params() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "it", + region: "IT", + }, + [ + { + identifier: "engine-1", + urls: { search: { params: [] } }, + }, + ], + "Should match the first variant with empty params." + ); +}); + +add_task(async function test_config_has_not_been_modified() { + Assert.deepEqual( + CONFIG, + CONFIG_CLONE, + "Should not modify the original test config after applying variant engines to the base engine." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_set_alias.js b/toolkit/components/search/tests/xpcshell/test_engine_set_alias.js new file mode 100644 index 0000000000..ec79fe6783 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_set_alias.js @@ -0,0 +1,132 @@ +"use strict"; + +add_setup(async function () { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_engine_set_alias() { + info("Set engine alias"); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: "b", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine1 = await Services.search.getEngineByName("bacon"); + Assert.ok(engine1.aliases.includes("b")); + engine1.alias = "a"; + Assert.equal(engine1.alias, "a"); + await extension.unload(); +}); + +add_task(async function test_engine_set_alias_with_left_space() { + info("Set engine alias with left space"); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: " a", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine2 = await Services.search.getEngineByName("bacon"); + Assert.ok(engine2.aliases.includes("a")); + engine2.alias = " c"; + Assert.equal(engine2.alias, "c"); + await extension.unload(); +}); + +add_task(async function test_engine_set_alias_with_right_space() { + info("Set engine alias with right space"); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: "c ", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine3 = await Services.search.getEngineByName("bacon"); + Assert.ok(engine3.aliases.includes("c")); + engine3.alias = "o "; + Assert.equal(engine3.alias, "o"); + await extension.unload(); +}); + +add_task(async function test_engine_set_alias_with_right_left_space() { + info("Set engine alias with left and right space"); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: " o ", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine4 = await Services.search.getEngineByName("bacon"); + Assert.ok(engine4.aliases.includes("o")); + engine4.alias = " n "; + Assert.equal(engine4.alias, "n"); + await extension.unload(); +}); + +add_task(async function test_engine_set_alias_with_space() { + info("Set engine alias with space"); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: " ", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine5 = await Services.search.getEngineByName("bacon"); + Assert.equal(engine5.alias, ""); + engine5.alias = "b"; + Assert.equal(engine5.alias, "b"); + engine5.alias = " "; + Assert.equal(engine5.alias, ""); + await extension.unload(); +}); + +add_task(async function test_engine_change_alias() { + let extension = await SearchTestUtils.installSearchExtension( + { + name: "bacon", + keyword: " o ", + search_url: "https://www.bacon.test/find", + }, + { skipUnload: true } + ); + let engine6 = await Services.search.getEngineByName("bacon"); + + let promise = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + engine6.alias = "ba"; + + await promise; + Assert.equal( + engine6.alias, + "ba", + "Should have correctly notified and changed the alias." + ); + + let observed = false; + Services.obs.addObserver(function observer(aSubject, aTopic, aData) { + observed = true; + }, SearchUtils.TOPIC_ENGINE_MODIFIED); + + engine6.alias = "ba"; + + Assert.equal(engine6.alias, "ba", "Should have not changed the alias"); + Assert.ok(!observed, "Should not have notified for no change in alias"); + + await extension.unload(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_encoding.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_encoding.js new file mode 100644 index 0000000000..c6f1099ddb --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_encoding.js @@ -0,0 +1,29 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const prefix = "https://example.com/?sourceId=Mozilla-search&search="; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +function testEncode(engine, charset, query, expected) { + engine.wrappedJSObject._queryCharset = charset; + + Assert.equal( + engine.getSubmission(query).uri.spec, + prefix + expected, + `Should have correctly encoded for ${charset}` + ); +} + +add_task(async function test_getSubmission_encoding() { + let engine = await Services.search.getEngineByName("Simple Engine"); + + testEncode(engine, "UTF-8", "caff\u00E8", "caff%C3%A8"); + testEncode(engine, "windows-1252", "caff\u00E8", "caff%E8"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_params.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_params.js new file mode 100644 index 0000000000..1be81a2e31 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_params.js @@ -0,0 +1,74 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +const searchTerms = "fxsearch"; +function checkSubstitution(url, prefix, engine, template, expected) { + url.template = prefix + template; + equal(engine.getSubmission(searchTerms).uri.spec, prefix + expected); +} + +add_task(async function test_paramSubstitution() { + let prefix = "https://example.com/?sourceId=Mozilla-search&search="; + let engine = await Services.search.getEngineByName("Simple Engine"); + let url = engine.wrappedJSObject._getURLOfType("text/html"); + equal(url.getSubmission("foo", engine).uri.spec, prefix + "foo"); + // Reset the engine parameters so we can have a clean template to use for + // the subsequent tests. + url.params = []; + + let check = checkSubstitution.bind(this, url, prefix, engine); + + // The same parameter can be used more than once. + check("{searchTerms}/{searchTerms}", searchTerms + "/" + searchTerms); + + // Optional parameters are replaced if we known them. + check("{searchTerms?}", searchTerms); + check("{unknownOptional?}", ""); + check("{unknownRequired}", "{unknownRequired}"); + + check("{language}", Services.locale.requestedLocale); + check("{language?}", Services.locale.requestedLocale); + + engine.wrappedJSObject._queryCharset = "UTF-8"; + check("{inputEncoding}", "UTF-8"); + check("{inputEncoding?}", "UTF-8"); + check("{outputEncoding}", "UTF-8"); + check("{outputEncoding?}", "UTF-8"); + + // 'Unsupported' parameters with hard coded values used only when the parameter is required. + check("{count}", "20"); + check("{count?}", ""); + check("{startIndex}", "1"); + check("{startIndex?}", ""); + check("{startPage}", "1"); + check("{startPage?}", ""); + + check("{moz:locale}", Services.locale.requestedLocale); +}); + +add_task(async function test_mozParamsFailForNonAppProvided() { + await SearchTestUtils.installSearchExtension(); + + let prefix = "https://example.com/?q="; + let engine = await Services.search.getEngineByName("Example"); + let url = engine.wrappedJSObject._getURLOfType("text/html"); + equal(url.getSubmission("foo", engine).uri.spec, prefix + "foo"); + // Reset the engine parameters so we can have a clean template to use for + // the subsequent tests. + url.params = []; + + let check = checkSubstitution.bind(this, url, prefix, engine); + + // Test moz: parameters (only supported for built-in engines, ie _isDefault == true). + check("{moz:locale}", "{moz:locale}"); + + await promiseAfterSettings(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_params_pref.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_pref.js new file mode 100644 index 0000000000..5283207394 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_pref.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Test that MozParam condition="pref" values used in search URLs are from the + * default branch, and that their special characters are URL encoded. */ + +"use strict"; + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const defaultBranch = Services.prefs.getDefaultBranch( + SearchUtils.BROWSER_SEARCH_PREF +); +const baseURL = "https://www.google.com/search?q=foo"; + +add_setup(async function () { + // The test engines used in this test need to be recognized as 'default' + // engines, or their MozParams will be ignored. + await SearchTestUtils.useTestEngines(); +}); + +add_task(async function test_pref_initial_value() { + defaultBranch.setCharPref("param.code", "good&id=unique"); + + // Preference params are only allowed to be modified on the user branch + // on nightly builds. For non-nightly builds, check that modifying on the + // normal branch doesn't work. + if (!AppConstants.NIGHTLY_BUILD) { + Services.prefs.setCharPref( + SearchUtils.BROWSER_SEARCH_PREF + "param.code", + "bad" + ); + } + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=good%26id%3Dunique", + "Should have got the submission URL with the correct code" + ); + + // Now clear the user-set preference. Having a user set preference means + // we don't get updates from the pref service of changes on the default + // branch. Normally, this won't be an issue, since we don't expect users + // to be playing with these prefs, and worst-case, they'll just get the + // actual change on restart. + Services.prefs.clearUserPref(SearchUtils.BROWSER_SEARCH_PREF + "param.code"); +}); + +add_task(async function test_pref_updated() { + // Update the pref without re-init nor restart. + defaultBranch.setCharPref("param.code", "supergood&id=unique123456"); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=supergood%26id%3Dunique123456", + "Should have got the submission URL with the updated code" + ); +}); + +add_task(async function test_pref_cleared() { + // Update the pref without re-init nor restart. + // Note you can't delete a preference from the default branch. + defaultBranch.setCharPref("param.code", ""); + + let engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL, + "Should have just the base URL after the pref was cleared" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus.js new file mode 100644 index 0000000000..e9b55ff3dc --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus.js @@ -0,0 +1,127 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Test that MozParam condition="pref" values used in search URLs can be set + by Nimbus, and that their special characters are URL encoded. */ + +"use strict"; + +const { NimbusFeatures } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); + +const baseURL = "https://www.google.com/search?q=foo"; + +let getVariableStub; +let updateStub; + +add_setup(async function () { + updateStub = sinon.stub(NimbusFeatures.search, "onUpdate"); + getVariableStub = sinon.stub(NimbusFeatures.search, "getVariable"); + sinon.stub(NimbusFeatures.search, "ready").resolves(); + + // The test engines used in this test need to be recognized as 'default' + // engines, or their MozParams will be ignored. + await SearchTestUtils.useTestEngines(); +}); + +add_task(async function test_pref_initial_value() { + // These values should match the nimbusParams below and the data/test/manifest.json + // search engine configuration + getVariableStub.withArgs("extraParams").returns([ + { + key: "code", + // The & and = in this parameter are to check that they are correctly + // encoded, and not treated as a separate parameter. + value: "good&id=unique", + }, + ]); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + Assert.ok( + updateStub.called, + "Should have called onUpdate to listen for future updates" + ); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=good%26id%3Dunique", + "Should have got the submission URL with the correct code" + ); +}); + +add_task(async function test_pref_updated() { + getVariableStub.withArgs("extraParams").returns([ + { + key: "code", + // The & and = in this parameter are to check that they are correctly + // encoded, and not treated as a separate parameter. + value: "supergood&id=unique123456", + }, + ]); + // Update the pref without re-init nor restart. + updateStub.firstCall.args[0](); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=supergood%26id%3Dunique123456", + "Should have got the submission URL with the updated code" + ); +}); + +add_task(async function test_multiple_params() { + getVariableStub.withArgs("extraParams").returns([ + { + key: "code", + value: "sng", + }, + { + key: "test", + value: "sup", + }, + ]); + // Update the pref without re-init nor restart. + updateStub.firstCall.args[0](); + + let engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=sng&test=sup", + "Should have got the submission URL with both parameters" + ); + + // Test removing just one of the parameters. + getVariableStub.withArgs("extraParams").returns([ + { + key: "code", + value: "sng", + }, + ]); + // Update the pref without re-init nor restart. + updateStub.firstCall.args[0](); + + engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=sng", + "Should have got the submission URL with one parameter" + ); +}); + +add_task(async function test_pref_cleared() { + // Update the pref without re-init nor restart. + // Note you can't delete a preference from the default branch. + getVariableStub.withArgs("extraParams").returns([]); + updateStub.firstCall.args[0](); + + let engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL, + "Should have just the base URL after the pref was cleared" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus_invalid.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus_invalid.js new file mode 100644 index 0000000000..e519a74c64 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_prefNimbus_invalid.js @@ -0,0 +1,87 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Test that if Nimbus provides invalid values for extraParams, the search + * service can still initialize and run. + * This is separate to test_getSubmission_params_prefNimbus so that initialization + * can be properly tested. + */ + +"use strict"; + +const { NimbusFeatures } = ChromeUtils.importESModule( + "resource://nimbus/ExperimentAPI.sys.mjs" +); + +const baseURL = "https://www.google.com/search?q=foo"; + +let getVariableStub; +let updateStub; + +add_setup(async function () { + consoleAllowList.push("Failed to load nimbus variables for extraParams"); + + updateStub = sinon.stub(NimbusFeatures.search, "onUpdate"); + getVariableStub = sinon.stub(NimbusFeatures.search, "getVariable"); + sinon.stub(NimbusFeatures.search, "ready").resolves(); + + // The test engines used in this test need to be recognized as 'default' + // engines, or their MozParams will be ignored. + await SearchTestUtils.useTestEngines(); +}); + +add_task(async function test_bad_nimbus_setting_on_init() { + getVariableStub.withArgs("extraParams").returns({ foo: "bar" }); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + Assert.ok( + updateStub.called, + "Should have called onUpdate to listen for future updates" + ); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL, + "Should have been able to get a submission URL" + ); +}); + +add_task(async function test_switch_to_good_nimbus_setting() { + // Switching to a good structure should provide the parameter. + getVariableStub.withArgs("extraParams").returns([ + { + key: "code", + // The & and = in this parameter are to check that they are correctly + // encoded, and not treated as a separate parameter. + value: "supergood&id=unique123456", + }, + ]); + + updateStub.firstCall.args[0](); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL + "&code=supergood%26id%3Dunique123456", + "Should have got the submission URL with the updated code" + ); +}); + +add_task(async function test_switch_back_to_bad_nimbus_setting() { + // Switching from a valid Nimbus setting to an invalid setting should fallback + // to a valid submission URL. + getVariableStub.withArgs("extraParams").returns({ bar: "foo" }); + + updateStub.firstCall.args[0](); + + const engine = Services.search.getEngineByName("engine-pref"); + Assert.equal( + engine.getSubmission("foo").uri.spec, + baseURL, + "Should have not have the extra parameters on the submission URL after a bad update" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_getSubmission_params_purpose.js b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_purpose.js new file mode 100644 index 0000000000..35a33280f6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_getSubmission_params_purpose.js @@ -0,0 +1,83 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that a search purpose can be specified and that query parameters for + * that purpose are included in the search URL. + */ + +"use strict"; + +add_setup(async function () { + // The test engines used in this test need to be recognized as 'default' + // engines, or their MozParams used to set the purpose will be ignored. + await SearchTestUtils.useTestEngines(); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_purpose() { + let engine = Services.search.getEngineByName("Test search engine"); + + function check_submission(aValue, aSearchTerm, aType, aPurpose) { + let submissionURL = engine.getSubmission(aSearchTerm, aType, aPurpose).uri + .spec; + let searchParams = new URLSearchParams(submissionURL.split("?")[1]); + if (aValue) { + Assert.equal(searchParams.get("channel"), aValue); + } else { + Assert.ok(!searchParams.has("channel")); + } + Assert.equal(searchParams.get("q"), aSearchTerm); + } + + check_submission("", "foo"); + check_submission("", "foo", null); + check_submission("", "foo", "text/html"); + check_submission("rcs", "foo", null, "contextmenu"); + check_submission("rcs", "foo", "text/html", "contextmenu"); + check_submission("fflb", "foo", null, "keyword"); + check_submission("fflb", "foo", "text/html", "keyword"); + check_submission("", "foo", "text/html", "invalid"); + + // Tests for a purpose on the search form (ie. empty query). + engine = Services.search.getEngineByName("engine-rel-searchform-purpose"); + + // See bug 1485508 + Assert.ok(!engine.searchForm.includes("?&")); + + // verify that the 'system' purpose falls back to the 'searchbar' purpose. + check_submission("sb", "foo", "text/html", "system"); + check_submission("sb", "foo", "text/html", "searchbar"); +}); + +add_task(async function test_purpose() { + let engine = Services.search.getEngineByName( + "Test search engine (Reordered)" + ); + + function check_submission(aValue, aSearchTerm, aType, aPurpose) { + let submissionURL = engine.getSubmission(aSearchTerm, aType, aPurpose).uri + .spec; + let searchParams = new URLSearchParams(submissionURL.split("?")[1]); + if (aValue) { + Assert.equal(searchParams.get("channel"), aValue); + } else { + Assert.ok(!searchParams.has("channel")); + } + Assert.equal(searchParams.get("q"), aSearchTerm); + } + + check_submission("", "foo"); + check_submission("", "foo", null); + check_submission("", "foo", "text/html"); + check_submission("rcs", "foo", null, "contextmenu"); + check_submission("rcs", "foo", "text/html", "contextmenu"); + check_submission("fflb", "foo", null, "keyword"); + check_submission("fflb", "foo", "text/html", "keyword"); + check_submission("", "foo", "text/html", "invalid"); + + // See bug 1485508 + Assert.ok(!engine.searchForm.includes("?&")); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_identifiers.js b/toolkit/components/search/tests/xpcshell/test_identifiers.js new file mode 100644 index 0000000000..4e39507d3a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_identifiers.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test of a search engine's telemetryId. + */ + +"use strict"; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines"); + await AddonTestUtils.promiseStartupManager(); + + const result = await Services.search.init(); + Assert.ok( + Components.isSuccessCode(result), + "Should have initialized the service" + ); + + useHttpServer(); + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); +}); + +function checkIdentifier(engineName, expectedIdentifier, expectedTelemetryId) { + const engine = Services.search.getEngineByName(engineName); + Assert.ok( + engine instanceof Ci.nsISearchEngine, + "Should be derived from nsISearchEngine" + ); + + Assert.equal( + engine.telemetryId, + expectedTelemetryId, + "Should have the correct telemetry Id" + ); + + // TODO: Bug 1877721 - We have 3 forms of identifiers which causes confusion, + // we can remove the identifier for nsISearchEngine. + Assert.equal( + engine.identifier, + expectedIdentifier, + "Should have the correct identifier" + ); +} + +add_task(async function test_from_profile() { + // An engine loaded from the profile directory won't have an identifier, + // because it's not built-in. + checkIdentifier(kTestEngineName, null, `other-${kTestEngineName}`); +}); + +add_task(async function test_from_telemetry_id() { + checkIdentifier("basic", "basic-telemetry", "basic-telemetry"); +}); + +add_task(async function test_from_webextension_id() { + // If not specified, the telemetry Id is derived from the WebExtension prefix, + // it should not use the WebExtension display name. + checkIdentifier("Simple Engine", "simple", "simple"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_ignorelist.js b/toolkit/components/search/tests/xpcshell/test_ignorelist.js new file mode 100644 index 0000000000..cd3a3a7238 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_ignorelist.js @@ -0,0 +1,72 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const kSearchEngineID1 = "ignorelist_test_engine1"; +const kSearchEngineID2 = "ignorelist_test_engine2"; +const kSearchEngineID3 = "ignorelist_test_engine3"; +const kSearchEngineURL1 = + "https://example.com/?search={searchTerms}&ignore=true"; +const kSearchEngineURL2 = + "https://example.com/?search={searchTerms}&IGNORE=TRUE"; +const kSearchEngineURL3 = "https://example.com/?search={searchTerms}"; +const kExtensionID = "searchignore@mozilla.com"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_ignoreList() { + await setupRemoteSettings(); + + Assert.ok( + !Services.search.isInitialized, + "Search service should not be initialized to begin with." + ); + + let updatePromise = SearchTestUtils.promiseSearchNotification( + "settings-update-complete" + ); + + await SearchTestUtils.installSearchExtension({ + name: kSearchEngineID1, + search_url: kSearchEngineURL1, + }); + + await updatePromise; + + let engine = Services.search.getEngineByName(kSearchEngineID1); + Assert.equal( + engine, + null, + "Engine with ignored search params should not exist" + ); + + await SearchTestUtils.installSearchExtension({ + name: kSearchEngineID2, + search_url: kSearchEngineURL2, + }); + + // An ignored engine shouldn't be available at all + engine = Services.search.getEngineByName(kSearchEngineID2); + Assert.equal( + engine, + null, + "Engine with ignored search params of a different case should not exist" + ); + + await SearchTestUtils.installSearchExtension({ + id: kExtensionID, + name: kSearchEngineID3, + search_url: kSearchEngineURL3, + }); + + // An ignored engine shouldn't be available at all + engine = Services.search.getEngineByName(kSearchEngineID3); + Assert.equal( + engine, + null, + "Engine with ignored extension id should not exist" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_ignorelist_update.js b/toolkit/components/search/tests/xpcshell/test_ignorelist_update.js new file mode 100644 index 0000000000..fc090d6ff7 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_ignorelist_update.js @@ -0,0 +1,89 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const kSearchEngineID1 = "ignorelist_test_engine1"; +const kSearchEngineID2 = "ignorelist_test_engine2"; +const kSearchEngineID3 = "ignorelist_test_engine3"; +const kSearchEngineURL1 = + "https://example.com/?search={searchTerms}&ignore=true"; +const kSearchEngineURL2 = + "https://example.com/?search={searchTerms}&IGNORE=TRUE"; +const kSearchEngineURL3 = "https://example.com/?search={searchTerms}"; +const kExtensionID = "searchignore@mozilla.com"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_ignoreList() { + Assert.ok( + !Services.search.isInitialized, + "Search service should not be initialized to begin with." + ); + + let updatePromise = SearchTestUtils.promiseSearchNotification( + "settings-update-complete" + ); + await SearchTestUtils.installSearchExtension({ + name: kSearchEngineID1, + search_url: kSearchEngineURL1, + }); + await SearchTestUtils.installSearchExtension({ + name: kSearchEngineID2, + search_url: kSearchEngineURL2, + }); + await SearchTestUtils.installSearchExtension({ + id: kExtensionID, + name: kSearchEngineID3, + search_url: kSearchEngineURL3, + }); + + // Ensure that the initial remote settings update from default values is + // complete. The defaults do not include the special inclusions inserted below. + await updatePromise; + + for (let engineName of [ + kSearchEngineID1, + kSearchEngineID2, + kSearchEngineID3, + ]) { + Assert.ok( + await Services.search.getEngineByName(engineName), + `Engine ${engineName} should be present` + ); + } + + // Simulate an ignore list update. + await RemoteSettings("hijack-blocklists").emit("sync", { + data: { + current: [ + { + id: "load-paths", + schema: 1553857697843, + last_modified: 1553859483588, + matches: ["[addon]searchignore@mozilla.com"], + }, + { + id: "submission-urls", + schema: 1553857697843, + last_modified: 1553859435500, + matches: ["ignore=true"], + }, + ], + }, + }); + + for (let engineName of [ + kSearchEngineID1, + kSearchEngineID2, + kSearchEngineID3, + ]) { + Assert.equal( + await Services.search.getEngineByName(engineName), + null, + `Engine ${engineName} should not be present` + ); + } +}); diff --git a/toolkit/components/search/tests/xpcshell/test_initialization.js b/toolkit/components/search/tests/xpcshell/test_initialization.js new file mode 100644 index 0000000000..c89f3cfcb3 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_initialization.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Tests that a delayed add-on manager start up does not affect the start up +// of the search service. + +"use strict"; + +const CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + orderHint: 30, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, +]; + +add_setup(() => { + do_get_profile(); + Services.fog.initializeFOG(); +}); + +add_task(async function test_initialization_delayed_addon_manager() { + let stub = await SearchTestUtils.useTestEngines("data", null, CONFIG); + // Wait until the search service gets its configuration before starting + // to initialise the add-on manager. This simulates the add-on manager + // starting late which used to cause the search service to fail to load any + // engines. + stub.callsFake(() => { + Services.tm.dispatchToMainThread(() => { + AddonTestUtils.promiseStartupManager(); + }); + return CONFIG; + }); + + await Services.search.init(); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Test engine shouldn't be the default anymore" + ); + + await assertGleanDefaultEngine({ + normal: { + engineId: "engine", + displayName: "Test search engine", + loadPath: [ + SearchUtils.newSearchConfigEnabled + ? "[app]engine@search.mozilla.org" + : "[addon]engine@search.mozilla.org", + ], + submissionUrl: "https://www.google.com/search?q=", + verified: "default", + }, + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_initialization_status_telemetry.js b/toolkit/components/search/tests/xpcshell/test_initialization_status_telemetry.js new file mode 100644 index 0000000000..fc62ded787 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_initialization_status_telemetry.js @@ -0,0 +1,89 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests telemetry is captured when search service initialization has failed or + * succeeded. + */ +const searchService = Services.search.wrappedJSObject; + +add_setup(async () => { + consoleAllowList.push("#init: failure initializing search:"); + await SearchTestUtils.useTestEngines("simple-engines"); + await AddonTestUtils.promiseStartupManager(); + Services.fog.initializeFOG(); +}); + +add_task(async function test_init_success_telemetry() { + Assert.equal( + searchService.isInitialized, + false, + "Search Service should not be initialized." + ); + + await Services.search.init(); + + Assert.equal( + searchService.hasSuccessfullyInitialized, + true, + "Search Service should have initialized successfully." + ); + + Assert.equal( + 1, + await Glean.searchService.initializationStatus.success.testGetValue(), + "Should have incremented init success by one." + ); +}); + +add_task(async function test_init_failure_telemetry() { + await startInitFailure("Settings"); + Assert.equal( + 1, + await Glean.searchService.initializationStatus.failedSettings.testGetValue(), + "Should have incremented get settings failure by one." + ); + + await startInitFailure("FetchEngines"); + Assert.equal( + 1, + await Glean.searchService.initializationStatus.failedFetchEngines.testGetValue(), + "Should have incremented fetch engines failure by one." + ); + + await startInitFailure("LoadEngines"); + Assert.equal( + 1, + await Glean.searchService.initializationStatus.failedLoadEngines.testGetValue(), + "Should have incremented load engines failure by one." + ); +}); + +async function startInitFailure(errorType) { + searchService.reset(); + searchService.errorToThrowInTest = errorType; + + Assert.equal( + searchService.isInitialized, + false, + "Search Service should not be initialized." + ); + + let regex = new RegExp( + `Fake ${errorType} error during search service initialization.` + ); + + await Assert.rejects( + Services.search.init(), + regex, + "Should have thrown an error on init." + ); + + await Assert.rejects( + Services.search.promiseInitialized, + regex, + "Should have rejected the promise." + ); + + searchService.errorToThrowInTest = null; +} diff --git a/toolkit/components/search/tests/xpcshell/test_initialization_with_region.js b/toolkit/components/search/tests/xpcshell/test_initialization_with_region.js new file mode 100644 index 0000000000..fb9cee5124 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_initialization_with_region.js @@ -0,0 +1,170 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const SEARCH_SERVICE_TOPIC = "browser-search-service"; +const SEARCH_ENGINE_TOPIC = "browser-search-engine-modified"; + +const CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + orderHint: 30, + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["FR"] }, + default: "yes", + defaultPrivate: "yes", + }, + ], + }, + { + webExtension: { + id: "engine-pref@search.mozilla.org", + name: "engine-pref", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "code", + condition: "pref", + pref: "code", + }, + { + name: "test", + condition: "pref", + pref: "test", + }, + ], + }, + orderHint: 20, + appliesTo: [ + { + included: { regions: ["FR"] }, + default: "yes", + defaultPrivate: "yes", + }, + ], + }, +]; + +// Default engine with no region defined. +const DEFAULT = "Test search engine"; +// Default engine with region set to FR. +const FR_DEFAULT = "engine-pref"; + +function listenFor(name, key) { + let notifyObserved = false; + let obs = (subject, topic, data) => { + if (data == key) { + notifyObserved = true; + } + }; + Services.obs.addObserver(obs, name); + + return () => { + Services.obs.removeObserver(obs, name); + return notifyObserved; + }; +} + +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.separatePrivateDefault", true); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + SearchTestUtils.useMockIdleService(); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); +}); + +// This tests what we expect is the normal startup route for a fresh profile - +// the search service initializes with no region details, then gets a region +// notified part way through / afterwards. +add_task(async function test_initialization_with_region() { + let reloadObserved = listenFor(SEARCH_SERVICE_TOPIC, "engines-reloaded"); + let initPromise; + + // Ensure the region lookup completes after init so the + // engines are reloaded + let srv = useHttpServer(); + srv.registerPathHandler("/fetch_region", async (req, res) => { + res.processAsync(); + await initPromise; + res.setStatusLine("1.1", 200, "OK"); + res.write(JSON.stringify({ country_code: "FR" })); + res.finish(); + }); + + Services.prefs.setCharPref( + "browser.region.network.url", + `http://localhost:${srv.identity.primaryPort}/fetch_region` + ); + + Region._setHomeRegion("", false); + Region.init(); + + initPromise = Services.search.init(); + await initPromise; + + let otherPromises = [ + // This test expects settings to be saved twice. + promiseAfterSettings().then(promiseAfterSettings), + SearchTestUtils.promiseSearchNotification( + "engine-default", + SEARCH_ENGINE_TOPIC + ), + ]; + + Assert.equal( + Services.search.defaultEngine.name, + DEFAULT, + "Test engine shouldn't be the default anymore" + ); + + await Promise.all(otherPromises); + + // Ensure that correct engine is being reported as the default. + Assert.equal( + Services.search.defaultEngine.name, + FR_DEFAULT, + "engine-pref should be the default in FR" + ); + Assert.equal( + (await Services.search.getDefaultPrivate()).name, + FR_DEFAULT, + "engine-pref should be the private default in FR" + ); + + Assert.ok(reloadObserved(), "Engines do reload with delayed region fetch"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_list_json_locale.js b/toolkit/components/search/tests/xpcshell/test_list_json_locale.js new file mode 100644 index 0000000000..75c77be22a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_list_json_locale.js @@ -0,0 +1,101 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Check default search engine is picked from list.json searchDefault */ + +"use strict"; + +add_setup(async function () { + await SearchTestUtils.useTestEngines(); + + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "de", + "fr", + ]; + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Region._setHomeRegion("US", false); +}); + +add_task(async function test_listJSONlocale() { + Services.locale.requestedLocales = ["de"]; + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + Assert.ok(Services.search.isInitialized, "search initialized"); + + let sortedEngines = await Services.search.getEngines(); + Assert.equal(sortedEngines.length, 1, "Should have only one engine"); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Should have the correct default engine" + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + // 'de' only displays google, so we'll be using the same engine as the + // normal default. + "Test search engine", + "Should have the correct private default engine" + ); +}); + +// Check that switching locale switches search engines +add_task(async function test_listJSONlocaleSwitch() { + await promiseSetLocale("fr"); + + Assert.ok(Services.search.isInitialized, "search initialized"); + + let sortedEngines = await Services.search.getEngines(); + Assert.deepEqual( + sortedEngines.map(e => e.name), + ["Test search engine", "engine-pref", "engine-resourceicon"], + "Should have the correct engine list" + ); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Should have the correct default engine" + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine-pref", + "Should have the correct private default engine" + ); +}); + +// Check that region overrides apply +add_task(async function test_listJSONRegionOverride() { + await promiseSetHomeRegion("RU"); + + Assert.ok(Services.search.isInitialized, "search initialized"); + + let sortedEngines = await Services.search.getEngines(); + Assert.deepEqual( + sortedEngines.map(e => e.name), + ["Test search engine", "engine-pref", "engine-chromeicon"], + "Should have the correct engine list" + ); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Should have the correct default engine" + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine-pref", + "Should have the correct private default engine" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_list_json_no_private_default.js b/toolkit/components/search/tests/xpcshell/test_list_json_no_private_default.js new file mode 100644 index 0000000000..8de8ad6a34 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_list_json_no_private_default.js @@ -0,0 +1,49 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// TODO: Test fallback to normal default when no private set at all. + +/* Check default search engine is picked from list.json searchDefault */ + +"use strict"; + +// Check that current engine matches with US searchDefault from list.json +add_task(async function test_searchDefaultEngineUS() { + await SearchTestUtils.useTestEngines("data1"); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + Assert.ok(Services.search.isInitialized, "search initialized"); + + Assert.equal( + Services.search.appDefaultEngine.name, + "engine1", + "Should have the expected engine as app default" + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have the expected engine as default" + ); + Assert.equal( + Services.search.appPrivateDefaultEngine.name, + "engine1", + "Should have the same engine for the app private default" + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine1", + "Should have the same engine for the private default" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_list_json_searchdefault.js b/toolkit/components/search/tests/xpcshell/test_list_json_searchdefault.js new file mode 100644 index 0000000000..38c7f302d2 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_list_json_searchdefault.js @@ -0,0 +1,70 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Check default search engine is picked from list.json searchDefault */ + +"use strict"; + +// Check that current engine matches with US searchDefault from list.json +add_task(async function test_searchDefaultEngineUS() { + await SearchTestUtils.useTestEngines(); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + Assert.ok(Services.search.isInitialized, "search initialized"); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Should have the expected engine as default." + ); + Assert.equal( + Services.search.appDefaultEngine.name, + "Test search engine", + "Should have the expected engine as the app default" + ); + + // First with the pref off to check using the existing values. + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + + Assert.equal( + Services.search.defaultPrivateEngine.name, + Services.search.defaultEngine.name, + "Should have the normal default engine when separate private browsing is off." + ); + Assert.equal( + Services.search.appPrivateDefaultEngine.name, + Services.search.appDefaultEngine.name, + "Should have the normal app engine when separate private browsing is off." + ); + + // Then with the pref on. + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine-pref", + "Should have the private default engine when separate private browsing is on." + ); + Assert.equal( + Services.search.appPrivateDefaultEngine.name, + "engine-pref", + "Should have the app private engine set correctly when separate private browsing is on." + ); + + Services.prefs.clearUserPref("browser.search.region"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_list_json_searchorder.js b/toolkit/components/search/tests/xpcshell/test_list_json_searchorder.js new file mode 100644 index 0000000000..a4933657be --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_list_json_searchorder.js @@ -0,0 +1,66 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Check default search engine is picked from list.json searchDefault */ + +"use strict"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + await SearchTestUtils.useTestEngines(); + await Services.search.init(); +}); + +async function checkOrder(expectedOrder) { + const sortedEngines = await Services.search.getEngines(); + Assert.deepEqual( + sortedEngines.map(s => s.name), + expectedOrder, + "Should have the expected engine order" + ); +} + +add_task(async function test_searchOrderJSON_no_separate_private() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + + await checkOrder([ + // Default engine + "Test search engine", + // Two engines listed in searchOrder. + "engine-resourceicon", + "engine-chromeicon", + // Rest of the engines in order. + "engine-pref", + "engine-rel-searchform-purpose", + "Test search engine (Reordered)", + ]); +}); + +add_task(async function test_searchOrderJSON_separate_private() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + await checkOrder([ + // Default engine + "Test search engine", + // Default private engine + "engine-pref", + // Two engines listed in searchOrder. + "engine-resourceicon", + "engine-chromeicon", + // Rest of the engines in order. + "engine-rel-searchform-purpose", + "Test search engine (Reordered)", + ]); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_location_timeout_xhr.js b/toolkit/components/search/tests/xpcshell/test_location_timeout_xhr.js new file mode 100644 index 0000000000..9f965271c2 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_location_timeout_xhr.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This is testing the long, last-resort XHR-based timeout for the location +// search. + +function startServer(continuePromise) { + let srv = new HttpServer(); + function lookupCountry(metadata, response) { + response.processAsync(); + // wait for our continuePromise to resolve before writing a valid + // response. + // This will be resolved after the timeout period, so we can check + // the behaviour in that case. + continuePromise.then(() => { + response.setStatusLine("1.1", 200, "OK"); + response.write('{"country_code" : "AU"}'); + response.finish(); + }); + } + srv.registerPathHandler("/lookup_country", lookupCountry); + srv.start(-1); + return srv; +} + +function verifyProbeSum(probe, sum) { + let histogram = Services.telemetry.getHistogramById(probe); + let snapshot = histogram.snapshot(); + equal(snapshot.sum, sum, probe); +} + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_location_timeout_xhr() { + let resolveContinuePromise; + let continuePromise = new Promise(resolve => { + resolveContinuePromise = resolve; + }); + + let server = startServer(continuePromise); + let url = + "http://localhost:" + server.identity.primaryPort + "/lookup_country"; + Services.prefs.setCharPref("browser.search.geoip.url", url); + // The timeout for the timer. + Services.prefs.setIntPref("browser.search.geoip.timeout", 10); + let promiseXHRStarted = SearchTestUtils.promiseSearchNotification( + "geoip-lookup-xhr-starting" + ); + await Services.search.init(); + ok( + !Services.prefs.prefHasUserValue("browser.search.region"), + "should be no region pref" + ); + // should be no result recorded at all. + checkCountryResultTelemetry(null); + + // should not have SEARCH_SERVICE_COUNTRY_FETCH_TIME_MS recorded as our + // test server is still blocked on our promise. + verifyProbeSum("SEARCH_SERVICE_COUNTRY_FETCH_TIME_MS", 0); + + promiseXHRStarted.then(xhr => { + // Set the timeout on the xhr object to an extremely low value, so it + // should timeout immediately. + xhr.timeout = 10; + // wait for the xhr timeout to fire. + SearchTestUtils.promiseSearchNotification("geoip-lookup-xhr-complete").then( + () => { + // should have the XHR timeout recorded. + checkCountryResultTelemetry(TELEMETRY_RESULT_ENUM.TIMEOUT); + // still should not have a report of how long the response took as we + // only record that on success responses. + verifyProbeSum("SEARCH_SERVICE_COUNTRY_FETCH_TIME_MS", 0); + // and we still don't know the country code or region. + ok( + !Services.prefs.prefHasUserValue("browser.search.region"), + "should be no region pref" + ); + + // unblock the server even though nothing is listening. + resolveContinuePromise(); + + return new Promise(resolve => { + server.stop(resolve); + }); + } + ); + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_maybereloadengine_order.js b/toolkit/components/search/tests/xpcshell/test_maybereloadengine_order.js new file mode 100644 index 0000000000..663b7205a7 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_maybereloadengine_order.js @@ -0,0 +1,88 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const TEST_CONFIG = [ + { + webExtension: { + id: "plainengine@search.mozilla.org", + name: "Plain", + search_url: "https://duckduckgo.com/", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + suggest_url: "https://ac.duckduckgo.com/ac/q={searchTerms}&type=list", + }, + appliesTo: [{ included: { everywhere: true } }], + }, + { + webExtension: { + id: "special-engine@search.mozilla.org", + name: "Special", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "client", + condition: "purpose", + purpose: "keyword", + value: "firefox-b-1-ab", + }, + { + name: "client", + condition: "purpose", + purpose: "searchbar", + value: "firefox-b-1", + }, + ], + suggest_url: + "https://www.google.com/complete/search?client=firefox&q={searchTerms}", + }, + appliesTo: [{ default: "yes", included: { regions: ["FR"] } }], + }, +]; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("test-extensions", null, TEST_CONFIG); + await AddonTestUtils.promiseStartupManager(); + + registerCleanupFunction(AddonTestUtils.promiseShutdownManager); +}); + +add_task(async function basic_multilocale_test() { + let resolver; + let initPromise = new Promise(resolve => (resolver = resolve)); + useCustomGeoServer("FR", initPromise); + + await Services.search.init(); + await Services.search.getAppProvidedEngines(); + resolver(); + await SearchTestUtils.promiseSearchNotification("engines-reloaded"); + + let engines = await Services.search.getAppProvidedEngines(); + + Assert.deepEqual( + engines.map(e => e._name), + ["Special", "Plain"], + "Special engine is default so should be first" + ); + + engines.forEach(engine => { + Assert.ok(!engine._metaData.order, "Order is not defined"); + }); + + Assert.equal( + Services.search.wrappedJSObject._settings.getMetaDataAttribute( + "useSavedOrder" + ), + false, + "We should not set the engine order during maybeReloadEngines" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_migrateWebExtensionEngine.js b/toolkit/components/search/tests/xpcshell/test_migrateWebExtensionEngine.js new file mode 100644 index 0000000000..562fd9191c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_migrateWebExtensionEngine.js @@ -0,0 +1,91 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const kExtensionID = "simple@tests.mozilla.org"; + +add_setup(async function () { + useHttpServer("opensearch"); + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines("data1"); + await Services.search.init(); +}); + +add_task(async function test_migrateLegacyEngine() { + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: gDataUrl + "simple.xml", + }); + + // Modify the loadpath so it looks like a legacy plugin loadpath + engine.wrappedJSObject._loadPath = `jar:[profile]/extensions/${kExtensionID}.xpi!/simple.xml`; + engine.wrappedJSObject._extensionID = null; + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + // This should replace the existing engine + let extension = await SearchTestUtils.installSearchExtension( + { + id: "simple", + name: "simple", + search_url: "https://example.com/", + }, + { skipUnload: true } + ); + + engine = Services.search.getEngineByName("simple"); + Assert.equal(engine.wrappedJSObject._loadPath, "[addon]" + kExtensionID); + Assert.equal(engine.wrappedJSObject._extensionID, kExtensionID); + + Assert.equal( + (await Services.search.getDefault()).name, + "simple", + "Should have kept the default engine the same" + ); + + await extension.unload(); +}); + +add_task(async function test_migrateLegacyEngineDifferentName() { + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: gDataUrl + "simple.xml", + }); + + // Modify the loadpath so it looks like an legacy plugin loadpath + engine.wrappedJSObject._loadPath = `jar:[profile]/extensions/${kExtensionID}.xpi!/simple.xml`; + engine.wrappedJSObject._extensionID = null; + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + // This should replace the existing engine - it has the same id, but a different name. + let extension = await SearchTestUtils.installSearchExtension( + { + id: "simple", + name: "simple search", + search_url: "https://example.com/", + }, + { skipUnload: true } + ); + + engine = Services.search.getEngineByName("simple"); + Assert.equal(engine, null, "Should have removed the old engine"); + + // The engine should have changed its name. + engine = Services.search.getEngineByName("simple search"); + Assert.equal(engine.wrappedJSObject._loadPath, "[addon]" + kExtensionID); + Assert.equal(engine.wrappedJSObject._extensionID, kExtensionID); + + Assert.equal( + (await Services.search.getDefault()).name, + "simple search", + "Should have made the new engine default" + ); + + await extension.unload(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_missing_engine.js b/toolkit/components/search/tests/xpcshell/test_missing_engine.js new file mode 100644 index 0000000000..3da9fe14a6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_missing_engine.js @@ -0,0 +1,126 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test is designed to check the search service keeps working if there's +// a built-in engine missing from the configuration. + +"use strict"; + +const GOOD_CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + }, +]; + +const BAD_CONFIG = [ + ...GOOD_CONFIG, + { + webExtension: { + id: "engine-missing@search.mozilla.org", + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + ], + }, +]; + +add_setup(async function () { + SearchTestUtils.useMockIdleService(); + await AddonTestUtils.promiseStartupManager(); + + // This test purposely attempts to load a missing engine. + consoleAllowList.push( + "Could not load engine engine-missing@search.mozilla.org" + ); +}); + +add_task(async function test_startup_with_missing() { + await SearchTestUtils.useTestEngines("data", null, BAD_CONFIG); + + const result = await Services.search.init(); + Assert.ok( + Components.isSuccessCode(result), + "Should have started the search service successfully." + ); + + const engines = await Services.search.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine"], + "Should have listed just the good engine" + ); +}); + +add_task(async function test_update_with_missing() { + let reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + + await RemoteSettings(SearchUtils.SETTINGS_KEY).emit("sync", { + data: { + current: GOOD_CONFIG, + }, + }); + + SearchTestUtils.idleService._fireObservers("idle"); + + await reloadObserved; + + const engines = await Services.search.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine"], + "Should have just the good engine" + ); + + reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + + await RemoteSettings(SearchUtils.SETTINGS_KEY).emit("sync", { + data: { + current: BAD_CONFIG, + }, + }); + + SearchTestUtils.idleService._fireObservers("idle"); + + await reloadObserved; + + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine"], + "Should still have just the good engine" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_nodb_pluschanges.js b/toolkit/components/search/tests/xpcshell/test_nodb_pluschanges.js new file mode 100644 index 0000000000..99b0da3900 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_nodb_pluschanges.js @@ -0,0 +1,52 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * test_nodb: Start search service without existing settings file. + * + * Ensure that : + * - nothing explodes; + * - if we change the order, search.json.mozlz4 is updated; + * - this search.json.mozlz4 can be parsed; + * - the order stored in search.json.mozlz4 is consistent. + * + * Notes: + * - we install the search engines of test "test_downloadAndAddEngines.js" + * to ensure that this test is independent from locale, commercial agreements + * and configuration of Firefox. + */ + +add_setup(async function () { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_nodb_pluschanges() { + let engine1 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + let engine2 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine2.xml`, + }); + await promiseAfterSettings(); + + let search = Services.search; + + await search.moveEngine(engine1, 0); + await search.moveEngine(engine2, 1); + + // This is needed to avoid some reentrency issues in nsSearchService. + info("Next step is forcing flush"); + await new Promise(resolve => executeSoon(resolve)); + + info("Forcing flush"); + let promiseCommit = promiseAfterSettings(); + search.QueryInterface(Ci.nsIObserver).observe(null, "quit-application", ""); + await promiseCommit; + info("Commit complete"); + + // Check that the entries are placed as specified correctly + let metadata = await promiseEngineMetadata(); + Assert.equal(metadata["Test search engine"].order, 1); + Assert.equal(metadata["A second test engine"].order, 2); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_notifications.js b/toolkit/components/search/tests/xpcshell/test_notifications.js new file mode 100644 index 0000000000..cda2888f65 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_notifications.js @@ -0,0 +1,124 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +let engine; +let appDefaultEngine; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + useHttpServer(); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + appDefaultEngine = await Services.search.getDefault(); +}); + +add_task(async function test_addingEngine_opensearch() { + const addEngineObserver = new SearchObserver( + [ + // engine-added + // Engine was added to the store by the search service. + SearchUtils.MODIFIED_TYPE.ADDED, + ], + SearchUtils.MODIFIED_TYPE.ADDED + ); + + await SearchTestUtils.promiseNewSearchEngine({ + url: gDataUrl + "engine.xml", + }); + + engine = await addEngineObserver.promise; + + let retrievedEngine = Services.search.getEngineByName("Test search engine"); + Assert.equal(engine, retrievedEngine); +}); + +add_task(async function test_addingEngine_webExtension() { + const addEngineObserver = new SearchObserver( + [ + // engine-added + // Engine was added to the store by the search service. + SearchUtils.MODIFIED_TYPE.ADDED, + ], + SearchUtils.MODIFIED_TYPE.ADDED + ); + + await SearchTestUtils.installSearchExtension({ + name: "Example Engine", + }); + + let webExtensionEngine = await addEngineObserver.promise; + + let retrievedEngine = Services.search.getEngineByName("Example Engine"); + Assert.equal(webExtensionEngine, retrievedEngine); +}); + +async function defaultNotificationTest( + setPrivateDefault, + expectNotificationForPrivate +) { + const defaultObserver = new SearchObserver([ + expectNotificationForPrivate + ? SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE + : SearchUtils.MODIFIED_TYPE.DEFAULT, + ]); + + Services.search[ + setPrivateDefault ? "defaultPrivateEngine" : "defaultEngine" + ] = engine; + await defaultObserver.promise; +} + +add_task(async function test_defaultEngine_notifications() { + await defaultNotificationTest(false, false); +}); + +add_task(async function test_defaultPrivateEngine_notifications() { + await defaultNotificationTest(true, true); +}); + +add_task( + async function test_defaultPrivateEngine_notifications_when_not_enabled() { + await Services.search.setDefault( + appDefaultEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + + await defaultNotificationTest(true, true); + } +); + +add_task(async function test_removeEngine() { + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + await Services.search.setDefaultPrivate( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + const removedObserver = new SearchObserver([ + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE, + SearchUtils.MODIFIED_TYPE.REMOVED, + ]); + + await Services.search.removeEngine(engine); + + await removedObserver; +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch.js b/toolkit/components/search/tests/xpcshell/test_opensearch.js new file mode 100644 index 0000000000..bc31e61275 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch.js @@ -0,0 +1,168 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests that OpenSearch engines are installed and set up correctly. + * + * Note: simple.xml, post.xml, suggestion.xml and suggestion-alternate.xml + * all use different namespaces to reflect the possibitities that may be + * installed. + * mozilla-ns.xml uses the mozilla namespace. + */ + +"use strict"; + +const tests = [ + { + file: "simple.xml", + name: "simple", + description: "A small test engine", + searchForm: "https://example.com/", + searchUrl: "https://example.com/search?q=foo", + }, + { + file: "post.xml", + name: "Post", + description: "", + // The POST method is not supported for `rel="searchform"` so we fallback + // to the `SearchForm` url. + searchForm: "http://engine-rel-searchform-post.xml/?search", + searchUrl: "https://example.com/post", + searchPostData: "searchterms=foo", + }, + { + file: "suggestion.xml", + name: "suggestion", + description: "A small engine with suggestions", + queryCharset: "windows-1252", + searchForm: "http://engine-rel-searchform.xml/?search", + searchUrl: "https://example.com/search?q=foo", + suggestUrl: "https://example.com/suggest?suggestion=foo", + }, + { + file: "suggestion-alternate.xml", + name: "suggestion-alternate", + description: "A small engine with suggestions", + searchForm: "https://example.com/", + searchUrl: "https://example.com/search?q=foo", + suggestUrl: "https://example.com/suggest?suggestion=foo", + }, + { + file: "mozilla-ns.xml", + name: "mozilla-ns", + description: "An engine using mozilla namespace", + searchForm: "https://example.com/", + // mozilla-ns.xml also specifies a MozParam. However, they are only + // valid for app-provided engines, and hence the param should not show + // here. + searchUrl: "https://example.com/search?q=foo", + }, + { + file: "searchform-invalid.xml", + name: "searchform-invalid", + description: "Bug 483086 Test 1", + // Should fall back to the root url, if the searchForm url is invalid. + searchForm: "http://mochi.test:8888", + searchUrl: + "http://mochi.test:8888/browser/browser/components/search/test/browser/?search&test=foo", + }, +]; + +add_setup(async function () { + Services.fog.initializeFOG(); + useHttpServer("opensearch"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +for (const test of tests) { + add_task(async () => { + info(`Testing ${test.file}`); + let promiseEngineAdded = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.ADDED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: gDataUrl + test.file, + }); + await promiseEngineAdded; + Assert.ok(engine, "Should have installed the engine."); + + Assert.equal(engine.name, test.name, "Should have the correct name"); + Assert.equal( + engine.description, + test.description, + "Should have a description" + ); + + Assert.equal( + engine.wrappedJSObject._loadPath, + `[http]localhost/${test.file}` + ); + + Assert.equal( + engine.queryCharset, + test.queryCharset ?? SearchUtils.DEFAULT_QUERY_CHARSET, + "Should have the expected query charset" + ); + + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + test.searchUrl, + "Should have the correct search url" + ); + + if (test.searchPostData) { + let sis = Cc["@mozilla.org/scriptableinputstream;1"].createInstance( + Ci.nsIScriptableInputStream + ); + sis.init(submission.postData); + let data = sis.read(submission.postData.available()); + Assert.equal( + decodeURIComponent(data), + test.searchPostData, + "Should have received the correct POST data" + ); + } else { + Assert.equal( + submission.postData, + null, + "Should have not received any POST data" + ); + } + + Assert.equal( + engine.searchForm, + test.searchForm, + "Should have the correct search form url" + ); + + submission = engine.getSubmission("foo", SearchUtils.URL_TYPE.SUGGEST_JSON); + if (test.suggestUrl) { + Assert.equal( + submission.uri.spec, + test.suggestUrl, + "Should have the correct suggest url" + ); + } else { + Assert.equal(submission, null, "Should not have a suggestion url"); + } + }); +} + +add_task(async function test_telemetry_reporting() { + // Use an engine from the previous tests. + let engine = Services.search.getEngineByName("simple"); + Services.search.defaultEngine = engine; + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-simple", + displayName: "simple", + loadPath: "[http]localhost/simple.xml", + submissionUrl: "blank:", + verified: "verified", + }, + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch_icon.js b/toolkit/components/search/tests/xpcshell/test_opensearch_icon.js new file mode 100644 index 0000000000..68cd9577c1 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch_icon.js @@ -0,0 +1,91 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +add_setup(async function () { + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +const ICON_TESTS = [ + { + name: "Big Icon", + image: "bigIcon.ico", + expected: "data:image/png;base64,", + }, + { + name: "Remote Icon", + image: "remoteIcon.ico", + expected: "data:image/x-icon;base64,", + }, + { + name: "SVG Icon", + image: "svgIcon.svg", + expected: "data:image/svg+xml;base64,", + }, +]; + +add_task(async function test_icon_types() { + for (let test of ICON_TESTS) { + info(`Testing ${test.name}`); + + let promiseEngineAdded = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.ADDED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + let promiseEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + const engineData = { + baseURL: gDataUrl, + image: test.image, + name: test.name, + method: "GET", + }; + // The easiest way to test adding the icon is via a generated xml, otherwise + // we have to somehow insert the address of the server into it. + SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify(engineData)}`, + }); + let engine = await promiseEngineAdded; + // Ensure this is a nsISearchEngine. + engine.QueryInterface(Ci.nsISearchEngine); + await promiseEngineChanged; + + Assert.ok(engine.getIconURL(), `${test.name} engine has an icon`); + Assert.ok( + engine.getIconURL().startsWith(test.expected), + `${test.name} iconURI starts with the expected information` + ); + } +}); + +add_task(async function test_multiple_icons_in_file() { + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineImages.xml`, + }); + + Assert.ok(engine.getIconURL().includes("ico16")); + Assert.ok(engine.getIconURL(16).includes("ico16")); + Assert.ok(engine.getIconURL(32).includes("ico32")); + Assert.ok(engine.getIconURL(74).includes("ico74")); + + info("Invalid dimensions should return null until bug 1655070 is fixed."); + Assert.equal(null, engine.getIconURL(50)); +}); + +add_task(async function test_icon_not_in_opensearch_file() { + let engineUrl = gDataUrl + "engine-fr.xml"; + let engine = await Services.search.addOpenSearchEngine( + engineUrl, + "data:image/x-icon;base64,ico16" + ); + + // Even though the icon wasn't specified inside the XML file, it should be + // available both in the iconURI attribute and with getIconURLBySize. + Assert.ok(engine.getIconURL(16).includes("ico16")); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch_icons_invalid.js b/toolkit/components/search/tests/xpcshell/test_opensearch_icons_invalid.js new file mode 100644 index 0000000000..6db13a0da8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch_icons_invalid.js @@ -0,0 +1,56 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Test that an installed engine can't use a resource URL for an icon */ + +"use strict"; + +add_setup(async function () { + let server = useHttpServer(""); + server.registerContentType("sjs", "sjs"); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_installedresourceicon() { + // Attempts to load a resource:// url as an icon. + let engine1 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}opensearch/resourceicon.xml`, + }); + // Attempts to load a chrome:// url as an icon. + let engine2 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}opensearch/chromeicon.xml`, + }); + + Assert.equal(undefined, engine1.getIconURL()); + Assert.equal(undefined, engine2.getIconURL()); +}); + +add_task(async function test_installedhttpplace() { + let observed = TestUtils.consoleMessageObserved(msg => { + return msg.wrappedJSObject.arguments[0].includes( + "Content type does not match expected" + ); + }); + + // The easiest way to test adding the icon is via a generated xml, otherwise + // we have to somehow insert the address of the server into it. + // Attempts to load a non-image page into an image icon. + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: + `${gDataUrl}data/engineMaker.sjs?` + + JSON.stringify({ + baseURL: gDataUrl, + image: "head_search.js", + name: "invalidicon", + method: "GET", + }), + }); + + await observed; + + Assert.equal( + undefined, + engine.getIconURL(), + "Should not have set an iconURI" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch_install_errors.js b/toolkit/components/search/tests/xpcshell/test_opensearch_install_errors.js new file mode 100644 index 0000000000..0f1f525c65 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch_install_errors.js @@ -0,0 +1,67 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test that various install failures are handled correctly. + */ + +add_setup(async function () { + useHttpServer("opensearch"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + // This test purposely attempts to load an invalid engine. + consoleAllowList.push("_onLoad: Failed to init engine!"); + consoleAllowList.push("Invalid search plugin due to namespace not matching"); +}); + +add_task(async function test_invalid_path_fails() { + await Assert.rejects( + Services.search.addOpenSearchEngine("http://invalid/data/engine.xml", null), + error => { + Assert.equal( + error.result, + Ci.nsISearchService.ERROR_DOWNLOAD_FAILURE, + "Should have returned download failure." + ); + return true; + }, + "Should fail to install an engine with an invalid path." + ); +}); + +add_task(async function test_install_duplicate_fails() { + let engine = await Services.search.addOpenSearchEngine( + gDataUrl + "simple.xml", + null + ); + Assert.equal(engine.name, "simple", "Should have installed the engine."); + + await Assert.rejects( + Services.search.addOpenSearchEngine(gDataUrl + "simple.xml", null), + error => { + Assert.equal( + error.result, + Ci.nsISearchService.ERROR_DUPLICATE_ENGINE, + "Should have returned duplicate failure." + ); + return true; + }, + "Should fail to install a duplicate engine." + ); +}); + +add_task(async function test_invalid_engine_from_dir() { + await Assert.rejects( + Services.search.addOpenSearchEngine(gDataUrl + "invalid.xml", null), + error => { + Assert.equal( + error.result, + Ci.nsISearchService.ERROR_ENGINE_CORRUPTED, + "Should have returned corruption failure." + ); + return true; + }, + "Should fail to install an invalid engine." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch_telemetry.js b/toolkit/components/search/tests/xpcshell/test_opensearch_telemetry.js new file mode 100644 index 0000000000..f666e0dd6c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch_telemetry.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. +http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const { promiseStartupManager, promiseShutdownManager } = AddonTestUtils; + +const openSearchEngineFiles = [ + "secure-and-securely-updated1.xml", + "secure-and-securely-updated2.xml", + "secure-and-securely-updated3.xml", + // An insecure search form should not affect telemetry. + "secure-and-securely-updated-insecure-form.xml", + "secure-and-insecurely-updated1.xml", + "secure-and-insecurely-updated2.xml", + "insecure-and-securely-updated1.xml", + "insecure-and-insecurely-updated1.xml", + "insecure-and-insecurely-updated2.xml", + "secure-and-no-update-url1.xml", + "insecure-and-no-update-url1.xml", + "secure-localhost.xml", + "secure-onionv2.xml", + "secure-onionv3.xml", +]; + +async function verifyTelemetry(probeNameFragment, engineCount, type) { + Services.telemetry.clearScalars(); + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertScalar( + TelemetryTestUtils.getProcessScalars("parent"), + `browser.searchinit.${probeNameFragment}`, + engineCount, + `Count of ${type} engines: ${engineCount}` + ); +} + +add_setup(async function () { + useHttpServer("opensearch"); + + await promiseStartupManager(); + await Services.search.init(); + + for (let file of openSearchEngineFiles) { + await SearchTestUtils.promiseNewSearchEngine({ url: gDataUrl + file }); + } + + registerCleanupFunction(async () => { + await promiseShutdownManager(); + }); +}); + +add_task(async function () { + verifyTelemetry("secure_opensearch_engine_count", 10, "secure"); + verifyTelemetry("insecure_opensearch_engine_count", 4, "insecure"); + verifyTelemetry("secure_opensearch_update_count", 5, "securely updated"); + verifyTelemetry("insecure_opensearch_update_count", 4, "insecurely updated"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_opensearch_update.js b/toolkit/components/search/tests/xpcshell/test_opensearch_update.js new file mode 100644 index 0000000000..ecbc1d3826 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_opensearch_update.js @@ -0,0 +1,120 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Test that user-set metadata isn't lost on engine update */ + +"use strict"; + +const KEYWORD = "keyword"; +let timerManager; + +add_setup(async function () { + let server = useHttpServer(""); + server.registerContentType("sjs", "sjs"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + timerManager = Cc["@mozilla.org/updates/timer-manager;1"].getService( + Ci.nsIUpdateTimerManager + ); +}); + +add_task(async function test_installEngine_with_updates_disabled() { + const engineData = { + baseURL: gDataUrl, + name: "test engine", + method: "GET", + updateFile: "opensearch/simple.xml", + }; + + Services.prefs.setBoolPref(SearchUtils.BROWSER_SEARCH_PREF + "update", false); + Assert.ok( + !("search-engine-update-timer" in timerManager.wrappedJSObject._timers), + "Should not have registered the update timer already" + ); + + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}data/engineMaker.sjs?${JSON.stringify(engineData)}`, + }); + + Assert.ok( + Services.search.getEngineByName("test engine"), + "Should have added the test engine." + ); + Assert.ok( + !("search-engine-update-timer" in timerManager.wrappedJSObject._timers), + "Should have not registered the update timer when updates are disabled" + ); +}); + +add_task(async function test_installEngine_with_updates_enabled() { + const engineData = { + baseURL: gDataUrl, + name: "original engine", + method: "GET", + updateFile: "opensearch/simple.xml", + }; + + Services.prefs.setBoolPref(SearchUtils.BROWSER_SEARCH_PREF + "update", true); + + Assert.ok( + !("search-engine-update-timer" in timerManager.wrappedJSObject._timers), + "Should not have registered the update timer already" + ); + + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}data/engineMaker.sjs?${JSON.stringify(engineData)}`, + }); + + Assert.ok( + "search-engine-update-timer" in timerManager.wrappedJSObject._timers, + "Should have registered the update timer" + ); + + engine.alias = KEYWORD; + await Services.search.moveEngine(engine, 0); + + Assert.ok( + !!Services.search.getEngineByName("original engine"), + "Should be able to get the engine by the original name" + ); + Assert.ok( + !Services.search.getEngineByName("simple"), + "Should not be able to get the engine by the new name" + ); +}); + +add_task(async function test_engineUpdate() { + const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000; + + let promiseUpdate = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + // set last update to 8 days ago, since the default interval is 7, then + // trigger an update + let engine = Services.search.getEngineByName("original engine"); + engine.wrappedJSObject.setAttr("updateexpir", Date.now() - ONE_DAY_IN_MS * 8); + Services.search.QueryInterface(Ci.nsITimerCallback).notify(null); + + await promiseUpdate; + + Assert.equal(engine.name, "simple", "Should have updated the engine's name"); + + Assert.equal(engine.alias, KEYWORD, "Should have kept the keyword"); + Assert.equal( + engine.wrappedJSObject.getAttr("order"), + 1, + "Should have kept the order" + ); + + Assert.ok( + !!Services.search.getEngineByName("simple"), + "Should be able to get the engine by the new name" + ); + Assert.ok( + !Services.search.getEngineByName("original engine"), + "Should not be able to get the engine by the old name" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_override_allowlist.js b/toolkit/components/search/tests/xpcshell/test_override_allowlist.js new file mode 100644 index 0000000000..f7acaee2f8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_override_allowlist.js @@ -0,0 +1,399 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests to ensure that when a user installs or uninstalls an add-on, + * we correctly handle the overriding of default and/or parameters + * according to the allowlist. + */ + +"use strict"; + +const kBaseURL = "https://example.com/"; +const kSearchEngineURL = `${kBaseURL}?q={searchTerms}&foo=myparams`; +const kOverriddenEngineName = "Simple Engine"; + +const allowlist = [ + { + thirdPartyId: "test@thirdparty.example.com", + overridesId: "simple@search.mozilla.org", + urls: [], + }, +]; + +const tests = [ + { + title: "test_not_changing_anything", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: "MozParamsTest2", + keyword: "MozSearch", + search_url: kSearchEngineURL, + }, + expected: { + switchToDefaultAllowed: false, + canInstallEngine: true, + overridesEngine: false, + }, + }, + { + title: "test_changing_default_engine", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kSearchEngineURL, + }, + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, + { + title: "test_changing_default_engine", + startupReason: "ADDON_ENABLE", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kSearchEngineURL, + }, + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, + { + title: "test_overriding_default_engine", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kSearchEngineURL, + }, + allowlistUrls: [ + { + search_url: kSearchEngineURL, + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: true, + searchUrl: kSearchEngineURL, + }, + }, + { + title: "test_overriding_default_engine_enable", + startupReason: "ADDON_ENABLE", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kSearchEngineURL, + }, + allowlistUrls: [ + { + search_url: kSearchEngineURL, + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: true, + searchUrl: kSearchEngineURL, + }, + }, + { + title: "test_overriding_default_engine_different_url", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kSearchEngineURL + "a", + }, + allowlistUrls: [ + { + search_url: kSearchEngineURL, + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, + { + title: "test_overriding_default_engine_get_params", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_url_get_params: "q={searchTerms}&enc=UTF-8", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_url_get_params: "q={searchTerms}&enc=UTF-8", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: true, + searchUrl: `${kBaseURL}?q={searchTerms}&enc=UTF-8`, + }, + }, + { + title: "test_overriding_default_engine_different_get_params", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_url_get_params: "q={searchTerms}&enc=UTF-8a", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_url_get_params: "q={searchTerms}&enc=UTF-8", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, + { + title: "test_overriding_default_engine_post_params", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_url_post_params: "q={searchTerms}&enc=UTF-8", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_url_post_params: "q={searchTerms}&enc=UTF-8", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: true, + searchUrl: `${kBaseURL}`, + postData: "q={searchTerms}&enc=UTF-8", + }, + }, + { + title: "test_overriding_default_engine_different_post_params", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_url_post_params: "q={searchTerms}&enc=UTF-8a", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_url_post_params: "q={searchTerms}&enc=UTF-8", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, + { + title: "test_overriding_default_engine_search_form", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_form: "https://example.com/form", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_form: "https://example.com/form", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: true, + searchUrl: `${kBaseURL}`, + searchForm: "https://example.com/form", + }, + }, + { + title: "test_overriding_default_engine_different_search_form", + startupReason: "ADDON_INSTALL", + search_provider: { + is_default: true, + name: kOverriddenEngineName, + keyword: "MozSearch", + search_url: kBaseURL, + search_form: "https://example.com/forma", + }, + allowlistUrls: [ + { + search_url: kBaseURL, + search_form: "https://example.com/form", + }, + ], + expected: { + switchToDefaultAllowed: true, + canInstallEngine: false, + overridesEngine: false, + }, + }, +]; + +let baseExtension; +let remoteSettingsStub; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + baseExtension = ExtensionTestUtils.loadExtension({ + manifest: { + browser_specific_settings: { + gecko: { + id: "test@thirdparty.example.com", + }, + }, + }, + useAddonManager: "permanent", + }); + await baseExtension.startup(); + + const settings = await RemoteSettings(SearchUtils.SETTINGS_ALLOWLIST_KEY); + remoteSettingsStub = sinon.stub(settings, "get").returns([]); + + registerCleanupFunction(async () => { + await baseExtension.unload(); + }); +}); + +for (const test of tests) { + add_task(async () => { + info(test.title); + + let extension = { + ...baseExtension, + startupReason: test.startupReason, + manifest: { + chrome_settings_overrides: { + search_provider: test.search_provider, + }, + }, + }; + + if (test.expected.overridesEngine) { + remoteSettingsStub.returns([ + { ...allowlist[0], urls: test.allowlistUrls }, + ]); + } + + let result = await Services.search.maybeSetAndOverrideDefault(extension); + Assert.equal( + result.canChangeToAppProvided, + test.expected.switchToDefaultAllowed, + "Should have returned the correct value for allowing switch to default or not." + ); + Assert.equal( + result.canInstallEngine, + test.expected.canInstallEngine, + "Should have returned the correct value for allowing to install the engine or not." + ); + + let engine = await Services.search.getEngineByName(kOverriddenEngineName); + Assert.equal( + !!engine.wrappedJSObject.getAttr("overriddenBy"), + test.expected.overridesEngine, + "Should have correctly overridden or not." + ); + + Assert.equal( + engine.telemetryId, + "simple" + (test.expected.overridesEngine ? "-addon" : ""), + "Should set the correct telemetry Id" + ); + + if (test.expected.overridesEngine) { + let submission = engine.getSubmission("{searchTerms}"); + Assert.equal( + decodeURI(submission.uri.spec), + test.expected.searchUrl, + "Should have set the correct url on an overriden engine" + ); + + if (test.expected.search_form) { + Assert.equal( + engine.wrappedJSObject._searchForm, + test.expected.searchForm, + "Should have overridden the search form." + ); + } + + if (test.expected.postData) { + let sis = Cc["@mozilla.org/scriptableinputstream;1"].createInstance( + Ci.nsIScriptableInputStream + ); + sis.init(submission.postData); + let data = sis.read(submission.postData.available()); + Assert.equal( + decodeURIComponent(data), + test.expected.postData, + "Should have overridden the postData" + ); + } + + // As we're not testing the WebExtension manager as well, + // set this engine as default so we can check the telemetry data. + let oldDefaultEngine = Services.search.defaultEngine; + Services.search.defaultEngine = engine; + + let engineInfo = Services.search.getDefaultEngineInfo(); + Assert.deepEqual( + engineInfo, + { + defaultSearchEngine: "simple-addon", + defaultSearchEngineData: { + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]simple@search.mozilla.org" + : "[addon]simple@search.mozilla.org", + name: "Simple Engine", + origin: "default", + submissionURL: test.expected.searchUrl.replace("{searchTerms}", ""), + }, + }, + "Should return the extended identifier and alternate submission url to telemetry" + ); + Services.search.defaultEngine = oldDefaultEngine; + + engine.wrappedJSObject.removeExtensionOverride(); + } + }); +} diff --git a/toolkit/components/search/tests/xpcshell/test_override_allowlist_switch.js b/toolkit/components/search/tests/xpcshell/test_override_allowlist_switch.js new file mode 100644 index 0000000000..cd51e7bdee --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_override_allowlist_switch.js @@ -0,0 +1,721 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests to ensure that we correctly switch and update engines when + * adding and removing application provided engines which overlap + * with engines in the override allow list. + */ + +"use strict"; + +const SEARCH_URL_BASE = "https://example.com/"; +const SEARCH_URL_PARAMS = `?sourceId=enterprise&q={searchTerms}`; +const ENGINE_NAME = "Simple Engine"; + +const ALLOWLIST = [ + { + thirdPartyId: "simpleengine@tests.mozilla.org", + overridesId: "simple@search.mozilla.org", + urls: [ + { search_url: SEARCH_URL_BASE, search_url_get_params: SEARCH_URL_PARAMS }, + ], + }, + { + thirdPartyId: "opensearch@search.mozilla.org", + engineName: ENGINE_NAME, + overridesId: "simple@search.mozilla.org", + urls: [ + { search_url: SEARCH_URL_BASE, search_url_get_params: SEARCH_URL_PARAMS }, + ], + }, +]; + +const CONFIG_SIMPLE_LOCALE_DE = [ + { + webExtension: { + id: "basic@search.mozilla.org", + name: "basic", + search_url: + "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + params: [ + { + name: "search", + value: "{searchTerms}", + }, + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + { + webExtension: { + id: "simple@search.mozilla.org", + name: "Simple Engine", + search_url: "https://example.com", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + { + name: "search", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { locales: { matches: ["de"] } }, + default: "no", + }, + ], + }, +]; + +const CONFIG_SIMPLE_LOCALE_DE_V2 = [ + { + recordType: "engine", + identifier: "basic", + base: { + name: "basic", + urls: { + search: { + base: "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + searchTermParamName: "search", + }, + }, + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "engine", + identifier: "simple", + base: { + name: "Simple Engine", + urls: { + search: { + base: "https://example.com", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + searchTermParamName: "search", + }, + }, + }, + variants: [ + { + environment: { locales: ["de"] }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "basic", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const CONFIG_SIMPLE_EVERYWHERE = [ + { + webExtension: { + id: "basic@search.mozilla.org", + name: "basic", + search_url: + "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + params: [ + { + name: "search", + value: "{searchTerms}", + }, + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + { + webExtension: { + id: "simple@search.mozilla.org", + name: "Simple Engine", + search_url: "https://example.com", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + { + name: "search", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "no", + }, + ], + }, +]; + +const CONFIG_SIMPLE_EVERYWHERE_V2 = [ + { + recordType: "engine", + identifier: "basic", + base: { + name: "basic", + urls: { + search: { + base: "https://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + searchTermParamName: "search", + }, + }, + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "engine", + identifier: "simple", + base: { + name: "Simple Engine", + urls: { + search: { + base: "https://example.com", + params: [ + { + name: "sourceId", + value: "Mozilla-search", + }, + ], + searchTermParamName: "search", + }, + }, + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "basic", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +let lastEngineId; +let extension; +let configStub; +let notificationBoxStub; + +add_setup(async function () { + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + SearchTestUtils.useMockIdleService(); + configStub = await SearchTestUtils.useTestEngines("simple-engines"); + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "en", + "de", + ]; + Services.locale.requestedLocales = ["en"]; + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + const settings = await RemoteSettings(SearchUtils.SETTINGS_ALLOWLIST_KEY); + sinon.stub(settings, "get").returns(ALLOWLIST); + + notificationBoxStub = sinon.stub( + Services.search.wrappedJSObject, + "_showRemovalOfSearchEngineNotificationBox" + ); + + consoleAllowList.push("Failed to load"); +}); + +/** + * Tests that overrides are correctly applied when the deployment of the app + * provided engine is extended into an area, or removed from an area, where a + * user has the WebExtension installed and set as default. + */ +add_task(async function test_app_provided_engine_deployment_extended() { + await assertCorrectlySwitchedWhenExtended(async () => { + info("Change configuration to include engine in user's environment"); + + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + }); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info("Change configuration to remove engine from user's environment"); + + await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_SIMPLE_LOCALE_DE); + configStub.returns(CONFIG_SIMPLE_LOCALE_DE); + }); +}); + +/** + * Tests that overrides are correctly applied when the deployment of the app + * provided engine is extended into an area, or removed from an area, where a + * user has the OpenSearch engine installed and set as default. + */ +add_task( + async function test_app_provided_engine_deployment_extended_opensearch() { + await assertCorrectlySwitchedWhenExtended(async () => { + info("Change configuration to include engine in user's environment"); + + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + }, true); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info("Change configuration to remove engine from user's environment"); + + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + }, true); + } +); + +add_task( + async function test_app_provided_engine_deployment_extended_restart_only() { + await assertCorrectlySwitchedWhenExtended(async () => { + info( + "Change configuration with restart to include engine in user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + }); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info( + "Change configuration with restart to remove engine from user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + // Ensure settings have been saved before the engines are added, so that + // we know we won't have race conditions when `addEnginesFromExtension` + // loads the settings itself. + await promiseAfterSettings(); + + // Simulate the add-on manager starting up and telling the + // search service about the add-on again. + let extensionData = { + ...extension.extension, + startupReason: "APP_STARTUP", + }; + await Services.search.addEnginesFromExtension(extensionData); + }); + + let settingsData = await promiseSettingsData(); + Assert.ok( + settingsData.engines.every(e => !e._metaData.overriddenBy), + "Should have cleared the overridden by flag after removal" + ); + } +); + +add_task( + async function test_app_provided_engine_deployment_extended_restart_only_startup_extension() { + await assertCorrectlySwitchedWhenExtended(async () => { + info( + "Change configuration with restart to include engine in user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + }); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info( + "Change configuration with restart to remove engine from user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + // Simulate the add-on manager starting up and telling the + // search service about the add-on again. + // + // In this test, it does this before init() is called, to + // simulate this being a startup extension. + let extensionData = { + ...extension.extension, + startupReason: "APP_STARTUP", + }; + await Services.search.addEnginesFromExtension(extensionData); + + await Services.search.init(); + }); + + let settingsData = await promiseSettingsData(); + Assert.ok( + settingsData.engines.every(e => !e._metaData.overriddenBy), + "Should have cleared the overridden by flag after removal" + ); + } +); + +add_task( + async function test_app_provided_engine_deployment_extended_opensearch_restart_only() { + await assertCorrectlySwitchedWhenExtended(async () => { + info( + "Change configuration with restart to include engine in user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_EVERYWHERE_V2 + : CONFIG_SIMPLE_EVERYWHERE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + }, true); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info( + "Change configuration with restart to remove engine from user's environment" + ); + + configStub.returns( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + }, true); + + let settingsData = await promiseSettingsData(); + Assert.ok( + settingsData.engines.every(e => !e._metaData.overriddenBy), + "Should have cleared the overridden by flag after removal" + ); + } +); + +/** + * Tests that overrides are correctly applied when the user's environment changes + * e.g. they have the WebExtension installed and change to a locale where the + * application provided engine is (or is not) available. + */ +add_task(async function test_user_environment_changes() { + await assertCorrectlySwitchedWhenExtended(async () => { + info("Change locale to de"); + + await promiseSetLocale("de"); + }); + + await assertCorrectlySwitchedWhenRemoved(async () => { + info("Change locale to en"); + + await promiseSetLocale("en"); + }); +}); + +/** + * Asserts that overrides are handled correctly when a WebExtension is + * installed, and an application provided engine is added for the user. + * + * This is designed to be used prior to assertCorrectlySwitchedWhenRemoved. + * + * @param {Function} changeFn + * A function that applies the change to cause the application provided + * engine to be added for the user. + * @param {boolean} testOpenSearch + * Set to true to test OpenSearch based engines. + */ +async function assertCorrectlySwitchedWhenExtended( + changeFn, + testOpenSearch = false +) { + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled + ? CONFIG_SIMPLE_LOCALE_DE_V2 + : CONFIG_SIMPLE_LOCALE_DE + ); + notificationBoxStub.resetHistory(); + + info( + `Install ${ + testOpenSearch ? "OpenSearch" : "WebExtension" + } based engine and set as default` + ); + + let engine; + if (testOpenSearch) { + engine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify({ + baseURL: SEARCH_URL_BASE, + queryString: SEARCH_URL_PARAMS, + name: ENGINE_NAME, + method: "GET", + })}`, + }); + } else { + extension = await SearchTestUtils.installSearchExtension( + { + name: ENGINE_NAME, + search_url: SEARCH_URL_BASE, + search_url_get_params: SEARCH_URL_PARAMS, + }, + { skipUnload: true } + ); + await extension.awaitStartup(); + + engine = Services.search.getEngineById( + "simpleengine@tests.mozilla.orgdefault" + ); + } + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + // Set a user defined alias. + engine.alias = "star"; + + await assertEngineCorrectlySet({ + expectedId: engine.id, + expectedAlias: "star", + appEngineOverriden: false, + }); + + await changeFn(); + + await assertEngineCorrectlySet({ + expectedId: "simple@search.mozilla.orgdefault", + expectedAlias: "star", + appEngineOverriden: true, + }); + Assert.ok( + notificationBoxStub.notCalled, + "Should not have attempted to display a notification box" + ); + + info("Test restarting search service ensure settings are kept."); + + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + if (!testOpenSearch) { + let extensionData = { + ...extension.extension, + startupReason: "APP_STARTUP", + }; + await Services.search.maybeSetAndOverrideDefault(extensionData); + } + + Assert.ok( + notificationBoxStub.notCalled, + "Should not have attempted to display a notification box" + ); + await assertEngineCorrectlySet({ + expectedId: "simple@search.mozilla.orgdefault", + expectedAlias: "star", + appEngineOverriden: true, + }); + + // Save lastEngineId for use in assertCorrectlySwitchedWhenRemoved. + lastEngineId = engine.id; +} + +/** + * Asserts that overrides are handled correctly when a WebExtension is + * installed and overriding an application provided engine, and then the + * application provided engine is removed from the user. + * + * This is designed to be used after to assertCorrectlySwitchedWhenExtended. + * + * @param {Function} changeFn + * A function that applies the change to cause the application provided + * engine to be removed for the user. + * @param {boolean} testOpenSearch + * Set to true to test OpenSearch based engines. + */ +async function assertCorrectlySwitchedWhenRemoved( + changeFn, + testOpenSearch = false +) { + notificationBoxStub.resetHistory(); + + await changeFn(); + + await assertEngineCorrectlySet({ + expectedId: lastEngineId, + expectedAlias: "star", + appEngineOverriden: false, + }); + + info("Test restarting search service to remove application provided engine"); + + await promiseAfterSettings(); + Services.search.wrappedJSObject.reset(); + + if (!testOpenSearch) { + let extensionData = { + ...extension.extension, + startupReason: "APP_STARTUP", + }; + await Services.search.addEnginesFromExtension(extensionData); + } + + await Services.search.init(); + + await assertEngineCorrectlySet({ + expectedId: lastEngineId, + expectedAlias: "star", + appEngineOverriden: false, + }); + + if (testOpenSearch) { + await Services.search.removeEngine( + Services.search.getEngineById(lastEngineId) + ); + } else { + await extension.unload(); + } +} + +async function assertEngineCorrectlySet({ + expectedAlias = "", + expectedId, + appEngineOverriden, +}) { + let engines = await Services.search.getEngines(); + Assert.equal( + engines.filter(e => e.name == ENGINE_NAME).length, + 1, + "Should only be one engine with matching name after changing configuration" + ); + + let defaultEngine = await Services.search.getDefault(); + Assert.equal( + defaultEngine.id, + expectedId, + "Should have kept the third party engine as default" + ); + Assert.equal( + decodeURI(defaultEngine.getSubmission("{searchTerms}").uri.spec), + SEARCH_URL_BASE + SEARCH_URL_PARAMS, + "Should have used the third party engine's URLs" + ); + Assert.equal( + !!defaultEngine.wrappedJSObject.getAttr("overriddenBy"), + appEngineOverriden, + "Should have correctly overridden or not." + ); + + Assert.equal( + defaultEngine.telemetryId, + appEngineOverriden ? "simple-addon" : "other-Simple Engine", + "Should set the correct telemetry Id" + ); + + Assert.equal( + defaultEngine.alias, + expectedAlias, + "Should have the correct alias" + ); +} diff --git a/toolkit/components/search/tests/xpcshell/test_parseSubmissionURL.js b/toolkit/components/search/tests/xpcshell/test_parseSubmissionURL.js new file mode 100644 index 0000000000..bee90dbb5b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_parseSubmissionURL.js @@ -0,0 +1,182 @@ +/* Any copyright is dedicated to the Public Domain. + * https://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests getAlternateDomains API. + */ + +"use strict"; + +add_task(async function setup() { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_parseSubmissionURL() { + let engine1 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + let engine2 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine-fr.xml`, + }); + + await SearchTestUtils.installSearchExtension({ + name: "bacon_addParam", + keyword: "bacon_addParam", + encoding: "windows-1252", + search_url: "https://www.bacon.test/find", + }); + await SearchTestUtils.installSearchExtension({ + name: "idn_addParam", + keyword: "idn_addParam", + search_url: "https://www.xn--bcher-kva.ch/search", + }); + let engine3 = Services.search.getEngineByName("bacon_addParam"); + let engine4 = Services.search.getEngineByName("idn_addParam"); + + // The following engine provides it's query keyword in + // its template in the form of q={searchTerms} + let engine5 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine2.xml`, + }); + + // The following engines cannot identify the search parameter. + await SearchTestUtils.installSearchExtension({ + name: "bacon", + keyword: "bacon", + search_url: "https://www.bacon.moz/search?q=", + search_url_get_params: "", + }); + + await Services.search.setDefault( + engine1, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + // Hide the default engines to prevent them from being used in the search. + for (let engine of await Services.search.getAppProvidedEngines()) { + await Services.search.removeEngine(engine); + } + + // Test the first engine, whose URLs use UTF-8 encoding. + // This also tests the query parameter in a different position not being the + // first parameter. + let url = "https://www.google.com/search?foo=bar&q=caff%C3%A8"; + let result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "caff\u00E8"); + + // The second engine uses a locale-specific domain that is an alternate domain + // of the first one, but the second engine should get priority when matching. + // The URL used with this engine uses ISO-8859-1 encoding instead. + url = "https://www.google.fr/search?q=caff%E8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine2); + Assert.equal(result.terms, "caff\u00E8"); + + // Test a domain that is an alternate domain of those defined. In this case, + // the first matching engine from the ordered list should be returned. + url = "https://www.google.co.uk/search?q=caff%C3%A8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "caff\u00E8"); + + // We support parsing URLs from a dynamically added engine. + url = "https://www.bacon.test/find?q=caff%E8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine, engine3); + Assert.equal(result.terms, "caff\u00E8"); + + // Test URLs with unescaped unicode characters. + url = "https://www.google.com/search?q=foo+b\u00E4r"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "foo b\u00E4r"); + + // Test search engines with unescaped IDNs. + url = "https://www.b\u00FCcher.ch/search?q=foo+bar"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine, engine4); + Assert.equal(result.terms, "foo bar"); + + // Test search engines with escaped IDNs. + url = "https://www.xn--bcher-kva.ch/search?q=foo+bar"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine, engine4); + Assert.equal(result.terms, "foo bar"); + + // Parsing of parameters from an engine template URL is not supported + // if no matching parameter value template is provided. + Assert.equal( + Services.search.parseSubmissionURL("https://www.bacon.moz/search?q=") + .engine, + null + ); + + // Parsing of parameters from an engine template URL is supported + // if a matching parameter value template is provided. + url = "https://duckduckgo.com/?foo=bar&q=caff%C3%A8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine5); + Assert.equal(result.terms, "caff\u00E8"); + + // If the search params are in the template, the query parameter + // doesn't need to be separated from the host by a slash, only by + // by a question mark. + url = "https://duckduckgo.com?foo=bar&q=caff%C3%A8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine5); + Assert.equal(result.terms, "caff\u00E8"); + + // HTTP and HTTPS schemes are interchangeable. + url = "https://www.google.com/search?q=caff%C3%A8"; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "caff\u00E8"); + + // Decoding search terms with multiple spaces should work. + result = Services.search.parseSubmissionURL( + "https://www.google.com/search?q=+with++spaces+" + ); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, " with spaces "); + + // Parsing search terms with ampersands should work. + result = Services.search.parseSubmissionURL( + "https://www.google.com/search?q=with%26ampersand" + ); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "with&ersand"); + + // Capitals in the path should work + result = Services.search.parseSubmissionURL( + "https://www.google.com/SEARCH?q=caps" + ); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, "caps"); + + // An empty query parameter should work the same. + url = "https://www.google.com/search?q="; + result = Services.search.parseSubmissionURL(url); + Assert.equal(result.engine.wrappedJSObject, engine1); + Assert.equal(result.terms, ""); + + // There should be no match when the path is different. + result = Services.search.parseSubmissionURL( + "https://www.google.com/search/?q=test" + ); + Assert.equal(result.engine, null); + Assert.equal(result.terms, ""); + + // There should be no match when the argument is different. + result = Services.search.parseSubmissionURL( + "https://www.google.com/search?q2=test" + ); + Assert.equal(result.engine, null); + Assert.equal(result.terms, ""); + + // There should be no match for URIs that are not HTTP or HTTPS. + result = Services.search.parseSubmissionURL("file://localhost/search?q=test"); + Assert.equal(result.engine, null); + Assert.equal(result.terms, ""); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_policyEngine.js b/toolkit/components/search/tests/xpcshell/test_policyEngine.js new file mode 100644 index 0000000000..e081e5fb1e --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_policyEngine.js @@ -0,0 +1,185 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests that Enterprise Policy Engines can be installed correctly. + */ + +"use strict"; + +const { EnterprisePolicyTesting } = ChromeUtils.importESModule( + "resource://testing-common/EnterprisePolicyTesting.sys.mjs" +); + +SearchSettings.SETTINGS_INVALIDATION_DELAY = 100; + +/** + * Loads a new enterprise policy, and re-initialise the search service + * with the new policy. Also waits for the search service to write the settings + * file to disk. + * + * @param {object} policy + * The enterprise policy to use. + */ +async function setupPolicyEngineWithJson(policy) { + Services.search.wrappedJSObject.reset(); + + await EnterprisePolicyTesting.setupPolicyEngineWithJson(policy); + + let settingsWritten = SearchTestUtils.promiseSearchNotification( + "write-settings-to-disk-complete" + ); + await Services.search.init(); + await settingsWritten; +} + +add_setup(async function () { + // This initializes the policy engine for xpcshell tests + let policies = Cc["@mozilla.org/enterprisepolicies;1"].getService( + Ci.nsIObserver + ); + policies.observe(null, "policies-startup", null); + + Services.fog.initializeFOG(); + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines(); + + SearchUtils.GENERAL_SEARCH_ENGINE_IDS = new Set([ + "engine-resourceicon@search.mozilla.org", + ]); +}); + +add_task(async function test_enterprise_policy_engine() { + await setupPolicyEngineWithJson({ + policies: { + SearchEngines: { + Add: [ + { + Name: "policy", + Description: "Test policy engine", + IconURL: "data:image/gif;base64,R0lGODl", + Alias: "p", + URLTemplate: "https://example.com?q={searchTerms}", + SuggestURLTemplate: "https://example.com/suggest/?q={searchTerms}", + }, + ], + }, + }, + }); + + let engine = Services.search.getEngineByName("policy"); + Assert.ok(engine, "Should have installed the engine."); + + Assert.equal(engine.name, "policy", "Should have the correct name"); + Assert.equal( + engine.description, + "Test policy engine", + "Should have a description" + ); + Assert.deepEqual(engine.aliases, ["p"], "Should have the correct alias"); + + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://example.com/?q=foo", + "Should have the correct search url" + ); + + submission = engine.getSubmission("foo", SearchUtils.URL_TYPE.SUGGEST_JSON); + Assert.equal( + submission.uri.spec, + "https://example.com/suggest/?q=foo", + "Should have the correct suggest url" + ); + + Services.search.defaultEngine = engine; + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-policy", + displayName: "policy", + loadPath: "[policy]", + submissionUrl: "blank:", + verified: "verified", + }, + }); +}); + +add_task(async function test_enterprise_policy_engine_hidden_persisted() { + // Set the engine alias, and wait for the settings to be written. + let settingsWritten = SearchTestUtils.promiseSearchNotification( + "write-settings-to-disk-complete" + ); + let engine = Services.search.getEngineByName("policy"); + engine.hidden = "p1"; + engine.alias = "p1"; + await settingsWritten; + + // This will reset and re-initialise the search service. + await setupPolicyEngineWithJson({ + policies: { + SearchEngines: { + Add: [ + { + Name: "policy", + Description: "Test policy engine", + IconURL: "data:image/gif;base64,R0lGODl", + Alias: "p", + URLTemplate: "https://example.com?q={searchTerms}", + SuggestURLTemplate: "https://example.com/suggest/?q={searchTerms}", + }, + ], + }, + }, + }); + + engine = Services.search.getEngineByName("policy"); + Assert.equal(engine.alias, "p1", "Should have retained the engine alias"); + Assert.ok(engine.hidden, "Should have kept the engine hidden"); +}); + +add_task(async function test_enterprise_policy_engine_remove() { + // This will reset and re-initialise the search service. + await setupPolicyEngineWithJson({ + policies: {}, + }); + + Assert.ok( + !Services.search.getEngineByName("policy"), + "Should not have the policy engine installed" + ); + + let settings = await promiseSettingsData(); + Assert.ok( + !settings.engines.find(e => e.name == "p1"), + "Should not have the engine settings stored" + ); +}); + +add_task(async function test_enterprise_policy_hidden_default() { + await setupPolicyEngineWithJson({ + policies: { + SearchEngines: { + Remove: ["Test search engine"], + }, + }, + }); + + Services.search.resetToAppDefaultEngine(); + + Assert.equal(Services.search.defaultEngine.name, "engine-resourceicon"); +}); + +add_task(async function test_enterprise_policy_default() { + await setupPolicyEngineWithJson({ + policies: { + SearchEngines: { + Default: "engine-pref", + }, + }, + }); + + Services.search.resetToAppDefaultEngine(); + + Assert.equal(Services.search.defaultEngine.name, "engine-pref"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_reload_engines.js b/toolkit/components/search/tests/xpcshell/test_reload_engines.js new file mode 100644 index 0000000000..ac63b62e59 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_reload_engines.js @@ -0,0 +1,436 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const CONFIG = [ + { + // Engine initially default, but the defaults will be changed to engine-pref. + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + defaultPrivate: "yes", + }, + { + included: { regions: ["FR"] }, + default: "no", + defaultPrivate: "no", + }, + ], + }, + { + // This will become defaults when region is changed to FR. + webExtension: { + id: "engine-pref@search.mozilla.org", + name: "engine-pref", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "code", + condition: "pref", + pref: "code", + }, + { + name: "test", + condition: "pref", + pref: "test", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { regions: ["FR"] }, + default: "yes", + defaultPrivate: "yes", + }, + ], + }, + { + // This engine will get an update when region is changed to FR. + webExtension: { + id: "engine-chromeicon@search.mozilla.org", + name: "engine-chromeicon", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { regions: ["FR"] }, + extraParams: [ + { name: "c", value: "my-test" }, + { name: "q1", value: "{searchTerms}" }, + ], + }, + ], + }, + { + // This engine will be removed when the region is changed to FR. + webExtension: { + id: "engine-rel-searchform-purpose@search.mozilla.org", + name: "engine-rel-searchform-purpose", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + { + name: "channel", + condition: "purpose", + purpose: "searchbar", + value: "sb", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["FR"] }, + }, + ], + }, + { + // This engine will be added when the region is changed to FR. + webExtension: { + id: "engine-reordered@search.mozilla.org", + name: "Test search engine (Reordered)", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { regions: ["FR"] }, + }, + ], + }, + { + // This engine will be re-ordered and have a changed name, when moved to FR. + webExtension: { + id: "engine-resourceicon@search.mozilla.org", + name: "engine-resourceicon", + search_url: "https://www.google.com/search", + searchProvider: { + en: { + name: "engine-resourceicon", + search_url: "https://www.google.com/search", + }, + gd: { + name: "engine-resourceicon-gd", + search_url: "https://www.google.com/search", + }, + }, + }, + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["FR"] }, + }, + { + included: { regions: ["FR"] }, + webExtension: { + locales: ["gd"], + }, + orderHint: 30, + }, + ], + }, + { + // This engine has the same name, but still should be replaced correctly. + webExtension: { + id: "engine-same-name@search.mozilla.org", + name: "engine-same-name", + search_url: "https://www.google.com/search?q={searchTerms}", + searchProvider: { + en: { + name: "engine-same-name", + search_url: "https://www.google.com/search?q={searchTerms}", + }, + gd: { + name: "engine-same-name", + search_url: "https://www.example.com/search?q={searchTerms}", + }, + }, + }, + appliesTo: [ + { + included: { everywhere: true }, + excluded: { regions: ["FR"] }, + }, + { + included: { regions: ["FR"] }, + webExtension: { + locales: ["gd"], + }, + }, + ], + }, +]; + +async function visibleEngines() { + return (await Services.search.getVisibleEngines()).map(e => e.identifier); +} + +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.separatePrivateDefault", true); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + + SearchTestUtils.useMockIdleService(); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); +}); + +// This is to verify that the loaded configuration matches what we expect for +// the test. +add_task(async function test_initial_config_correct() { + Region._setHomeRegion("", false); + + await Services.search.init(); + + const installedEngines = await Services.search.getAppProvidedEngines(); + Assert.deepEqual( + installedEngines.map(e => e.identifier), + [ + "engine", + "engine-chromeicon", + "engine-pref", + "engine-rel-searchform-purpose", + "engine-resourceicon", + "engine-same-name", + ], + "Should have the correct list of engines installed." + ); + + Assert.equal( + (await Services.search.getDefault()).identifier, + "engine", + "Should have loaded the expected default engine" + ); + + Assert.equal( + (await Services.search.getDefaultPrivate()).identifier, + "engine", + "Should have loaded the expected private default engine" + ); +}); + +add_task(async function test_config_updated_engine_changes() { + // Update the config. + const reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + const defaultEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + const defaultPrivateEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT_PRIVATE, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + const enginesAdded = []; + const enginesModified = []; + const enginesRemoved = []; + + function enginesObs(subject, topic, data) { + if (data == SearchUtils.MODIFIED_TYPE.ADDED) { + enginesAdded.push(subject.QueryInterface(Ci.nsISearchEngine).identifier); + } else if (data == SearchUtils.MODIFIED_TYPE.CHANGED) { + enginesModified.push( + subject.QueryInterface(Ci.nsISearchEngine).identifier + ); + } else if (data == SearchUtils.MODIFIED_TYPE.REMOVED) { + enginesRemoved.push(subject.QueryInterface(Ci.nsISearchEngine).name); + } + } + Services.obs.addObserver(enginesObs, SearchUtils.TOPIC_ENGINE_MODIFIED); + + Region._setHomeRegion("FR", false); + + await Services.search.wrappedJSObject._maybeReloadEngines(); + + await reloadObserved; + Services.obs.removeObserver(enginesObs, SearchUtils.TOPIC_ENGINE_MODIFIED); + + Assert.deepEqual( + enginesAdded, + ["engine-resourceicon-gd", "engine-reordered"], + "Should have added the correct engines" + ); + + Assert.deepEqual( + enginesModified.sort(), + ["engine", "engine-chromeicon", "engine-pref", "engine-same-name-gd"], + "Should have modified the expected engines" + ); + + Assert.deepEqual( + enginesRemoved, + ["engine-rel-searchform-purpose", "engine-resourceicon"], + "Should have removed the expected engine" + ); + + const installedEngines = await Services.search.getAppProvidedEngines(); + + Assert.deepEqual( + installedEngines.map(e => e.identifier), + [ + "engine-pref", + "engine-resourceicon-gd", + "engine-chromeicon", + "engine-same-name-gd", + "engine", + "engine-reordered", + ], + "Should have the correct list of engines installed in the expected order." + ); + + const newDefault = await defaultEngineChanged; + Assert.equal( + newDefault.QueryInterface(Ci.nsISearchEngine).name, + "engine-pref", + "Should have correctly notified the new default engine" + ); + + const newDefaultPrivate = await defaultPrivateEngineChanged; + Assert.equal( + newDefaultPrivate.QueryInterface(Ci.nsISearchEngine).name, + "engine-pref", + "Should have correctly notified the new default private engine" + ); + + const engineWithParams = await Services.search.getEngineByName( + "engine-chromeicon" + ); + Assert.equal( + engineWithParams.getSubmission("test").uri.spec, + "https://www.google.com/search?c=my-test&q1=test", + "Should have updated the parameters" + ); + + const engineWithSameName = await Services.search.getEngineByName( + "engine-same-name" + ); + Assert.equal( + engineWithSameName.getSubmission("test").uri.spec, + "https://www.example.com/search?q=test", + "Should have correctly switched to the engine of the same name" + ); + + Assert.equal( + Services.search.wrappedJSObject._settings.getMetaDataAttribute( + "useSavedOrder" + ), + false, + "Should not have set the useSavedOrder preference" + ); +}); + +add_task(async function test_user_settings_persist() { + let reload = SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Region._setHomeRegion(""); + await reload; + + Assert.ok( + (await visibleEngines()).includes("engine-rel-searchform-purpose"), + "Rel Searchform engine should be included by default" + ); + + let settingsFileWritten = promiseAfterSettings(); + let engine = await Services.search.getEngineByName( + "engine-rel-searchform-purpose" + ); + await Services.search.removeEngine(engine); + await settingsFileWritten; + + Assert.ok( + !(await visibleEngines()).includes("engine-rel-searchform-purpose"), + "Rel Searchform engine has been removed" + ); + + reload = SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Region._setHomeRegion("FR"); + await reload; + + reload = SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Region._setHomeRegion(""); + await reload; + + Assert.ok( + !(await visibleEngines()).includes("engine-rel-searchform-purpose"), + "Rel Searchform removal should be remembered" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_reload_engines_duplicate.js b/toolkit/components/search/tests/xpcshell/test_reload_engines_duplicate.js new file mode 100644 index 0000000000..3614370a8c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_reload_engines_duplicate.js @@ -0,0 +1,169 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests reloading engines when a user has an engine installed that is the + * same name as an application provided engine being added to the user's set + * of engines. + * + * Ensures that settings are not automatically taken across. + */ + +"use strict"; + +const CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + { + webExtension: { + id: "engine-pref@search.mozilla.org", + name: "engine-pref", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { regions: ["FR"] }, + }, + ], + }, +]; + +const CONFIG_V2 = [ + { + recordType: "engine", + identifier: "engine", + base: { + name: "Test search engine", + urls: { + search: { + base: "https://www.google.com/search", + params: [], + searchTermParamName: "q", + }, + }, + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-pref", + base: { + name: "engine-pref", + urls: { + search: { + base: "https://www.google.com/search", + params: [], + searchTermParamName: "q", + }, + }, + }, + variants: [ + { + environment: { regions: ["FR"] }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "engine", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +add_setup(async () => { + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + + // We use a region that doesn't install `engine-pref` by default so that we can + // manually install it first (like when a user installs a browser add-on), and + // then test what happens when we switch regions to one which would install + // `engine-pref`. + Region._setHomeRegion("US", false); + + await SearchTestUtils.useTestEngines( + "data", + null, + SearchUtils.newSearchConfigEnabled ? CONFIG_V2 : CONFIG + ); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_reload_engines_with_duplicate() { + let engines = await Services.search.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine"], + "Should have the expected default engines" + ); + // Simulate a user installing a search engine that shares the same name as an + // application provided search engine not currently installed in their browser. + let engine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify({ + baseURL: gDataUrl, + name: "engine-pref", + method: "GET", + })}`, + }); + + engine.alias = "testEngine"; + + let engineId = engine.id; + + Region._setHomeRegion("FR", false); + + await Services.search.wrappedJSObject._maybeReloadEngines(); + + Assert.ok( + !(await Services.search.getEngineById(engineId)), + "Should not have added the duplicate engine" + ); + + engines = await Services.search.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine", "engine-pref"], + "Should have the expected default engines" + ); + + let enginePref = await Services.search.getEngineByName("engine-pref"); + + Assert.equal( + enginePref.alias, + "", + "Should not have copied the alias from the duplicate engine" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_reload_engines_experiment.js b/toolkit/components/search/tests/xpcshell/test_reload_engines_experiment.js new file mode 100644 index 0000000000..ea5cdee3e5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_reload_engines_experiment.js @@ -0,0 +1,166 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const CONFIG = [ + { + // Just a basic engine that won't be changed. + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + { + // This engine will have the locale swapped when the experiment is set. + webExtension: { + id: "engine-same-name@search.mozilla.org", + default_locale: "en", + searchProvider: { + en: { + name: "engine-same-name", + search_url: "https://www.google.com/search?q={searchTerms}", + }, + gd: { + name: "engine-same-name", + search_url: "https://www.example.com/search?q={searchTerms}", + }, + }, + }, + appliesTo: [ + { + included: { everywhere: true }, + webExtension: { + locales: ["en"], + }, + }, + { + included: { everywhere: true }, + webExtension: { + locales: ["gd"], + }, + experiment: "xpcshell", + }, + ], + }, +]; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); +}); + +// This is to verify that the loaded configuration matches what we expect for +// the test. +add_task(async function test_initial_config_correct() { + await Services.search.init(); + + const installedEngines = await Services.search.getAppProvidedEngines(); + Assert.deepEqual( + installedEngines.map(e => e.identifier), + ["engine", "engine-same-name-en"], + "Should have the correct list of engines installed." + ); + + Assert.equal( + (await Services.search.getDefault()).identifier, + "engine", + "Should have loaded the expected default engine" + ); +}); + +add_task(async function test_config_updated_engine_changes() { + // Update the config. + const reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + const enginesAdded = []; + const enginesModified = []; + const enginesRemoved = []; + + function enginesObs(subject, topic, data) { + if (data == SearchUtils.MODIFIED_TYPE.ADDED) { + enginesAdded.push(subject.QueryInterface(Ci.nsISearchEngine).identifier); + } else if (data == SearchUtils.MODIFIED_TYPE.CHANGED) { + enginesModified.push( + subject.QueryInterface(Ci.nsISearchEngine).identifier + ); + } else if (data == SearchUtils.MODIFIED_TYPE.REMOVED) { + enginesRemoved.push(subject.QueryInterface(Ci.nsISearchEngine).name); + } + } + Services.obs.addObserver(enginesObs, SearchUtils.TOPIC_ENGINE_MODIFIED); + + Services.prefs.setCharPref( + SearchUtils.BROWSER_SEARCH_PREF + "experiment", + "xpcshell" + ); + + await reloadObserved; + Services.obs.removeObserver(enginesObs, SearchUtils.TOPIC_ENGINE_MODIFIED); + + Assert.deepEqual(enginesAdded, [], "Should have added the correct engines"); + + Assert.deepEqual( + enginesModified.sort(), + ["engine", "engine-same-name-gd"], + "Should have modified the expected engines" + ); + + Assert.deepEqual( + enginesRemoved, + [], + "Should have removed the expected engine" + ); + + const installedEngines = await Services.search.getAppProvidedEngines(); + + Assert.deepEqual( + installedEngines.map(e => e.identifier), + ["engine", "engine-same-name-gd"], + "Should have the correct list of engines installed in the expected order." + ); + + const engineWithSameName = await Services.search.getEngineByName( + "engine-same-name" + ); + Assert.equal( + engineWithSameName.getSubmission("test").uri.spec, + "https://www.example.com/search?q=test", + "Should have correctly switched to the engine of the same name" + ); + + Assert.equal( + Services.search.wrappedJSObject._settings.getMetaDataAttribute( + "useSavedOrder" + ), + false, + "Should not have set the useSavedOrder preference" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_reload_engines_locales.js b/toolkit/components/search/tests/xpcshell/test_reload_engines_locales.js new file mode 100644 index 0000000000..6fa277655d --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_reload_engines_locales.js @@ -0,0 +1,128 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests reloading engines when changing the in-use locale of a WebExtension, + * where the name of the engine changes as well. + */ + +"use strict"; + +const CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + { + webExtension: { + id: "engine-diff-name@search.mozilla.org", + default_locale: "en", + searchProvider: { + en: { + name: "engine-diff-name-en", + search_url: "https://en.wikipedia.com/search", + }, + gd: { + name: "engine-diff-name-gd", + search_url: "https://gd.wikipedia.com/search", + }, + }, + }, + appliesTo: [ + { + included: { everywhere: true }, + excluded: { locales: { matches: ["gd"] } }, + }, + { + included: { locales: { matches: ["gd"] } }, + webExtension: { + locales: ["gd"], + }, + }, + ], + }, +]; + +add_setup(async () => { + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "en", + "gd", + ]; + Services.locale.requestedLocales = ["gd"]; + + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_config_updated_engine_changes() { + let engines = await Services.search.getEngines(); + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine", "engine-diff-name-gd"], + "Should have the correct engines installed" + ); + + let engine = await Services.search.getEngineByName("engine-diff-name-gd"); + Assert.equal( + engine.name, + "engine-diff-name-gd", + "Should have the correct engine name" + ); + Assert.equal( + engine.getSubmission("test").uri.spec, + "https://gd.wikipedia.com/search", + "Should have the gd search url" + ); + + await promiseSetLocale("en"); + + engines = await Services.search.getEngines(); + Assert.deepEqual( + engines.map(e => e.name), + ["Test search engine", "engine-diff-name-en"], + "Should have the correct engines installed after locale change" + ); + + engine = await Services.search.getEngineByName("engine-diff-name-en"); + Assert.equal( + engine.name, + "engine-diff-name-en", + "Should have the correct engine name" + ); + Assert.equal( + engine.getSubmission("test").uri.spec, + "https://en.wikipedia.com/search", + "Should have the en search url" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_remove_engine_notification_box.js b/toolkit/components/search/tests/xpcshell/test_remove_engine_notification_box.js new file mode 100644 index 0000000000..0222334255 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_remove_engine_notification_box.js @@ -0,0 +1,393 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const CONFIG = [ + { + // Engine initially default, but the defaults will be changed to engine-pref. + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + { + included: { regions: ["FR"] }, + default: "no", + }, + ], + }, + { + // This will become defaults when region is changed to FR. + webExtension: { + id: "engine-pref@search.mozilla.org", + name: "engine-pref", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + }, + { + included: { regions: ["FR"] }, + default: "yes", + }, + ], + }, +]; + +const CONFIG_UPDATED = CONFIG.filter(r => + r.webExtension.id.startsWith("engine-pref") +); + +let stub; +let settingsFilePath; +let userSettings; + +add_setup(async function () { + SearchSettings.SETTINGS_INVALIDATION_DELAY = 100; + SearchTestUtils.useMockIdleService(); + await SearchTestUtils.useTestEngines("data", null, CONFIG); + await AddonTestUtils.promiseStartupManager(); + + stub = sinon.stub( + await Services.search.wrappedJSObject, + "_showRemovalOfSearchEngineNotificationBox" + ); + + settingsFilePath = PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME); + + Region._setHomeRegion("", false); + + let promiseSaved = promiseAfterSettings(); + await Services.search.init(); + await promiseSaved; + + userSettings = await Services.search.wrappedJSObject._settings.get(); +}); + +// Verify the loaded configuration matches what we expect for the test. +add_task(async function test_initial_config_correct() { + const installedEngines = await Services.search.getAppProvidedEngines(); + Assert.deepEqual( + installedEngines.map(e => e.identifier), + ["engine", "engine-pref"], + "Should have the correct list of engines installed." + ); + + Assert.equal( + (await Services.search.getDefault()).identifier, + "engine", + "Should have loaded the expected default engine" + ); +}); + +add_task(async function test_metadata_undefined() { + let defaultEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + info("Update region to FR."); + Region._setHomeRegion("FR", false); + + let settings = structuredClone(userSettings); + settings.metaData = undefined; + await reloadEngines(settings); + Assert.ok( + stub.notCalled, + "_reloadEngines should not have shown the notification box." + ); + + settings = structuredClone(userSettings); + settings.metaData = undefined; + await loadEngines(settings); + Assert.ok( + stub.notCalled, + "_loadEngines should not have shown the notification box." + ); + + const newDefault = await defaultEngineChanged; + Assert.equal( + newDefault.QueryInterface(Ci.nsISearchEngine).name, + "engine-pref", + "Should have correctly notified the new default engine." + ); +}); + +add_task(async function test_metadata_changed() { + let metaDataProperties = [ + "locale", + "region", + "channel", + "experiment", + "distroID", + ]; + + for (let name of metaDataProperties) { + let settings = structuredClone(userSettings); + settings.metaData[name] = "test"; + await assert_metadata_changed(settings); + } +}); + +add_task(async function test_default_engine_unchanged() { + let currentEngineName = + Services.search.wrappedJSObject._getEngineDefault(false).name; + + Assert.equal( + currentEngineName, + "Test search engine", + "Default engine should be unchanged." + ); + + await reloadEngines(structuredClone(userSettings)); + Assert.ok( + stub.notCalled, + "_reloadEngines should not have shown the notification box." + ); + + await loadEngines(structuredClone(userSettings)); + Assert.ok( + stub.notCalled, + "_loadEngines should not have shown the notification box." + ); +}); + +add_task(async function test_new_current_engine_is_undefined() { + consoleAllowList.push("No default engine"); + let settings = structuredClone(userSettings); + let getEngineDefaultStub = sinon.stub( + await Services.search.wrappedJSObject, + "_getEngineDefault" + ); + getEngineDefaultStub.returns(undefined); + + await loadEngines(settings); + Assert.ok( + stub.notCalled, + "_loadEngines should not have shown the notification box." + ); + + getEngineDefaultStub.restore(); +}); + +add_task(async function test_current_engine_is_null() { + Services.search.wrappedJSObject._currentEngine = null; + + await reloadEngines(structuredClone(userSettings)); + Assert.ok( + stub.notCalled, + "_reloadEngines should not have shown the notification box." + ); + + let settings = structuredClone(userSettings); + settings.metaData.current = null; + await loadEngines(settings); + Assert.ok( + stub.notCalled, + "_loadEngines should not have shown the notification box." + ); +}); + +add_task(async function test_default_changed_and_metadata_unchanged_exists() { + info("Update region to FR to change engine."); + Region._setHomeRegion("FR", false); + + info("Set user settings metadata to the same properties as cached metadata."); + await Services.search.wrappedJSObject._fetchEngineSelectorEngines(); + userSettings.metaData = { + ...Services.search.wrappedJSObject._settings.getSettingsMetaData(), + appDefaultEngine: "Test search engine", + }; + + await reloadEngines(structuredClone(userSettings)); + Assert.ok( + stub.notCalled, + "_reloadEngines should not show the notification box as the engine still exists." + ); + + // Reset. + Region._setHomeRegion("US", false); + await reloadEngines(structuredClone(userSettings)); +}); + +add_task(async function test_default_engine_changed_and_metadata_unchanged() { + info("Update region to FR to change engine."); + Region._setHomeRegion("FR", false); + + const defaultEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + info("Set user settings metadata to the same properties as cached metadata."); + await Services.search.wrappedJSObject._fetchEngineSelectorEngines(); + userSettings.metaData = { + ...Services.search.wrappedJSObject._settings.getSettingsMetaData(), + appDefaultEngineId: "engine@search.mozilla.orgdefault", + }; + + // Update config by removing the app default engine + await setConfigToLoad(CONFIG_UPDATED); + + await reloadEngines(structuredClone(userSettings)); + Assert.ok( + stub.calledOnce, + "_reloadEngines should show the notification box." + ); + + Assert.deepEqual( + stub.firstCall.args, + ["Test search engine", "engine-pref"], + "_showRemovalOfSearchEngineNotificationBox should display " + + "'Test search engine' as the engine removed and 'engine-pref' as the new " + + "default engine." + ); + + const newDefault = await defaultEngineChanged; + Assert.equal( + newDefault.QueryInterface(Ci.nsISearchEngine).name, + "engine-pref", + "Should have correctly notified the new default engine" + ); + + info("Reset userSettings.metaData.current engine."); + let settings = structuredClone(userSettings); + settings.metaData.current = Services.search.wrappedJSObject._currentEngine; + + await loadEngines(settings); + Assert.ok(stub.calledTwice, "_loadEngines should show the notification box."); + + Assert.deepEqual( + stub.secondCall.args, + ["Test search engine", "engine-pref"], + "_showRemovalOfSearchEngineNotificationBox should display " + + "'Test search engine' as the engine removed and 'engine-pref' as the new " + + "default engine." + ); +}); + +add_task(async function test_app_default_engine_changed_on_start_up() { + let settings = structuredClone(userSettings); + + // Set the current engine to "" so we can use the app default engine as + // default + settings.metaData.current = ""; + + // Update config by removing the app default engine + await setConfigToLoad(CONFIG_UPDATED); + + await loadEngines(settings); + Assert.ok( + stub.calledThrice, + "_loadEngines should show the notification box." + ); +}); + +add_task(async function test_app_default_engine_change_start_up_still_exists() { + stub.resetHistory(); + let settings = structuredClone(userSettings); + + // Set the current engine to "" so we can use the app default engine as + // default + settings.metaData.current = ""; + settings.metaData.appDefaultEngine = "Test search engine"; + + await setConfigToLoad(CONFIG); + + await loadEngines(settings); + Assert.ok( + stub.notCalled, + "_loadEngines should not show the notification box." + ); +}); + +async function setConfigToLoad(config) { + let searchSettingsObj = await RemoteSettings(SearchUtils.SETTINGS_KEY); + // Restore the get method in order to stub it again in useTestEngines + searchSettingsObj.get.restore(); + Services.search.wrappedJSObject.resetEngineSelector(); + await SearchTestUtils.useTestEngines("data", null, config); +} + +function writeSettings(settings) { + return IOUtils.writeJSON(settingsFilePath, settings, { compress: true }); +} + +async function reloadEngines(settings) { + let promiseSaved = promiseAfterSettings(); + + await Services.search.wrappedJSObject._reloadEngines(settings); + + await promiseSaved; +} + +async function loadEngines(settings) { + await writeSettings(settings); + + let promiseSaved = promiseAfterSettings(); + + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + await promiseSaved; +} + +async function assert_metadata_changed(settings) { + info("Update region."); + Region._setHomeRegion("FR", false); + await reloadEngines(settings); + Region._setHomeRegion("", false); + + let defaultEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.DEFAULT, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + await reloadEngines(settings); + Assert.ok( + stub.notCalled, + "_reloadEngines should not have shown the notification box." + ); + + let newDefault = await defaultEngineChanged; + Assert.equal( + newDefault.QueryInterface(Ci.nsISearchEngine).name, + "Test search engine", + "Should have correctly notified the new default engine." + ); + + Region._setHomeRegion("FR", false); + await reloadEngines(settings); + Region._setHomeRegion("", false); + + await loadEngines(settings); + Assert.ok( + stub.notCalled, + "_loadEngines should not have shown the notification box." + ); + + Assert.equal( + Services.search.defaultEngine.name, + "Test search engine", + "Should have correctly notified the new default engine." + ); +} diff --git a/toolkit/components/search/tests/xpcshell/test_remove_profile_engine.js b/toolkit/components/search/tests/xpcshell/test_remove_profile_engine.js new file mode 100644 index 0000000000..6cf73851ce --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_remove_profile_engine.js @@ -0,0 +1,46 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// This test is to ensure that we remove xml files from searchplugins/ in the +// profile directory when a user removes the actual engine from their profile. + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function run_test() { + // Copy an engine to [profile]/searchplugin/ + let dir = do_get_profile().clone(); + dir.append("searchplugins"); + if (!dir.exists()) { + dir.create(dir.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY); + } + do_get_file("data/engine.xml").copyTo(dir, "test-search-engine.xml"); + + let file = dir.clone(); + file.append("test-search-engine.xml"); + Assert.ok(file.exists()); + + let data = await readJSONFile(do_get_file("data/search-legacy.json")); + + // Put the filePath inside the settings file, to simulate what a pre-58 version + // of Firefox would have done. + for (let engine of data.engines) { + if (engine._name == "Test search engine") { + engine.filePath = file.path; + } + } + + await promiseSaveSettingsData(data); + + await Services.search.init(); + + // test the engine is loaded ok. + let engine = Services.search.getEngineByName("Test search engine"); + Assert.notEqual(engine, null, "Should have found the engine"); + + // remove the engine and verify the file has been removed too. + await Services.search.removeEngine(engine); + Assert.ok(!file.exists(), "Should have removed the file."); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_save_sorted_engines.js b/toolkit/components/search/tests/xpcshell/test_save_sorted_engines.js new file mode 100644 index 0000000000..480ee6d11a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_save_sorted_engines.js @@ -0,0 +1,67 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Ensure that metadata are stored correctly on disk after: + * - moving an engine + * - removing an engine + * - adding a new engine + * + * Notes: + * - we install the search engines of test "test_downloadAndAddEngines.js" + * to ensure that this test is independent from locale, commercial agreements + * and configuration of Firefox. + */ + +add_setup(async function () { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_save_sorted_engines() { + let engine1 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + let engine2 = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine2.xml`, + }); + await promiseAfterSettings(); + + let search = Services.search; + + // Test moving the engines + await search.moveEngine(engine1, 0); + await search.moveEngine(engine2, 1); + + // Changes should be commited immediately + await promiseAfterSettings(); + info("Commit complete after moveEngine"); + + // Check that the entries are placed as specified correctly + let metadata = await promiseEngineMetadata(); + Assert.equal(metadata["Test search engine"].order, 1); + Assert.equal(metadata["A second test engine"].order, 2); + + // Test removing an engine + search.removeEngine(engine1); + await promiseAfterSettings(); + info("Commit complete after removeEngine"); + + // Check that the order of the remaining engine was updated correctly + metadata = await promiseEngineMetadata(); + Assert.equal(metadata["A second test engine"].order, 1); + + // Test adding a new engine + await SearchTestUtils.installSearchExtension({ + name: "foo", + keyword: "foo", + }); + + let engine = Services.search.getEngineByName("foo"); + await promiseAfterSettings(); + info("Commit complete after addEngineWithDetails"); + + metadata = await promiseEngineMetadata(); + Assert.ok(engine.aliases.includes("foo")); + Assert.ok(metadata.foo.order > 0); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest.js new file mode 100644 index 0000000000..48769be41e --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest.js @@ -0,0 +1,901 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ +/* eslint-disable mozilla/no-arbitrary-setTimeout */ + +/** + * Testing search suggestions from SearchSuggestionController.jsm. + */ + +"use strict"; + +const { FormHistory } = ChromeUtils.importESModule( + "resource://gre/modules/FormHistory.sys.mjs" +); +const { SearchSuggestionController } = ChromeUtils.importESModule( + "resource://gre/modules/SearchSuggestionController.sys.mjs" +); +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const ENGINE_NAME = "other"; +const SEARCH_TELEMETRY_LATENCY = "SEARCH_SUGGESTIONS_LATENCY_MS"; + +// We must make sure the FormHistoryStartup component is +// initialized in order for it to respond to FormHistory +// requests from nsFormAutoComplete.js. +var formHistoryStartup = Cc[ + "@mozilla.org/satchel/form-history-startup;1" +].getService(Ci.nsIObserver); +formHistoryStartup.observe(null, "profile-after-change", null); + +var getEngine, postEngine, unresolvableEngine, alternateJSONEngine; + +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + // These tests intentionally test broken connections. + consoleAllowList = consoleAllowList.concat([ + "Non-200 status or empty HTTP response: 404", + "Non-200 status or empty HTTP response: 500", + "SearchSuggestionController found an unexpected string value", + "HTTP request timeout", + "HTTP error", + ]); + + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + + await AddonTestUtils.promiseStartupManager(); + + let getEngineData = { + baseURL: gDataUrl, + name: "GET suggestion engine", + method: "GET", + }; + + let postEngineData = { + baseURL: gDataUrl, + name: "POST suggestion engine", + method: "POST", + }; + + let unresolvableEngineData = { + baseURL: "http://example.invalid/", + name: "Offline suggestion engine", + method: "GET", + }; + + let alternateJSONSuggestEngineData = { + baseURL: gDataUrl, + name: "Alternative JSON suggestion type", + method: "GET", + alternativeJSONType: true, + }; + + getEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify(getEngineData)}`, + }); + postEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify(postEngineData)}`, + }); + unresolvableEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify(unresolvableEngineData)}`, + }); + alternateJSONEngine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify( + alternateJSONSuggestEngineData + )}`, + }); + + registerCleanupFunction(async () => { + // Remove added form history entries + await updateSearchHistory("remove", null); + Services.prefs.clearUserPref("browser.search.suggest.enabled"); + }); +}); + +// Begin tests + +add_task(async function simple_no_result_promise() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("no remote", false, getEngine); + Assert.equal(result.term, "no remote"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function simple_remote_no_local_result() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, getEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "Mozilla"); + Assert.equal(result.remote[1].value, "modern"); + Assert.equal(result.remote[2].value, "mom"); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function simple_remote_no_local_result_telemetry() { + Services.telemetry.clearScalars(); + + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + await controller.fetch("mo", false, getEngine); + + let scalars = {}; + const key = "browser.search.data_transferred"; + + await TestUtils.waitForCondition(() => { + scalars = + Services.telemetry.getSnapshotForKeyedScalars("main", false).parent || {}; + return key in scalars; + }, "should have the expected keyed scalars"); + + const scalar = scalars[key]; + Assert.ok(`sggt-${ENGINE_NAME}` in scalar, "correct telemetry category"); + Assert.notEqual(scalar[`sggt-${ENGINE_NAME}`], 0, "bandwidth logged"); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function simple_remote_no_local_result_alternative_type() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, alternateJSONEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "Mozilla"); + Assert.equal(result.remote[1].value, "modern"); + Assert.equal(result.remote[2].value, "mom"); +}); + +add_task(async function remote_term_case_mismatch() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("Query Case Mismatch", false, getEngine); + Assert.equal(result.term, "Query Case Mismatch"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "Query Case Mismatch"); +}); + +add_task(async function simple_local_no_remote_result() { + await updateSearchHistory("bump", "no remote entries"); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("no remote", false, getEngine); + Assert.equal(result.term, "no remote"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "no remote entries"); + Assert.equal(result.remote.length, 0); + + await updateSearchHistory("remove", "no remote entries"); +}); + +add_task(async function simple_non_ascii() { + await updateSearchHistory("bump", "I ❤️ XUL"); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("I ❤️", false, getEngine); + Assert.equal(result.term, "I ❤️"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "I ❤️ XUL"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "I ❤️ Mozilla"); +}); + +add_task(async function both_local_remote_result_dedupe() { + await updateSearchHistory("bump", "Mozilla"); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, getEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "Mozilla"); + Assert.equal(result.remote.length, 2); + Assert.equal(result.remote[0].value, "modern"); + Assert.equal(result.remote[1].value, "mom"); +}); + +add_task(async function POST_both_local_remote_result_dedupe() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, postEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "Mozilla"); + Assert.equal(result.remote.length, 2); + Assert.equal(result.remote[0].value, "modern"); + Assert.equal(result.remote[1].value, "mom"); +}); + +add_task(async function both_local_remote_result_dedupe2() { + await updateSearchHistory("bump", "mom"); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, getEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 2); + Assert.equal(result.local[0].value, "mom"); + Assert.equal(result.local[1].value, "Mozilla"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "modern"); +}); + +add_task(async function both_local_remote_result_dedupe3() { + // All of the server entries also exist locally + await updateSearchHistory("bump", "modern"); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("mo", false, getEngine); + Assert.equal(result.term, "mo"); + Assert.equal(result.local.length, 3); + Assert.equal(result.local[0].value, "modern"); + Assert.equal(result.local[1].value, "mom"); + Assert.equal(result.local[2].value, "Mozilla"); + Assert.equal(result.remote.length, 0); +}); + +add_task(async function valid_tail_results() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("tail query", false, getEngine); + Assert.equal(result.term, "tail query"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "tail query normal"); + Assert.ok(!result.remote[0].matchPrefix); + Assert.ok(!result.remote[0].tail); + Assert.equal(result.remote[1].value, "tail query tail 1"); + Assert.equal(result.remote[1].matchPrefix, "… "); + Assert.equal(result.remote[1].tail, "tail 1"); + Assert.equal(result.remote[2].value, "tail query tail 2"); + Assert.equal(result.remote[2].matchPrefix, "… "); + Assert.equal(result.remote[2].tail, "tail 2"); +}); + +add_task(async function alt_tail_results() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("tailalt query", false, getEngine); + Assert.equal(result.term, "tailalt query"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "tailalt query normal"); + Assert.ok(!result.remote[0].matchPrefix); + Assert.ok(!result.remote[0].tail); + Assert.equal(result.remote[1].value, "tailalt query tail 1"); + Assert.equal(result.remote[1].matchPrefix, "… "); + Assert.equal(result.remote[1].tail, "tail 1"); + Assert.equal(result.remote[2].value, "tailalt query tail 2"); + Assert.equal(result.remote[2].matchPrefix, "… "); + Assert.equal(result.remote[2].tail, "tail 2"); +}); + +add_task(async function invalid_tail_results() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("tailjunk query", false, getEngine); + Assert.equal(result.term, "tailjunk query"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "tailjunk query normal"); + Assert.ok(!result.remote[0].matchPrefix); + Assert.ok(!result.remote[0].tail); + Assert.equal(result.remote[1].value, "tailjunk query tail 1"); + Assert.ok(!result.remote[1].matchPrefix); + Assert.ok(!result.remote[1].tail); + Assert.equal(result.remote[2].value, "tailjunk query tail 2"); + Assert.ok(!result.remote[2].matchPrefix); + Assert.ok(!result.remote[2].tail); +}); + +add_task(async function too_few_tail_results() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("tailjunk few query", false, getEngine); + Assert.equal(result.term, "tailjunk few query"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "tailjunk few query normal"); + Assert.ok(!result.remote[0].matchPrefix); + Assert.ok(!result.remote[0].tail); + Assert.equal(result.remote[1].value, "tailjunk few query tail 1"); + Assert.ok(!result.remote[1].matchPrefix); + Assert.ok(!result.remote[1].tail); + Assert.equal(result.remote[2].value, "tailjunk few query tail 2"); + Assert.ok(!result.remote[2].matchPrefix); + Assert.ok(!result.remote[2].tail); +}); + +add_task(async function empty_rich_results() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("richempty query", false, getEngine); + Assert.equal(result.term, "richempty query"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[0].value, "richempty query normal"); + Assert.ok(!result.remote[0].matchPrefix); + Assert.ok(!result.remote[0].tail); + Assert.equal(result.remote[1].value, "richempty query tail 1"); + Assert.ok(!result.remote[1].matchPrefix); + Assert.ok(!result.remote[1].tail); + Assert.equal(result.remote[2].value, "richempty query tail 2"); + Assert.ok(!result.remote[2].matchPrefix); + Assert.ok(!result.remote[2].tail); +}); + +add_task(async function tail_offset_index() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("tail tail 1 t", false, getEngine); + Assert.equal(result.term, "tail tail 1 t"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 3); + Assert.equal(result.remote[1].value, "tail tail 1 t tail 1"); + Assert.equal(result.remote[1].matchPrefix, "… "); + Assert.equal(result.remote[1].tail, "tail 1"); + Assert.equal(result.remote[1].tailOffsetIndex, 14); +}); + +add_task(async function fetch_twice_in_a_row() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + // Two entries since the first will match the first fetch but not the second. + await updateSearchHistory("bump", "delay local"); + await updateSearchHistory("bump", "delayed local"); + + let controller = new SearchSuggestionController(); + let resultPromise1 = controller.fetch("delay", false, getEngine); + + // A second fetch while the server is still waiting to return results leads to an abort. + let resultPromise2 = controller.fetch("delayed ", false, getEngine); + await resultPromise1.then(results => Assert.equal(null, results)); + + let result = await resultPromise2; + Assert.equal(result.term, "delayed "); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "delayed local"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "delayed "); + + // Only the second fetch's latency should be recorded since the first fetch + // was aborted and latencies for aborted fetches are not recorded. + assertLatencyHistogram(histogram, true); +}); + +add_task(async function both_identical_with_more_than_max_results() { + // Add letters A through Z to form history which will match the server + for ( + let charCode = "A".charCodeAt(); + charCode <= "Z".charCodeAt(); + charCode++ + ) { + await updateSearchHistory( + "bump", + "letter " + String.fromCharCode(charCode) + ); + } + + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 7; + controller.maxRemoteResults = 10; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 7); + for (let i = 0; i < controller.maxLocalResults; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.local.length + result.remote.length, 10); + for (let i = 0; i < result.remote.length; i++) { + Assert.equal( + result.remote[i].value, + "letter " + + String.fromCharCode("A".charCodeAt() + controller.maxLocalResults + i) + ); + } +}); + +add_task(async function noremote_maxLocal() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 2; // (should be ignored because no remote results) + controller.maxRemoteResults = 0; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 26); + for (let i = 0; i < result.local.length; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, false); +}); + +add_task(async function someremote_maxLocal() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 2; + controller.maxRemoteResults = 4; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 2); + for (let i = 0; i < result.local.length; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.remote.length, 2); + // "A" and "B" will have been de-duped, start at C for remote results + for (let i = 0; i < result.remote.length; i++) { + Assert.equal( + result.remote[i].value, + "letter " + String.fromCharCode("C".charCodeAt() + i) + ); + } + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function one_of_each() { + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 1; + controller.maxRemoteResults = 2; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "letter A"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "letter B"); +}); + +add_task(async function local_result_returned_remote_result_disabled() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + Services.prefs.setBoolPref("browser.search.suggest.enabled", false); + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 1; + controller.maxRemoteResults = 1; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 26); + for (let i = 0; i < 26; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.remote.length, 0); + assertLatencyHistogram(histogram, false); + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); +}); + +add_task( + async function local_result_returned_remote_result_disabled_after_creation_of_controller() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 1; + controller.maxRemoteResults = 1; + Services.prefs.setBoolPref("browser.search.suggest.enabled", false); + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 26); + for (let i = 0; i < 26; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.remote.length, 0); + assertLatencyHistogram(histogram, false); + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + } +); + +add_task( + async function one_of_each_disabled_before_creation_enabled_after_creation_of_controller() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + Services.prefs.setBoolPref("browser.search.suggest.enabled", false); + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 1; + controller.maxRemoteResults = 2; + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "letter A"); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "letter B"); + + assertLatencyHistogram(histogram, true); + + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + } +); + +add_task(async function one_local_zero_remote() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 1; + controller.maxRemoteResults = 0; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 26); + for (let i = 0; i < 26; i++) { + Assert.equal( + result.local[i].value, + "letter " + String.fromCharCode("A".charCodeAt() + i) + ); + } + Assert.equal(result.remote.length, 0); + assertLatencyHistogram(histogram, false); +}); + +add_task(async function zero_local_one_remote() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 0; + controller.maxRemoteResults = 1; + let result = await controller.fetch("letter ", false, getEngine); + Assert.equal(result.term, "letter "); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "letter A"); + assertLatencyHistogram(histogram, true); +}); + +add_task(async function stop_search() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + let controller = new SearchSuggestionController(result => { + do_throw("The callback shouldn't be called after stop()"); + }); + let resultPromise = controller.fetch("mo", false, getEngine); + controller.stop(); + await resultPromise.then(result => { + Assert.equal(null, result); + }); + assertLatencyHistogram(histogram, false); +}); + +add_task(async function empty_searchTerm() { + // Empty searches don't go to the server but still get form history. + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + let controller = new SearchSuggestionController(); + let result = await controller.fetch("", false, getEngine); + Assert.equal(result.term, ""); + Assert.ok(!!result.local.length); + Assert.equal(result.remote.length, 0); + assertLatencyHistogram(histogram, false); +}); + +add_task(async function slow_timeout() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + // Make the server return suggestions on a delay longer than the timeout of + // the suggestion controller. + let delayMs = 3 * SearchSuggestionController.REMOTE_TIMEOUT_DEFAULT; + let searchString = `delay${delayMs} `; + + // Add a local result. + let localValue = searchString + " local result"; + await updateSearchHistory("bump", localValue); + + // Do a search. The remote fetch should time out but the local result should + // be returned. + let controller = new SearchSuggestionController(); + let result = await controller.fetch(searchString, false, getEngine); + Assert.equal(result.term, searchString); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, localValue); + Assert.equal(result.remote.length, 0); + + // The remote fetch isn't done yet, so the latency histogram should not be + // updated. + assertLatencyHistogram(histogram, false); + + // Wait for the remote fetch to finish. + await new Promise(r => setTimeout(r, delayMs)); + + // Now the latency histogram should be updated. + assertLatencyHistogram(histogram, true); +}); + +add_task(async function slow_timeout_2() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + // Make the server return suggestions on a delay longer the timeout of the + // suggestion controller. + let delayMs = 3 * SearchSuggestionController.REMOTE_TIMEOUT_DEFAULT; + let searchString = `delay${delayMs} `; + + // Add a local result. + let localValue = searchString + " local result"; + await updateSearchHistory("bump", localValue); + + // Do two searches using the same controller. Both times, the remote fetches + // should time out and only the local result should be returned. The second + // search should abort the remote fetch of the first search, and the remote + // fetch of the second search should be ongoing when the second search + // finishes. + let controller = new SearchSuggestionController(); + for (let i = 0; i < 2; i++) { + let result = await controller.fetch(searchString, false, getEngine); + Assert.equal(result.term, searchString); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, localValue); + Assert.equal(result.remote.length, 0); + } + + // The remote fetch of the second search isn't done yet, so the latency + // histogram should not be updated. + assertLatencyHistogram(histogram, false); + + // Wait for the second remote fetch to finish. + await new Promise(r => setTimeout(r, delayMs)); + + // Now the latency histogram should be updated, and only the remote fetch of + // the second search should be recorded. + assertLatencyHistogram(histogram, true); +}); + +add_task(async function slow_stop() { + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + // Make the server return suggestions on a delay longer the timeout of the + // suggestion controller. + let delayMs = 3 * SearchSuggestionController.REMOTE_TIMEOUT_DEFAULT; + let searchString = `delay${delayMs} `; + + // Do a search but stop it before it finishes. Wait a tick before stopping it + // to better simulate the real world. + let controller = new SearchSuggestionController(); + let resultPromise = controller.fetch(searchString, false, getEngine); + await TestUtils.waitForTick(); + controller.stop(); + let result = await resultPromise; + Assert.equal(result, null, "No result should be returned"); + + // The remote fetch should have been aborted by stopping the controller, but + // wait for the timeout period just to make sure it's done. + await new Promise(r => setTimeout(r, delayMs)); + + // Since the latencies of aborted fetches are not recorded, the latency + // histogram should not be updated. + assertLatencyHistogram(histogram, false); +}); + +// Error handling + +add_task(async function remote_term_mismatch() { + await updateSearchHistory("bump", "Query Mismatch Entry"); + + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("Query Mismatch", false, getEngine); + Assert.equal(result.term, "Query Mismatch"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "Query Mismatch Entry"); + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function http_404() { + await updateSearchHistory("bump", "HTTP 404 Entry"); + + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("HTTP 404", false, getEngine); + Assert.equal(result.term, "HTTP 404"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "HTTP 404 Entry"); + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function http_500() { + await updateSearchHistory("bump", "HTTP 500 Entry"); + + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch("HTTP 500", false, getEngine); + Assert.equal(result.term, "HTTP 500"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "HTTP 500 Entry"); + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, true); +}); + +add_task(async function invalid_response_does_not_throw() { + let controller = new SearchSuggestionController(); + // Although the server will return invalid json, the error is handled by + // the suggestion controller, and so we receive no results. + let result = await controller.fetch("invalidJSON", false, getEngine); + Assert.equal(result.term, "invalidJSON"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 0); +}); + +add_task(async function invalid_content_type_treated_as_json() { + let controller = new SearchSuggestionController(); + // An invalid content type is overridden as we expect all the responses to + // be JSON. + let result = await controller.fetch("invalidContentType", false, getEngine); + Assert.equal(result.term, "invalidContentType"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 1); + Assert.equal(result.remote[0].value, "invalidContentType response"); +}); + +add_task(async function unresolvable_server() { + await updateSearchHistory("bump", "Unresolvable Server Entry"); + + let histogram = TelemetryTestUtils.getAndClearKeyedHistogram( + SEARCH_TELEMETRY_LATENCY + ); + + let controller = new SearchSuggestionController(); + let result = await controller.fetch( + "Unresolvable Server", + false, + unresolvableEngine + ); + Assert.equal(result.term, "Unresolvable Server"); + Assert.equal(result.local.length, 1); + Assert.equal(result.local[0].value, "Unresolvable Server Entry"); + Assert.equal(result.remote.length, 0); + + assertLatencyHistogram(histogram, true); +}); + +// Exception handling + +add_task(async function missing_pb() { + Assert.throws(() => { + let controller = new SearchSuggestionController(); + controller.fetch("No privacy"); + }, /priva/i); +}); + +add_task(async function missing_engine() { + Assert.throws(() => { + let controller = new SearchSuggestionController(); + controller.fetch("No engine", false); + }, /engine/i); +}); + +add_task(async function invalid_engine() { + Assert.throws(() => { + let controller = new SearchSuggestionController(); + controller.fetch("invalid engine", false, {}); + }, /engine/i); +}); + +add_task(async function no_results_requested() { + Assert.throws(() => { + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 0; + controller.maxRemoteResults = 0; + controller.fetch("No results requested", false, getEngine); + }, /result/i); +}); + +add_task(async function minus_one_results_requested() { + Assert.throws(() => { + let controller = new SearchSuggestionController(); + controller.maxLocalResults = -1; + controller.fetch("-1 results requested", false, getEngine); + }, /result/i); +}); + +add_task(async function test_userContextId() { + let controller = new SearchSuggestionController(); + controller._fetchRemote = function ( + searchTerm, + engine, + privateMode, + userContextId + ) { + Assert.equal(userContextId, 1); + return Promise.withResolvers(); + }; + + controller.fetch("test", false, getEngine, 1); +}); + +// Non-English characters + +add_task(async function suggestions_contain_escaped_unicode() { + let controller = new SearchSuggestionController(); + let result = await controller.fetch("stü", false, getEngine); + Assert.equal(result.term, "stü"); + Assert.equal(result.local.length, 0); + Assert.equal(result.remote.length, 2); + Assert.equal(result.remote[0].value, "stühle"); + Assert.equal(result.remote[1].value, "stüssy"); +}); + +// Helpers + +function updateSearchHistory(operation, value) { + return FormHistory.update({ + op: operation, + fieldname: "searchbar-history", + value, + }); +} + +function assertLatencyHistogram(histogram, shouldRecord) { + let snapshot = histogram.snapshot(); + info("Checking latency snapshot: " + JSON.stringify(snapshot)); + + // Build a map from engine ID => number of non-zero values recorded for it. + let valueCountByEngineId = Object.entries(snapshot).reduce( + (memo, [key, data]) => { + memo[key] = Object.values(data.values).filter(v => v != 0); + return memo; + }, + {} + ); + + let expected = shouldRecord ? { [ENGINE_NAME]: [1] } : {}; + Assert.deepEqual( + valueCountByEngineId, + expected, + shouldRecord ? "Latency histogram updated" : "Latency histogram not updated" + ); +} diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js new file mode 100644 index 0000000000..cfa9e56144 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js @@ -0,0 +1,143 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Test that search suggestions from SearchSuggestionController.jsm don't store + * cookies. + */ + +"use strict"; + +const { SearchSuggestionController } = ChromeUtils.importESModule( + "resource://gre/modules/SearchSuggestionController.sys.mjs" +); + +// We must make sure the FormHistoryStartup component is +// initialized in order for it to respond to FormHistory +// requests from nsFormAutoComplete.js. +var formHistoryStartup = Cc[ + "@mozilla.org/satchel/form-history-startup;1" +].getService(Ci.nsIObserver); +formHistoryStartup.observe(null, "profile-after-change", null); + +function countCacheEntries() { + info("Enumerating cache entries"); + return new Promise(resolve => { + let storage = Services.cache2.diskCacheStorage( + Services.loadContextInfo.default + ); + storage.asyncVisitStorage( + { + onCacheStorageInfo(num, consumption) { + this._num = num; + }, + onCacheEntryInfo(uri) { + info("Found cache entry: " + uri.asciiSpec); + }, + onCacheEntryVisitCompleted() { + resolve(this._num || 0); + }, + }, + true /* Do walk entries */ + ); + }); +} + +function countCookieEntries() { + info("Enumerating cookies"); + let cookies = Services.cookies.cookies; + let cookieCount = 0; + for (let cookie of cookies) { + info( + "Cookie:" + cookie.rawHost + " " + JSON.stringify(cookie.originAttributes) + ); + cookieCount++; + break; + } + return cookieCount; +} + +let engines; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + Services.prefs.setBoolPref("browser.search.suggest.enabled.private", true); + + registerCleanupFunction(async () => { + // Clean up all the data. + await new Promise(resolve => + Services.clearData.deleteData(Ci.nsIClearDataService.CLEAR_ALL, resolve) + ); + Services.prefs.clearUserPref("browser.search.suggest.enabled"); + Services.prefs.clearUserPref("browser.search.suggest.enabled.private"); + }); + + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + + let unicodeName = ["\u30a8", "\u30c9"].join(""); + engines = [ + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify({ + baseURL: gDataUrl, + name: unicodeName, + method: "GET", + })}`, + }), + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify({ + baseURL: gDataUrl, + name: "engine two", + method: "GET", + })}`, + }), + ]; + + // Clean up all the data. + await new Promise(resolve => + Services.clearData.deleteData(Ci.nsIClearDataService.CLEAR_ALL, resolve) + ); + Assert.equal(await countCacheEntries(), 0, "The cache should be empty"); + Assert.equal(await countCookieEntries(), 0, "Should not find any cookie"); +}); + +add_task(async function test_private_mode() { + await test_engine(true); +}); +add_task(async function test_normal_mode() { + await test_engine(false); +}); + +async function test_engine(privateMode) { + info(`Testing ${privateMode ? "private" : "normal"} mode`); + let controller = new SearchSuggestionController(); + let result = await controller.fetch("no results", privateMode, engines[0]); + Assert.equal(result.local.length, 0, "Should have no local suggestions"); + Assert.equal(result.remote.length, 0, "Should have no remote suggestions"); + + result = await controller.fetch("cookie", privateMode, engines[1]); + Assert.equal(result.local.length, 0, "Should have no local suggestions"); + Assert.equal(result.remote.length, 0, "Should have no remote suggestions"); + Assert.equal(await countCacheEntries(), 0, "The cache should be empty"); + Assert.equal(await countCookieEntries(), 0, "Should not find any cookie"); + + let firstPartyDomain1 = controller.firstPartyDomains.get(engines[0].name); + Assert.ok( + /^[\.a-z0-9-]+\.search\.suggestions\.mozilla/.test(firstPartyDomain1), + "Check firstPartyDomain1" + ); + + let firstPartyDomain2 = controller.firstPartyDomains.get(engines[1].name); + Assert.ok( + /^[\.a-z0-9-]+\.search\.suggestions\.mozilla/.test(firstPartyDomain2), + "Check firstPartyDomain2" + ); + + Assert.notEqual( + firstPartyDomain1, + firstPartyDomain2, + "Check firstPartyDomain id unique per engine" + ); +} diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest_extraParams.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest_extraParams.js new file mode 100644 index 0000000000..8ecf4b02f3 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest_extraParams.js @@ -0,0 +1,57 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const TEST_CONFIG = [ + { + webExtension: { + id: "get@search.mozilla.org", + name: "Get Engine", + search_url: "https://example.com", + search_url_get_params: "webExtension=1&search={searchTerms}", + suggest_url: "https://example.com", + suggest_url_get_params: "webExtension=1&suggest={searchTerms}", + }, + appliesTo: [{ included: { everywhere: true } }], + suggestExtraParams: [ + { + name: "custom_param", + pref: "test_pref_param", + condition: "pref", + }, + ], + }, +]; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("method-extensions", null, TEST_CONFIG); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_custom_suggest_param() { + let engine = Services.search.getEngineByName("Get Engine"); + Assert.notEqual(engine, null, "Should have found an engine"); + + let submissionSuggest = engine.getSubmission( + "bar", + SearchUtils.URL_TYPE.SUGGEST_JSON + ); + Assert.equal( + submissionSuggest.uri.spec, + "https://example.com/?webExtension=1&suggest=bar", + "Suggest URLs should match" + ); + + let defaultBranch = Services.prefs.getDefaultBranch("browser.search."); + defaultBranch.setCharPref("param.test_pref_param", "good"); + + let nextSubmissionSuggest = engine.getSubmission( + "bar", + SearchUtils.URL_TYPE.SUGGEST_JSON + ); + Assert.equal( + nextSubmissionSuggest.uri.spec, + "https://example.com/?custom_param=good&webExtension=1&suggest=bar", + "Suggest URLs should include custom param" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js new file mode 100644 index 0000000000..063f3ada49 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js @@ -0,0 +1,54 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Test that search suggestions from SearchSuggestionController.jsm operate + * correctly in private mode. + */ + +"use strict"; + +const { SearchSuggestionController } = ChromeUtils.importESModule( + "resource://gre/modules/SearchSuggestionController.sys.mjs" +); + +let engine; + +add_setup(async function () { + Services.prefs.setBoolPref("browser.search.suggest.enabled", true); + + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + + await AddonTestUtils.promiseStartupManager(); + + const engineData = { + baseURL: gDataUrl, + name: "GET suggestion engine", + method: "GET", + }; + + engine = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engineMaker.sjs?${JSON.stringify(engineData)}`, + }); +}); + +add_task(async function test_suggestions_in_private_mode_enabled() { + Services.prefs.setBoolPref("browser.search.suggest.enabled.private", true); + + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 0; + controller.maxRemoteResults = 1; + let result = await controller.fetch("mo", true, engine); + Assert.equal(result.remote.length, 1); +}); + +add_task(async function test_suggestions_in_private_mode_disabled() { + Services.prefs.setBoolPref("browser.search.suggest.enabled.private", false); + + let controller = new SearchSuggestionController(); + controller.maxLocalResults = 0; + controller.maxRemoteResults = 1; + let result = await controller.fetch("mo", true, engine); + Assert.equal(result.remote.length, 0); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_searchTermFromResult.js b/toolkit/components/search/tests/xpcshell/test_searchTermFromResult.js new file mode 100644 index 0000000000..700c6a3450 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchTermFromResult.js @@ -0,0 +1,350 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Tests searchTermFromResult API. + */ + +let defaultEngine; + +// The test string contains special characters to ensure +// that they are encoded/decoded properly. +const TERM = "c;,?:@&=+$-_.!~*'()# d\u00E8f"; +const TERM_ENCODED = "c%3B%2C%3F%3A%40%26%3D%2B%24-_.!~*'()%23+d%C3%A8f"; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data", null, [ + { + webExtension: { + id: "engine-purposes@search.mozilla.org", + name: "Test Engine With Purposes", + search_url: "https://www.example.com/search", + params: [ + { + name: "form", + condition: "purpose", + purpose: "keyword", + value: "MOZKEYWORD", + }, + { + name: "form", + condition: "purpose", + purpose: "contextmenu", + value: "MOZCONTEXT", + }, + { + name: "form", + condition: "purpose", + purpose: "newtab", + value: "MOZNEWTAB", + }, + { + name: "form", + condition: "purpose", + purpose: "searchbar", + value: "MOZSEARCHBAR", + }, + { + name: "form", + condition: "purpose", + purpose: "homepage", + value: "MOZHOMEPAGE", + }, + { + name: "pc", + value: "FIREFOX", + }, + { + name: "channel", + condition: "pref", + pref: "testChannelEnabled", + }, + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [ + { + included: { everywhere: true }, + default: "yes", + }, + ], + }, + ]); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + + defaultEngine = Services.search.getEngineByName("Test Engine With Purposes"); +}); + +add_task(async function test_searchTermFromResult_withAllPurposes() { + for (let purpose of Object.values(SearchUtils.PARAM_PURPOSES)) { + let uri = defaultEngine.getSubmission(TERM, null, purpose).uri; + let searchTerm = defaultEngine.searchTermFromResult(uri); + Assert.equal( + searchTerm, + TERM, + `Should return the correct url for purpose: ${purpose}` + ); + } +}); + +add_task(async function test_searchTermFromResult() { + // Internationalized Domain Name search engine. + await SearchTestUtils.installSearchExtension({ + name: "idn_addParam", + keyword: "idn_addParam", + search_url: "https://www.xn--bcher-kva.ch/search", + }); + let engineEscapedIDN = Services.search.getEngineByName("idn_addParam"); + + // Setup server for french engine. + await useHttpServer(); + + // For ISO-8859-1 encoding testing. + let engineISOCharset = await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine-fr.xml`, + }); + + // For Windows-1252 encoding testing. + await SearchTestUtils.installSearchExtension({ + name: "bacon_addParam", + keyword: "bacon_addParam", + encoding: "windows-1252", + search_url: "https://www.bacon.test/find", + }); + let engineWinCharset = Services.search.getEngineByName("bacon_addParam"); + + // Verify getValidEngineUrl returns a URL that can return a search term. + let testUrl = getValidEngineUrl(); + Assert.equal( + getTerm(testUrl), + TERM, + "Should get term from a url generated by getSubmission." + ); + + testUrl = getValidEngineUrl(); + testUrl.pathname = "/SEARCH"; + Assert.equal( + getTerm(testUrl), + TERM, + "Should get term even if path is not the same case as the engine." + ); + + let url = `https://www.xn--bcher-kva.ch/search?q=${TERM_ENCODED}`; + Assert.equal( + getTerm(url, engineEscapedIDN), + TERM, + "Should get term from IDNs urls." + ); + + url = `http://www.google.fr/search?q=caf%E8+au+lait&ie=iso-8859-1&oe=iso-8859-1`; + Assert.equal( + getTerm(url, engineISOCharset), + "caf\u00E8 au lait", + "Should get term from ISO-8859-1 encoded url containing a search term." + ); + + url = `http://www.google.fr/search?&ie=iso-8859-1&oe=iso-8859-1&q=`; + Assert.equal( + getTerm(url, engineISOCharset), + "", + "Should get a blank string from ISO-8859-1 encoded url missing a search term" + ); + + url = "https://www.bacon.test/find?q=caf%E8+au+lait"; + Assert.equal( + getTerm(url, engineWinCharset), + "caf\u00E8 au lait", + "Should get term from Windows-1252 encoded url containing a search term." + ); + + url = "https://www.bacon.test/find?q="; + Assert.equal( + getTerm(url, engineWinCharset), + "", + "Should get a blank string from Windows-1252 encoded url missing a search term." + ); + + url = "about:blank"; + Assert.equal(getTerm(url), "", "Should get a blank string from about:blank."); + + url = "about:newtab"; + Assert.equal( + getTerm(url), + "", + "Should get a blank string from about:newtab." + ); +}); + +// Use a version of the url that should return a term and make minute +// modifications that should cause it to return a blank value. +add_task(async function test_searchTermFromResult_blank() { + let url = getValidEngineUrl(); + url.searchParams.set("hello", "world"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from url containing query param name not recognized by the engine." + ); + + url = getValidEngineUrl(); + url.protocol = "http"; + Assert.equal( + getTerm(url), + "", + "Should get a blank string from url that has a different scheme from the engine." + ); + + url = getValidEngineUrl(); + url.protocol = "http"; + Assert.equal( + getTerm(url), + "", + "Should get a blank string from url that has a different path from the engine." + ); + + url = getValidEngineUrl(); + url.host = "images.example.com"; + Assert.equal( + getTerm(url), + "", + "Should get a blank string from url that has a different host from the engine." + ); + + url = getValidEngineUrl(); + url.host = "example.com"; + Assert.equal( + getTerm(url), + "", + "Should get a blank string from url that has a different host from the engine." + ); + + url = getValidEngineUrl(); + url.searchParams.set("form", "MOZUNKNOWN"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from a url that has an un-recognized form value." + ); + + url = getValidEngineUrl(); + url.searchParams.set("q", ""); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from a url with a missing search query value." + ); + + url = getValidEngineUrl(); + url.searchParams.delete("q"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from a url with a missing search query name." + ); + + url = getValidEngineUrl(); + url.searchParams.delete("pc"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from a url with a missing a query parameter." + ); + + url = getValidEngineUrl(); + url.searchParams.delete("form"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string from a url with a missing a query parameter." + ); +}); + +add_task(async function test_searchTermFromResult_prefParam() { + const defaultBranch = Services.prefs.getDefaultBranch( + SearchUtils.BROWSER_SEARCH_PREF + ); + + defaultBranch.setCharPref("param.testChannelEnabled", "yes"); + + let url = getValidEngineUrl(true); + Assert.equal(getTerm(url), TERM, "Should get term after pref is turned on."); + + url.searchParams.delete("channel"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string if pref is on and channel param is missing." + ); + + defaultBranch.setCharPref("param.testChannelEnabled", ""); + url = getValidEngineUrl(true); + Assert.equal(getTerm(url), TERM, "Should get term after pref is turned off."); + + url.searchParams.set("channel", "yes"); + Assert.equal( + getTerm(url), + "", + "Should get a blank string if pref is turned off but channel param is present." + ); +}); + +// searchTermFromResult attempts to look into the template of a search +// engine if query params aren't present in the url.params, so make sure +// it works properly and fails gracefully. +add_task(async function test_searchTermFromResult_paramsInSearchUrl() { + await SearchTestUtils.installSearchExtension({ + name: "engine_params_in_search_url", + search_url: "https://example.com/?q={searchTerms}&pc=firefox", + search_url_get_params: "", + }); + let testEngine = Services.search.getEngineByName( + "engine_params_in_search_url" + ); + let url = `https://example.com/?q=${TERM_ENCODED}&pc=firefox`; + Assert.equal( + getTerm(url, testEngine), + TERM, + "Should get term from an engine with params in its search url." + ); + + url = `https://example.com/?q=${TERM_ENCODED}`; + Assert.equal( + getTerm(url, testEngine), + "", + "Should get a blank string when not all params are present." + ); + + await SearchTestUtils.installSearchExtension({ + name: "engine_params_in_search_url_without_delimiter", + search_url: "https://example.com/q={searchTerms}", + search_url_get_params: "", + }); + testEngine = Services.search.getEngineByName( + "engine_params_in_search_url_without_delimiter" + ); + url = `https://example.com/?q=${TERM_ENCODED}&pc=firefox&page=1`; + Assert.equal( + getTerm(url, testEngine), + "", + "Should get a blank string from an engine with no params and no delimiter in its url." + ); +}); + +function getTerm(url, searchEngine = defaultEngine) { + return searchEngine.searchTermFromResult(Services.io.newURI(url.toString())); +} + +// Return a new instance of a submission URL so that it can modified +// and tested again. Allow callers to force the cache to update, especially +// if the engine is expected to have updated. +function getValidEngineUrl(updateCache = false) { + if (updateCache || !this._submissionUrl) { + this._submissionUrl = defaultEngine.getSubmission(TERM, null).uri.spec; + } + return new URL(this._submissionUrl); +} diff --git a/toolkit/components/search/tests/xpcshell/test_searchUrlDomain.js b/toolkit/components/search/tests/xpcshell/test_searchUrlDomain.js new file mode 100644 index 0000000000..ea7db5e516 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_searchUrlDomain.js @@ -0,0 +1,21 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests searchUrlDomain API. + */ + +"use strict"; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data", null); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_resultDomain() { + await Services.search.init(); + + let engine = Services.search.getEngineByName("Test search engine"); + + Assert.equal(engine.searchUrlDomain, "www.google.com"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_selectedEngine.js b/toolkit/components/search/tests/xpcshell/test_selectedEngine.js new file mode 100644 index 0000000000..f42670156c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_selectedEngine.js @@ -0,0 +1,155 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const kDefaultEngineName = "engine1"; + +add_setup(async function () { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines("data1"); + Assert.ok(!Services.search.isInitialized); + Services.prefs.setBoolPref( + "browser.search.removeEngineInfobar.enabled", + false + ); +}); + +// Check that the default engine matches the defaultenginename pref +add_task(async function test_defaultEngine() { + await Services.search.init(); + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); +}); + +// Setting the search engine should be persisted across restarts. +add_task(async function test_persistAcrossRestarts() { + // Set the engine through the API. + await Services.search.setDefault( + Services.search.getEngineByName(kTestEngineName), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal(Services.search.defaultEngine.name, kTestEngineName); + await promiseAfterSettings(); + + // Check that the a hash was saved. + let metadata = await promiseGlobalMetadata(); + Assert.equal(metadata.defaultEngineIdHash.length, 44); + + // Re-init and check the engine is still the same. + Services.search.wrappedJSObject.reset(); + await Services.search.init(true); + Assert.equal(Services.search.defaultEngine.name, kTestEngineName); + + // Cleanup (set the engine back to default). + Services.search.resetToAppDefaultEngine(); + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); +}); + +// An engine set without a valid hash should be ignored. +add_task(async function test_ignoreInvalidHash() { + // Set the engine through the API. + await Services.search.setDefault( + Services.search.getEngineByName(kTestEngineName), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal(Services.search.defaultEngine.name, kTestEngineName); + await promiseAfterSettings(); + + // Then mess with the file (make the hash invalid). + let metadata = await promiseGlobalMetadata(); + metadata.defaultEngineIdHash = "invalid"; + await promiseSaveGlobalMetadata(metadata); + + // Re-init the search service, and check that the json file is ignored. + Services.search.wrappedJSObject.reset(); + await Services.search.init(true); + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); +}); + +// Resetting the engine to the default should remove the saved value. +add_task(async function test_settingToDefault() { + // Set the engine through the API. + await Services.search.setDefault( + Services.search.getEngineByName(kTestEngineName), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal(Services.search.defaultEngine.name, kTestEngineName); + await promiseAfterSettings(); + + // Check that the current engine was saved. + let metadata = await promiseGlobalMetadata(); + let currentEngine = Services.search.getEngineByName(kTestEngineName); + Assert.equal(metadata.defaultEngineId, currentEngine.id); + + // Then set the engine back to the default through the API. + await Services.search.setDefault( + Services.search.getEngineByName(kDefaultEngineName), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + await promiseAfterSettings(); + + // Check that the current engine is no longer saved in the JSON file. + metadata = await promiseGlobalMetadata(); + Assert.equal(metadata.defaultEngineId, ""); +}); + +add_task(async function test_resetToOriginalDefaultEngine() { + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); + + await Services.search.setDefault( + Services.search.getEngineByName(kTestEngineName), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal(Services.search.defaultEngine.name, kTestEngineName); + await promiseAfterSettings(); + + Services.search.resetToAppDefaultEngine(); + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); + await promiseAfterSettings(); +}); + +add_task(async function test_fallback_kept_after_restart() { + // Set current engine to a default engine that isn't the original default. + let builtInEngines = await Services.search.getAppProvidedEngines(); + let nonDefaultBuiltInEngine; + for (let engine of builtInEngines) { + if (engine.name != kDefaultEngineName) { + nonDefaultBuiltInEngine = engine; + break; + } + } + await Services.search.setDefault( + nonDefaultBuiltInEngine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + Assert.equal( + Services.search.defaultEngine.name, + nonDefaultBuiltInEngine.name + ); + await promiseAfterSettings(); + + // Remove that engine... + await Services.search.removeEngine(nonDefaultBuiltInEngine); + // The engine being a default (built-in) one, it should be hidden + // rather than actually removed. + Assert.ok(nonDefaultBuiltInEngine.hidden); + + // Using the defaultEngine getter should force a fallback to the + // original default engine. + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); + + // Restoring the default engines should unhide our built-in test + // engine, but not change the value of defaultEngine. + Services.search.restoreDefaultEngines(); + Assert.ok(!nonDefaultBuiltInEngine.hidden); + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); + await promiseAfterSettings(); + + // After a restart, the defaultEngine value should still be unchanged. + Services.search.wrappedJSObject.reset(); + await Services.search.init(true); + Assert.equal(Services.search.defaultEngine.name, kDefaultEngineName); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_sendSubmissionURL.js b/toolkit/components/search/tests/xpcshell/test_sendSubmissionURL.js new file mode 100644 index 0000000000..0a41a47621 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_sendSubmissionURL.js @@ -0,0 +1,90 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests covering sending submission URLs for major engines + */ + +const SUBMISSION_YES = [ + ["Google1 Test", "https://www.google.com/search", "q={searchTerms}"], + ["Google2 Test", "https://www.google.co.uk/search", "q={searchTerms}"], + ["Yahoo1 Test", "https://search.yahoo.com/search", "p={searchTerms}"], + ["Yahoo2 Test", "https://uk.search.yahoo.com/search", "p={searchTerms}"], + ["AOL1 Test", "https://search.aol.com/aol/search", "q={searchTerms}"], + ["AOL2 Test", "https://search.aol.co.uk/aol/search", "q={searchTerms}"], + ["Yandex1 Test", "https://yandex.ru/search/", "text={searchTerms}"], + ["Yandex2 Test", "https://yandex.com/search/", "text={searchTerms}"], + ["Ask1 Test", "https://www.ask.com/web", "q={searchTerms}"], + ["Ask2 Test", "https://fr.ask.com/web", "q={searchTerms}"], + ["Bing Test", "https://www.bing.com/search", "q={searchTerms}"], + [ + "Startpage Test", + "https://www.startpage.com/do/search", + "query={searchTerms}", + ], + ["DuckDuckGo Test", "https://duckduckgo.com/", "q={searchTerms}"], + ["Baidu Test", "https://www.baidu.com/s", "wd={searchTerms}"], +]; + +const SUBMISSION_NO = [ + ["Other1 Test", "https://example.com", "q={searchTerms}"], + ["Other2 Test", "https://googlebutnotgoogle.com", "q={searchTerms}"], +]; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); +}); + +async function addAndMakeDefault(name, search_url, search_url_get_params) { + await SearchTestUtils.installSearchExtension({ + name, + search_url, + search_url_get_params, + }); + + let engine = Services.search.getEngineByName(name); + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + return engine; +} + +add_task(async function test_submission_url_matching() { + Assert.ok(!Services.search.isInitialized); + let engineInfo; + let engine; + + for (let [name, searchURL, searchParams] of SUBMISSION_YES) { + engine = await addAndMakeDefault(name, searchURL, searchParams); + engineInfo = Services.search.getDefaultEngineInfo(); + Assert.equal( + engineInfo.defaultSearchEngineData.submissionURL, + (searchURL + "?" + searchParams).replace("{searchTerms}", "") + ); + await Services.search.removeEngine(engine); + } + + for (let [name, searchURL, searchParams] of SUBMISSION_NO) { + engine = await addAndMakeDefault(name, searchURL, searchParams); + engineInfo = Services.search.getDefaultEngineInfo(); + Assert.equal(engineInfo.defaultSearchEngineData.submissionURL, null); + await Services.search.removeEngine(engine); + } +}); + +add_task(async function test_submission_url_built_in() { + const engine = await Services.search.getEngineByName("engine1"); + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + const engineInfo = Services.search.getDefaultEngineInfo(); + Assert.equal( + engineInfo.defaultSearchEngineData.submissionURL, + "https://1.example.com/search?q=", + "Should have given the submission url for a built-in engine." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings.js b/toolkit/components/search/tests/xpcshell/test_settings.js new file mode 100644 index 0000000000..33995b49fd --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings.js @@ -0,0 +1,612 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test initializing from the search settings. + */ + +"use strict"; + +const legacyUseSavedOrderPrefName = + SearchUtils.BROWSER_SEARCH_PREF + "useDBForOrder"; + +var settingsTemplate; + +/** + * Test reading from search.json.mozlz4 + */ +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +async function loadSettingsFile(settingsFile, setVersion, setHashes) { + settingsTemplate = await readJSONFile(do_get_file(settingsFile)); + if (setVersion) { + settingsTemplate.version = SearchUtils.SETTINGS_VERSION; + } + + if (setHashes) { + settingsTemplate.metaData.hash = SearchUtils.getVerificationHash( + settingsTemplate.metaData.current + ); + settingsTemplate.metaData.privateHash = SearchUtils.getVerificationHash( + settingsTemplate.metaData.private + ); + } + + delete settingsTemplate.visibleDefaultEngines; + + await promiseSaveSettingsData(settingsTemplate); +} + +/** + * Start the search service and confirm the engine properties match the expected values. + * + * @param {string} settingsFile + * The path to the settings file to use. + * @param {boolean} setVersion + * True if to set the version in the copied settings file. + * @param {boolean} expectedUseDBValue + * The value expected for the `useSavedOrder` metadata attribute. + */ +async function checkLoadSettingProperties( + settingsFile, + setVersion, + expectedUseDBValue +) { + info("init search service"); + let ss = Services.search.wrappedJSObject; + + await loadSettingsFile(settingsFile, setVersion); + + const settingsFileWritten = promiseAfterSettings(); + + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let engines = await ss.getEngines(); + + Assert.equal( + engines[0].name, + "engine1", + "Should have loaded the correct first engine" + ); + Assert.equal(engines[0].alias, "testAlias", "Should have set the alias"); + Assert.equal(engines[0].hidden, false, "Should have not hidden the engine"); + Assert.equal(engines[0].id, "engine1@search.mozilla.orgdefault"); + + Assert.equal( + engines[1].name, + "engine2", + "Should have loaded the correct second engine" + ); + Assert.equal(engines[1].alias, "", "Should have not set the alias"); + Assert.equal(engines[1].hidden, true, "Should have hidden the engine"); + Assert.equal(engines[1].id, "engine2@search.mozilla.orgdefault"); + + // The extra engine is the second in the list. + isSubObjectOf(EXPECTED_ENGINE.engine, engines[2]); + Assert.ok(engines[2].id, "test-addon-id@mozilla.orgdefault"); + + let engineFromSS = ss.getEngineByName(EXPECTED_ENGINE.engine.name); + Assert.ok(!!engineFromSS); + isSubObjectOf(EXPECTED_ENGINE.engine, engineFromSS); + + Assert.equal( + engineFromSS.getSubmission("foo").uri.spec, + "http://www.google.com/search?q=foo", + "Should have the correct URL with no mozparams" + ); + + Assert.equal( + ss._settings.getMetaDataAttribute("useSavedOrder"), + expectedUseDBValue, + "Should have set the useSavedOrder metadata correctly." + ); + + let migratedSettingsFile = await promiseSettingsData(); + + Assert.equal( + migratedSettingsFile.engines[0].id, + "engine1@search.mozilla.orgdefault" + ); + + removeSettingsFile(); +} + +add_task(async function test_legacy_setting_engine_properties() { + Services.prefs.setBoolPref(legacyUseSavedOrderPrefName, true); + + let legacySettings = await readJSONFile( + do_get_file("data/search-legacy.json") + ); + + // Assert the engine ids have not been migrated yet + for (let engine of legacySettings.engines) { + Assert.ok(!("id" in engine)); + } + Assert.ok(!("defaultEngineId" in legacySettings.metaData)); + Assert.ok(!("privateDefaultEngineId" in legacySettings.metaData)); + + await checkLoadSettingProperties("data/search-legacy.json", false, true); + + Assert.ok( + !Services.prefs.prefHasUserValue(legacyUseSavedOrderPrefName), + "Should have cleared the legacy pref." + ); +}); + +add_task( + async function test_legacy_setting_migration_with_undefined_metaData_current_and_private() { + let ss = Services.search.wrappedJSObject; + + await loadSettingsFile("data/search-legacy.json", false); + const settingsFileWritten = promiseAfterSettings(); + + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let migratedSettingsFile = await promiseSettingsData(); + + Assert.equal( + migratedSettingsFile.metaData.defaultEngineId, + "", + "When there is no metaData.current attribute in settings file, the migration should set the defaultEngineId to an empty string." + ); + Assert.equal( + migratedSettingsFile.metaData.privateDefaultEngineId, + "", + "When there is no metaData.private attribute in settings file, the migration should set the privateDefaultEngineId to an empty string." + ); + + removeSettingsFile(); + } +); + +add_task( + async function test_legacy_setting_migration_with_correct_metaData_current_and_private_hashes() { + let ss = Services.search.wrappedJSObject; + + await loadSettingsFile( + "data/search-legacy-correct-default-engine-hashes.json", + false, + true + ); + const settingsFileWritten = promiseAfterSettings(); + + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let migratedSettingsFile = await promiseSettingsData(); + + Assert.equal( + migratedSettingsFile.metaData.defaultEngineId, + "engine2@search.mozilla.orgdefault", + "When the metaData.current and associated hash are correct, the migration should set the defaultEngineId to the engine id." + ); + Assert.equal( + migratedSettingsFile.metaData.privateDefaultEngineId, + "engine2@search.mozilla.orgdefault", + "When the metaData.private and associated hash are correct, the migration should set the privateDefaultEngineId to the private engine id." + ); + + removeSettingsFile(); + } +); + +add_task( + async function test_legacy_setting_migration_with_incorrect_metaData_current_and_private_hashes_app_provided() { + let ss = Services.search.wrappedJSObject; + + // Here we are testing correct migration for the case that a user has set + // their default engine to an application provided engine (but not the app + // default). + // + // In this case we should ignore invalid hashes for the default engines, + // and allow the select default to remain. This covers the case where + // a user has copied a profile from a different directory. + // See SearchService._getEngineDefault for more details. + + await loadSettingsFile( + "data/search-legacy-wrong-default-engine-hashes.json", + false, + false + ); + const settingsFileWritten = promiseAfterSettings(); + + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let migratedSettingsFile = await promiseSettingsData(); + + Assert.equal( + migratedSettingsFile.metaData.defaultEngineId, + "engine2@search.mozilla.orgdefault", + "Should ignore invalid metaData.hash when the default engine is application provided." + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have the correct engine set as default" + ); + + Assert.equal( + migratedSettingsFile.metaData.privateDefaultEngineId, + "engine2@search.mozilla.orgdefault", + "Should ignore invalid metaData.privateHash when the default private engine is application provided." + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine2", + "Should have the correct engine set as default private" + ); + + removeSettingsFile(); + } +); + +add_task( + async function test_legacy_setting_migration_with_incorrect_metaData_current_and_private_hashes_third_party() { + let ss = Services.search.wrappedJSObject; + + // This test is checking that if the user has set a third-party engine as + // default, and the verification hash is invalid, then we do not copy + // the default engine setting. + + await loadSettingsFile( + "data/search-legacy-wrong-third-party-engine-hashes.json", + false, + false + ); + const settingsFileWritten = promiseAfterSettings(); + + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let migratedSettingsFile = await promiseSettingsData(); + + Assert.equal( + migratedSettingsFile.metaData.defaultEngineId, + "", + "Should reset the default engine when metaData.hash is invalid and the engine is not application provided." + ); + Assert.equal( + Services.search.defaultEngine.name, + "engine1", + "Should have reset the default engine" + ); + + Assert.equal( + migratedSettingsFile.metaData.privateDefaultEngineId, + "", + "Should reset the default engine when metaData.privateHash is invalid and the engine is not application provided." + ); + Assert.equal( + Services.search.defaultPrivateEngine.name, + "engine1", + "Should have reset the default private engine" + ); + + removeSettingsFile(); + } +); + +add_task(async function test_current_setting_engine_properties() { + await checkLoadSettingProperties("data/search.json", true, false); +}); + +add_task(async function test_settings_metadata_properties() { + let ss = Services.search.wrappedJSObject; + + await loadSettingsFile("data/search.json"); + + const settingsFileWritten = promiseAfterSettings(); + await ss.reset(); + await Services.search.init(); + + await settingsFileWritten; + + let metaDataProperties = [ + "locale", + "region", + "channel", + "experiment", + "distroID", + ]; + + for (let name of metaDataProperties) { + Assert.notEqual( + ss._settings.getMetaDataAttribute(`${name}`), + undefined, + `Search settings should have ${name} property defined.` + ); + } + + removeSettingsFile(); +}); + +add_task(async function test_settings_write_when_settings_changed() { + let ss = Services.search.wrappedJSObject; + await loadSettingsFile("data/search.json"); + + const settingsFileWritten = promiseAfterSettings(); + await ss.reset(); + await Services.search.init(); + await settingsFileWritten; + + Assert.ok( + ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings and cached settings should be the same after search service initializaiton." + ); + + const settingsFileWritten2 = promiseAfterSettings(); + ss._settings.setMetaDataAttribute("value", "test"); + + Assert.ok( + !ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings should differ from cached settings after a new attribute is set." + ); + + await settingsFileWritten2; + info("Settings write complete"); + + Assert.ok( + ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings and cached settings should be the same after new attribte on settings is written." + ); + + removeSettingsFile(); +}); + +add_task(async function test_set_and_get_engine_metadata_attribute() { + let ss = Services.search.wrappedJSObject; + await loadSettingsFile("data/search.json"); + + const settingsFileWritten = promiseAfterSettings(); + await ss.reset(); + await Services.search.init(); + await settingsFileWritten; + + let engines = await ss.getEngines(); + const settingsFileWritten2 = promiseAfterSettings(); + ss._settings.setEngineMetaDataAttribute(engines[0].name, "value", "test"); + await settingsFileWritten2; + + Assert.equal( + "test", + ss._settings.getEngineMetaDataAttribute(engines[0].name, "value"), + `${engines[0].name}'s metadata property "value" should be set as "test" after calling getEngineMetaDataAttribute.` + ); + + let userSettings = await ss._settings.get(); + let engine = userSettings.engines.find(e => e._name == engines[0].name); + + Assert.equal( + "test", + engine._metaData.value, + `${engines[0].name}'s metadata property "value" should be set as "test" from settings file.` + ); + + removeSettingsFile(); +}); + +add_task( + async function test_settings_write_prevented_when_settings_unchanged() { + let ss = Services.search.wrappedJSObject; + await loadSettingsFile("data/search.json"); + + const settingsFileWritten = promiseAfterSettings(); + await ss.reset(); + await Services.search.init(); + await settingsFileWritten; + + Assert.ok( + ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings and cached settings should be the same after search service initializaiton." + ); + + // Update settings. + const settingsFileWritten2 = promiseAfterSettings(); + ss._settings.setMetaDataAttribute("value", "test"); + + Assert.ok( + !ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings should differ from cached settings after a new attribute is set." + ); + await settingsFileWritten2; + + // Set the same attribute as before to ensure there was no change. + // Settings write should be prevented. + let promiseWritePrevented = SearchTestUtils.promiseSearchNotification( + "write-prevented-when-settings-unchanged" + ); + ss._settings.setMetaDataAttribute("value", "test"); + + Assert.ok( + ss._settings.isCurrentAndCachedSettingsEqual(), + "Settings and cached settings should be the same." + ); + await promiseWritePrevented; + + removeSettingsFile(); + } +); + +/** + * Test that the JSON settings written in the profile is correct. + */ +add_task(async function test_settings_write() { + let ss = Services.search.wrappedJSObject; + info("test settings writing"); + + await loadSettingsFile("data/search.json"); + + const settingsFileWritten = promiseAfterSettings(); + await ss.reset(); + await Services.search.init(); + await settingsFileWritten; + + let settingsData = await promiseSettingsData(); + + // Remove buildID and locale, as they are no longer used. + delete settingsTemplate.buildID; + delete settingsTemplate.locale; + + for (let engine of settingsTemplate.engines) { + // Remove _shortName from the settings template, as it is no longer supported, + // but older settings used to have it, so we keep it in the template as an + // example. + if ("_shortName" in engine) { + delete engine._shortName; + } + if ("_urls" in engine) { + // Only app-provided engines support purpose, others do not, + // so filter them out of the expected template. + for (let urls of engine._urls) { + urls.params = urls.params.filter(p => !p.purpose); + // resultDomain is also no longer supported. + if ("resultDomain" in urls) { + delete urls.resultDomain; + } + } + } + // Remove queryCharset, if it is the same as the default, as we don't save + // it in that case. + if (engine?.queryCharset == SearchUtils.DEFAULT_QUERY_CHARSET) { + delete engine.queryCharset; + } + } + + // Note: the file is copied with an old version number, which should have + // been updated on write. + settingsTemplate.version = SearchUtils.SETTINGS_VERSION; + + isSubObjectOf(settingsTemplate, settingsData, (prop, value) => { + if (prop != "_iconURL" && prop != "{}") { + return false; + } + // Skip items that are to do with icons for extensions, as we can't + // control the uuid. + return value.startsWith("moz-extension://"); + }); +}); + +async function settings_write_check(disableFn) { + let ss = Services.search.wrappedJSObject; + + sinon.stub(ss._settings, "_write").returns(Promise.resolve()); + + // Simulate the search service being initialized. + disableFn(true); + + ss._settings.setMetaDataAttribute("value", "test"); + + Assert.ok( + ss._settings._write.notCalled, + "Should not have attempted to _write" + ); + + // Wait for two periods of the normal delay to ensure we still do not write. + await new Promise(r => + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + setTimeout(r, SearchSettings.SETTNGS_INVALIDATION_DELAY * 2) + ); + + Assert.ok( + ss._settings._write.notCalled, + "Should not have attempted to _write" + ); + + disableFn(false); + + await TestUtils.waitForCondition( + () => ss._settings._write.calledOnce, + "Should attempt to write the settings." + ); + + sinon.restore(); +} + +add_task(async function test_settings_write_prevented_during_init() { + await settings_write_check(disable => { + let status = disable ? "success" : "failed"; + Services.search.wrappedJSObject.forceInitializationStatusForTests(status); + }); +}); + +add_task(async function test_settings_write_prevented_during_reload() { + await settings_write_check( + disable => (Services.search.wrappedJSObject._reloadingEngines = disable) + ); +}); + +var EXPECTED_ENGINE = { + engine: { + name: "Test search engine", + alias: "", + description: "A test search engine (based on Google search)", + searchForm: "http://www.google.com/", + wrappedJSObject: { + _extensionID: "test-addon-id@mozilla.org", + _iconURL: + "data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAA" + + "AIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9F" + + "sfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2F" + + "Ptft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2Fgg" + + "M%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F" + + "%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJ" + + "vvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%" + + "2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%" + + "2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%" + + "2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%" + + "2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYS" + + "BHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWc" + + "TxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4j" + + "wA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsgg" + + "A7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7" + + "kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%" + + "2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFE" + + "MwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%" + + "2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCT" + + "IYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesA" + + "AN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOc" + + "AAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v" + + "8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA", + _urls: [ + { + type: "application/x-suggestions+json", + method: "GET", + template: + "http://suggestqueries.google.com/complete/search?output=firefox&client=firefox" + + "&hl={moz:locale}&q={searchTerms}", + params: "", + }, + { + type: "text/html", + method: "GET", + template: "http://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + purpose: undefined, + }, + ], + }, + ], + }, + }, +}; diff --git a/toolkit/components/search/tests/xpcshell/test_settings_broken.js b/toolkit/components/search/tests/xpcshell/test_settings_broken.js new file mode 100644 index 0000000000..dcc197d802 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_broken.js @@ -0,0 +1,131 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test initializing from broken search settings. This is one where the engines + * array for some reason has lost all the default engines, but retained either + * one or two, or a user-supplied engine. We don't know why this happens, but + * we have seen it (bug 1578807). + */ + +"use strict"; + +const { getAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); + +const enginesSettings = { + version: SearchUtils.SETTINGS_VERSION, + buildID: "TBD", + appVersion: "TBD", + locale: "en-US", + metaData: { + searchDefault: "Test search engine", + searchDefaultHash: "TBD", + // Intentionally in the past, but shouldn't actually matter for this test. + searchDefaultExpir: 1567694909002, + current: "", + hash: "TBD", + visibleDefaultEngines: + "engine,engine-pref,engine-rel-searchform-purpose,engine-chromeicon,engine-resourceicon,engine-reordered", + visibleDefaultEnginesHash: "TBD", + }, + engines: [ + // This is a user-installed engine - the only one that was listed due to the + // original issue. + { + _name: "A second test engine", + _shortName: "engine2", + _loadPath: "[profile]/searchplugins/engine2.xml", + description: "A second test search engine (based on DuckDuckGo)", + _iconURL: + "data:image/x-icon;base64,AAABAAEAEBAQAAEABAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAEAgQAhIOEAMjHyABIR0gA6ejpAGlqaQCpqKkAKCgoAPz9/AAZGBkAmJiYANjZ2ABXWFcAent6ALm6uQA8OjwAiIiIiIiIiIiIiI4oiL6IiIiIgzuIV4iIiIhndo53KIiIiB/WvXoYiIiIfEZfWBSIiIEGi/foqoiIgzuL84i9iIjpGIoMiEHoiMkos3FojmiLlUipYliEWIF+iDe0GoRa7D6GPbjcu1yIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + _iconMapObj: { + '{"width":16,"height":16}': + "data:image/x-icon;base64,AAABAAEAEBAQAAEABAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAEAgQAhIOEAMjHyABIR0gA6ejpAGlqaQCpqKkAKCgoAPz9/AAZGBkAmJiYANjZ2ABXWFcAent6ALm6uQA8OjwAiIiIiIiIiIiIiI4oiL6IiIiIgzuIV4iIiIhndo53KIiIiB/WvXoYiIiIfEZfWBSIiIEGi/foqoiIgzuL84i9iIjpGIoMiEHoiMkos3FojmiLlUipYliEWIF+iDe0GoRa7D6GPbjcu1yIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }, + _isBuiltin: false, + _metaData: { + order: 1, + }, + _urls: [ + { + template: "https://duckduckgo.com/?q={searchTerms}", + rels: [], + resultDomain: "duckduckgo.com", + params: [], + }, + ], + queryCharset: "UTF-8", + filePath: "TBD", + }, + ], +}; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + // Allow telemetry probes which may otherwise be disabled for some applications (e.g. Thunderbird) + Services.prefs.setBoolPref( + "toolkit.telemetry.testing.overrideProductsCheck", + true + ); + + await SearchTestUtils.useTestEngines(); + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + Services.locale.availableLocales = ["en-US"]; + Services.locale.requestedLocales = ["en-US"]; + + // We dynamically generate the hashes because these depend on the profile. + enginesSettings.metaData.searchDefaultHash = SearchUtils.getVerificationHash( + enginesSettings.metaData.searchDefault + ); + enginesSettings.metaData.hash = SearchUtils.getVerificationHash( + enginesSettings.metaData.current + ); + enginesSettings.metaData.visibleDefaultEnginesHash = + SearchUtils.getVerificationHash( + enginesSettings.metaData.visibleDefaultEngines + ); + const appInfo = getAppInfo(); + enginesSettings.buildID = appInfo.platformBuildID; + enginesSettings.appVersion = appInfo.version; + + await IOUtils.writeJSON( + PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME), + enginesSettings, + { compress: true } + ); +}); + +add_task(async function test_cached_engine_properties() { + info("init search service"); + + const initResult = await Services.search.init(); + + info("init'd search service"); + Assert.ok( + Components.isSuccessCode(initResult), + "Should have successfully created the search service" + ); + + const engines = await Services.search.getEngines(); + + const expectedEngines = [ + // Default engines + "Test search engine", + // Rest of engines in order + "engine-resourceicon", + "engine-chromeicon", + "engine-pref", + "engine-rel-searchform-purpose", + "Test search engine (Reordered)", + "A second test engine", + ]; + + Assert.deepEqual( + engines.map(e => e.name), + expectedEngines, + "Should have the expected default engines" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_duplicate.js b/toolkit/components/search/tests/xpcshell/test_settings_duplicate.js new file mode 100644 index 0000000000..20093be6a0 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_duplicate.js @@ -0,0 +1,146 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test initializing with an engine that's a duplicate of an app-provided + * engine. + */ + +"use strict"; + +const { getAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); + +const DUPLICATE_ENGINE_ID = "f3094ab7-3302-4d5b-9f79-ea92c9a49f87"; + +const enginesSettings = { + version: SearchUtils.SETTINGS_VERSION, + buildID: "TBD", + appVersion: "TBD", + locale: "en-US", + metaData: { + searchDefault: "Test search engine", + searchDefaultHash: "TBD", + // Intentionally in the past, but shouldn't actually matter for this test. + searchDefaultExpir: 1567694909002, + current: "", + hash: "TBD", + visibleDefaultEngines: + "engine,engine-pref,engine-rel-searchform-purpose,engine-chromeicon,engine-resourceicon,engine-reordered", + visibleDefaultEnginesHash: "TBD", + }, + engines: [ + { + id: "engine1@search.mozilla.orgdefault", + _metaData: { alias: null }, + _isAppProvided: true, + _name: "engine1", + }, + { + id: "engine2@search.mozilla.orgdefault", + _metaData: { alias: null }, + _isAppProvided: true, + _name: "engine2", + }, + // This is a user-installed engine - the only one that was listed due to the + // original issue. + { + id: DUPLICATE_ENGINE_ID, + _name: "engine1", + _shortName: "engine1", + _loadPath: "[https]oldduplicateversion", + description: "An old near duplicate version of engine1", + _iconURL: + "data:image/x-icon;base64,AAABAAEAEBAQAAEABAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAEAgQAhIOEAMjHyABIR0gA6ejpAGlqaQCpqKkAKCgoAPz9/AAZGBkAmJiYANjZ2ABXWFcAent6ALm6uQA8OjwAiIiIiIiIiIiIiI4oiL6IiIiIgzuIV4iIiIhndo53KIiIiB/WvXoYiIiIfEZfWBSIiIEGi/foqoiIgzuL84i9iIjpGIoMiEHoiMkos3FojmiLlUipYliEWIF+iDe0GoRa7D6GPbjcu1yIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + _iconMapObj: { + '{"width":16,"height":16}': + "data:image/x-icon;base64,AAABAAEAEBAQAAEABAAoAQAAFgAAACgAAAAQAAAAIAAAAAEABAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAEAgQAhIOEAMjHyABIR0gA6ejpAGlqaQCpqKkAKCgoAPz9/AAZGBkAmJiYANjZ2ABXWFcAent6ALm6uQA8OjwAiIiIiIiIiIiIiI4oiL6IiIiIgzuIV4iIiIhndo53KIiIiB/WvXoYiIiIfEZfWBSIiIEGi/foqoiIgzuL84i9iIjpGIoMiEHoiMkos3FojmiLlUipYliEWIF+iDe0GoRa7D6GPbjcu1yIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }, + _metaData: { + order: 1, + }, + _urls: [ + { + template: "https://example.com/?myquery={searchTerms}", + rels: [], + resultDomain: "example.com", + params: [], + }, + ], + queryCharset: "UTF-8", + filePath: "TBD", + }, + ], +}; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + // Allow telemetry probes which may otherwise be disabled for some applications (e.g. Thunderbird) + Services.prefs.setBoolPref( + "toolkit.telemetry.testing.overrideProductsCheck", + true + ); + + await SearchTestUtils.useTestEngines("data1"); + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + Services.locale.availableLocales = ["en-US"]; + Services.locale.requestedLocales = ["en-US"]; + + // We dynamically generate the hashes because these depend on the profile. + enginesSettings.metaData.searchDefaultHash = SearchUtils.getVerificationHash( + enginesSettings.metaData.searchDefault + ); + enginesSettings.metaData.hash = SearchUtils.getVerificationHash( + enginesSettings.metaData.current + ); + enginesSettings.metaData.visibleDefaultEnginesHash = + SearchUtils.getVerificationHash( + enginesSettings.metaData.visibleDefaultEngines + ); + let appInfo = getAppInfo(); + enginesSettings.buildID = appInfo.platformBuildID; + enginesSettings.appVersion = appInfo.version; + + await IOUtils.write( + PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME), + new TextEncoder().encode(JSON.stringify(enginesSettings)), + { compress: true } + ); + + consoleAllowList.push("Failed to load"); +}); + +add_task(async function test_cached_duplicate() { + info("init search service"); + + let initResult = await Services.search.init(); + + info("init'd search service"); + Assert.ok( + Components.isSuccessCode(initResult), + "Should have successfully created the search service" + ); + + let engine = Services.search.getEngineByName("engine1"); + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://1.example.com/search?q=foo", + "Should have not changed the app provided engine." + ); + + Assert.ok( + !(await Services.search.getEngineById(DUPLICATE_ENGINE_ID)), + "Should not have added the duplicate engine" + ); + + let engines = await Services.search.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["engine1", "engine2"], + "Should have the expected default engines" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_good.js b/toolkit/components/search/tests/xpcshell/test_settings_good.js new file mode 100644 index 0000000000..b954b94038 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_good.js @@ -0,0 +1,103 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test initializing from good search settings. + */ + +"use strict"; + +const { getAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); + +const enginesSettings = { + version: SearchUtils.SETTINGS_VERSION, + buildID: "TBD", + appVersion: "TBD", + locale: "en-US", + metaData: { + searchDefault: "Test search engine", + searchDefaultHash: "TBD", + // Intentionally in the past, but shouldn't actually matter for this test. + searchDefaultExpir: 1567694909002, + // We use the second engine here so that the user's default is set + // to something different, and hence so that we exercise the appropriate + // code paths. + defaultEngineId: "engine2@search.mozilla.orgdefault", + defaultEngineIdHash: "TBD", + visibleDefaultEngines: "engine1,engine2", + visibleDefaultEnginesHash: "TBD", + }, + engines: [ + { + _metaData: { alias: null }, + _isAppProvided: true, + _name: "engine1", + }, + { + _metaData: { alias: null }, + _isAppProvided: true, + _name: "engine2", + }, + ], +}; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + // Allow telemetry probes which may otherwise be disabled for some applications (e.g. Thunderbird) + Services.prefs.setBoolPref( + "toolkit.telemetry.testing.overrideProductsCheck", + true + ); + + await SearchTestUtils.useTestEngines("data1"); + Services.prefs.setCharPref(SearchUtils.BROWSER_SEARCH_PREF + "region", "US"); + Services.locale.availableLocales = ["en-US"]; + Services.locale.requestedLocales = ["en-US"]; + + // We dynamically generate the hashes because these depend on the profile. + enginesSettings.metaData.searchDefaultHash = SearchUtils.getVerificationHash( + enginesSettings.metaData.searchDefault + ); + enginesSettings.metaData.defaultEngineIdHash = + SearchUtils.getVerificationHash(enginesSettings.metaData.defaultEngineId); + enginesSettings.metaData.visibleDefaultEnginesHash = + SearchUtils.getVerificationHash( + enginesSettings.metaData.visibleDefaultEngines + ); + const appInfo = getAppInfo(); + enginesSettings.buildID = appInfo.platformBuildID; + enginesSettings.appVersion = appInfo.version; + + await IOUtils.writeJSON( + PathUtils.join(PathUtils.profileDir, SETTINGS_FILENAME), + enginesSettings, + { compress: true } + ); +}); + +add_task(async function test_cached_engine_properties() { + info("init search service"); + + const initResult = await Services.search.init(); + + info("init'd search service"); + Assert.ok( + Components.isSuccessCode(initResult), + "Should have successfully created the search service" + ); + + const engines = await Services.search.getEngines(); + Assert.equal( + Services.search.defaultEngine.name, + "engine2", + "Should have the expected default engine" + ); + Assert.deepEqual( + engines.map(e => e.name), + ["engine1", "engine2"], + "Should have the expected application provided engines" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_ignorelist.js b/toolkit/components/search/tests/xpcshell/test_settings_ignorelist.js new file mode 100644 index 0000000000..edc8eb12b8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_ignorelist.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test initializing from the search settings. + */ + +"use strict"; + +var { getAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); + +var settingsTemplate; + +/** + * Test reading from search.json.mozlz4 + */ +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + await setupRemoteSettings(); + + settingsTemplate = await readJSONFile( + do_get_file("data/search_ignorelist.json") + ); + settingsTemplate.buildID = getAppInfo().platformBuildID; + + await promiseSaveSettingsData(settingsTemplate); +}); + +/** + * Start the search service and confirm the settings were reset + */ +add_task(async function test_settings_rest() { + info("init search service"); + + let updatePromise = SearchTestUtils.promiseSearchNotification( + "settings-update-complete" + ); + + let result = await Services.search.init(); + + Assert.ok( + Components.isSuccessCode(result), + "Search service should be successfully initialized" + ); + await updatePromise; + + const engines = await Services.search.getEngines(); + + // Engine list will have been reset to the default, + // Not the one engine in the settings. + // It should have more than one engine. + Assert.greater( + engines.length, + 1, + "Should have more than one engine in the list" + ); + + removeSettingsFile(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_migration_hideOneOffs.js b/toolkit/components/search/tests/xpcshell/test_settings_migration_hideOneOffs.js new file mode 100644 index 0000000000..a35ef367d7 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_migration_hideOneOffs.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Loads the settings file and ensures it has not already been migrated. + * + * @param {string} settingsFile The settings file to load + */ +async function loadSettingsFile(settingsFile) { + let settingsTemplate = await readJSONFile(do_get_file(settingsFile)); + + Assert.less( + settingsTemplate.version, + 7, + "Should be a version older than when hideOneOffs was moved into settings" + ); + for (let engine of settingsTemplate.engines) { + Assert.ok(!("id" in engine)); + } + + await promiseSaveSettingsData(settingsTemplate); +} + +/** + * Test reading from search.json.mozlz4 + */ +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_migration_from_pre_ids() { + await loadSettingsFile("data/search-legacy.json"); + + Services.prefs.setStringPref("browser.search.hiddenOneOffs", "engine1"); + + const settingsFileWritten = promiseAfterSettings(); + + await Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + await settingsFileWritten; + + const engine1 = await Services.search.getEngineByName("engine1"); + const engine2 = await Services.search.getEngineByName("engine2"); + + Assert.ok( + engine1.hideOneOffButton, + "Should have hideOneOffButton set to true" + ); + Assert.ok( + !engine2.hideOneOffButton, + "Should have hideOneOffButton set to false" + ); + + Assert.ok( + !Services.prefs.prefHasUserValue("browser.search.hiddenOneOffs"), + "HiddenOneOffs pref is cleared" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_migration_ids.js b/toolkit/components/search/tests/xpcshell/test_settings_migration_ids.js new file mode 100644 index 0000000000..8123ccc4ab --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_migration_ids.js @@ -0,0 +1,118 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test migration of user, enterprise policy and OpenSearch engines + * from when engines were referenced by name rather than id. + * + * Add-ons and default engine ids are tested in test_settings.js. + */ + +"use strict"; + +const { EnterprisePolicyTesting } = ChromeUtils.importESModule( + "resource://testing-common/EnterprisePolicyTesting.sys.mjs" +); + +const enterprisePolicy = { + policies: { + SearchEngines: { + Add: [ + { + Name: "Policy", + Encoding: "windows-1252", + URLTemplate: "http://example.com/?q={searchTerms}", + }, + ], + }, + }, +}; + +/** + * Loads the settings file and ensures it has not already been migrated. + * + * @param {string} settingsFile The settings file to load + */ +async function loadSettingsFile(settingsFile) { + let settingsTemplate = await readJSONFile(do_get_file(settingsFile)); + + Assert.less( + settingsTemplate.version, + 7, + "Should be a version older than when indexing engines by id was introduced" + ); + for (let engine of settingsTemplate.engines) { + Assert.ok(!("id" in engine)); + } + + await promiseSaveSettingsData(settingsTemplate); +} + +/** + * Test reading from search.json.mozlz4 + */ +add_setup(async function () { + // This initializes the policy engine for xpcshell tests + let policies = Cc["@mozilla.org/enterprisepolicies;1"].getService( + Ci.nsIObserver + ); + policies.observe(null, "policies-startup", null); + + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); + await EnterprisePolicyTesting.setupPolicyEngineWithJson(enterprisePolicy); + // Setting the enterprise policy starts the search service initialising, + // so we wait for that to complete before starting the test. + await Services.search.init(); +}); + +/** + * Tests that an installed engine matches the expected data. + * + * @param {object} expectedData The expected data for the engine + */ +async function assertInstalledEngineMatches(expectedData) { + let engine = await Services.search.getEngineByName(expectedData.name); + + Assert.ok(engine, `Should have found the ${expectedData.type} engine`); + if (expectedData.idLength) { + Assert.equal( + engine.id.length, + expectedData.idLength, + "Should have been given an id" + ); + } else { + Assert.equal(engine.id, expectedData.id, "Should have the expected id"); + } + Assert.equal(engine.alias, expectedData.alias, "Should have kept the alias"); +} + +add_task(async function test_migration_from_pre_ids() { + await loadSettingsFile("data/search-legacy-no-ids.json"); + + const settingsFileWritten = promiseAfterSettings(); + + await Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + await settingsFileWritten; + + await assertInstalledEngineMatches({ + type: "OpenSearch", + name: "Bugzilla@Mozilla", + idLength: 36, + alias: "bugzillaAlias", + }); + await assertInstalledEngineMatches({ + type: "Enterprise Policy", + name: "Policy", + id: "policy-Policy", + alias: "PolicyAlias", + }); + await assertInstalledEngineMatches({ + type: "User", + name: "User", + idLength: 36, + alias: "UserAlias", + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_migration_loadPath.js b/toolkit/components/search/tests/xpcshell/test_settings_migration_loadPath.js new file mode 100644 index 0000000000..d4193f7a17 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_migration_loadPath.js @@ -0,0 +1,126 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test migration load path for user, enterprise policy and add-on + * engines. + */ + +"use strict"; + +const { EnterprisePolicyTesting } = ChromeUtils.importESModule( + "resource://testing-common/EnterprisePolicyTesting.sys.mjs" +); + +const enterprisePolicy = { + policies: { + SearchEngines: { + Add: [ + { + Name: "Policy", + Encoding: "windows-1252", + URLTemplate: "http://example.com/?q={searchTerms}", + }, + ], + }, + }, +}; + +add_setup(async function () { + // This initializes the policy engine for xpcshell tests + let policies = Cc["@mozilla.org/enterprisepolicies;1"].getService( + Ci.nsIObserver + ); + policies.observe(null, "policies-startup", null); + + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); + await EnterprisePolicyTesting.setupPolicyEngineWithJson(enterprisePolicy); + // Setting the enterprise policy starts the search service initialising, + // so we wait for that to complete before starting the test, we can + // then also add an extra add-on engine. + await Services.search.init(); + let settingsFileWritten = promiseAfterSettings(); + await SearchTestUtils.installSearchExtension(); + await settingsFileWritten; +}); + +/** + * Loads the settings file and ensures it has not already been migrated. + */ +add_task(async function test_load_and_check_settings() { + let settingsTemplate = await readJSONFile( + do_get_file("data/search-legacy-old-loadPaths.json") + ); + + Assert.less( + settingsTemplate.version, + 8, + "Should be a version older than when indexing engines by id was introduced" + ); + let engine = settingsTemplate.engines.find(e => e.id == "policy-Policy"); + Assert.equal( + engine._loadPath, + "[other]addEngineWithDetails:set-via-policy", + "Should have a old style load path for the policy engine" + ); + engine = settingsTemplate.engines.find( + e => e.id == "bbc163e7-7b1a-47aa-a32c-c59062de2754" + ); + Assert.equal( + engine._loadPath, + "[other]addEngineWithDetails:set-via-user", + "Should have a old style load path for the user engine" + ); + engine = settingsTemplate.engines.find( + e => e.id == "example@tests.mozilla.orgdefault" + ); + Assert.equal( + engine._loadPath, + "[other]addEngineWithDetails:example@tests.mozilla.org", + "Should have a old style load path for the add-on engine" + ); + + await promiseSaveSettingsData(settingsTemplate); +}); + +/** + * Tests that an installed engine matches the expected data. + * + * @param {object} expectedData The expected data for the engine + */ +async function assertInstalledEngineMatches(expectedData) { + let engine = await Services.search.getEngineByName(expectedData.name); + + Assert.ok(engine, `Should have found the ${expectedData.type} engine`); + Assert.equal( + engine.wrappedJSObject._loadPath, + expectedData.loadPath, + "Should have migrated the loadPath" + ); +} + +add_task(async function test_migration_from_pre_ids() { + const settingsFileWritten = promiseAfterSettings(); + + await Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + await settingsFileWritten; + + await assertInstalledEngineMatches({ + type: "Policy", + name: "Policy", + loadPath: "[policy]", + }); + await assertInstalledEngineMatches({ + type: "User", + name: "User", + loadPath: "[user]", + }); + await assertInstalledEngineMatches({ + type: "Add-on", + name: "Example", + loadPath: "[addon]example@tests.mozilla.org", + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_none.js b/toolkit/components/search/tests/xpcshell/test_settings_none.js new file mode 100644 index 0000000000..dbd42f2ee5 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_none.js @@ -0,0 +1,50 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * test_nosettings: Start search engine + * - without search.json.mozlz4 + * + * Ensure that : + * - nothing explodes; + * - search.json.mozlz4 is created. + */ + +add_setup(async function () { + useHttpServer(); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_nosettings() { + let search = Services.search; + + let afterSettingsPromise = promiseAfterSettings(); + + await search.init(); + + // Check that the settings is created at startup + await afterSettingsPromise; + + // Check that search.json.mozlz4 has been created. + let settingsFile = do_get_profile().clone(); + settingsFile.append(SETTINGS_FILENAME); + Assert.ok(settingsFile.exists()); + + await SearchTestUtils.promiseNewSearchEngine({ + url: `${gDataUrl}engine.xml`, + }); + + info("Engine has been added, let's wait for the settings to be built"); + await promiseAfterSettings(); + + info("Searching test engine in settings"); + let settings = await promiseSettingsData(); + let found = false; + for (let engine of settings.engines) { + if (engine._name == "Test search engine") { + found = true; + break; + } + } + Assert.ok(found); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_obsolete.js b/toolkit/components/search/tests/xpcshell/test_settings_obsolete.js new file mode 100644 index 0000000000..85022ac931 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_obsolete.js @@ -0,0 +1,83 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test removing obsolete engine types on upgrade of settings. + */ + +"use strict"; + +async function loadSettingsFile(settingsFile, name) { + let settings = await readJSONFile(do_get_file(settingsFile)); + + settings.metaData.current = name; + settings.metaData.hash = SearchUtils.getVerificationHash(name); + + await promiseSaveSettingsData(settings); +} + +/** + * Start the search service and confirm the engine properties match the expected values. + * + * @param {string} settingsFile + * The path to the settings file to use. + * @param {string} engineName + * The engine name that should be default and is being removed. + */ +async function checkLoadSettingProperties(settingsFile, engineName) { + await loadSettingsFile(settingsFile, engineName); + + const settingsFileWritten = promiseAfterSettings(); + let ss = new SearchService(); + let result = await ss.init(); + + Assert.ok( + Components.isSuccessCode(result), + "Should have successfully initialized the search service" + ); + + await settingsFileWritten; + + let engines = await ss.getEngines(); + + Assert.deepEqual( + engines.map(e => e.name), + ["engine1", "engine2"], + "Should have only loaded the app-provided engines" + ); + + Assert.equal( + (await Services.search.getDefault()).name, + "engine1", + "Should have used the configured default engine" + ); + + removeSettingsFile(); + ss._removeObservers(); +} + +/** + * Test reading from search.json.mozlz4 + */ +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await AddonTestUtils.promiseStartupManager(); +}); + +add_task(async function test_obsolete_distribution_engine() { + await checkLoadSettingProperties( + "data/search-obsolete-distribution.json", + "Distribution" + ); +}); + +add_task(async function test_obsolete_langpack_engine() { + await checkLoadSettingProperties( + "data/search-obsolete-langpack.json", + "Langpack" + ); +}); + +add_task(async function test_obsolete_app_engine() { + await checkLoadSettingProperties("data/search-obsolete-app.json", "App"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_persist.js b/toolkit/components/search/tests/xpcshell/test_settings_persist.js new file mode 100644 index 0000000000..bed6771554 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_settings_persist.js @@ -0,0 +1,106 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const CONFIG_DEFAULT = [ + { + webExtension: { + id: "plainengine@search.mozilla.org", + name: "Plain", + search_url: "https://duckduckgo.com/", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [{ included: { everywhere: true } }], + }, + { + webExtension: { + id: "special-engine@search.mozilla.org", + name: "Special", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [{ included: { everywhere: true } }], + }, +]; + +const CONFIG_UPDATED = CONFIG_DEFAULT.filter(r => + r.webExtension.id.startsWith("plainengine") +); + +async function startup() { + let settingsFileWritten = promiseAfterSettings(); + let ss = new SearchService(); + await AddonTestUtils.promiseRestartManager(); + await ss.init(false); + await settingsFileWritten; + return ss; +} + +async function updateConfig(config) { + const settings = await RemoteSettings(SearchUtils.SETTINGS_KEY); + settings.get.restore(); + sinon.stub(settings, "get").returns(config); +} + +async function visibleEngines(ss) { + return (await ss.getVisibleEngines()).map(e => e._name); +} + +add_setup(async function () { + await SearchTestUtils.useTestEngines("test-extensions", null, CONFIG_DEFAULT); + registerCleanupFunction(AddonTestUtils.promiseShutdownManager); + await AddonTestUtils.promiseStartupManager(); + // This is only needed as otherwise events will not be properly notified + // due to https://searchfox.org/mozilla-central/source/toolkit/components/search/SearchUtils.jsm#186 + let settingsFileWritten = promiseAfterSettings(); + await Services.search.init(false); + Services.search.wrappedJSObject._removeObservers(); + await settingsFileWritten; +}); + +add_task(async function () { + let ss = await startup(); + Assert.ok( + (await visibleEngines(ss)).includes("Special"), + "Should have both engines on first startup" + ); + + let settingsFileWritten = promiseAfterSettings(); + let engine = await ss.getEngineByName("Special"); + await ss.removeEngine(engine); + await settingsFileWritten; + + Assert.ok( + !(await visibleEngines(ss)).includes("Special"), + "Special has been remove, only Plain should remain" + ); + + ss._removeObservers(); + updateConfig(CONFIG_UPDATED); + ss = await startup(); + + Assert.ok( + !(await visibleEngines(ss)).includes("Special"), + "Updated to new configuration that doesnt have Special" + ); + + ss._removeObservers(); + updateConfig(CONFIG_DEFAULT); + ss = await startup(); + + Assert.ok( + !(await visibleEngines(ss)).includes("Special"), + "Configuration now includes Special but we should remember its removal" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_sort_orders-no-hints.js b/toolkit/components/search/tests/xpcshell/test_sort_orders-no-hints.js new file mode 100644 index 0000000000..d2a2d7dd93 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_sort_orders-no-hints.js @@ -0,0 +1,69 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Check the correct default engines are picked from the configuration list, + * when we have some with the same orderHint, and some without any. + */ + +"use strict"; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + await SearchTestUtils.useTestEngines( + "data", + null, + ( + await readJSONFile(do_get_file("data/engines-no-order-hint.json")) + ).data + ); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); +}); + +async function checkOrder(type, expectedOrder) { + // Reset the sorted list. + Services.search.wrappedJSObject._cachedSortedEngines = null; + + const sortedEngines = await Services.search[type](); + Assert.deepEqual( + sortedEngines.map(s => s.name), + expectedOrder, + `Should have the expected engine order from ${type}` + ); +} + +add_task(async function test_engine_sort_with_non_builtins_sort() { + await SearchTestUtils.installSearchExtension({ name: "nonbuiltin1" }); + + // As we've added an engine, the pref will have been set to true, but + // we do really want to test the default sort. + Services.search.wrappedJSObject._settings.setMetaDataAttribute( + "useSavedOrder", + false + ); + + const EXPECTED_ORDER = [ + // Default engine. + "Test search engine", + // Alphabetical order for the two with orderHint = 1000. + "engine-chromeicon", + "engine-rel-searchform-purpose", + // Alphabetical order for the remaining engines without orderHint. + "engine-pref", + "engine-resourceicon", + "Test search engine (Reordered)", + ]; + + // We should still have the same built-in engines listed. + await checkOrder("getAppProvidedEngines", EXPECTED_ORDER); + + const expected = [...EXPECTED_ORDER]; + // This is inserted in alphabetical order for the last three. + expected.splice(expected.length - 1, 0, "nonbuiltin1"); + await checkOrder("getEngines", expected); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_sort_orders.js b/toolkit/components/search/tests/xpcshell/test_sort_orders.js new file mode 100644 index 0000000000..836b47ec1e --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_sort_orders.js @@ -0,0 +1,91 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Check the correct default engines are picked from the configuration list, + * and have the correct orders. + */ + +"use strict"; + +const EXPECTED_ORDER = [ + // Default engines + "Test search engine", + "engine-pref", + // Now the engines in orderHint order. + "engine-resourceicon", + "engine-chromeicon", + "engine-rel-searchform-purpose", + "Test search engine (Reordered)", +]; + +add_setup(async function () { + await AddonTestUtils.promiseStartupManager(); + + await SearchTestUtils.useTestEngines(); + + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "gd", + ]; + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); +}); + +async function checkOrder(type, expectedOrder) { + // Reset the sorted list. + Services.search.wrappedJSObject._cachedSortedEngines = null; + + const sortedEngines = await Services.search[type](); + Assert.deepEqual( + sortedEngines.map(s => s.name), + expectedOrder, + `Should have the expected engine order from ${type}` + ); +} + +add_task(async function test_engine_sort_only_builtins() { + await checkOrder("getAppProvidedEngines", EXPECTED_ORDER); + await checkOrder("getEngines", EXPECTED_ORDER); +}); + +add_task(async function test_engine_sort_with_non_builtins_sort() { + await SearchTestUtils.installSearchExtension({ name: "nonbuiltin1" }); + + // As we've added an engine, the pref will have been set to true, but + // we do really want to test the default sort. + Services.search.wrappedJSObject._settings.setMetaDataAttribute( + "useSavedOrder", + false + ); + + // We should still have the same built-in engines listed. + await checkOrder("getAppProvidedEngines", EXPECTED_ORDER); + + const expected = [...EXPECTED_ORDER]; + expected.splice(EXPECTED_ORDER.length, 0, "nonbuiltin1"); + await checkOrder("getEngines", expected); +}); + +add_task(async function test_engine_sort_with_locale() { + await promiseSetLocale("gd"); + + const expected = [ + "engine-resourceicon-gd", + "engine-pref", + "engine-rel-searchform-purpose", + "engine-chromeicon", + "Test search engine (Reordered)", + ]; + + await checkOrder("getAppProvidedEngines", expected); + expected.push("nonbuiltin1"); + await checkOrder("getEngines", expected); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_telemetry_event_default.js b/toolkit/components/search/tests/xpcshell/test_telemetry_event_default.js new file mode 100644 index 0000000000..84bd1be9cc --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_telemetry_event_default.js @@ -0,0 +1,517 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests for the default engine telemetry event that can be tested via xpcshell, + * related to changing or selecting a different configuration. + * Other tests are typically in browser mochitests. + */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs", +}); + +const BASE_CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [{ included: { everywhere: true } }], + default: "yes", + }, +]; +const MAIN_CONFIG = [ + { + webExtension: { + id: "engine@search.mozilla.org", + name: "Test search engine", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "channel", + condition: "purpose", + purpose: "contextmenu", + value: "rcs", + }, + { + name: "channel", + condition: "purpose", + purpose: "keyword", + value: "fflb", + }, + ], + suggest_url: + "https://suggestqueries.google.com/complete/search?output=firefox&client=firefox&hl={moz:locale}&q={searchTerms}", + }, + appliesTo: [{ included: { everywhere: true } }], + default: "no", + }, + { + webExtension: { + id: "engine-chromeicon@search.mozilla.org", + + name: "engine-chromeicon", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + ], + }, + appliesTo: [{ included: { everywhere: true } }], + default: "yes-if-no-other", + }, + { + webExtension: { + id: "engine-fr@search.mozilla.org", + name: "Test search engine (fr)", + search_url: "https://www.google.fr/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "ie", + value: "iso-8859-1", + }, + { + name: "oe", + value: "iso-8859-1", + }, + ], + }, + appliesTo: [ + { included: { everywhere: true } }, + { + included: { locales: { matches: ["fr"] } }, + excluded: { regions: ["DE"] }, + default: "yes", + }, + ], + default: "no", + }, + { + webExtension: { + id: "engine-pref@search.mozilla.org", + name: "engine-pref", + search_url: "https://www.google.com/search", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "code", + condition: "pref", + pref: "code", + }, + { + name: "test", + condition: "pref", + pref: "test", + }, + ], + }, + appliesTo: [ + { included: { everywhere: true } }, + { included: { regions: ["DE"] }, default: "yes" }, + ], + default: "no", + }, + { + webExtension: { + id: "engine2@search.mozilla.org", + name: "A second test engine", + search_url: "https://duckduckgo.com/?q={searchTerms}", + }, + appliesTo: [ + { included: { everywhere: true } }, + { included: { everywhere: true }, experiment: "test1", default: "yes" }, + ], + default: "no", + }, +]; + +const testSearchEngine = { + id: "engine", + name: "Test search engine", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine@search.mozilla.org" + : "[addon]engine@search.mozilla.org", + submissionURL: "https://www.google.com/search?q=", +}; +const testChromeIconEngine = { + id: "engine-chromeicon", + name: "engine-chromeicon", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-chromeicon@search.mozilla.org" + : "[addon]engine-chromeicon@search.mozilla.org", + + submissionURL: "https://www.google.com/search?q=", +}; +const testFrEngine = { + id: "engine-fr", + name: "Test search engine (fr)", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-fr@search.mozilla.org" + : "[addon]engine-fr@search.mozilla.org", + submissionURL: "https://www.google.fr/search?q=&ie=iso-8859-1&oe=iso-8859-1", +}; +const testPrefEngine = { + id: "engine-pref", + name: "engine-pref", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine-pref@search.mozilla.org" + : "[addon]engine-pref@search.mozilla.org", + submissionURL: "https://www.google.com/search?q=", +}; +const testEngine2 = { + id: "engine2", + name: "A second test engine", + loadPath: SearchUtils.newSearchConfigEnabled + ? "[app]engine2@search.mozilla.org" + : "[addon]engine2@search.mozilla.org", + submissionURL: "https://duckduckgo.com/?q=", +}; + +function clearTelemetry() { + Services.telemetry.clearEvents(); + Services.fog.testResetFOG(); +} + +async function checkTelemetry( + source, + prevEngine, + newEngine, + checkPrivate = false, + additionalEventsExpected = false +) { + // TODO Bug 1876178 - Improve engine change telemetry. + // When we reload engines due to a config change, we update the engines as + // they may have changed, we don't track if any attribute has actually changed + // from previous, and so we send out an update regardless. This is why in + // this test we test for the additional `engine-update` event that's recorded. + // In future, we should be more specific about when to record the event and + // so only one event is captured and not two. + let additionalEvent = [ + { + object: checkPrivate ? "change_private" : "change_default", + value: "engine-update", + extra: { + prev_id: prevEngine?.id ?? "", + new_id: prevEngine?.id ?? "", + new_name: prevEngine?.name ?? "", + new_load_path: prevEngine?.loadPath ?? "", + // Telemetry has a limit of 80 characters. + new_sub_url: prevEngine?.submissionURL.slice(0, 80) ?? "", + }, + }, + ]; + + TelemetryTestUtils.assertEvents( + [ + ...(additionalEventsExpected ? additionalEvent : []), + { + object: checkPrivate ? "change_private" : "change_default", + value: source, + extra: { + prev_id: prevEngine?.id ?? "", + new_id: newEngine?.id ?? "", + new_name: newEngine?.name ?? "", + new_load_path: newEngine?.loadPath ?? "", + // Telemetry has a limit of 80 characters. + new_sub_url: newEngine?.submissionURL.slice(0, 80) ?? "", + }, + }, + ], + { category: "search", method: "engine" } + ); + + let snapshot; + if (checkPrivate) { + snapshot = await Glean.searchEnginePrivate.changed.testGetValue(); + } else { + snapshot = await Glean.searchEngineDefault.changed.testGetValue(); + } + + if (additionalEventsExpected) { + delete snapshot[0].timestamp; + Assert.deepEqual( + snapshot[0], + { + category: checkPrivate + ? "search.engine.private" + : "search.engine.default", + name: "changed", + extra: { + change_source: "engine-update", + previous_engine_id: prevEngine?.id ?? "", + new_engine_id: prevEngine?.id ?? "", + new_display_name: prevEngine?.name ?? "", + new_load_path: prevEngine?.loadPath ?? "", + new_submission_url: prevEngine?.submissionURL ?? "", + }, + }, + "Should have received the correct event details" + ); + snapshot.shift(); + } + + delete snapshot[0].timestamp; + Assert.deepEqual( + snapshot[0], + { + category: checkPrivate + ? "search.engine.private" + : "search.engine.default", + name: "changed", + extra: { + change_source: source, + previous_engine_id: prevEngine?.id ?? "", + new_engine_id: newEngine?.id ?? "", + new_display_name: newEngine?.name ?? "", + new_load_path: newEngine?.loadPath ?? "", + new_submission_url: newEngine?.submissionURL ?? "", + }, + }, + "Should have received the correct event details" + ); +} + +let getVariableStub; + +add_setup(async () => { + Region._setHomeRegion("US", false); + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "en", + "fr", + ]; + Services.locale.requestedLocales = ["en"]; + + sinon.spy(NimbusFeatures.searchConfiguration, "onUpdate"); + sinon.stub(NimbusFeatures.searchConfiguration, "ready").resolves(); + getVariableStub = sinon.stub( + NimbusFeatures.searchConfiguration, + "getVariable" + ); + getVariableStub.returns(null); + + SearchTestUtils.useMockIdleService(); + Services.fog.initializeFOG(); + sinon.stub( + Services.search.wrappedJSObject, + "_showRemovalOfSearchEngineNotificationBox" + ); + + await SearchTestUtils.useTestEngines("data", null, BASE_CONFIG); + await AddonTestUtils.promiseStartupManager(); + + await Services.search.init(); +}); + +add_task(async function test_configuration_changes_default() { + clearTelemetry(); + + await SearchTestUtils.updateRemoteSettingsConfig(MAIN_CONFIG); + + await checkTelemetry( + "config", + testSearchEngine, + testChromeIconEngine, + false, + true + ); +}); + +add_task(async function test_experiment_changes_default() { + clearTelemetry(); + + let reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + getVariableStub.callsFake(name => (name == "experiment" ? "test1" : null)); + NimbusFeatures.searchConfiguration.onUpdate.firstCall.args[0](); + await reloadObserved; + + await checkTelemetry( + "experiment", + testChromeIconEngine, + testEngine2, + false, + true + ); + + // Reset the stub so that we are no longer in an experiment. + getVariableStub.returns(null); +}); + +add_task(async function test_locale_changes_default() { + clearTelemetry(); + + let reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Services.locale.requestedLocales = ["fr"]; + await reloadObserved; + + await checkTelemetry("locale", testEngine2, testFrEngine, false, true); +}); + +add_task(async function test_region_changes_default() { + clearTelemetry(); + + let reloadObserved = + SearchTestUtils.promiseSearchNotification("engines-reloaded"); + Region._setHomeRegion("DE", true); + await reloadObserved; + + await checkTelemetry("region", testFrEngine, testPrefEngine, false, true); +}); + +add_task(async function test_user_changes_separate_private_pref() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + true + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + await Services.search.setDefaultPrivate( + Services.search.getEngineByName("engine-chromeicon"), + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + Assert.notEqual( + await Services.search.getDefault(), + await Services.search.getDefaultPrivate(), + "Should have different engines for the pre-condition" + ); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + false + ); + + clearTelemetry(); + + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + false + ); + + await checkTelemetry("user_private_split", testChromeIconEngine, null, true); + + getVariableStub.returns(null); +}); + +add_task(async function test_experiment_with_separate_default_notifies() { + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault.ui.enabled", + false + ); + Services.prefs.setBoolPref( + SearchUtils.BROWSER_SEARCH_PREF + "separatePrivateDefault", + true + ); + + clearTelemetry(); + + getVariableStub.callsFake(name => + name == "seperatePrivateDefaultUIEnabled" ? true : null + ); + NimbusFeatures.searchConfiguration.onUpdate.firstCall.args[0](); + + await checkTelemetry("experiment", null, testChromeIconEngine, true); + + clearTelemetry(); + + // Reset the stub so that we are no longer in an experiment. + getVariableStub.returns(null); + NimbusFeatures.searchConfiguration.onUpdate.firstCall.args[0](); + + await checkTelemetry("experiment", testChromeIconEngine, null, true); +}); + +add_task(async function test_default_engine_update() { + clearTelemetry(); + let extension = await SearchTestUtils.installSearchExtension( + { + name: "engine", + id: "engine@tests.mozilla.org", + search_url_get_params: `q={searchTerms}&version=1.0`, + search_url: "https://www.google.com/search", + version: "1.0", + }, + { skipUnload: true } + ); + let engine = Services.search.getEngineByName("engine"); + + Assert.ok(!!engine, "Should have loaded the engine"); + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + clearTelemetry(); + + let promiseChanged = TestUtils.topicObserved( + "browser-search-engine-modified", + (eng, verb) => verb == "engine-changed" + ); + let manifest = SearchTestUtils.createEngineManifest({ + name: "Bar", + id: "engine@tests.mozilla.org", + search_url_get_params: `q={searchTerms}&version=2.0`, + search_url: "https://www.google.com/search", + version: "2.0", + }); + + await extension.upgrade({ + useAddonManager: "permanent", + manifest, + }); + await AddonTestUtils.waitForSearchProviderStartup(extension); + await promiseChanged; + + const defaultEngineData = { + id: engine.telemetryId, + name: "Bar", + loadPath: engine.wrappedJSObject._loadPath, + submissionURL: "https://www.google.com/search?q=&version=2.0", + }; + await checkTelemetry("engine-update", defaultEngineData, defaultEngineData); + await extension.unload(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_userEngine.js b/toolkit/components/search/tests/xpcshell/test_userEngine.js new file mode 100644 index 0000000000..7d7fa18c0a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_userEngine.js @@ -0,0 +1,56 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Tests that User Engines can be installed correctly. + */ + +"use strict"; + +add_setup(async function () { + Services.fog.initializeFOG(); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); +}); + +add_task(async function test_user_engine() { + let promiseEngineAdded = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.ADDED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + await Services.search.addUserEngine( + "user", + "https://example.com/user?q={searchTerms}", + "u" + ); + await promiseEngineAdded; + + let engine = Services.search.getEngineByName("user"); + Assert.ok(engine, "Should have installed the engine."); + + Assert.equal(engine.name, "user", "Should have the correct name"); + Assert.equal(engine.description, null, "Should not have a description"); + Assert.deepEqual(engine.aliases, ["u"], "Should have the correct alias"); + + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://example.com/user?q=foo", + "Should have the correct search url" + ); + + submission = engine.getSubmission("foo", SearchUtils.URL_TYPE.SUGGEST_JSON); + Assert.equal(submission, null, "Should not have a suggest url"); + + Services.search.defaultEngine = engine; + + await assertGleanDefaultEngine({ + normal: { + engineId: "other-user", + displayName: "user", + loadPath: "[user]", + submissionUrl: "blank:", + verified: "verified", + }, + }); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_validate_engines.js b/toolkit/components/search/tests/xpcshell/test_validate_engines.js new file mode 100644 index 0000000000..8a25216057 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_validate_engines.js @@ -0,0 +1,34 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Ensure all the engines defined in the configuration are valid by +// creating a refined configuration that includes all the engines everywhere. + +"use strict"; + +const { SearchService } = ChromeUtils.importESModule( + "resource://gre/modules/SearchService.sys.mjs" +); + +const ss = new SearchService(); + +add_task(async function test_validate_engines() { + let settings = RemoteSettings(SearchUtils.SETTINGS_KEY); + let config = await settings.get(); + config = config.map(e => { + return { + appliesTo: [ + { + included: { + everywhere: true, + }, + }, + ], + webExtension: e.webExtension, + }; + }); + + sinon.stub(settings, "get").returns(config); + await AddonTestUtils.promiseStartupManager(); + await ss.init(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_validate_manifests.js b/toolkit/components/search/tests/xpcshell/test_validate_manifests.js new file mode 100644 index 0000000000..8a267296c8 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_validate_manifests.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { ExtensionData } = ChromeUtils.importESModule( + "resource://gre/modules/Extension.sys.mjs" +); + +const SEARCH_EXTENSIONS_PATH = "resource://search-extensions"; + +function getFileURI(resourceURI) { + let resHandler = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + let filePath = resHandler.resolveURI(Services.io.newURI(resourceURI)); + return Services.io.newURI(filePath); +} + +async function getSearchExtensions() { + // Fetching the root will give us the directory listing which we can parse + // for each file name + let list = await fetch(`${SEARCH_EXTENSIONS_PATH}/`).then(req => req.text()); + return list + .split("\n") + .slice(2) + .reduce((acc, line) => { + let parts = line.split(" "); + if (parts.length > 2 && !parts[1].endsWith(".json")) { + // When the directory listing comes from omni jar each engine + // has a trailing slash (engine/) which we dont get locally, or want. + acc.push(parts[1].split("/")[0]); + } + return acc; + }, []); +} + +add_task(async function test_validate_manifest() { + let searchExtensions = await getSearchExtensions(); + ok( + !!searchExtensions.length, + `Found ${searchExtensions.length} search extensions` + ); + for (const xpi of searchExtensions) { + info(`loading: ${SEARCH_EXTENSIONS_PATH}/${xpi}/`); + let fileURI = getFileURI(`${SEARCH_EXTENSIONS_PATH}/${xpi}/`); + let extension = new ExtensionData(fileURI, false); + await extension.loadManifest(); + let locales = await extension.promiseLocales(); + for (let locale of locales.keys()) { + try { + let manifest = await extension.getLocalizedManifest(locale); + ok(!!manifest, `parsed manifest ${xpi.leafName} in ${locale}`); + } catch (e) { + ok( + false, + `FAIL manifest for ${xpi.leafName} in locale ${locale} failed ${e} :: ${e.stack}` + ); + } + } + } +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_builtin_upgrade.js b/toolkit/components/search/tests/xpcshell/test_webextensions_builtin_upgrade.js new file mode 100644 index 0000000000..763338dba6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_builtin_upgrade.js @@ -0,0 +1,264 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// Enable SCOPE_APPLICATION for builtin testing. Default in tests is only SCOPE_PROFILE. +// AddonManager.SCOPE_PROFILE | AddonManager.SCOPE_APPLICATION == 5; +Services.prefs.setIntPref("extensions.enabledScopes", 5); + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +const TEST_CONFIG = [ + { + webExtension: { + id: "multilocale@search.mozilla.org", + locales: ["af", "an"], + }, + appliesTo: [{ included: { everywhere: true } }], + }, + { + webExtension: { + id: "plainengine@search.mozilla.org", + }, + appliesTo: [{ included: { everywhere: true } }], + params: { + searchUrlGetParams: [ + { + name: "config", + value: "applied", + }, + ], + }, + }, +]; + +async function getEngineNames() { + let engines = await Services.search.getEngines(); + return engines.map(engine => engine._name); +} + +function makePlainExtension(version, name = "Plain") { + return { + useAddonManager: "permanent", + manifest: { + name, + version, + browser_specific_settings: { + gecko: { + id: "plainengine@search.mozilla.org", + }, + }, + chrome_settings_overrides: { + search_provider: { + name, + search_url: "https://duckduckgo.com/", + params: [ + { + name: "q", + value: "{searchTerms}", + }, + { + name: "t", + condition: "purpose", + purpose: "contextmenu", + value: "ffcm", + }, + { + name: "t", + condition: "purpose", + purpose: "keyword", + value: "ffab", + }, + { + name: "t", + condition: "purpose", + purpose: "searchbar", + value: "ffsb", + }, + { + name: "t", + condition: "purpose", + purpose: "homepage", + value: "ffhp", + }, + { + name: "t", + condition: "purpose", + purpose: "newtab", + value: "ffnt", + }, + ], + suggest_url: "https://ac.duckduckgo.com/ac/q={searchTerms}&type=list", + }, + }, + }, + }; +} + +function makeMultiLocaleExtension(version) { + return { + useAddonManager: "permanent", + manifest: { + name: "__MSG_searchName__", + version, + browser_specific_settings: { + gecko: { + id: "multilocale@search.mozilla.org", + }, + }, + default_locale: "an", + chrome_settings_overrides: { + search_provider: { + name: "__MSG_searchName__", + search_url: "__MSG_searchUrl__", + }, + }, + }, + files: { + "_locales/af/messages.json": { + searchUrl: { + message: `https://example.af/?q={searchTerms}&version=${version}`, + description: "foo", + }, + searchName: { + message: `Multilocale AF`, + description: "foo", + }, + }, + "_locales/an/messages.json": { + searchUrl: { + message: `https://example.an/?q={searchTerms}&version=${version}`, + description: "foo", + }, + searchName: { + message: `Multilocale AN`, + description: "foo", + }, + }, + }, + }; +} + +add_setup( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function () { + await SearchTestUtils.useTestEngines("test-extensions", null, TEST_CONFIG); + await promiseStartupManager(); + + registerCleanupFunction(promiseShutdownManager); + await Services.search.init(); + } +); + +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function basic_multilocale_test() { + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain", + ]); + + let ext = ExtensionTestUtils.loadExtension(makeMultiLocaleExtension("2.0")); + await ext.startup(); + await AddonTestUtils.waitForSearchProviderStartup(ext); + + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain", + ]); + + let engine = await Services.search.getEngineByName("Multilocale AF"); + Assert.equal( + engine.getSubmission("test").uri.spec, + "https://example.af/?q=test&version=2.0", + "Engine got update" + ); + engine = await Services.search.getEngineByName("Multilocale AN"); + Assert.equal( + engine.getSubmission("test").uri.spec, + "https://example.an/?q=test&version=2.0", + "Engine got update" + ); + + await ext.unload(); + } +); + +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function upgrade_with_configuration_change_test() { + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain", + ]); + + let engine = await Services.search.getEngineByName("Plain"); + Assert.ok(engine.isAppProvided); + Assert.equal( + engine.getSubmission("test").uri.spec, + // This test engine specifies the q and t params in its search_url, therefore + // we get both those and the extra parameter specified in the test config. + "https://duckduckgo.com/?q=test&t=ffsb&config=applied", + "Should have the configuration applied before update." + ); + + let ext = ExtensionTestUtils.loadExtension(makePlainExtension("2.0")); + await ext.startup(); + await AddonTestUtils.waitForSearchProviderStartup(ext); + + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain", + ]); + + engine = await Services.search.getEngineByName("Plain"); + Assert.equal( + engine.getSubmission("test").uri.spec, + // This test engine specifies the q and t params in its search_url, therefore + // we get both those and the extra parameter specified in the test config. + "https://duckduckgo.com/?q=test&t=ffsb&config=applied", + "Should still have the configuration applied after update." + ); + + await ext.unload(); + } +); + +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_upgrade_with_name_change() { + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain", + ]); + + let ext = ExtensionTestUtils.loadExtension( + makePlainExtension("2.0", "Plain2") + ); + await ext.startup(); + await AddonTestUtils.waitForSearchProviderStartup(ext); + + Assert.deepEqual(await getEngineNames(), [ + "Multilocale AF", + "Multilocale AN", + "Plain2", + ]); + + let engine = await Services.search.getEngineByName("Plain2"); + Assert.equal( + engine.getSubmission("test").uri.spec, + // This test engine specifies the q and t params in its search_url, therefore + // we get both those and the extra parameter specified in the test config. + "https://duckduckgo.com/?q=test&t=ffsb&config=applied", + "Should still have the configuration applied after update." + ); + + await ext.unload(); + } +); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_install.js b/toolkit/components/search/tests/xpcshell/test_webextensions_install.js new file mode 100644 index 0000000000..6183d6f95a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_install.js @@ -0,0 +1,221 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +let gBaseUrl; + +async function getEngineNames() { + let engines = await Services.search.getEngines(); + return engines.map(engine => engine._name); +} + +add_setup(async function () { + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + gBaseUrl = `http://localhost:${server.identity.primaryPort}/`; + + await SearchTestUtils.useTestEngines("test-extensions"); + await promiseStartupManager(); + + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "af", + ]; + + registerCleanupFunction(async () => { + await promiseShutdownManager(); + Services.prefs.clearUserPref("browser.search.region"); + }); +}); + +add_task(async function basic_install_test() { + await Services.search.init(); + await promiseAfterSettings(); + + // On first boot, we get the configuration defaults + Assert.deepEqual(await getEngineNames(), ["Plain", "Special"]); + + // User installs a new search engine + let extension = await SearchTestUtils.installSearchExtension( + { + encoding: "windows-1252", + }, + { skipUnload: true } + ); + Assert.deepEqual((await getEngineNames()).sort(), [ + "Example", + "Plain", + "Special", + ]); + + let engine = await Services.search.getEngineByName("Example"); + Assert.equal( + engine.wrappedJSObject.queryCharset, + "windows-1252", + "Should have the correct charset" + ); + + // User uninstalls their engine + await extension.awaitStartup(); + await extension.unload(); + await promiseAfterSettings(); + Assert.deepEqual(await getEngineNames(), ["Plain", "Special"]); +}); + +add_task(async function test_install_duplicate_engine() { + let name = "Plain"; + consoleAllowList.push(`An engine called ${name} already exists`); + let extension = await SearchTestUtils.installSearchExtension( + { + name, + search_url: "https://example.com/plain", + }, + { skipUnload: true } + ); + + let engine = await Services.search.getEngineByName("Plain"); + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://duckduckgo.com/?q=foo&t=ffsb", + "Should have not changed the app provided engine." + ); + + // User uninstalls their engine + await extension.unload(); +}); + +add_task( + // Not needed for new configuration. + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function basic_multilocale_test() { + await promiseSetHomeRegion("an"); + + Assert.deepEqual(await getEngineNames(), [ + "Plain", + "Special", + "Multilocale AN", + ]); + } +); + +add_task( + // Not needed for new configuration. + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function complex_multilocale_test() { + await promiseSetHomeRegion("af"); + + Assert.deepEqual(await getEngineNames(), [ + "Plain", + "Special", + "Multilocale AF", + "Multilocale AN", + ]); + } +); + +add_task( + // Not needed for new configuration. + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_manifest_selection() { + // Sets the home region without updating. + Region._setHomeRegion("an", false); + await promiseSetLocale("af"); + + let engine = await Services.search.getEngineByName("Multilocale AN"); + Assert.ok( + engine.getIconURL().endsWith("favicon-an.ico"), + "Should have the correct favicon for an extension of one locale using a different locale." + ); + Assert.equal( + engine.description, + "A enciclopedia Libre", + "Should have the correct engine name for an extension of one locale using a different locale." + ); + } +); + +add_task(async function test_load_favicon_invalid() { + let observed = TestUtils.consoleMessageObserved(msg => { + return msg.wrappedJSObject.arguments[0].includes( + "Content type does not match expected" + ); + }); + + // User installs a new search engine + let extension = await SearchTestUtils.installSearchExtension( + { + favicon_url: `${gBaseUrl}/head_search.js`, + }, + { skipUnload: true } + ); + + await observed; + + let engine = await Services.search.getEngineByName("Example"); + Assert.equal(null, engine.getIconURL(), "Should not have set an iconURI"); + + // User uninstalls their engine + await extension.awaitStartup(); + await extension.unload(); + await promiseAfterSettings(); +}); + +add_task(async function test_load_favicon_invalid_redirect() { + let observed = TestUtils.consoleMessageObserved(msg => { + return msg.wrappedJSObject.arguments[0].includes( + "Content type does not match expected" + ); + }); + + // User installs a new search engine + let extension = await SearchTestUtils.installSearchExtension( + { + favicon_url: `${gDataUrl}/iconsRedirect.sjs?type=invalid`, + }, + { skipUnload: true } + ); + + await observed; + + let engine = await Services.search.getEngineByName("Example"); + Assert.equal(null, engine.getIconURL(), "Should not have set an iconURI"); + + // User uninstalls their engine + await extension.awaitStartup(); + await extension.unload(); + await promiseAfterSettings(); +}); + +add_task(async function test_load_favicon_redirect() { + let promiseEngineChanged = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + // User installs a new search engine + let extension = await SearchTestUtils.installSearchExtension( + { + favicon_url: `${gDataUrl}/iconsRedirect.sjs`, + }, + { skipUnload: true } + ); + + let engine = await Services.search.getEngineByName("Example"); + + await promiseEngineChanged; + + Assert.ok(engine.getIconURL(), "Should have set an iconURI"); + Assert.ok( + engine.getIconURL().startsWith("data:image/x-icon;base64,"), + "Should have saved the expected content type for the icon" + ); + + // User uninstalls their engine + await extension.awaitStartup(); + await extension.unload(); + await promiseAfterSettings(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_language_switch.js b/toolkit/components/search/tests/xpcshell/test_webextensions_language_switch.js new file mode 100644 index 0000000000..6a781bba5b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_language_switch.js @@ -0,0 +1,110 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +add_setup(async function () { + Services.locale.availableLocales = [ + ...Services.locale.availableLocales, + "en", + "de", + "fr", + ]; + Services.locale.requestedLocales = ["en"]; + + await SearchTestUtils.useTestEngines("data1"); + await promiseStartupManager(); + await Services.search.init(); + await promiseAfterSettings(); + + registerCleanupFunction(promiseShutdownManager); +}); + +add_task(async function test_language_switch_changes_name() { + await SearchTestUtils.installSearchExtension( + { + name: "__MSG_engineName__", + id: "engine@tests.mozilla.org", + search_url_get_params: `q={searchTerms}&version=1.0`, + default_locale: "en", + version: "1.0", + }, + { skipUnload: false }, + { + "_locales/en/messages.json": { + engineName: { + message: "English Name", + description: "The Name", + }, + }, + "_locales/fr/messages.json": { + engineName: { + message: "French Name", + description: "The Name", + }, + }, + } + ); + + let engine = Services.search.getEngineById("engine@tests.mozilla.orgdefault"); + Assert.ok(!!engine, "Should have loaded the engine"); + Assert.equal( + engine.name, + "English Name", + "Should have loaded the English version of the name" + ); + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + let promiseChanged = TestUtils.topicObserved( + "browser-search-engine-modified", + (eng, verb) => verb == "engine-changed" + ); + + await promiseSetLocale("fr"); + + await promiseChanged; + + engine = Services.search.getEngineById("engine@tests.mozilla.orgdefault"); + Assert.ok(!!engine, "Should still be available"); + Assert.equal( + engine.name, + "French Name", + "Should have updated to the French version of the name" + ); + + Assert.equal( + (await Services.search.getDefault()).id, + engine.id, + "Should have kept the default engine the same" + ); + + promiseChanged = TestUtils.topicObserved( + "browser-search-engine-modified", + (eng, verb) => verb == "engine-changed" + ); + + // Check for changing to a locale the add-on doesn't have. + await promiseSetLocale("de"); + + await promiseChanged; + + engine = Services.search.getEngineById("engine@tests.mozilla.orgdefault"); + Assert.ok(!!engine, "Should still be available"); + Assert.equal( + engine.name, + "English Name", + "Should have fallen back to the default locale (English) version of the name" + ); + + Assert.equal( + (await Services.search.getDefault()).id, + engine.id, + "Should have kept the default engine the same" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_migrate_to.js b/toolkit/components/search/tests/xpcshell/test_webextensions_migrate_to.js new file mode 100644 index 0000000000..cb44ebde84 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_migrate_to.js @@ -0,0 +1,66 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Test migrating legacy add-on engines in background. + */ + +"use strict"; + +add_setup(async function () { + useHttpServer("opensearch"); + await AddonTestUtils.promiseStartupManager(); + await SearchTestUtils.useTestEngines("data1"); + + let data = await readJSONFile(do_get_file("data/search-migration.json")); + + await promiseSaveSettingsData(data); + + await Services.search.init(); + + // We need the extension installed for this test, but we do not want to + // trigger the functions that happen on installation, so stub that out. + // The manifest already has details of this engine. + let oldFunc = Services.search.wrappedJSObject.addEnginesFromExtension; + Services.search.wrappedJSObject.addEnginesFromExtension = () => {}; + + // Add the add-on so add-on manager has a valid item. + await SearchTestUtils.installSearchExtension({ + id: "simple", + name: "simple search", + search_url: "https://example.com/", + }); + + Services.search.wrappedJSObject.addEnginesFromExtension = oldFunc; +}); + +add_task(async function test_migrateLegacyEngineDifferentName() { + await Services.search.init(); + + let engine = Services.search.getEngineByName("simple"); + Assert.ok(engine, "Should have the legacy add-on engine."); + + // Set this engine as default, the new engine should become the default + // after migration. + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + engine = Services.search.getEngineByName("simple search"); + Assert.ok(engine, "Should have the WebExtension engine."); + + await Services.search.runBackgroundChecks(); + + engine = Services.search.getEngineByName("simple"); + Assert.ok(!engine, "Should have removed the legacy add-on engine"); + + engine = Services.search.getEngineByName("simple search"); + Assert.ok(engine, "Should have kept the WebExtension engine."); + + Assert.equal( + (await Services.search.getDefault()).name, + engine.name, + "Should have switched to the WebExtension engine as default." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_normandy_upgrade.js b/toolkit/components/search/tests/xpcshell/test_webextensions_normandy_upgrade.js new file mode 100644 index 0000000000..4beb614510 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_normandy_upgrade.js @@ -0,0 +1,106 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +SearchTestUtils.initXPCShellAddonManager(this, "system"); + +async function restart() { + Services.search.wrappedJSObject.reset(); + await AddonTestUtils.promiseRestartManager(); + await Services.search.init(false); +} + +const CONFIG_DEFAULT = [ + { + webExtension: { id: "plainengine@search.mozilla.org" }, + appliesTo: [{ included: { everywhere: true } }], + }, +]; + +const CONFIG_UPDATED = [ + { + webExtension: { id: "plainengine@search.mozilla.org" }, + appliesTo: [{ included: { everywhere: true } }], + }, + { + webExtension: { id: "example@search.mozilla.org" }, + appliesTo: [{ included: { everywhere: true } }], + }, +]; + +async function getEngineNames() { + let engines = await Services.search.getAppProvidedEngines(); + return engines.map(engine => engine._name); +} + +add_setup( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function () { + await SearchTestUtils.useTestEngines( + "test-extensions", + null, + CONFIG_DEFAULT + ); + await AddonTestUtils.promiseStartupManager(); + registerCleanupFunction(AddonTestUtils.promiseShutdownManager); + SearchTestUtils.useMockIdleService(); + await Services.search.init(); + } +); + +// Test the situation where we receive an updated configuration +// that references an engine that doesnt exist locally as it +// will be installed by Normandy. +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_config_before_normandy() { + // Ensure initial default setup. + await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_DEFAULT); + await restart(); + Assert.deepEqual(await getEngineNames(), ["Plain"]); + // Updated configuration references nonexistant engine. + await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_UPDATED); + Assert.deepEqual( + await getEngineNames(), + ["Plain"], + "Updated engine hasnt been installed yet" + ); + // Normandy then installs the engine. + let addon = await SearchTestUtils.installSystemSearchExtension(); + Assert.deepEqual( + await getEngineNames(), + ["Plain", "Example"], + "Both engines are now enabled" + ); + await addon.unload(); + } +); + +// Test the situation where we receive a newly installed +// engine from Normandy followed by the update to the +// configuration that uses that engine. +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_normandy_before_config() { + // Ensure initial default setup. + await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_DEFAULT); + await restart(); + Assert.deepEqual(await getEngineNames(), ["Plain"]); + // Normandy installs the enigne. + let addon = await SearchTestUtils.installSystemSearchExtension(); + Assert.deepEqual( + await getEngineNames(), + ["Plain"], + "Normandy engine ignored as not in config yet" + ); + // Configuration is updated to use the engine. + await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_UPDATED); + Assert.deepEqual( + await getEngineNames(), + ["Plain", "Example"], + "Both engines are now enabled" + ); + await addon.unload(); + } +); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_startup_duplicate.js b/toolkit/components/search/tests/xpcshell/test_webextensions_startup_duplicate.js new file mode 100644 index 0000000000..3e42fdee13 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_startup_duplicate.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const lazy = {}; + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +ChromeUtils.defineESModuleGetters(lazy, { + ExtensionTestUtils: + "resource://testing-common/ExtensionXPCShellUtils.sys.mjs", +}); + +add_setup(async function () { + let server = useHttpServer(); + server.registerContentType("sjs", "sjs"); + await SearchTestUtils.useTestEngines("test-extensions"); + await promiseStartupManager(); + + registerCleanupFunction(async () => { + await promiseShutdownManager(); + }); +}); + +add_task(async function test_install_duplicate_engine_startup() { + let name = "Plain"; + let id = "plain@tests.mozilla.org"; + consoleAllowList.push( + `#installExtensionEngine failed for ${id}`, + `An engine called ${name} already exists` + ); + // Do not use SearchTestUtils.installSearchExtension, as we need to manually + // start the search service after installing the extension. + let extensionInfo = { + useAddonManager: "permanent", + files: {}, + manifest: SearchTestUtils.createEngineManifest({ + name, + search_url: "https://example.com/plain", + }), + }; + + let extension = lazy.ExtensionTestUtils.loadExtension(extensionInfo); + await extension.startup(); + + await Services.search.init(); + + await AddonTestUtils.waitForSearchProviderStartup(extension); + let engine = await Services.search.getEngineByName(name); + let submission = engine.getSubmission("foo"); + Assert.equal( + submission.uri.spec, + "https://duckduckgo.com/?q=foo&t=ffsb", + "Should have not changed the app provided engine." + ); + + await extension.unload(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_startup_remove.js b/toolkit/components/search/tests/xpcshell/test_webextensions_startup_remove.js new file mode 100644 index 0000000000..a357320c7b --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_startup_remove.js @@ -0,0 +1,83 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const ENGINE_ID = "enginetest@example.com"; +let xpi; +let profile = do_get_profile().clone(); + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + xpi = AddonTestUtils.createTempWebExtensionFile({ + manifest: { + version: "1.0", + browser_specific_settings: { + gecko: { id: ENGINE_ID }, + }, + chrome_settings_overrides: { + search_provider: { + name: "Test Engine", + search_url: `https://example.com/?q={searchTerms}`, + }, + }, + }, + }); + await AddonTestUtils.manuallyInstall(xpi); +}); + +add_task(async function test_removeAddonOnStartup() { + // First startup the add-on manager and ensure the engine is installed. + await AddonTestUtils.promiseStartupManager(); + let promise = promiseAfterSettings(); + await Services.search.init(); + + let engine = Services.search.getEngineByName("Test Engine"); + let allEngines = await Services.search.getEngines(); + + Assert.ok(!!engine, "Should have installed the test engine"); + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + await promise; + + await AddonTestUtils.promiseShutdownManager(); + + // Now remove it, reset the search service and start up the add-on manager. + // Note: the saved settings will have the engine in. If this didn't work, + // the engine would still be present. + await IOUtils.remove( + PathUtils.join(profile.path, "extensions", `${ENGINE_ID}.xpi`) + ); + + let removePromise = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.REMOVED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + Services.search.wrappedJSObject.reset(); + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + await removePromise; + + Assert.ok( + !Services.search.getEngineByName("Test Engine"), + "Should have removed the test engine" + ); + + let newEngines = await Services.search.getEngines(); + Assert.deepEqual( + newEngines.map(e => e.name), + allEngines.map(e => e.name).filter(n => n != "Test Engine"), + "Should no longer have the test engine in the full list" + ); + let newDefault = await Services.search.getDefault(); + Assert.equal( + newDefault.name, + "engine1", + "Should have changed the default engine back to the configuration default" + ); + + await AddonTestUtils.promiseShutdownManager(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_upgrade.js b/toolkit/components/search/tests/xpcshell/test_webextensions_upgrade.js new file mode 100644 index 0000000000..bfd789780a --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_upgrade.js @@ -0,0 +1,188 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("data1"); + await promiseStartupManager(); + await Services.search.init(); + await promiseAfterSettings(); + + registerCleanupFunction(promiseShutdownManager); +}); + +add_task(async function test_basic_upgrade() { + let extension = await SearchTestUtils.installSearchExtension( + { + version: "1.0", + search_url_get_params: `q={searchTerms}&version=1.0`, + keyword: "foo", + }, + { skipUnload: true } + ); + + let engine = await Services.search.getEngineByAlias("foo"); + Assert.ok(engine, "Can fetch engine with alias"); + engine.alias = "testing"; + + engine = await Services.search.getEngineByAlias("testing"); + Assert.ok(engine, "Can fetch engine by alias"); + let params = engine.getSubmission("test").uri.query.split("&"); + Assert.ok(params.includes("version=1.0"), "Correct version installed"); + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + let promiseChanged = TestUtils.topicObserved( + "browser-search-engine-modified", + (eng, verb) => verb == "engine-changed" + ); + + let manifest = SearchTestUtils.createEngineManifest({ + version: "2.0", + search_url_get_params: `q={searchTerms}&version=2.0`, + keyword: "bar", + }); + await extension.upgrade({ + useAddonManager: "permanent", + manifest, + }); + await AddonTestUtils.waitForSearchProviderStartup(extension); + await promiseChanged; + + engine = await Services.search.getEngineByAlias("testing"); + Assert.ok(engine, "Engine still has alias set"); + + params = engine.getSubmission("test").uri.query.split("&"); + Assert.ok(params.includes("version=2.0"), "Correct version installed"); + + Assert.equal( + Services.search.defaultEngine.name, + "Example", + "Should have retained the same default engine" + ); + + await extension.unload(); + await promiseAfterSettings(); +}); + +add_task(async function test_upgrade_changes_name() { + let extension = await SearchTestUtils.installSearchExtension( + { + name: "engine", + id: "engine@tests.mozilla.org", + search_url_get_params: `q={searchTerms}&version=1.0`, + version: "1.0", + }, + { skipUnload: true } + ); + + let engine = Services.search.getEngineByName("engine"); + Assert.ok(!!engine, "Should have loaded the engine"); + + await Services.search.setDefault( + engine, + Ci.nsISearchService.CHANGE_REASON_UNKNOWN + ); + + // When we add engines currently, we normally force using the saved order. + // Reset that here, so we can check the order is reset in the case this + // is a application provided engine change. + Services.search.wrappedJSObject._settings.setMetaDataAttribute( + "useSavedOrder", + false + ); + Services.search.getEngineByName("engine1").wrappedJSObject._orderHint = null; + Services.search.getEngineByName("engine2").wrappedJSObject._orderHint = null; + + Assert.deepEqual( + (await Services.search.getVisibleEngines()).map(e => e.name), + ["engine1", "engine2", "engine"], + "Should have the expected order initially" + ); + + let promiseChanged = TestUtils.topicObserved( + "browser-search-engine-modified", + (eng, verb) => verb == "engine-changed" + ); + + let manifest = SearchTestUtils.createEngineManifest({ + name: "Bar", + id: "engine@tests.mozilla.org", + search_url_get_params: `q={searchTerms}&version=2.0`, + version: "2.0", + }); + await extension.upgrade({ + useAddonManager: "permanent", + manifest, + }); + await AddonTestUtils.waitForSearchProviderStartup(extension); + + await promiseChanged; + + engine = Services.search.getEngineByName("Bar"); + Assert.ok(!!engine, "Should be able to get the new engine"); + + Assert.equal( + (await Services.search.getDefault()).name, + "Bar", + "Should have kept the default engine the same" + ); + + Assert.deepEqual( + (await Services.search.getVisibleEngines()).map(e => e.name), + // Expected order: Default, then others in alphabetical. + ["engine1", "Bar", "engine2"], + "Should have updated the engine order" + ); + + await extension.unload(); + await promiseAfterSettings(); +}); + +add_task(async function test_upgrade_to_existing_name_not_allowed() { + let extension = await SearchTestUtils.installSearchExtension( + { + name: "engine", + search_url_get_params: `q={searchTerms}&version=1.0`, + version: "1.0", + }, + { skipUnload: true } + ); + + let engine = Services.search.getEngineByName("engine"); + Assert.ok(!!engine, "Should have loaded the engine"); + + let promise = AddonTestUtils.waitForSearchProviderStartup(extension); + let name = "engine1"; + consoleAllowList.push(`An engine called ${name} already exists`); + let manifest = SearchTestUtils.createEngineManifest({ + name, + search_url_get_params: `q={searchTerms}&version=2.0`, + version: "2.0", + }); + await extension.upgrade({ + useAddonManager: "permanent", + manifest, + }); + await promise; + + Assert.equal( + Services.search.getEngineByName("engine1").getSubmission("").uri.spec, + "https://1.example.com/", + "Should have not changed the original engine" + ); + + console.log((await Services.search.getEngines()).map(e => e.name)); + + engine = Services.search.getEngineByName("engine"); + Assert.ok(!!engine, "Should still be able to get the engine by the old name"); + + await extension.unload(); + await promiseAfterSettings(); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_webextensions_valid.js b/toolkit/components/search/tests/xpcshell/test_webextensions_valid.js new file mode 100644 index 0000000000..36ba8155bf --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_webextensions_valid.js @@ -0,0 +1,199 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { TelemetryTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TelemetryTestUtils.sys.mjs" +); + +const { promiseShutdownManager, promiseStartupManager } = AddonTestUtils; + +let extension; +let extensionPostData; +let oldRemoveEngineFunc; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines"); + await promiseStartupManager(); + + Services.telemetry.canRecordExtended = true; + + await Services.search.init(); + await promiseAfterSettings(); + + extension = await SearchTestUtils.installSearchExtension( + {}, + { skipUnload: true } + ); + extensionPostData = await SearchTestUtils.installSearchExtension( + { + name: "PostData", + search_url_post_params: "?q={searchTerms}&post=1", + }, + { skipUnload: true } + ); + await extension.awaitStartup(); + await extensionPostData.awaitStartup(); + + // For these tests, stub-out the removeEngine function, so that when we + // remove it from the add-on manager, the engine is left in the search + // settings. + oldRemoveEngineFunc = Services.search.wrappedJSObject.removeEngine.bind( + Services.search.wrappedJSObject + ); + Services.search.wrappedJSObject.removeEngine = () => {}; + + registerCleanupFunction(async () => { + await promiseShutdownManager(); + }); +}); + +add_task(async function test_valid_extensions_do_nothing() { + Services.telemetry.clearScalars(); + + Assert.ok( + Services.search.getEngineByName("Example"), + "Should have installed the engine" + ); + Assert.ok( + !!Services.search.getEngineByName("PostData"), + "Should have installed the PostData engine" + ); + + await Services.search.runBackgroundChecks(); + + let scalars = TelemetryTestUtils.getProcessScalars("parent", true, true); + + Assert.deepEqual(scalars, {}, "Should not have recorded any issues"); +}); + +add_task(async function test_different_name() { + Services.telemetry.clearScalars(); + + let engine = Services.search.getEngineByName("Example"); + + engine.wrappedJSObject._name = "Example Test"; + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extension.id, + 5 + ); + + engine.wrappedJSObject._name = "Example"; +}); + +add_task(async function test_different_url() { + Services.telemetry.clearScalars(); + + let engine = Services.search.getEngineByName("Example"); + + engine.wrappedJSObject._urls = []; + engine.wrappedJSObject._setUrls({ + search_url: "https://example.com/123", + search_url_get_params: "?q={searchTerms}", + }); + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extension.id, + 6 + ); +}); + +add_task(async function test_different_url_post_data() { + Services.telemetry.clearScalars(); + + let engine = Services.search.getEngineByName("PostData"); + + engine.wrappedJSObject._urls = []; + engine.wrappedJSObject._setUrls({ + search_url: "https://example.com/123", + search_url_post_params: "?q={searchTerms}", + }); + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extensionPostData.id, + 6 + ); +}); + +add_task(async function test_extension_no_longer_specifies_engine() { + Services.telemetry.clearScalars(); + + let extensionInfo = { + useAddonManager: "permanent", + manifest: { + version: "2.0", + browser_specific_settings: { + gecko: { + id: "example@tests.mozilla.org", + }, + }, + }, + }; + + await extension.upgrade(extensionInfo); + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extension.id, + 4 + ); +}); + +add_task(async function test_disabled_extension() { + // We don't clear scalars between tests to ensure the scalar gets set + // to the new value, rather than added. + + // Disable the extension, this won't remove the search engine because we've + // stubbed removeEngine. + await extension.addon.disable(); + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extension.id, + 2 + ); + + extension.addon.enable(); + await extension.awaitStartup(); +}); + +add_task(async function test_missing_extension() { + // We don't clear scalars between tests to ensure the scalar gets set + // to the new value, rather than added. + + let extensionId = extension.id; + // Remove the extension, this won't remove the search engine because we've + // stubbed removeEngine. + await extension.unload(); + + await Services.search.runBackgroundChecks(); + + TelemetryTestUtils.assertKeyedScalar( + TelemetryTestUtils.getProcessScalars("parent", true, true), + "browser.searchinit.engine_invalid_webextension", + extensionId, + 1 + ); + + await oldRemoveEngineFunc(Services.search.getEngineByName("Example")); +}); diff --git a/toolkit/components/search/tests/xpcshell/xpcshell.toml b/toolkit/components/search/tests/xpcshell/xpcshell.toml new file mode 100644 index 0000000000..47847c93de --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/xpcshell.toml @@ -0,0 +1,306 @@ +[DEFAULT] +firefox-appdir = "browser" +head = "head_search.js" +dupe-manifest = "" +tags = "searchmain" +skip-if = ["os == 'android'"] +prefs = ["browser.search.removeEngineInfobar.enabled=true"] + +support-files = [ + "data/engine.xml", + "data/engine/manifest.json", + "data/engine2.xml", + "data/engine2/manifest.json", + "data/engine-app/manifest.json", + "data/engine-diff-name/manifest.json", + "data/engine-diff-name/_locales/en/messages.json", + "data/engine-diff-name/_locales/gd/messages.json", + "data/engine-fr.xml", + "data/engine-fr/manifest.json", + "data/engine-reordered/manifest.json", + "data/engineMaker.sjs", + "data/engine-pref/manifest.json", + "data/engine-rel-searchform-purpose/manifest.json", + "data/engine-system-purpose/manifest.json", + "data/engineImages.xml", + "data/engine-chromeicon/manifest.json", + "data/engine-purposes/manifest.json", + "data/engine-resourceicon/manifest.json", + "data/engine-resourceicon/_locales/en/messages.json", + "data/engine-resourceicon/_locales/gd/messages.json", + "data/engine-same-name/manifest.json", + "data/engine-same-name/_locales/en/messages.json", + "data/engine-same-name/_locales/gd/messages.json", + "data/engines-no-order-hint.json", + "data/engines.json", + "data/iconsRedirect.sjs", + "data/search.json", + "data/search-legacy.json", + "data/search-legacy-correct-default-engine-hashes.json", + "data/search-legacy-no-ids.json", + "data/search-legacy-old-loadPaths.json", + "data/search-legacy-wrong-default-engine-hashes.json", + "data/search-legacy-wrong-third-party-engine-hashes.json", + "data/search-obsolete-app.json", + "data/search-obsolete-distribution.json", + "data/search-obsolete-langpack.json", + "data/searchSuggestions.sjs", + "data/geolookup-extensions/multilocale/favicon.ico", + "data/geolookup-extensions/multilocale/manifest.json", + "data/geolookup-extensions/multilocale/_locales/af/messages.json", + "data/geolookup-extensions/multilocale/_locales/an/messages.json", + "data1/engine1/manifest.json", + "data1/engine2/manifest.json", + "data1/exp2/manifest.json", + "data1/exp3/manifest.json", + "data1/engines.json", + "data1/search-config-v2.json", + "simple-engines/engines.json", + "simple-engines/search-config-v2.json", + "simple-engines/basic/manifest.json", + "simple-engines/simple/manifest.json", + "test-extensions/engines.json", + "test-extensions/plainengine/favicon.ico", + "test-extensions/plainengine/manifest.json", + "test-extensions/special-engine/favicon.ico", + "test-extensions/special-engine/manifest.json", + "test-extensions/multilocale/favicon-af.ico", + "test-extensions/multilocale/favicon-an.ico", + "test-extensions/multilocale/manifest.json", + "test-extensions/multilocale/_locales/af/messages.json", + "test-extensions/multilocale/_locales/an/messages.json", +] + +["test_SearchStaticData.js"] + +["test_appDefaultEngine.js"] + +["test_async.js"] + +["test_config_engine_params.js"] +support-files = [ + "method-extensions/get/manifest.json", + "method-extensions/post/manifest.json", + "method-extensions/engines.json", +] + +["test_defaultEngine.js"] + +["test_defaultEngine_experiments.js"] + +["test_defaultEngine_fallback.js"] + +["test_defaultPrivateEngine.js"] + +["test_engine_alias.js"] + +["test_engine_ids.js"] + +["test_engine_multiple_alias.js"] + +["test_engine_old_selector.js"] + +["test_engine_old_selector_application.js"] + +["test_engine_old_selector_application_distribution.js"] + +["test_engine_old_selector_application_name.js"] + +["test_engine_old_selector_order.js"] + +["test_engine_old_selector_override.js"] + +["test_engine_old_selector_remote_settings.js"] +tags = "remotesettings searchmain" + +["test_engine_old_selector_remote_override.js"] + +["test_engine_selector_engine_orders.js"] + +["test_engine_selector_defaults.js"] + +["test_engine_selector_environment.js"] + +["test_engine_selector_variants.js"] + +["test_engine_set_alias.js"] + +["test_getSubmission_encoding.js"] + +["test_getSubmission_params.js"] + +["test_getSubmission_params_pref.js"] + +["test_getSubmission_params_prefNimbus.js"] + +["test_getSubmission_params_prefNimbus_invalid.js"] + +["test_getSubmission_params_purpose.js"] + +["test_identifiers.js"] + +["test_ignorelist.js"] + +["test_ignorelist_update.js"] + +["test_initialization.js"] + +["test_initialization_status_telemetry.js"] + +["test_initialization_with_region.js"] + +["test_list_json_locale.js"] + +["test_list_json_no_private_default.js"] + +["test_list_json_searchdefault.js"] + +["test_list_json_searchorder.js"] + +["test_maybereloadengine_order.js"] + +["test_migrateWebExtensionEngine.js"] + +["test_missing_engine.js"] + +["test_nodb_pluschanges.js"] + +["test_notifications.js"] + +["test_opensearch.js"] +support-files = [ + "opensearch/mozilla-ns.xml", + "opensearch/post.xml", + "opensearch/searchform-invalid.xml", + "opensearch/simple.xml", + "opensearch/suggestion.xml", + "opensearch/suggestion-alternate.xml", +] + +["test_opensearch_icon.js"] +support-files = [ + "data/bigIcon.ico", + "data/remoteIcon.ico", + "data/svgIcon.svg", +] + +["test_opensearch_icons_invalid.js"] +support-files = [ + "opensearch/chromeicon.xml", + "opensearch/resourceicon.xml", +] + +["test_opensearch_install_errors.js"] +support-files = ["opensearch/invalid.xml"] + +["test_opensearch_telemetry.js"] +support-files = [ + "opensearch/secure-and-securely-updated1.xml", + "opensearch/secure-and-securely-updated2.xml", + "opensearch/secure-and-securely-updated3.xml", + "opensearch/secure-and-securely-updated-insecure-form.xml", + "opensearch/secure-and-insecurely-updated1.xml", + "opensearch/secure-and-insecurely-updated2.xml", + "opensearch/insecure-and-securely-updated1.xml", + "opensearch/insecure-and-insecurely-updated1.xml", + "opensearch/insecure-and-insecurely-updated2.xml", + "opensearch/secure-and-no-update-url1.xml", + "opensearch/insecure-and-no-update-url1.xml", + "opensearch/secure-localhost.xml", + "opensearch/secure-onionv2.xml", + "opensearch/secure-onionv3.xml", +] + +["test_opensearch_update.js"] + +["test_override_allowlist.js"] + +["test_override_allowlist_switch.js"] + +["test_parseSubmissionURL.js"] + +["test_policyEngine.js"] + +["test_reload_engines.js"] + +["test_reload_engines_duplicate.js"] + +["test_reload_engines_experiment.js"] + +["test_reload_engines_locales.js"] + +["test_remove_engine_notification_box.js"] + +["test_remove_profile_engine.js"] + +["test_save_sorted_engines.js"] + +["test_searchSuggest.js"] + +["test_searchSuggest_cookies.js"] + +["test_searchSuggest_extraParams.js"] + +["test_searchSuggest_private.js"] + +["test_searchTermFromResult.js"] + +["test_searchUrlDomain.js"] + +["test_selectedEngine.js"] + +["test_sendSubmissionURL.js"] + +["test_settings.js"] + +["test_settings_broken.js"] + +["test_settings_duplicate.js"] + +["test_settings_good.js"] + +["test_settings_ignorelist.js"] +support-files = ["data/search_ignorelist.json"] + +["test_settings_migration_hideOneOffs.js"] + +["test_settings_migration_ids.js"] + +["test_settings_migration_loadPath.js"] + +["test_settings_none.js"] + +["test_settings_obsolete.js"] + +["test_settings_persist.js"] + +["test_sort_orders-no-hints.js"] + +["test_sort_orders.js"] + +["test_telemetry_event_default.js"] + +["test_userEngine.js"] + +["test_validate_engines.js"] + +["test_validate_manifests.js"] + +["test_webextensions_builtin_upgrade.js"] + +["test_webextensions_install.js"] + +["test_webextensions_language_switch.js"] + +["test_webextensions_migrate_to.js"] +support-files = ["data/search-migration.json"] + +["test_webextensions_normandy_upgrade.js"] + +["test_webextensions_startup_duplicate.js"] + +["test_webextensions_startup_remove.js"] + +["test_webextensions_upgrade.js"] + +["test_webextensions_valid.js"] |
