diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-15 03:35:49 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-15 03:35:49 +0000 |
commit | d8bbc7858622b6d9c278469aab701ca0b609cddf (patch) | |
tree | eff41dc61d9f714852212739e6b3738b82a2af87 /toolkit/components/search | |
parent | Releasing progress-linux version 125.0.3-1~progress7.99u1. (diff) | |
download | firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.tar.xz firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.zip |
Merging upstream version 126.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'toolkit/components/search')
31 files changed, 1478 insertions, 210 deletions
diff --git a/toolkit/components/search/.eslintrc.js b/toolkit/components/search/.eslintrc.js deleted file mode 100644 index 9aafb4a214..0000000000 --- a/toolkit/components/search/.eslintrc.js +++ /dev/null @@ -1,9 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -module.exports = { - extends: ["plugin:mozilla/require-jsdoc"], -}; diff --git a/toolkit/components/search/AppProvidedSearchEngine.sys.mjs b/toolkit/components/search/AppProvidedSearchEngine.sys.mjs index 7401ba115c..ed815b96d1 100644 --- a/toolkit/components/search/AppProvidedSearchEngine.sys.mjs +++ b/toolkit/components/search/AppProvidedSearchEngine.sys.mjs @@ -9,6 +9,8 @@ import { EngineURL, } from "resource://gre/modules/SearchEngine.sys.mjs"; +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + const lazy = {}; ChromeUtils.defineESModuleGetters(lazy, { @@ -16,14 +18,56 @@ ChromeUtils.defineESModuleGetters(lazy, { SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", }); +XPCOMUtils.defineLazyServiceGetter( + lazy, + "idleService", + "@mozilla.org/widget/useridleservice;1", + "nsIUserIdleService" +); + +// After the user has been idle for 30s, we'll update icons if we need to. +const ICON_UPDATE_ON_IDLE_DELAY = 30; + /** * Handles loading application provided search engine icons from remote settings. */ class IconHandler { - #iconList = null; + /** + * The remote settings client for the search engine icons. + * + * @type {?RemoteSettingsClient} + */ #iconCollection = null; /** + * The list of icon records from the remote settings collection. + * + * @type {?object[]} + */ + #iconList = null; + + /** + * A flag that indicates if we have queued an idle observer to update icons. + * + * @type {boolean} + */ + #queuedIdle = false; + + /** + * A map of pending updates that need to be applied to the engines. This is + * keyed via record id, so that if multiple updates are queued for the same + * record, then we will only update the engine once. + * + * @type {Map<string, object>} + */ + #pendingUpdatesMap = new Map(); + + constructor() { + this.#iconCollection = lazy.RemoteSettings("search-config-icons"); + this.#iconCollection.on("sync", this._onIconListUpdated.bind(this)); + } + + /** * Returns the icon for the record that matches the engine identifier * and the preferred width. * @@ -40,14 +84,9 @@ class IconHandler { await this.#getIconList(); } - let iconRecords = this.#iconList.filter(r => { - return r.engineIdentifiers.some(i => { - if (i.endsWith("*")) { - return engineIdentifier.startsWith(i.slice(0, -1)); - } - return engineIdentifier == i; - }); - }); + let iconRecords = this.#iconList.filter(r => + this._identifierMatches(engineIdentifier, r.engineIdentifiers) + ); if (!iconRecords.length) { console.warn("No icon found for", engineIdentifier); @@ -66,28 +105,110 @@ class IconHandler { } } - let iconURL; + let iconData; try { - iconURL = await this.#iconCollection.attachments.get(iconRecord); + iconData = await this.#iconCollection.attachments.get(iconRecord); } catch (ex) { console.error(ex); - return null; } - if (!iconURL) { + if (!iconData) { console.warn("Unable to find the icon for", engineIdentifier); + // Queue an update in case we haven't downloaded it yet. + this.#pendingUpdatesMap.set(iconRecord.id, iconRecord); + this.#maybeQueueIdle(); return null; } + + if (iconData.record.last_modified != iconRecord.last_modified) { + // The icon we have stored is out of date, queue an update so that we'll + // download the new icon. + this.#pendingUpdatesMap.set(iconRecord.id, iconRecord); + this.#maybeQueueIdle(); + } return URL.createObjectURL( - new Blob([iconURL.buffer]), + new Blob([iconData.buffer]), iconRecord.attachment.mimetype ); } + QueryInterface = ChromeUtils.generateQI(["nsIObserver"]); + + /** + * Called when there is an update queued and the user has been observed to be + * idle for ICON_UPDATE_ON_IDLE_DELAY seconds. + * + * This will always download new icons (added or updated), even if there is + * no current engine that matches the identifiers. This is to ensure that we + * have pre-populated the cache if the engine is added later for this user. + * + * We do not handle deletes, as remote settings will handle the cleanup of + * removed records. We also do not expect the case where an icon is removed + * for an active engine. + * + * @param {nsISupports} subject + * The subject of the observer. + * @param {string} topic + * The topic of the observer. + */ + async observe(subject, topic) { + if (topic != "idle") { + return; + } + + this.#queuedIdle = false; + lazy.idleService.removeIdleObserver(this, ICON_UPDATE_ON_IDLE_DELAY); + + // Update the icon list, in case engines will call getIcon() again. + await this.#getIconList(); + + let appProvidedEngines = await Services.search.getAppProvidedEngines(); + for (let record of this.#pendingUpdatesMap.values()) { + let iconData; + try { + iconData = await this.#iconCollection.attachments.download(record); + } catch (ex) { + console.error("Could not download new icon", ex); + continue; + } + + for (let engine of appProvidedEngines) { + await engine.maybeUpdateIconURL( + record.engineIdentifiers, + URL.createObjectURL( + new Blob([iconData.buffer]), + record.attachment.mimetype + ) + ); + } + } + + this.#pendingUpdatesMap.clear(); + } + + /** + * Checks if the identifier matches any of the engine identifiers. + * + * @param {string} identifier + * The identifier of the engine. + * @param {string[]} engineIdentifiers + * The list of engine identifiers to match against. This can include + * wildcards at the end of strings. + * @returns {boolean} + * Returns true if the identifier matches any of the engine identifiers. + */ + _identifierMatches(identifier, engineIdentifiers) { + return engineIdentifiers.some(i => { + if (i.endsWith("*")) { + return identifier.startsWith(i.slice(0, -1)); + } + return identifier == i; + }); + } + /** * Obtains the icon list from the remote settings collection. */ async #getIconList() { - this.#iconCollection = lazy.RemoteSettings("search-config-icons"); try { this.#iconList = await this.#iconCollection.get(); } catch (ex) { @@ -98,6 +219,41 @@ class IconHandler { console.error("Failed to obtain search engine icon list records"); } } + + /** + * Called via a callback when remote settings updates the icon list. This + * stores potential updates and queues an idle observer to apply them. + * + * @param {object} payload + * The payload from the remote settings collection. + * @param {object} payload.data + * The payload data from the remote settings collection. + * @param {object[]} payload.data.created + * The list of created records. + * @param {object[]} payload.data.updated + * The list of updated records. + */ + async _onIconListUpdated({ data: { created, updated } }) { + created.forEach(record => { + this.#pendingUpdatesMap.set(record.id, record); + }); + for (let record of updated) { + if (record.new) { + this.#pendingUpdatesMap.set(record.new.id, record.new); + } + } + this.#maybeQueueIdle(); + } + + /** + * Queues an idle observer if there are pending updates. + */ + #maybeQueueIdle() { + if (this.#pendingUpdatesMap && !this.#queuedIdle) { + this.#queuedIdle = true; + lazy.idleService.addIdleObserver(this, ICON_UPDATE_ON_IDLE_DELAY); + } + } } /** @@ -113,19 +269,28 @@ export class AppProvidedSearchEngine extends SearchEngine { static iconHandler = new IconHandler(); /** - * @typedef {?Promise<string>} - * A promise for the blob URL of the icon. We save the promise to avoid - * reentrancy issues. + * A promise for the blob URL of the icon. We save the promise to avoid + * reentrancy issues. + * + * @type {?Promise<string>} */ #blobURLPromise = null; /** - * @typedef {?string} - * The identifier from the configuration. + * The identifier from the configuration. + * + * @type {?string} */ #configurationId = null; /** + * Whether or not this is a general purpose search engine. + * + * @type {boolean} + */ + #isGeneralPurposeSearchEngine = false; + + /** * @param {object} options * The options for this search engine. * @param {object} options.config @@ -231,11 +396,15 @@ export class AppProvidedSearchEngine extends SearchEngine { return true; } + /** + * Whether or not this engine is a "general" search engine, e.g. is it for + * generally searching the web, or does it have a specific purpose like + * shopping. + * + * @returns {boolean} + */ get isGeneralPurposeEngine() { - return !!( - this._extensionID && - lazy.SearchUtils.GENERAL_SEARCH_ENGINE_IDS.has(this._extensionID) - ); + return this.#isGeneralPurposeSearchEngine; } /** @@ -258,6 +427,36 @@ export class AppProvidedSearchEngine extends SearchEngine { } /** + * This will update the icon URL for the search engine if the engine + * identifier matches the given engine identifiers. + * + * @param {string[]} engineIdentifiers + * The engine identifiers to check against. + * @param {string} blobURL + * The new icon URL for the search engine. + */ + async maybeUpdateIconURL(engineIdentifiers, blobURL) { + // TODO: Bug 1875912. Once newSearchConfigEnabled has been enabled, we will + // be able to use `this.id` instead of `this.#configurationId`. At that + // point, `IconHandler._identifierMatches` can be made into a private + // function, as this if statement can be handled within `IconHandler.observe`. + if ( + !AppProvidedSearchEngine.iconHandler._identifierMatches( + this.#configurationId, + engineIdentifiers + ) + ) { + return; + } + if (this.#blobURLPromise) { + URL.revokeObjectURL(await this.#blobURLPromise); + this.#blobURLPromise = null; + } + this.#blobURLPromise = Promise.resolve(blobURL); + lazy.SearchUtils.notifyAction(this, lazy.SearchUtils.MODIFIED_TYPE.CHANGED); + } + + /** * Creates a JavaScript object that represents this engine. * * @returns {object} @@ -283,6 +482,12 @@ export class AppProvidedSearchEngine extends SearchEngine { #init(engineConfig) { this._orderHint = engineConfig.orderHint; this._telemetryId = engineConfig.identifier; + this.#isGeneralPurposeSearchEngine = + engineConfig.classification == "general"; + + if (engineConfig.charset) { + this._queryCharset = engineConfig.charset; + } if (engineConfig.telemetrySuffix) { this._telemetryId += `-${engineConfig.telemetrySuffix}`; diff --git a/toolkit/components/search/SearchEngineSelector.sys.mjs b/toolkit/components/search/SearchEngineSelector.sys.mjs index 0c9fb7cb70..acc1044ced 100644 --- a/toolkit/components/search/SearchEngineSelector.sys.mjs +++ b/toolkit/components/search/SearchEngineSelector.sys.mjs @@ -261,20 +261,24 @@ export class SearchEngineSelector { continue; } - let variants = - config.variants?.filter(variant => - this.#matchesUserEnvironment(variant, userEnv) - ) ?? []; + let variant = config.variants?.findLast(variant => + this.#matchesUserEnvironment(variant, userEnv) + ); - if (!variants.length) { + if (!variant) { continue; } + let subVariant = variant.subVariants?.findLast(subVariant => + this.#matchesUserEnvironment(subVariant, userEnv) + ); + let engine = structuredClone(config.base); engine.identifier = config.identifier; + engine = this.#deepCopyObject(engine, variant); - for (let variant of variants) { - engine = this.#deepCopyObject(engine, variant); + if (subVariant) { + engine = this.#deepCopyObject(engine, subVariant); } for (let override of this._configurationOverrides) { @@ -359,6 +363,10 @@ export class SearchEngineSelector { continue; } + if (["subVariants"].includes(key)) { + continue; + } + if (typeof source[key] == "object" && !Array.isArray(source[key])) { if (key in target) { this.#deepCopyObject(target[key], source[key]); @@ -431,14 +439,15 @@ export class SearchEngineSelector { user.version ) && this.#matchesChannel(config.environment.channels, user.channel) && - this.#matchesApplication(config.environment.applications, user.appName) + this.#matchesApplication(config.environment.applications, user.appName) && + !this.#hasDeviceType(config.environment) ); } /** * @param {string} userDistro * The distribution from the user's environment. - * @param {Array} configDistro + * @param {string[]} configDistro * An array of distributions for the particular environment in the config. * @returns {boolean} * True if the user's distribution is included in the config distribution @@ -497,7 +506,7 @@ export class SearchEngineSelector { } /** - * @param {Array} configChannels + * @param {string[]} configChannels * Release channels such as nightly, beta, release, esr. * @param {string} userChannel * The user's channel. @@ -514,7 +523,7 @@ export class SearchEngineSelector { } /** - * @param {Array} configApps + * @param {string[]} configApps * The applications such as firefox, firefox-android, firefox-ios, * focus-android, and focus-ios. * @param {string} userApp @@ -532,6 +541,21 @@ export class SearchEngineSelector { } /** + * Generally the device type option should only be used when the application + * is selected to be on an android or iOS based product. However, we support + * rejecting if this is non-empty in case of future requirements that we haven't + * predicted. + * + * @param {object} environment + * An environment section from the engine configuration. + * @returns {boolean} + * Returns true if there is a device type section and it is not empty. + */ + #hasDeviceType(environment) { + return !!environment.deviceType?.length; + } + + /** * Determines whether the region and locale constraints in the config * environment applies to a user given what region and locale they are using. * diff --git a/toolkit/components/search/SearchService.sys.mjs b/toolkit/components/search/SearchService.sys.mjs index b9de8e0bb3..a645cc05e8 100644 --- a/toolkit/components/search/SearchService.sys.mjs +++ b/toolkit/components/search/SearchService.sys.mjs @@ -173,7 +173,7 @@ class ParseSubmissionResult { /** * String containing the sought terms. This can be an empty string in case no - * terms were specified or the URL does not represent a search submission.* + * terms were specified or the URL does not represent a search submission. * * @type {string} */ @@ -967,6 +967,10 @@ export class SearchService { ); } + getAlternateDomains(domain) { + return lazy.SearchStaticData.getAlternateDomains(domain); + } + /** * This is a nsITimerCallback for the timerManager notification that is * registered for handling updates to search engines. Only OpenSearch engines @@ -991,9 +995,10 @@ export class SearchService { /** * A deferred promise that is resolved when initialization has finished. * + * Resolved when initalization has successfully finished, and rejected if it + * has failed. + * * @type {Promise} - * Resolved when initalization has successfully finished, and rejected if it - * has failed. */ #initDeferredPromise = Promise.withResolvers(); @@ -1078,10 +1083,10 @@ export class SearchService { * engine, as suggested by the configuration. * For the legacy configuration, this is the user visible name. * - * @type {object} - * * This is prefixed with _ rather than # because it is * called in a test. + * + * @type {object} */ _searchDefault = null; @@ -1113,6 +1118,16 @@ export class SearchService { #startupRemovedExtensions = new Set(); /** + * Used in #parseSubmissionMap + * + * @typedef {object} submissionMapEntry + * @property {nsISearchEngine} engine + * The search engine. + * @property {string} termsParameterName + * The search term parameter name. + */ + + /** * This map is built lazily after the available search engines change. It * allows quick parsing of an URL representing a search submission into the * search engine name and original terms. @@ -1120,12 +1135,7 @@ export class SearchService { * The keys are strings containing the domain name and lowercase path of the * engine submission, for example "www.google.com/search". * - * The values are objects with these properties: - * { - * engine: The associated nsISearchEngine. - * termsParameterName: Name of the URL parameter containing the search - * terms, for example "q". - * } + * @type {Map<string, submissionMapEntry>|null} */ #parseSubmissionMap = null; diff --git a/toolkit/components/search/SearchSettings.sys.mjs b/toolkit/components/search/SearchSettings.sys.mjs index fed0dd1808..e355316595 100644 --- a/toolkit/components/search/SearchSettings.sys.mjs +++ b/toolkit/components/search/SearchSettings.sys.mjs @@ -94,11 +94,14 @@ export class SearchSettings { #settings = null; /** - * @type {object} A deep copy of #settings. - * #cachedSettings is updated when we read the settings from disk and when - * we write settings to disk. #cachedSettings is compared with #settings - * before we do a write to disk. If there's no change to the settings - * attributes, then we don't write the settings to disk. + * #cachedSettings is updated when we read the settings from disk and when + * we write settings to disk. #cachedSettings is compared with #settings + * before we do a write to disk. If there's no change to the settings + * attributes, then we don't write the settings to disk. + * + * This is a deep copy of #settings. + * + * @type {object} */ #cachedSettings = {}; diff --git a/toolkit/components/search/SearchSuggestions.sys.mjs b/toolkit/components/search/SearchSuggestions.sys.mjs index 32cd25a0cd..4a43975576 100644 --- a/toolkit/components/search/SearchSuggestions.sys.mjs +++ b/toolkit/components/search/SearchSuggestions.sys.mjs @@ -4,8 +4,9 @@ const lazy = {}; ChromeUtils.defineESModuleGetters(lazy, { - FormAutoCompleteResult: "resource://gre/modules/FormAutoComplete.sys.mjs", - FormHistoryClient: "resource://gre/modules/FormAutoComplete.sys.mjs", + FormHistoryAutoCompleteResult: + "resource://gre/modules/FormHistoryAutoComplete.sys.mjs", + FormHistoryClient: "resource://gre/modules/FormHistoryAutoComplete.sys.mjs", SearchSuggestionController: "resource://gre/modules/SearchSuggestionController.sys.mjs", @@ -29,7 +30,7 @@ class SuggestAutoComplete { /** * Notifies the front end of new results. * - * @param {FormAutoCompleteResult} result + * @param {FormHistoryAutoCompleteResult} result * Any previous form history result. * @private */ @@ -168,7 +169,7 @@ class SuggestAutoComplete { ...historyEntry, }) ); - let autoCompleteResult = new lazy.FormAutoCompleteResult( + let autoCompleteResult = new lazy.FormHistoryAutoCompleteResult( client, formHistoryEntries, this.#suggestionController.formHistoryParam, diff --git a/toolkit/components/search/SearchUtils.sys.mjs b/toolkit/components/search/SearchUtils.sys.mjs index 27c8f8ad03..1262bb28b1 100644 --- a/toolkit/components/search/SearchUtils.sys.mjs +++ b/toolkit/components/search/SearchUtils.sys.mjs @@ -434,12 +434,12 @@ XPCOMUtils.defineLazyPreferenceGetter( false ); -XPCOMUtils.defineLazyPreferenceGetter( - SearchUtils, - "newSearchConfigEnabled", - "browser.search.newSearchConfig.enabled", - false -); +ChromeUtils.defineLazyGetter(SearchUtils, "newSearchConfigEnabled", () => { + return Services.prefs.getBoolPref( + "browser.search.newSearchConfig.enabled", + false + ); +}); // Can't use defineLazyPreferenceGetter because we want the value // from the default branch diff --git a/toolkit/components/search/docs/SearchEngineConfiguration.rst b/toolkit/components/search/docs/SearchEngineConfiguration.rst index c782f9f7c3..ca6009a0ef 100644 --- a/toolkit/components/search/docs/SearchEngineConfiguration.rst +++ b/toolkit/components/search/docs/SearchEngineConfiguration.rst @@ -9,19 +9,23 @@ user's region and locale. Configuration Management ======================== -The application stores a dump of the configuration that is used for first -initialisation. Subsequent updates to the configuration are either updates to the -static dump, or they may be served via remote servers. +The configuration is delivered and managed via `remote settings`_. There are +:searchfox:`dumps <services/settings/dumps/main/>` of the configuration +that are shipped with the application, for use on first startup of a fresh profile, +or when a client has not been able to receive remote settings updates for +whatever reason. -The mechanism of delivering the settings dumps to the Search Service is -`the remote settings`_. - -Remote settings ---------------- +Remote Settings Bucket +---------------------- The remote settings bucket for the search engine configuration list is -``search-config``. The version that is currently being delivered -to clients can be `viewed live`_. +``search-config-v2``. The version that is currently being delivered +to clients can be `viewed live`_. There are additional remote settings buckets +with information for each search engine. These buckets are listed below. + +- `search-config-icons`_ is a mapping of icons to a search engine. +- `search-config-overrides-v2`_ contains information that may override engines + properties in search-config-v2. Configuration Schema ==================== @@ -30,43 +34,13 @@ The configuration format is defined via a `JSON schema`_. The search engine configuration schema is `stored in mozilla-central`_ and is uploaded to the Remote Settings server at convenient times after it changes. -An outline of the schema may be found on the `Search Configuration Schema`_ page. - -Updating Search Engine WebExtensions -==================================== - -Updates for application provided search engine WebExtensions are provided via -`Normandy`_. - -It is likely that updates for search engine WebExtensions will be -received separately to configuration updates which may or may not be directly -related. As a result several situations may occur: - - - The updated WebExtension is for an app-provided engine already in-use by - the user. - - - In this case, the search service will apply the changes to the - app-provided engine's data. - - - A WebExtension addition/update is for an app-provided engine that is not - in-use by the user, or not in the configuration. - - - In this case, the search service will ignore the WebExtension. - - If the configuration (search or user) is updated later and the - new engine is added, then the Search Service will start to use the - new engine. - - - A configuration update is received that needs a WebExtension that is - not found locally. - - - In this case, the search service will ignore the missing engine and - continue without it. - - When the WebExtension is delivered, the search engine will then be - installed and added. +An outline of the schemas may be found on the `Search Configuration Schema`_ page. -.. _the remote settings: /services/settings/index.html +.. _remote settings: /services/settings/index.html .. _JSON schema: https://json-schema.org/ .. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ .. _Search Configuration Schema: SearchConfigurationSchema.html -.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records -.. _Normandy: /toolkit/components/normandy/normandy/services.html +.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config-v2/records +.. _search-config-icons: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config-icons/records +.. _search-config-overrides-v2: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config-overrides-v2/records +.. _search-default-override-allowlist: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-default-override-allowlist/records diff --git a/toolkit/components/search/docs/SearchEngineConfigurationArchive.rst b/toolkit/components/search/docs/SearchEngineConfigurationArchive.rst new file mode 100644 index 0000000000..a6aba91a42 --- /dev/null +++ b/toolkit/components/search/docs/SearchEngineConfigurationArchive.rst @@ -0,0 +1,72 @@ +====================================== +Search Engine Configuration (Archived) +====================================== + +The search engine configuration is a mapping that is used to determine the +list of search engines for each user. The mapping is primarily based on the +user's region and locale. + +Configuration Management +======================== + +The application stores a dump of the configuration that is used for first +initialisation. Subsequent updates to the configuration are either updates to the +static dump, or they may be served via remote servers. + +The mechanism of delivering the settings dumps to the Search Service is +`the remote settings`_. + +Remote settings +--------------- + +The remote settings bucket for the search engine configuration list is +``search-config``. The version that is currently being delivered +to clients can be `viewed live`_. + +Configuration Schema +==================== + +The configuration format is defined via a `JSON schema`_. The search engine +configuration schema is `stored in mozilla-central`_ and is uploaded to the +Remote Settings server at convenient times after it changes. + +An outline of the schema may be found on the `Search Configuration Schema`_ page. + +Updating Search Engine WebExtensions +==================================== + +Updates for application provided search engine WebExtensions are provided via +`Normandy`_. + +It is likely that updates for search engine WebExtensions will be +received separately to configuration updates which may or may not be directly +related. As a result several situations may occur: + + - The updated WebExtension is for an app-provided engine already in-use by + the user. + + - In this case, the search service will apply the changes to the + app-provided engine's data. + + - A WebExtension addition/update is for an app-provided engine that is not + in-use by the user, or not in the configuration. + + - In this case, the search service will ignore the WebExtension. + - If the configuration (search or user) is updated later and the + new engine is added, then the Search Service will start to use the + new engine. + + - A configuration update is received that needs a WebExtension that is + not found locally. + + - In this case, the search service will ignore the missing engine and + continue without it. + - When the WebExtension is delivered, the search engine will then be + installed and added. + +.. _the remote settings: /services/settings/index.html +.. _JSON schema: https://json-schema.org/ +.. _stored in mozilla-central: https://searchfox.org/mozilla-central/source/toolkit/components/search/schema/ +.. _Search Configuration Schema: SearchConfigurationSchema.html +.. _viewed live: https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/search-config/records +.. _Normandy: /toolkit/components/normandy/normandy/services.html diff --git a/toolkit/components/search/docs/index.rst b/toolkit/components/search/docs/index.rst index d8d4bc8d8f..10ae8118bf 100644 --- a/toolkit/components/search/docs/index.rst +++ b/toolkit/components/search/docs/index.rst @@ -32,6 +32,14 @@ Contents Preferences Telemetry +Contents for search-config (archived) +===================================== + +.. toctree:: + :maxdepth: 2 + + SearchEngineConfigurationArchive + API Reference ------------- diff --git a/toolkit/components/search/nsISearchService.idl b/toolkit/components/search/nsISearchService.idl index 4421c25974..b59e9a9399 100644 --- a/toolkit/components/search/nsISearchService.idl +++ b/toolkit/components/search/nsISearchService.idl @@ -290,13 +290,13 @@ interface nsISearchService : nsISupports * @return |true| if the search service is now initialized, |false| if * initialization has not been triggered yet. */ - readonly attribute bool isInitialized; + readonly attribute boolean isInitialized; /** * Determine whether initialization has been completed successfully. * */ - readonly attribute bool hasSuccessfullyInitialized; + readonly attribute boolean hasSuccessfullyInitialized; /** @@ -548,4 +548,14 @@ interface nsISearchService : nsISupports * "https://www.google.com/search?q=terms". */ nsISearchParseSubmissionResult parseSubmissionURL(in AString url); + + /** + * Returns a list of alternate domains for a given search engine domain. + * + * @param domain + * The domain of the search engine. + * @returns {Array} + * An array which contains all alternate domains. + */ + Array<ACString> getAlternateDomains(in ACString domain); }; diff --git a/toolkit/components/search/schema/search-config-v2-schema.json b/toolkit/components/search/schema/search-config-v2-schema.json index cfd0124fa3..7b335ed2a6 100644 --- a/toolkit/components/search/schema/search-config-v2-schema.json +++ b/toolkit/components/search/schema/search-config-v2-schema.json @@ -102,13 +102,12 @@ }, "applications": { "title": "Application Identifiers", - "description": "The application(s) this section applies to (default/not specified is everywhere).", + "description": "The application(s) this section applies to (default/not specified is everywhere). `firefox` relates to Firefox Desktop.", "type": "array", "items": { "type": "string", "pattern": "^[a-z-]{0,100}$", "enum": [ - "", "firefox", "firefox-android", "firefox-ios", @@ -129,12 +128,23 @@ "description": "The maximum application version this section applies to (less-than comparison).", "type": "string", "pattern": "^[0-9a-z.]{0,20}$" + }, + "deviceType": { + "title": "Device Type", + "description": "The device type(s) this section applies to. On desktop when this property is specified and non-empty, the associated section will be ignored.", + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z-]{0,100}$", + "enum": ["smartphone", "tablet"] + }, + "uniqueItems": true } } }, "partnerCode": { "title": "Partner Code", - "description": "The partner code for the engine or variant. This will be inserted into parameters which include '{pc}'", + "description": "The partner code for the engine or variant. This will be inserted into parameters which include '{partnerCode}'", "type": "string", "pattern": "^[a-zA-Z0-9-_]*$" }, @@ -191,7 +201,7 @@ }, "value": { "title": "Value", - "description": "The parameter value, this may be a static value, or additionally contain a parameter replacement, e.g. {inputEncoding}. For the partner code parameter, this field should be {pc}.", + "description": "The parameter value, this may be a static value, or additionally contain a parameter replacement, e.g. {inputEncoding}. For the partner code parameter, this field should be {partnerCode}.", "type": "string", "pattern": "^[a-zA-Z0-9-_{}]*$" }, @@ -327,7 +337,7 @@ }, "variants": { "title": "Variants", - "description": "This section describes variations of this search engine that may occur depending on the user's environment. If multiple sections match a user's environment, then all matching sections are applied cumulatively in the order in the array.", + "description": "This section describes variations of this search engine that may occur depending on the user's environment. The last variant that matches the user's environment will be applied to the engine, subvariants may also be applied.", "type": "array", "items": { "type": "object", @@ -345,18 +355,46 @@ }, "telemetrySuffix": { "title": "Telemetry Suffix", - "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`. There should always be a suffix supplied if the partner code is different.", + "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`. There should always be a suffix supplied if the partner code is different for a reason other than being on a different platform.", "type": "string", "pattern": "^[a-zA-Z0-9-]*$" }, "urls": { "$ref": "#/definitions/urls" + }, + "subVariants": { + "title": "Subvariants", + "description": "This section describes subvariations of this search engine that may occur depending on the user's environment. The last subvariant that matches the user's environment will be applied to the engine.", + "type": "array", + "items": { + "type": "object", + "properties": { + "environment": { + "$ref": "#/definitions/environment" + }, + "partnerCode": { + "$ref": "#/definitions/partnerCode" + }, + "optional": { + "title": "Optional", + "description": "This search engine is presented as an option that the user may enable. It is not included in the initial list of search engines. If not specified, defaults to false.", + "type": "boolean" + }, + "telemetrySuffix": { + "title": "Telemetry Suffix", + "description": "Suffix that is appended to the search engine identifier following a dash, i.e. `<identifier>-<suffix>`. There should always be a suffix supplied if the partner code is different for a reason other than being on a different platform.", + "type": "string", + "pattern": "^[a-zA-Z0-9-]*$" + }, + "urls": { + "$ref": "#/definitions/urls" + } + }, + "required": ["environment"] + } } }, - "required": ["environment"], - "dependencies": { - "partnerCode": ["telemetrySuffix"] - } + "required": ["environment"] } } }, diff --git a/toolkit/components/search/tests/xpcshell/data/search-config-v2.json b/toolkit/components/search/tests/xpcshell/data/search-config-v2.json index 569e16dfe4..bca7cc3bdf 100644 --- a/toolkit/components/search/tests/xpcshell/data/search-config-v2.json +++ b/toolkit/components/search/tests/xpcshell/data/search-config-v2.json @@ -112,6 +112,7 @@ "recordType": "engine", "identifier": "engine-resourceicon", "base": { + "classification": "general", "name": "engine-resourceicon", "urls": { "search": { @@ -151,6 +152,7 @@ "recordType": "engine", "identifier": "engine-reordered", "base": { + "classification": "general", "name": "Test search engine (Reordered)", "urls": { "search": { diff --git a/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json b/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json index 98bdfa26ff..ac5f7f77cd 100644 --- a/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json +++ b/toolkit/components/search/tests/xpcshell/data1/search-config-v2.json @@ -9,7 +9,7 @@ "searchTermParamName": "q" } }, - "classification": "unknown" + "classification": "general" }, "variants": [{ "environment": { "allRegionsAndLocales": true } }], "identifier": "engine1", @@ -24,7 +24,7 @@ "searchTermParamName": "q" } }, - "classification": "unknown" + "classification": "general" }, "variants": [{ "environment": { "allRegionsAndLocales": true } }], "identifier": "engine2", @@ -39,7 +39,7 @@ "searchTermParamName": "q" } }, - "classification": "unknown" + "classification": "general" }, "variants": [ { @@ -58,7 +58,7 @@ "searchTermParamName": "q" } }, - "classification": "unknown" + "classification": "general" }, "variants": [ { diff --git a/toolkit/components/search/tests/xpcshell/head_search.js b/toolkit/components/search/tests/xpcshell/head_search.js index 1c0504e277..72bc90185c 100644 --- a/toolkit/components/search/tests/xpcshell/head_search.js +++ b/toolkit/components/search/tests/xpcshell/head_search.js @@ -307,6 +307,101 @@ async function setupRemoteSettings() { } /** + * Reads the specified file from the data directory and returns its contents as + * an Uint8Array. + * + * @param {string} filename + * The name of the file to read. + * @returns {Promise<Uint8Array>} + * The contents of the file in an Uint8Array. + */ +async function getFileDataBuffer(filename) { + return IOUtils.read(PathUtils.join(do_get_cwd().path, "data", filename)); +} + +/** + * Creates a mock attachment record for use in remote settings related tests. + * + * @param {object} item + * An object containing the details of the attachment. + * @param {string} item.filename + * The name of the attachmnet file in the data directory. + * @param {string[]} item.engineIdentifiers + * The engine identifiers for the attachment. + * @param {number} item.imageSize + * The size of the image. + * @param {string} [item.id] + * The ID to use for the record. If not provided, a new UUID will be generated. + * @param {number} [item.lastModified] + * The last modified time for the record. Defaults to the current time. + */ +async function mockRecordWithAttachment({ + filename, + engineIdentifiers, + imageSize, + id = Services.uuid.generateUUID().toString(), + lastModified = Date.now(), +}) { + let buffer = await getFileDataBuffer(filename); + + // Generate a hash. + let hasher = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + hasher.init(Ci.nsICryptoHash.SHA256); + hasher.update(buffer, buffer.length); + + let hash = hasher.finish(false); + hash = Array.from(hash, (_, i) => + ("0" + hash.charCodeAt(i).toString(16)).slice(-2) + ).join(""); + + let record = { + id, + engineIdentifiers, + imageSize, + attachment: { + hash, + location: `${filename}`, + filename, + size: buffer.byteLength, + mimetype: "application/json", + }, + last_modified: lastModified, + }; + + let attachment = { + record, + blob: new Blob([buffer]), + }; + + return { record, attachment }; +} + +/** + * Inserts an attachment record into the remote settings collection. + * + * @param {RemoteSettingsClient} client + * The remote settings client to use. + * @param {object} item + * An object containing the details of the attachment - see mockRecordWithAttachment. + * @param {boolean} [addAttachmentToCache] + * Whether to add the attachment file to the cache. Defaults to true. + */ +async function insertRecordIntoCollection( + client, + item, + addAttachmentToCache = true +) { + let { record, attachment } = await mockRecordWithAttachment(item); + await client.db.create(record); + if (addAttachmentToCache) { + await client.attachments.cacheImpl.set(record.id, attachment); + } + await client.db.importChanges({}, record.last_modified); +} + +/** * Helper function that sets up a server and respnds to region * fetch requests. * diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js b/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js index 72c4d4f04f..c58ac1c25b 100644 --- a/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js @@ -115,7 +115,7 @@ class SearchConfigTest { async setup(version = "42.0") { if (SearchUtils.newSearchConfigEnabled) { updateAppInfo({ - name: "XPCShell", + name: "firefox", ID: "xpcshell@tests.mozilla.org", version, platformVersion: version, diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js index 4024385729..eab28a7ba9 100644 --- a/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_qwant.js @@ -14,6 +14,9 @@ const test = new SearchConfigTest({ { locales: ["fr"], }, + { + regions: ["be", "ch", "es", "fr", "it", "nl"], + }, ], }, details: [ @@ -28,9 +31,30 @@ const test = new SearchConfigTest({ }); add_setup(async function () { - await test.setup(); + if (SearchUtils.newSearchConfigEnabled) { + await test.setup(); + } else { + await test.setup("124.0"); + } }); add_task(async function test_searchConfig_qwant() { await test.run(); }); + +add_task( + { skip_if: () => SearchUtils.newSearchConfigEnabled }, + async function test_searchConfig_qwant_pre124() { + const version = "123.0"; + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + version, + version + ); + // For pre-124, Qwant is not available in the extra regions. + test._config.available.included.pop(); + + await test.run(); + } +); diff --git a/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js index 86686b62f7..f727d60719 100644 --- a/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js +++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_searchconfig_validates.js @@ -172,6 +172,32 @@ add_task( add_task( { skip_if: () => !SearchUtils.newSearchConfigEnabled }, + async function test_search_config_valid_partner_codes() { + delete SearchUtils.newSearchConfigEnabled; + SearchUtils.newSearchConfigEnabled = true; + + let selector = new SearchEngineSelector(() => {}); + + for (let entry of await selector.getEngineConfiguration()) { + if (entry.recordType == "engine") { + for (let variant of entry.variants) { + if ( + "partnerCode" in variant && + "distributions" in variant.environment + ) { + Assert.ok( + variant.telemetrySuffix, + `${entry.identifier} should have a telemetrySuffix when a distribution is specified with a partnerCode.` + ); + } + } + } + } + } +); + +add_task( + { skip_if: () => !SearchUtils.newSearchConfigEnabled }, async function test_search_config_override_validates_to_schema() { let selector = new SearchEngineSelector(() => {}); diff --git a/toolkit/components/search/tests/xpcshell/test_appProvided_engine.js b/toolkit/components/search/tests/xpcshell/test_appProvided_engine.js new file mode 100644 index 0000000000..56297a9d2c --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_appProvided_engine.js @@ -0,0 +1,182 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests that ensure application provided engines have all base fields set up + * correctly from the search configuration. + */ + +"use strict"; + +let CONFIG = [ + { + identifier: "testEngine", + recordType: "engine", + base: { + aliases: ["testEngine1", "testEngine2"], + charset: "EUC-JP", + classification: "general", + name: "testEngine name", + partnerCode: "pc", + urls: { + search: { + base: "https://example.com/1", + // Method defaults to GET + params: [ + { name: "partnerCode", value: "{partnerCode}" }, + { name: "starbase", value: "Regula I" }, + { name: "experiment", value: "Genesis" }, + { + name: "accessPoint", + searchAccessPoint: { + addressbar: "addressbar", + contextmenu: "contextmenu", + homepage: "homepage", + newtab: "newtab", + searchbar: "searchbar", + }, + }, + ], + searchTermParamName: "search", + }, + suggestions: { + base: "https://example.com/2", + method: "POST", + searchTermParamName: "suggestions", + }, + trending: { + base: "https://example.com/3", + searchTermParamName: "trending", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + identifier: "testOtherValuesEngine", + recordType: "engine", + base: { + classification: "unknown", + name: "testOtherValuesEngine name", + urls: { + search: { + base: "https://example.com/1", + searchTermParamName: "search", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + recordType: "defaultEngines", + globalDefault: "engine_no_initial_icon", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +add_setup(async function () { + await SearchTestUtils.useTestEngines("simple-engines", null, CONFIG); + await Services.search.init(); +}); + +add_task(async function test_engine_with_all_params_set() { + let engine = Services.search.getEngineById( + "testEngine@search.mozilla.orgdefault" + ); + Assert.ok(engine, "Should have found the engine"); + + Assert.equal( + engine.name, + "testEngine name", + "Should have the correct engine name" + ); + Assert.deepEqual( + engine.aliases, + ["@testEngine1", "@testEngine2"], + "Should have the correct aliases" + ); + Assert.ok( + engine.isGeneralPurposeEngine, + "Should be a general purpose engine" + ); + Assert.equal( + engine.wrappedJSObject.queryCharset, + "EUC-JP", + "Should have the correct encoding" + ); + + let submission = engine.getSubmission("test"); + Assert.equal( + submission.uri.spec, + "https://example.com/1?partnerCode=pc&starbase=Regula%20I&experiment=Genesis&accessPoint=searchbar&search=test", + "Should have the correct search URL" + ); + Assert.ok(!submission.postData, "Should not have postData for a GET url"); + + let suggestSubmission = engine.getSubmission( + "test", + SearchUtils.URL_TYPE.SUGGEST_JSON + ); + Assert.equal( + suggestSubmission.uri.spec, + "https://example.com/2", + "Should have the correct suggestion URL" + ); + Assert.equal( + suggestSubmission.postData.data.data, + "suggestions=test", + "Should have the correct postData for a POST URL" + ); + + let trendingSubmission = engine.getSubmission( + "test", + SearchUtils.URL_TYPE.TRENDING_JSON + ); + Assert.equal( + trendingSubmission.uri.spec, + "https://example.com/3?trending=test" + ); + Assert.ok(!submission.postData, "Should not have postData for a GET url"); +}); + +add_task(async function test_engine_with_some_params_set() { + let engine = Services.search.getEngineById( + "testOtherValuesEngine@search.mozilla.orgdefault" + ); + Assert.ok(engine, "Should have found the engine"); + + Assert.equal( + engine.name, + "testOtherValuesEngine name", + "Should have the correct engine name" + ); + Assert.deepEqual(engine.aliases, [], "Should have no aliases"); + Assert.ok( + !engine.isGeneralPurposeEngine, + "Should not be a general purpose engine" + ); + Assert.equal( + engine.wrappedJSObject.queryCharset, + "UTF-8", + "Should default to UTF-8 charset" + ); + Assert.equal( + engine.getSubmission("test").uri.spec, + "https://example.com/1?search=test", + "Should have the correct search URL" + ); + Assert.equal( + engine.getSubmission("test", SearchUtils.URL_TYPE.SUGGEST_JSON), + null, + "Should not have a suggestions URL" + ); + Assert.equal( + engine.getSubmission("test", SearchUtils.URL_TYPE.TRENDING_JSON), + null, + "Should not have a trending URL" + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_appProvided_icons.js b/toolkit/components/search/tests/xpcshell/test_appProvided_icons.js index e4d8033993..f6cc8b2415 100644 --- a/toolkit/components/search/tests/xpcshell/test_appProvided_icons.js +++ b/toolkit/components/search/tests/xpcshell/test_appProvided_icons.js @@ -53,7 +53,11 @@ let TESTS = [ icons: [ { filename: "remoteIcon.ico", - engineIdentifiers: ["engine_non_default_sized_icon"], + engineIdentifiers: [ + // This also tests multiple engine idenifiers works. + "enterprise_shuttle", + "engine_non_default_sized_icon", + ], imageSize: 32, }, ], @@ -77,64 +81,6 @@ let TESTS = [ }, ]; -async function getFileDataBuffer(filename) { - let data = await IOUtils.read( - PathUtils.join(do_get_cwd().path, "data", filename) - ); - return new TextEncoder().encode(data).buffer; -} - -async function mockRecordWithAttachment({ - filename, - engineIdentifiers, - imageSize, -}) { - let buffer = await getFileDataBuffer(filename); - - let stream = Cc["@mozilla.org/io/arraybuffer-input-stream;1"].createInstance( - Ci.nsIArrayBufferInputStream - ); - stream.setData(buffer, 0, buffer.byteLength); - - // Generate a hash. - let hasher = Cc["@mozilla.org/security/hash;1"].createInstance( - Ci.nsICryptoHash - ); - hasher.init(Ci.nsICryptoHash.SHA256); - hasher.updateFromStream(stream, -1); - let hash = hasher.finish(false); - hash = Array.from(hash, (_, i) => - ("0" + hash.charCodeAt(i).toString(16)).slice(-2) - ).join(""); - - let record = { - id: Services.uuid.generateUUID().toString(), - engineIdentifiers, - imageSize, - attachment: { - hash, - location: `main-workspace/search-config-icons/${filename}`, - filename, - size: buffer.byteLength, - mimetype: "application/json", - }, - }; - - let attachment = { - record, - blob: new Blob([buffer]), - }; - - return { record, attachment }; -} - -async function insertRecordIntoCollection(client, db, item) { - let { record, attachment } = await mockRecordWithAttachment(item); - await db.create(record); - await client.attachments.cacheImpl.set(record.id, attachment); - await db.importChanges({}, Date.now()); -} - add_setup(async function () { let client = RemoteSettings("search-config-icons"); let db = client.db; @@ -159,10 +105,7 @@ add_setup(async function () { if ("icons" in test) { for (let icon of test.icons) { - await insertRecordIntoCollection(client, db, { - ...icon, - id: test.engineId, - }); + await insertRecordIntoCollection(client, { ...icon }); } } } diff --git a/toolkit/components/search/tests/xpcshell/test_appProvided_icons_updates.js b/toolkit/components/search/tests/xpcshell/test_appProvided_icons_updates.js new file mode 100644 index 0000000000..4159b9f0fb --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_appProvided_icons_updates.js @@ -0,0 +1,324 @@ +/* Any copyright is dedicated to the Public Domain. +https://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests to ensure that icons for application provided engines are correctly + * updated from remote settings. + */ + +"use strict"; + +// A skeleton configuration that gets filled in from TESTS during `add_setup`. +let CONFIG = [ + { + identifier: "engine_no_initial_icon", + recordType: "engine", + base: { + name: "engine_no_initial_icon name", + urls: { + search: { + base: "https://example.com/1", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + identifier: "engine_icon_updates", + recordType: "engine", + base: { + name: "engine_icon_updates name", + urls: { + search: { + base: "https://example.com/2", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + identifier: "engine_icon_not_local", + recordType: "engine", + base: { + name: "engine_icon_not_local name", + urls: { + search: { + base: "https://example.com/3", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + identifier: "engine_icon_out_of_date", + recordType: "engine", + base: { + name: "engine_icon_out_of_date name", + urls: { + search: { + base: "https://example.com/4", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + recordType: "defaultEngines", + globalDefault: "engine_no_initial_icon", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +async function assertIconMatches(actualIconData, expectedIcon) { + let expectedBuffer = new Uint8Array(await getFileDataBuffer(expectedIcon)); + + Assert.equal( + actualIconData.length, + expectedBuffer.length, + "Should have received matching buffer lengths for the expected icon" + ); + Assert.ok( + actualIconData.every((value, index) => value === expectedBuffer[index]), + "Should have received matching data for the expected icon" + ); +} + +async function assertEngineIcon(engineName, expectedIcon) { + let engine = Services.search.getEngineByName(engineName); + let engineIconURL = await engine.getIconURL(16); + + if (expectedIcon) { + Assert.notEqual( + engineIconURL, + null, + "Should have an icon URL for the engine." + ); + + let response = await fetch(engineIconURL); + let buffer = new Uint8Array(await response.arrayBuffer()); + + await assertIconMatches(buffer, expectedIcon); + } else { + Assert.equal( + engineIconURL, + null, + "Should not have an icon URL for the engine." + ); + } +} + +let originalIconId = Services.uuid.generateUUID().toString(); +let client; + +add_setup(async function setup() { + useHttpServer(); + SearchTestUtils.useMockIdleService(); + + client = RemoteSettings("search-config-icons"); + await client.db.clear(); + + sinon.stub(client.attachments, "_baseAttachmentsURL").returns(gDataUrl); + + // Add some initial records and attachments into the remote settings collection. + await insertRecordIntoCollection(client, { + id: originalIconId, + filename: "remoteIcon.ico", + // This uses a wildcard match to test the icon is still applied correctly. + engineIdentifiers: ["engine_icon_upd*"], + imageSize: 16, + }); + // This attachment is not cached, so we don't have it locally. + await insertRecordIntoCollection( + client, + { + id: Services.uuid.generateUUID().toString(), + filename: "bigIcon.ico", + engineIdentifiers: [ + // This also tests multiple engine idenifiers works. + "enterprise", + "next_generation", + "engine_icon_not_local", + ], + imageSize: 16, + }, + false + ); + + // Add a record that is out of date, and update it with a newer one, but don't + // cache the attachment for the new one. + let outOfDateRecordId = Services.uuid.generateUUID().toString(); + await insertRecordIntoCollection( + client, + { + id: outOfDateRecordId, + filename: "remoteIcon.ico", + engineIdentifiers: ["engine_icon_out_of_date"], + imageSize: 16, + // 10 minutes ago. + lastModified: Date.now() - 600000, + }, + true + ); + let { record } = await mockRecordWithAttachment({ + id: outOfDateRecordId, + filename: "bigIcon.ico", + engineIdentifiers: ["engine_icon_out_of_date"], + imageSize: 16, + }); + await client.db.update(record); + await client.db.importChanges({}, record.lastModified); + + await SearchTestUtils.useTestEngines("simple-engines", null, CONFIG); + await Services.search.init(); + + // Testing that an icon is not local generates a `Could not find {id}...` + // message. + consoleAllowList.push("Could not find"); +}); + +add_task(async function test_icon_added_unknown_engine() { + // If the engine is unknown, and this is a new icon, we should still download + // the icon, in case the engine is added to the configuration later. + let newIconId = Services.uuid.generateUUID().toString(); + + let mock = await mockRecordWithAttachment({ + id: newIconId, + filename: "bigIcon.ico", + engineIdentifiers: ["engine_unknown"], + imageSize: 16, + }); + await client.db.update(mock.record, Date.now()); + + await client.emit("sync", { + data: { + current: [mock.record], + created: [mock.record], + updated: [], + deleted: [], + }, + }); + + SearchTestUtils.idleService._fireObservers("idle"); + + let icon; + await TestUtils.waitForCondition(async () => { + try { + icon = await client.attachments.get(mock.record); + } catch (ex) { + // Do nothing. + } + return !!icon; + }, "Should have loaded the icon into the attachments store."); + + await assertIconMatches(new Uint8Array(icon.buffer), "bigIcon.ico"); +}); + +add_task(async function test_icon_added_existing_engine() { + // If the engine is unknown, and this is a new icon, we should still download + // it, in case the engine is added to the configuration later. + let newIconId = Services.uuid.generateUUID().toString(); + + let mock = await mockRecordWithAttachment({ + id: newIconId, + filename: "bigIcon.ico", + engineIdentifiers: ["engine_no_initial_icon"], + imageSize: 16, + }); + await client.db.update(mock.record, Date.now()); + + let promiseEngineUpdated = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + await client.emit("sync", { + data: { + current: [mock.record], + created: [mock.record], + updated: [], + deleted: [], + }, + }); + + SearchTestUtils.idleService._fireObservers("idle"); + + await promiseEngineUpdated; + await assertEngineIcon("engine_no_initial_icon name", "bigIcon.ico"); +}); + +add_task(async function test_icon_updated() { + // Test that when an update for an engine icon is received, the engine is + // correctly updated. + + // Check the engine has the expected icon to start with. + await assertEngineIcon("engine_icon_updates name", "remoteIcon.ico"); + + // Update the icon for the engine. + let mock = await mockRecordWithAttachment({ + id: originalIconId, + filename: "bigIcon.ico", + engineIdentifiers: ["engine_icon_upd*"], + imageSize: 16, + }); + await client.db.update(mock.record, Date.now()); + + let promiseEngineUpdated = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + + await client.emit("sync", { + data: { + current: [mock.record], + created: [], + updated: [{ new: mock.record }], + deleted: [], + }, + }); + SearchTestUtils.idleService._fireObservers("idle"); + + await promiseEngineUpdated; + await assertEngineIcon("engine_icon_updates name", "bigIcon.ico"); +}); + +add_task(async function test_icon_not_local() { + // Tests that a download is queued and triggered when the icon for an engine + // is not in either the local dump nor the cache. + + await assertEngineIcon("engine_icon_not_local name", null); + + // A download should have been queued, so fire idle to trigger it. + let promiseEngineUpdated = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + SearchTestUtils.idleService._fireObservers("idle"); + await promiseEngineUpdated; + + await assertEngineIcon("engine_icon_not_local name", "bigIcon.ico"); +}); + +add_task(async function test_icon_out_of_date() { + // Tests that a download is queued and triggered when the icon for an engine + // is not in either the local dump nor the cache. + + await assertEngineIcon("engine_icon_out_of_date name", "remoteIcon.ico"); + + // A download should have been queued, so fire idle to trigger it. + let promiseEngineUpdated = SearchTestUtils.promiseSearchNotification( + SearchUtils.MODIFIED_TYPE.CHANGED, + SearchUtils.TOPIC_ENGINE_MODIFIED + ); + SearchTestUtils.idleService._fireObservers("idle"); + await promiseEngineUpdated; + + await assertEngineIcon("engine_icon_out_of_date name", "bigIcon.ico"); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js b/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js index 12cb6568e7..2f269cc016 100644 --- a/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js +++ b/toolkit/components/search/tests/xpcshell/test_defaultEngine_fallback.js @@ -17,6 +17,15 @@ let appDefault; let appPrivateDefault; +async function getSearchConfig() { + let workDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile); + let configFileName = + "file://" + PathUtils.join(workDir.path, "data", "search-config-v2.json"); + + let response = await fetch(configFileName); + return response.json(); +} + add_setup(async function () { useHttpServer(); await SearchTestUtils.useTestEngines(); @@ -292,10 +301,33 @@ add_task(async function test_default_fallback_remove_default_no_visible() { add_task( async function test_default_fallback_remove_default_no_visible_or_general() { - // Reset. Services.search.restoreDefaultEngines(); - Services.search.defaultEngine = Services.search.defaultPrivateEngine = - appPrivateDefault; + + // For this test, we need to change any general search engines to unknown, + // so that we can test what happens in the unlikely event that there are no + // general search engines. + if (SearchUtils.newSearchConfigEnabled) { + let searchConfig = await getSearchConfig(); + for (let entry of searchConfig.data) { + if ( + entry.recordType == "engine" && + entry.base.classification == "general" + ) { + entry.base.classification = "unknown"; + } + } + const settings = await RemoteSettings(SearchUtils.SETTINGS_KEY); + settings.get.returns(searchConfig.data); + Services.search.wrappedJSObject.reset(); + await Services.search.init(); + + appPrivateDefault = await Services.search.getDefaultPrivate(); + + Services.search.defaultEngine = appPrivateDefault; + } else { + Services.search.defaultEngine = Services.search.defaultPrivateEngine = + appPrivateDefault; + } // Remove all but the default engine. let visibleEngines = await Services.search.getVisibleEngines(); @@ -310,7 +342,9 @@ add_task( "Should only have one visible engine" ); - SearchUtils.GENERAL_SEARCH_ENGINE_IDS.clear(); + if (!SearchUtils.newSearchConfigEnabled) { + SearchUtils.GENERAL_SEARCH_ENGINE_IDS.clear(); + } const observer = new SearchObserver( [ diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js index bf56984cde..401392b955 100644 --- a/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_environment.js @@ -388,6 +388,55 @@ const CONFIG_VERSIONS = [ }, ]; +const CONFIG_DEVICE_TYPE_LAYOUT = [ + { + recordType: "engine", + identifier: "engine-no-device-type", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-single-device-type", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + deviceType: ["tablet"], + }, + }, + ], + }, + { + recordType: "engine", + identifier: "engine-multiple-device-type", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + deviceType: ["tablet", "smartphone"], + }, + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + const engineSelector = new SearchEngineSelector(); let settings; let settingOverrides; @@ -793,3 +842,15 @@ add_task(async function test_engine_selector_does_not_match_optional_engines() { "Should match engines where optional flag is false or undefined" ); }); + +add_task(async function test_engine_selector_match_device_type() { + await assertActualEnginesEqualsExpected( + CONFIG_DEVICE_TYPE_LAYOUT, + { + locale: "en-CA", + region: "CA", + }, + ["engine-no-device-type"], + "Should only match engines with no device type." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_subvariants.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_subvariants.js new file mode 100644 index 0000000000..ed61362ca6 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_subvariants.js @@ -0,0 +1,142 @@ +/* Any copyright is dedicated to the Public Domain. +https://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const CONFIG = [ + { + recordType: "engine", + identifier: "engine-1", + base: {}, + variants: [ + { + environment: { + allRegionsAndLocales: true, + }, + partnerCode: "variant-partner-code", + subVariants: [ + { + environment: { regions: ["CA", "FR"] }, + telemetrySuffix: "subvariant-telemetry", + }, + { + environment: { regions: ["GB", "FR"] }, + partnerCode: "subvariant-partner-code", + }, + ], + }, + ], + }, + { + recordType: "defaultEngines", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + +const engineSelector = new SearchEngineSelector(); +let settings; +let configStub; + +/** + * This function asserts if the actual engines returned equals the expected + * engines. + * + * @param {object} config + * A fake search config containing engines. + * @param {object} userEnv + * A fake user's environment including locale and region, experiment, etc. + * @param {Array} expectedEngines + * The array of expected engines to be returned from the fake config. + * @param {string} message + * The assertion message. + */ +async function assertActualEnginesEqualsExpected( + config, + userEnv, + expectedEngines, + message +) { + engineSelector._configuration = null; + configStub.returns(config); + let { engines } = await engineSelector.fetchEngineConfiguration(userEnv); + + Assert.deepEqual(engines, expectedEngines, message); +} + +add_setup(async function () { + settings = await RemoteSettings(SearchUtils.NEW_SETTINGS_KEY); + configStub = sinon.stub(settings, "get"); +}); + +add_task(async function test_no_subvariants_match() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "fi", + region: "FI", + }, + [ + { + identifier: "engine-1", + partnerCode: "variant-partner-code", + }, + ], + "Should match no subvariants." + ); +}); + +add_task(async function test_matching_subvariant_with_properties() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-GB", + region: "GB", + }, + [ + { + identifier: "engine-1", + partnerCode: "subvariant-partner-code", + }, + ], + "Should match subvariant with subvariant properties." + ); +}); + +add_task(async function test_matching_variant_and_subvariant_with_properties() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "en-CA", + region: "CA", + }, + [ + { + identifier: "engine-1", + partnerCode: "variant-partner-code", + telemetrySuffix: "subvariant-telemetry", + }, + ], + "Should match subvariant with subvariant properties." + ); +}); + +add_task(async function test_matching_two_subvariant_with_properties() { + await assertActualEnginesEqualsExpected( + CONFIG, + { + locale: "fr", + region: "FR", + }, + [ + { + identifier: "engine-1", + partnerCode: "subvariant-partner-code", + }, + ], + "Should match the last subvariant with subvariant properties." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js b/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js index 0fd57f2094..51a7f0de09 100644 --- a/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js +++ b/toolkit/components/search/tests/xpcshell/test_engine_selector_variants.js @@ -122,7 +122,7 @@ add_task(async function test_no_variants_match() { ); }); -add_task(async function test_match_and_apply_all_variants() { +add_task(async function test_match_and_apply_last_variants() { await assertActualEnginesEqualsExpected( CONFIG, { @@ -133,11 +133,10 @@ add_task(async function test_match_and_apply_all_variants() { { identifier: "engine-1", urls: { search: { params: [{ name: "partner-code", value: "foo" }] } }, - telemetrySuffix: "telemetry", searchTermParamName: "search-param", }, ], - "Should match all variants and apply each variant property cumulatively." + "Should match and apply last variant." ); }); diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest.js index 4a30eb741a..d24970534f 100644 --- a/toolkit/components/search/tests/xpcshell/test_searchSuggest.js +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest.js @@ -3,7 +3,7 @@ /* eslint-disable mozilla/no-arbitrary-setTimeout */ /** - * Testing search suggestions from SearchSuggestionController.jsm. + * Testing search suggestions from SearchSuggestionController.sys.mjs. */ "use strict"; @@ -23,7 +23,7 @@ const SEARCH_TELEMETRY_LATENCY = "SEARCH_SUGGESTIONS_LATENCY_MS"; // We must make sure the FormHistoryStartup component is // initialized in order for it to respond to FormHistory -// requests from nsFormAutoComplete.js. +// requests from FormHistoryAutoComplete.sys.mjs. var formHistoryStartup = Cc[ "@mozilla.org/satchel/form-history-startup;1" ].getService(Ci.nsIObserver); diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js index 042c74d86a..bb4dda9e3f 100644 --- a/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest_cookies.js @@ -2,7 +2,7 @@ * http://creativecommons.org/publicdomain/zero/1.0/ */ /** - * Test that search suggestions from SearchSuggestionController.jsm don't store + * Test that search suggestions from SearchSuggestionController.sys.mjs don't store * cookies. */ @@ -14,7 +14,7 @@ const { SearchSuggestionController } = ChromeUtils.importESModule( // We must make sure the FormHistoryStartup component is // initialized in order for it to respond to FormHistory -// requests from nsFormAutoComplete.js. +// requests from FormHistoryAutoComplete.sys.mjs. var formHistoryStartup = Cc[ "@mozilla.org/satchel/form-history-startup;1" ].getService(Ci.nsIObserver); diff --git a/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js b/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js index 063f3ada49..b7750c0f30 100644 --- a/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js +++ b/toolkit/components/search/tests/xpcshell/test_searchSuggest_private.js @@ -2,7 +2,7 @@ * http://creativecommons.org/publicdomain/zero/1.0/ */ /** - * Test that search suggestions from SearchSuggestionController.jsm operate + * Test that search suggestions from SearchSuggestionController.sys.mjs operate * correctly in private mode. */ diff --git a/toolkit/components/search/tests/xpcshell/test_search_config_v2_nimbus.js b/toolkit/components/search/tests/xpcshell/test_search_config_v2_nimbus.js new file mode 100644 index 0000000000..56746a614f --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/test_search_config_v2_nimbus.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. +https://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Test to verify search-config-v2 preference is correctly toggled via a Nimbus + variable. */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + SearchTestUtils: "resource://testing-common/SearchTestUtils.sys.mjs", + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + ExperimentFakes: "resource://testing-common/NimbusTestUtils.sys.mjs", + ExperimentManager: "resource://nimbus/lib/ExperimentManager.sys.mjs", + AppProvidedSearchEngine: + "resource://gre/modules/AppProvidedSearchEngine.sys.mjs", +}); + +add_task(async function test_nimbus_experiment_enabled() { + Assert.equal( + Services.prefs.getBoolPref("browser.search.newSearchConfig.enabled"), + false, + "newSearchConfig.enabled PREF should initially be false." + ); + + await ExperimentManager.onStartup(); + await ExperimentAPI.ready(); + + let doExperimentCleanup = await ExperimentFakes.enrollWithFeatureConfig( + { + featureId: "search", + value: { + newSearchConfigEnabled: true, + }, + }, + { isRollout: true } + ); + + Assert.equal( + Services.prefs.getBoolPref("browser.search.newSearchConfig.enabled"), + true, + "After toggling the Nimbus variable, the current value of newSearchConfig.enabled PREF should be true." + ); + + Assert.equal( + SearchUtils.newSearchConfigEnabled, + true, + "After toggling the Nimbus variable, newSearchConfig.enabled should be cached as true." + ); + + await AddonTestUtils.promiseStartupManager(); + await Services.search.init(); + await SearchTestUtils.useTestEngines(); + + let { engines: engines2 } = + await Services.search.wrappedJSObject._fetchEngineSelectorEngines(); + + Assert.ok( + engines2.some(engine => engine.identifier), + "Engines in the search-config-v2 format should have an identifier." + ); + + let appProvidedEngines = + await Services.search.wrappedJSObject.getAppProvidedEngines(); + + Assert.ok( + appProvidedEngines.every( + engine => engine instanceof AppProvidedSearchEngine + ), + "All application provided engines for search-config-v2 should be instances of AppProvidedSearchEngine." + ); + + await doExperimentCleanup(); + + Assert.equal( + Services.prefs.getBoolPref("browser.search.newSearchConfig.enabled"), + false, + "After experiment unenrollment, the newSearchConfig.enabled should be false." + ); + + Assert.equal( + SearchUtils.newSearchConfigEnabled, + true, + "After experiment unenrollment, newSearchConfig.enabled should be cached as true." + ); +}); diff --git a/toolkit/components/search/tests/xpcshell/test_settings_persist.js b/toolkit/components/search/tests/xpcshell/test_settings_persist.js index 5c2cbd85c4..e3310a1fa2 100644 --- a/toolkit/components/search/tests/xpcshell/test_settings_persist.js +++ b/toolkit/components/search/tests/xpcshell/test_settings_persist.js @@ -81,7 +81,7 @@ add_setup(async function () { registerCleanupFunction(AddonTestUtils.promiseShutdownManager); await AddonTestUtils.promiseStartupManager(); // This is only needed as otherwise events will not be properly notified - // due to https://searchfox.org/mozilla-central/source/toolkit/components/search/SearchUtils.jsm#186 + // due to https://searchfox.org/mozilla-central/rev/5f0a7ca8968ac5cef8846e1d970ef178b8b76dcc/toolkit/components/search/SearchSettings.sys.mjs#41-42 let settingsFileWritten = promiseAfterSettings(); await Services.search.init(false); Services.search.wrappedJSObject._removeObservers(); diff --git a/toolkit/components/search/tests/xpcshell/xpcshell.toml b/toolkit/components/search/tests/xpcshell/xpcshell.toml index 7dd023cbec..899ac2d711 100644 --- a/toolkit/components/search/tests/xpcshell/xpcshell.toml +++ b/toolkit/components/search/tests/xpcshell/xpcshell.toml @@ -78,9 +78,18 @@ support-files = [ ["test_appDefaultEngine.js"] +["test_appProvided_engine.js"] +prefs = ["browser.search.newSearchConfig.enabled=true"] +support-files = [ + "../../schema/search-config-v2-schema.json", +] + ["test_appProvided_icons.js"] prefs = ["browser.search.newSearchConfig.enabled=true"] +["test_appProvided_icons_updates.js"] +prefs = ["browser.search.newSearchConfig.enabled=true"] + ["test_async.js"] ["test_config_engine_params.js"] @@ -128,6 +137,8 @@ tags = "remotesettings searchmain" ["test_engine_selector_environment.js"] +["test_engine_selector_subvariants.js"] + ["test_engine_selector_variants.js"] ["test_engine_set_alias.js"] @@ -254,6 +265,10 @@ support-files = [ ["test_searchUrlDomain.js"] +["test_search_config_v2_nimbus.js"] +prefs = ["browser.search.newSearchConfig.enabled=false"] +skip-if = ["appname == 'thunderbird'"] # Test relies on normandy. + ["test_selectedEngine.js"] ["test_sendSubmissionURL.js"] |