diff options
Diffstat (limited to 'browser/components/search')
59 files changed, 3210 insertions, 515 deletions
diff --git a/browser/components/search/DomainToCategoriesMap.worker.mjs b/browser/components/search/DomainToCategoriesMap.worker.mjs new file mode 100644 index 0000000000..07dc52cfb8 --- /dev/null +++ b/browser/components/search/DomainToCategoriesMap.worker.mjs @@ -0,0 +1,101 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { PromiseWorker } from "resource://gre/modules/workers/PromiseWorker.mjs"; + +/** + * Boilerplate to connect with the main thread PromiseWorker. + */ +const worker = new PromiseWorker.AbstractWorker(); +worker.dispatch = function (method, args = []) { + return agent[method](...args); +}; +worker.postMessage = function (message, ...transfers) { + self.postMessage(message, ...transfers); +}; +worker.close = function () { + self.close(); +}; + +self.addEventListener("message", msg => worker.handleMessage(msg)); +self.addEventListener("unhandledrejection", function (error) { + throw error.reason; +}); + +/** + * Stores and manages the Domain-to-Categories Map. + */ +class Agent { + /** + * @type {Map<string, Array<number>>} Hashes mapped to categories and values. + */ + #map = new Map(); + + /** + * Converts data from the array directly into a Map. + * + * @param {Array<ArrayBuffer>} fileContents Files + * @returns {boolean} Returns whether the Map contains results. + */ + populateMap(fileContents) { + this.#map.clear(); + + for (let fileContent of fileContents) { + let obj; + try { + obj = JSON.parse(new TextDecoder().decode(fileContent)); + } catch (ex) { + return false; + } + for (let objKey in obj) { + if (Object.hasOwn(obj, objKey)) { + this.#map.set(objKey, obj[objKey]); + } + } + } + return this.#map.size > 0; + } + + /** + * Retrieves scores for the hash from the map. + * + * @param {string} hash Key to look up in the map. + * @returns {Array<number>} + */ + getScores(hash) { + if (this.#map.has(hash)) { + return this.#map.get(hash); + } + return []; + } + + /** + * Empties the internal map. + * + * @returns {boolean} + */ + emptyMap() { + this.#map.clear(); + return true; + } + + /** + * Test only function to allow the map to contain information without + * having to go through Remote Settings. + * + * @param {object} obj The data to directly import into the Map. + * @returns {boolean} Whether the map contains values. + */ + overrideMapForTests(obj) { + this.#map.clear(); + for (let objKey in obj) { + if (Object.hasOwn(obj, objKey)) { + this.#map.set(objKey, obj[objKey]); + } + } + return this.#map.size > 0; + } +} + +const agent = new Agent(); diff --git a/browser/components/search/SearchOneOffs.sys.mjs b/browser/components/search/SearchOneOffs.sys.mjs index 0459af092a..9f0245a6be 100644 --- a/browser/components/search/SearchOneOffs.sys.mjs +++ b/browser/components/search/SearchOneOffs.sys.mjs @@ -14,7 +14,7 @@ const EMPTY_ADD_ENGINES = []; /** * Defines the search one-off button elements. These are displayed at the bottom * of the address bar and search bar. The address bar buttons are a subclass in - * browser/components/urlbar/UrlbarSearchOneOffs.jsm. If you are adding a new + * browser/components/urlbar/UrlbarSearchOneOffs.sys.mjs. If you are adding a new * subclass, see "Methods for subclasses to override" below. */ export class SearchOneOffs { @@ -466,7 +466,7 @@ export class SearchOneOffs { this.settingsButton.id = origin + "-anon-search-settings"; let engines = (await this.getEngineInfo()).engines; - this._rebuildEngineList(engines, addEngines); + await this._rebuildEngineList(engines, addEngines); } /** @@ -477,14 +477,14 @@ export class SearchOneOffs { * @param {Array} addEngines * The engines that can be added. */ - _rebuildEngineList(engines, addEngines) { + async _rebuildEngineList(engines, addEngines) { for (let i = 0; i < engines.length; ++i) { let engine = engines[i]; let button = this.document.createXULElement("button"); button.engine = engine; button.id = this._buttonIDForEngine(engine); let iconURL = - engine.getIconURL() || + (await engine.getIconURL()) || "chrome://browser/skin/search-engine-placeholder.png"; button.setAttribute("image", iconURL); button.setAttribute("class", "searchbar-engine-one-off-item"); @@ -981,7 +981,7 @@ export class SearchOneOffs { this.handleSearchCommand(event, engine); } - _on_command(event) { + async _on_command(event) { let target = event.target; if (target == this.settingsButton) { @@ -1043,7 +1043,7 @@ export class SearchOneOffs { // search engine first. Doing this as opposed to rebuilding all the // one-off buttons avoids flicker. let iconURL = - currentEngine.getIconURL() || + (await currentEngine.getIconURL()) || "chrome://browser/skin/search-engine-placeholder.png"; button.setAttribute("image", iconURL); button.setAttribute("tooltiptext", currentEngine.name); diff --git a/browser/components/search/SearchSERPTelemetry.sys.mjs b/browser/components/search/SearchSERPTelemetry.sys.mjs index 00105241bb..fa593be08c 100644 --- a/browser/components/search/SearchSERPTelemetry.sys.mjs +++ b/browser/components/search/SearchSERPTelemetry.sys.mjs @@ -7,6 +7,7 @@ import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; const lazy = {}; ChromeUtils.defineESModuleGetters(lazy, { + BasePromiseWorker: "resource://gre/modules/PromiseWorker.sys.mjs", BrowserSearchTelemetry: "resource:///modules/BrowserSearchTelemetry.sys.mjs", PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", Region: "resource://gre/modules/Region.sys.mjs", @@ -94,6 +95,10 @@ XPCOMUtils.defineLazyPreferenceGetter( export const SearchSERPTelemetryUtils = { ACTIONS: { CLICKED: "clicked", + // specific to cookie banner + CLICKED_ACCEPT: "clicked_accept", + CLICKED_REJECT: "clicked_reject", + CLICKED_MORE_OPTIONS: "clicked_more_options", EXPANDED: "expanded", SUBMITTED: "submitted", }, @@ -103,6 +108,7 @@ export const SearchSERPTelemetryUtils = { AD_LINK: "ad_link", AD_SIDEBAR: "ad_sidebar", AD_SITELINK: "ad_sitelink", + COOKIE_BANNER: "cookie_banner", INCONTENT_SEARCHBOX: "incontent_searchbox", NON_ADS_LINK: "non_ads_link", REFINED_SEARCH_BUTTONS: "refined_search_buttons", @@ -403,6 +409,10 @@ class TelemetryHandler { ); } + newProvider.ignoreLinkRegexps = provider.ignoreLinkRegexps?.length + ? provider.ignoreLinkRegexps.map(r => new RegExp(r)) + : []; + newProvider.nonAdsLinkRegexps = provider.nonAdsLinkRegexps?.length ? provider.nonAdsLinkRegexps.map(r => new RegExp(r)) : []; @@ -412,6 +422,9 @@ class TelemetryHandler { regexp: new RegExp(provider.shoppingTab.regexp), }; } + + newProvider.nonAdsLinkQueryParamNames = + provider.nonAdsLinkQueryParamNames ?? []; return newProvider; }); this._contentHandler._searchProviderInfo = this._searchProviderInfo; @@ -429,8 +442,8 @@ class TelemetryHandler { this._contentHandler._reportPageWithAdImpressions(info, browser); } - reportPageDomains(info, browser) { - this._contentHandler._reportPageDomains(info, browser); + async reportPageDomains(info, browser) { + await this._contentHandler._reportPageDomains(info, browser); } reportPageImpression(info, browser) { @@ -1212,7 +1225,7 @@ class ContentHandler { ); } - observe(aSubject, aTopic, aData) { + observe(aSubject, aTopic) { switch (aTopic) { case "http-on-stop-request": this._reportChannelBandwidth(aSubject); @@ -1330,6 +1343,11 @@ class ContentHandler { let originURL = wrappedChannel.originURI?.spec; let url = wrappedChannel.finalURL; + + if (info.ignoreLinkRegexps.some(r => r.test(url))) { + return; + } + // Some channels re-direct by loading pages that return 200. The result // is the channel will have an originURL that changes from the SERP to // either a nonAdsRegexp or an extraAdServersRegexps. This is typical @@ -1434,6 +1452,30 @@ class ContentHandler { let startFindComponent = Cu.now(); let parsedUrl = new URL(url); + + // Organic links may contain query param values mapped to links shown + // on the SERP at page load. If a stored component depends on that + // value, we need to be able to recover it or else we'll always consider + // it a non_ads_link. + if ( + info.nonAdsLinkQueryParamNames.length && + info.nonAdsLinkRegexps.some(r => r.test(url)) + ) { + let newParsedUrl; + for (let key of info.nonAdsLinkQueryParamNames) { + let paramValue = parsedUrl.searchParams.get(key); + if (paramValue) { + try { + newParsedUrl = /^https?:\/\//.test(paramValue) + ? new URL(paramValue) + : new URL(paramValue, parsedUrl.origin); + break; + } catch (e) {} + } + } + parsedUrl = newParsedUrl ?? parsedUrl; + } + // Determine the component type of the link. let type; for (let [ @@ -1629,8 +1671,8 @@ class ContentHandler { * * @param {object} info * The search provider infomation for the page. - * @param {string} info.type - * The component type that was clicked on. + * @param {string} info.target + * The target component that was interacted with. * @param {string} info.action * The action taken on the page. * @param {object} browser @@ -1643,22 +1685,23 @@ class ContentHandler { } let telemetryState = item.browserTelemetryStateMap.get(browser); let impressionId = telemetryState?.impressionId; - if (info.type && impressionId) { + if (info.target && impressionId) { lazy.logConsole.debug(`Recorded page action:`, { impressionId: telemetryState.impressionId, - type: info.type, + target: info.target, action: info.action, }); Glean.serp.engagement.record({ impression_id: impressionId, action: info.action, - target: info.type, + target: info.target, }); impressionIdsWithoutEngagementsSet.delete(impressionId); // In-content searches are not be categorized with a type, so they will // not be picked up in the network processes. if ( - info.type == SearchSERPTelemetryUtils.COMPONENTS.INCONTENT_SEARCHBOX && + info.target == + SearchSERPTelemetryUtils.COMPONENTS.INCONTENT_SEARCHBOX && info.action == SearchSERPTelemetryUtils.ACTIONS.SUBMITTED ) { telemetryState.searchBoxSubmitted = true; @@ -1667,6 +1710,7 @@ class ContentHandler { SearchSERPTelemetryUtils.INCONTENT_SOURCES.SEARCHBOX ); } + Services.obs.notifyObservers(null, "reported-page-with-action"); } else { lazy.logConsole.warn( "Expected to report a", @@ -1712,23 +1756,23 @@ class ContentHandler { } /** - * Initiates the categorization and reporting of domains extracted from - * SERPs. - * - * @param {object} info - * The search provider infomation for the page. - * @param {Set} info.nonAdDomains - The non-ad domains extracted from the page. - * @param {Set} info.adDomains - The ad domains extracted from the page. - * @param {object} browser - * The browser associated with the page. - */ - _reportPageDomains(info, browser) { + * Initiates the categorization and reporting of domains extracted from + * SERPs. + * + * @param {object} info + * The search provider infomation for the page. + * @param {Set} info.nonAdDomains + The non-ad domains extracted from the page. + * @param {Set} info.adDomains + The ad domains extracted from the page. + * @param {object} browser + * The browser associated with the page. + */ + async _reportPageDomains(info, browser) { let item = this._findItemForBrowser(browser); let telemetryState = item.browserTelemetryStateMap.get(browser); if (lazy.serpEventTelemetryCategorization && telemetryState) { - let result = SearchSERPCategorization.maybeCategorizeSERP( + let result = await SearchSERPCategorization.maybeCategorizeSERP( info.nonAdDomains, info.adDomains, item.info.provider @@ -1808,12 +1852,10 @@ class SERPCategorizer { * Domains from organic results extracted from the page. * @param {Set} adDomains * Domains from ad results extracted from the page. - * @param {string} provider - * The provider associated with the page. * @returns {CategorizationResult | null} * The final categorization result. Returns null if the map was empty. */ - maybeCategorizeSERP(nonAdDomains, adDomains, provider) { + async maybeCategorizeSERP(nonAdDomains, adDomains) { // Per DS, if the map was empty (e.g. because of a technical issue // downloading the data), we shouldn't report telemetry. // Thus, there is no point attempting to categorize the SERP. @@ -1822,15 +1864,13 @@ class SERPCategorizer { } let resultsToReport = {}; - let processedDomains = this.processDomains(nonAdDomains, provider); - let results = this.applyCategorizationLogic(processedDomains); + let results = await this.applyCategorizationLogic(nonAdDomains); resultsToReport.organic_category = results.category; resultsToReport.organic_num_domains = results.num_domains; resultsToReport.organic_num_unknown = results.num_unknown; resultsToReport.organic_num_inconclusive = results.num_inconclusive; - processedDomains = this.processDomains(adDomains, provider); - results = this.applyCategorizationLogic(processedDomains); + results = await this.applyCategorizationLogic(adDomains); resultsToReport.sponsored_category = results.category; resultsToReport.sponsored_num_domains = results.num_domains; resultsToReport.sponsored_num_unknown = results.num_unknown; @@ -1851,22 +1891,18 @@ class SERPCategorizer { * The final categorization results. Keys are: "category", "num_domains", * "num_unknown" and "num_inconclusive". */ - applyCategorizationLogic(domains) { + async applyCategorizationLogic(domains) { let domainInfo = {}; let domainsCount = 0; let unknownsCount = 0; let inconclusivesCount = 0; - // Per a request from Data Science, we need to limit the number of domains - // categorized to 10 non-ad domains and 10 ad domains. - domains = new Set( - [...domains].slice(0, CATEGORIZATION_SETTINGS.MAX_DOMAINS_TO_CATEGORIZE) - ); - for (let domain of domains) { domainsCount++; - let categoryCandidates = SearchSERPDomainToCategoriesMap.get(domain); + let categoryCandidates = await SearchSERPDomainToCategoriesMap.get( + domain + ); if (!categoryCandidates.length) { unknownsCount++; @@ -1919,65 +1955,6 @@ class SERPCategorizer { }; } - /** - * Processes raw domains extracted from the SERP into their final form before - * categorization. - * - * @param {Set} domains - * The domains extracted from the page. - * @param {string} provider - * The provider associated with the page. - * @returns {Set} processedDomains - * The final set of processed domains for a page. - */ - processDomains(domains, provider) { - let processedDomains = new Set(); - - for (let domain of domains) { - // Don't include domains associated with the search provider. - if ( - domain.startsWith(`${provider}.`) || - domain.includes(`.${provider}.`) - ) { - continue; - } - let domainWithoutSubdomains = this.#stripDomainOfSubdomains(domain); - // We may have come across the same domain twice, once with www. prefixed - // and another time without. - if ( - domainWithoutSubdomains && - !processedDomains.has(domainWithoutSubdomains) - ) { - processedDomains.add(domainWithoutSubdomains); - } - } - - return processedDomains; - } - - /** - * Helper to strip domains of any subdomains. - * - * @param {string} domain - * The domain to strip of any subdomains. - * @returns {object} browser - * The given domain with any subdomains removed. - */ - #stripDomainOfSubdomains(domain) { - let tld; - // Can throw an exception if the input has too few domain levels. - try { - tld = Services.eTLD.getKnownPublicSuffixFromHost(domain); - } catch (ex) { - return ""; - } - - let domainWithoutTLD = domain.substring(0, domain.length - tld.length); - let secondLevelDomain = domainWithoutTLD.split(".").at(-2); - - return secondLevelDomain ? `${secondLevelDomain}.${tld}` : ""; - } - #chooseRandomlyFrom(categories) { let randIdx = Math.floor(Math.random() * categories.length); return categories[randIdx]; @@ -2075,7 +2052,7 @@ class CategorizationEventScheduler { this.#init = false; } - observe(subject, topic, data) { + observe(subject, topic) { switch (topic) { case "idle": lazy.logConsole.debug("Triggering all callbacks due to idle."); @@ -2167,17 +2144,13 @@ class CategorizationRecorder { */ /** - * Maps domain to categories, with data synced with Remote Settings. + * Maps domain to categories, with its data synced using Remote Settings. The + * data is downloaded from Remote Settings and stored in a map in a worker + * thread to avoid processing the data from the attachments from occupying + * the main thread. */ class DomainToCategoriesMap { /** - * Contains the domain to category scores. - * - * @type {Object<string, Array<DomainCategoryScore>> | null} - */ - #map = null; - - /** * Latest version number of the attachments. * * @type {number | null} @@ -2222,6 +2195,17 @@ class DomainToCategoriesMap { #downloadRetries = 0; /** + * Whether the mappings are empty. + */ + #empty = true; + + /** + * @type {BasePromiseWorker|null} Worker used to access the raw domain + * to categories map data. + */ + #worker = null; + + /** * Runs at application startup with startup idle tasks. If the SERP * categorization preference is enabled, it creates a Remote Settings * client to listen to updates, and populates the map. @@ -2231,14 +2215,18 @@ class DomainToCategoriesMap { return; } lazy.logConsole.debug("Initializing domain-to-categories map."); - this.#setupClientAndMap(); + this.#worker = new lazy.BasePromiseWorker( + "resource:///modules/DomainToCategoriesMap.worker.mjs", + { type: "module" } + ); + await this.#setupClientAndMap(); this.#init = true; } uninit() { if (this.#init) { lazy.logConsole.debug("Un-initializing domain-to-categories map."); - this.#clearClientAndMap(); + this.#clearClientAndWorker(); this.#cancelAndNullifyTimer(); this.#init = false; } @@ -2252,16 +2240,16 @@ class DomainToCategoriesMap { * An array containing categories and their respective score. If no record * for the domain is available, return an empty array. */ - get(domain) { + async get(domain) { if (this.empty) { return []; } - lazy.gCryptoHash.init(lazy.gCryptoHash.MD5); + lazy.gCryptoHash.init(lazy.gCryptoHash.SHA256); let bytes = new TextEncoder().encode(domain); lazy.gCryptoHash.update(bytes, domain.length); let hash = lazy.gCryptoHash.finish(true); - let rawValues = this.#map[hash] ?? []; - if (rawValues.length) { + let rawValues = await this.#worker.post("getScores", [hash]); + if (rawValues?.length) { let output = []; // Transform data into a more readable format. // [x, y] => { category: x, score: y } @@ -2292,7 +2280,7 @@ class DomainToCategoriesMap { * @returns {boolean} */ get empty() { - return !this.#map; + return this.#empty; } /** @@ -2303,8 +2291,11 @@ class DomainToCategoriesMap { * An object where the key is a hashed domain and the value is an array * containing an arbitrary number of DomainCategoryScores. */ - overrideMapForTests(domainToCategoriesMap) { - this.#map = domainToCategoriesMap; + async overrideMapForTests(domainToCategoriesMap) { + let hasResults = await this.#worker.post("overrideMapForTests", [ + domainToCategoriesMap, + ]); + this.#empty = !hasResults; } async #setupClientAndMap() { @@ -2321,7 +2312,7 @@ class DomainToCategoriesMap { await this.#clearAndPopulateMap(records); } - #clearClientAndMap() { + #clearClientAndWorker() { if (this.#client) { lazy.logConsole.debug("Removing Remote Settings client."); this.#client.off("sync", this.#onSettingsSync); @@ -2330,11 +2321,16 @@ class DomainToCategoriesMap { this.#downloadRetries = 0; } - if (this.#map) { + if (!this.#empty) { lazy.logConsole.debug("Clearing domain-to-categories map."); - this.#map = null; + this.#empty = true; this.#version = null; } + + if (this.#worker) { + this.#worker.terminate(); + this.#worker = null; + } } /** @@ -2394,11 +2390,11 @@ class DomainToCategoriesMap { * */ async #clearAndPopulateMap(records) { - // Set map to null so that if there are errors in the downloads, consumers - // will be able to know whether the map has information. Once we've - // successfully downloaded attachments and are parsing them, a non-null - // object will be created. - this.#map = null; + // Empty map so that if there are errors in the download process, callers + // querying the map won't use information we know is already outdated. + await this.#worker.post("emptyMap"); + + this.#empty = true; this.#version = null; this.#cancelAndNullifyTimer(); @@ -2408,6 +2404,7 @@ class DomainToCategoriesMap { } let fileContents = []; + let start = Cu.now(); for (let record of records) { let result; // Downloading attachments can fail. @@ -2420,10 +2417,13 @@ class DomainToCategoriesMap { } fileContents.push(result.buffer); } + ChromeUtils.addProfilerMarker( + "SearchSERPTelemetry.#clearAndPopulateMap", + start, + "Download attachments." + ); - // All attachments should have the same version number. If for whatever - // reason they don't, we should only use the attachments with the latest - // version. + // Attachments should have a version number. this.#version = this.#retrieveLatestVersion(records); if (!this.#version) { @@ -2431,37 +2431,28 @@ class DomainToCategoriesMap { return; } - // Queue the series of assignments. - for (let i = 0; i < fileContents.length; ++i) { - let buffer = fileContents[i]; - Services.tm.idleDispatchToMainThread(() => { - let start = Cu.now(); - let json; - try { - json = JSON.parse(new TextDecoder().decode(buffer)); - } catch (ex) { - // TODO: If there was an error decoding the buffer, we may want to - // dispatch an error in telemetry or try again. - return; - } - ChromeUtils.addProfilerMarker( - "SearchSERPTelemetry.#clearAndPopulateMap", - start, - "Convert buffer to JSON." - ); - if (!this.#map) { - this.#map = {}; - } - Object.assign(this.#map, json); - lazy.logConsole.debug("Updated domain-to-categories map."); - if (i == fileContents.length - 1) { - Services.obs.notifyObservers( - null, - "domain-to-categories-map-update-complete" - ); - } - }); - } + Services.tm.idleDispatchToMainThread(async () => { + start = Cu.now(); + let hasResults; + try { + hasResults = await this.#worker.post("populateMap", [fileContents]); + } catch (ex) { + console.error(ex); + } + + this.#empty = !hasResults; + + ChromeUtils.addProfilerMarker( + "SearchSERPTelemetry.#clearAndPopulateMap", + start, + "Convert contents to JSON." + ); + lazy.logConsole.debug("Updated domain-to-categories map."); + Services.obs.notifyObservers( + null, + "domain-to-categories-map-update-complete" + ); + }); } #cancelAndNullifyTimer() { diff --git a/browser/components/search/content/autocomplete-popup.js b/browser/components/search/content/autocomplete-popup.js index 2c84bf8cd7..c5a33348ff 100644 --- a/browser/components/search/content/autocomplete-popup.js +++ b/browser/components/search/content/autocomplete-popup.js @@ -18,7 +18,7 @@ constructor() { super(); - this.addEventListener("popupshowing", event => { + this.addEventListener("popupshowing", () => { // First handle deciding if we are showing the reduced version of the // popup containing only the preferences button. We do this if the // glass icon has been clicked if the text field is empty. @@ -47,7 +47,7 @@ ); }); - this.addEventListener("popuphiding", event => { + this.addEventListener("popuphiding", () => { this._oneOffButtons.removeEventListener( "SelectedOneOffButtonChanged", this @@ -232,7 +232,7 @@ } } - let uri = engine.getIconURL(); + let uri = await engine.getIconURL(); if (uri) { this.setAttribute("src", uri); } else { diff --git a/browser/components/search/content/contentSearchUI.js b/browser/components/search/content/contentSearchUI.js index 9c7387d364..f3e54db1f3 100644 --- a/browser/components/search/content/contentSearchUI.js +++ b/browser/components/search/content/contentSearchUI.js @@ -557,11 +557,11 @@ this.ContentSearchUIController = (function () { } }, - _onMsgFocusInput(event) { + _onMsgFocusInput() { this.input.focus(); }, - _onMsgBlur(event) { + _onMsgBlur() { this.input.blur(); this._hideSuggestions(); }, diff --git a/browser/components/search/content/searchbar.js b/browser/components/search/content/searchbar.js index 986a1b4d82..c872236472 100644 --- a/browser/components/search/content/searchbar.js +++ b/browser/components/search/content/searchbar.js @@ -53,7 +53,7 @@ this._setupEventListeners(); let searchbar = this; this.observer = { - observe(aEngine, aTopic, aVerb) { + observe(aEngine, aTopic) { if (aTopic == "browser-search-engine-modified") { // Make sure the engine list is refetched next time it's needed searchbar._engines = null; @@ -115,7 +115,7 @@ window.requestIdleCallback(() => { Services.search .init() - .then(aStatus => { + .then(() => { // Bail out if the binding's been destroyed if (!this._initialized) { return; @@ -470,7 +470,7 @@ } _setupEventListeners() { - this.addEventListener("click", event => { + this.addEventListener("click", () => { this._maybeSelectAll(); }); @@ -484,17 +484,17 @@ true ); - this.addEventListener("input", event => { + this.addEventListener("input", () => { this.updateGoButtonVisibility(); }); - this.addEventListener("drop", event => { + this.addEventListener("drop", () => { this.updateGoButtonVisibility(); }); this.addEventListener( "blur", - event => { + () => { // Reset the flag since we can't capture enter keyup event if the event happens // after moving the focus. this._needBrowserFocusAtEnterKeyUp = false; @@ -508,7 +508,7 @@ this.addEventListener( "focus", - event => { + () => { // Speculatively connect to the current engine's search URI (and // suggest URI, if different) to reduce request latency this.currentEngine.speculativeConnect({ @@ -576,7 +576,7 @@ } _setupTextboxEventListeners() { - this.textbox.addEventListener("input", event => { + this.textbox.addEventListener("input", () => { this.textbox.popup.removeAttribute("showonlysettings"); }); @@ -826,7 +826,7 @@ } }; - this.textbox.onkeyup = event => { + this.textbox.onkeyup = () => { // Pressing Enter key while pressing Meta key, and next, even when // releasing Enter key before releasing Meta key, the keyup event is not // fired. Therefore, if Enter keydown is detecting, continue the post diff --git a/browser/components/search/metrics.yaml b/browser/components/search/metrics.yaml index 4faff64e3e..12fd44a0e2 100644 --- a/browser/components/search/metrics.yaml +++ b/browser/components/search/metrics.yaml @@ -202,9 +202,11 @@ serp: - https://bugzilla.mozilla.org/show_bug.cgi?id=1814773 - https://bugzilla.mozilla.org/show_bug.cgi?id=1816730 - https://bugzilla.mozilla.org/show_bug.cgi?id=1816735 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849371 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1814773 - https://bugzilla.mozilla.org/show_bug.cgi?id=1816730 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849371 data_sensitivity: - interaction notification_emails: @@ -216,7 +218,13 @@ serp: action: description: > The action taken on the page. - Possible values are `clicked`, `expanded`, and `submitted`. + Possible values are: + `clicked`, + `clicked_accept`, + `clicked_reject`, + `clicked_more_options`, + `expanded`, + `submitted`. type: string target: description: > @@ -227,6 +235,7 @@ serp: `ad_link`, `ad_sidebar`, `ad_sitelink`, + `cookie_banner`, `incontent_searchbox`, `non_ads_link`, `refined_search_buttons`, @@ -240,8 +249,10 @@ serp: bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=1816728 - https://bugzilla.mozilla.org/show_bug.cgi?id=1816729 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849371 data_reviews: - https://bugzilla.mozilla.org/show_bug.cgi?id=1816728 + - https://bugzilla.mozilla.org/show_bug.cgi?id=1849371 data_sensitivity: - interaction notification_emails: @@ -258,6 +269,7 @@ serp: `ad_link`, `ad_sidebar`, `ad_sitelink`, + `cookie_banner`, `refined_search_buttons`, `shopping_tab`. Defaults to `ad_link`. diff --git a/browser/components/search/moz.build b/browser/components/search/moz.build index 9f89090aa2..0289f32979 100644 --- a/browser/components/search/moz.build +++ b/browser/components/search/moz.build @@ -6,6 +6,7 @@ EXTRA_JS_MODULES += [ "BrowserSearchTelemetry.sys.mjs", + "DomainToCategoriesMap.worker.mjs", "SearchOneOffs.sys.mjs", "SearchSERPTelemetry.sys.mjs", "SearchUIUtils.sys.mjs", diff --git a/browser/components/search/pings.yaml b/browser/components/search/pings.yaml index 727204e3fa..891d0e43ac 100644 --- a/browser/components/search/pings.yaml +++ b/browser/components/search/pings.yaml @@ -20,3 +20,27 @@ search-with: - https://bugzilla.mozilla.org/show_bug.cgi?id=1870138 notification_emails: - mkaply@mozilla.com + +serp-categorization: + description: | + A ping representing a series of SERP loads that have been categorized. Does + not contain `client_id`. Is sent after a threshold of SERP loads is reached. + reasons: + startup: | + Submitted as one of the startup idle tasks. + inactivity: | + Submitted after 2 minutes of uninterrupted activity, followed by inactivity. + threshold_reached: | + Submitted after 10 SERPs have been categorized. + include_client_id: false + send_if_empty: false + metadata: + include_info_sections: false + use_ohttp: true + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1868476 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=1868476 + notification_emails: + - fx-search-telemetry@mozilla.com + - rev-data@mozilla.com diff --git a/browser/components/search/schema/search-telemetry-schema.json b/browser/components/search/schema/search-telemetry-schema.json index b985ae0802..50b6e124fc 100644 --- a/browser/components/search/schema/search-telemetry-schema.json +++ b/browser/components/search/schema/search-telemetry-schema.json @@ -251,6 +251,15 @@ "description": "The matching regular expression." } }, + "nonAdsLinkQueryParamNames": { + "type": "array", + "title": "Non-ads link query param names", + "description": "Query param names present in non-ads link that recover the link that will be redirected to.", + "items": { + "type": "string", + "string": "The query param name to examine." + } + }, "nonAdsLinkRegexps": { "type": "array", "title": "Non-ads link matching regular expressions", @@ -366,7 +375,7 @@ "description": "The query to inspect all elements on the SERP." }, "method": { - "enum": ["data-attribute"], + "enum": ["dataAttribute"], "description": "The extraction method used for the query." }, "options": { @@ -399,19 +408,37 @@ "queryParamKey": { "type": "string", "description": "The query parameter key to inspect in the href." + }, + "queryParamValueIsHref": { + "type": "boolean", + "description": "Whether the query param value is expected to contain an href." } }, "required": ["queryParamKey"] } }, "required": ["selectors", "method"] + }, + { + "type": "object", + "properties": { + "selectors": { + "type": "string", + "description": "The query to use to inspect all elements on the SERP." + }, + "method": { + "enum": ["textContent"], + "description": "The extraction method to use for the query." + } + }, + "required": ["selectors", "method"] } ] + }, + "skipCount": { + "title": "Skip Count", + "description": "Whether to skip reporting of the count of these elements to ad_impressions. Defaults to false.", + "type": "boolean" } - }, - "skipCount": { - "title": "Skip Count", - "description": "Whether to skip reporting of the count of these elements to ad_impressions. Defaults to false.", - "type": "boolean" } } diff --git a/browser/components/search/test/browser/browser_426329.js b/browser/components/search/test/browser/browser_426329.js index 093c793048..c793f6c27e 100644 --- a/browser/components/search/test/browser/browser_426329.js +++ b/browser/components/search/test/browser/browser_426329.js @@ -292,7 +292,7 @@ add_task(async function testClearHistory() { function promiseObserver(topic) { return new Promise(resolve => { - let obs = (aSubject, aTopic, aData) => { + let obs = (aSubject, aTopic) => { Services.obs.removeObserver(obs, aTopic); resolve(aSubject); }; diff --git a/browser/components/search/test/browser/browser_contentSearch.js b/browser/components/search/test/browser/browser_contentSearch.js index 7b9328fb94..07753927f1 100644 --- a/browser/components/search/test/browser/browser_contentSearch.js +++ b/browser/components/search/test/browser/browser_contentSearch.js @@ -50,6 +50,14 @@ add_setup(async function () { await SearchTestUtils.promiseNewSearchEngine({ url: getRootDirectory(gTestPath) + "testEngine_chromeicon.xml", }); + + // Install a WebExtension based engine to allow testing passing of plain + // URIs (moz-extension://) to the content process. + await SearchTestUtils.installSearchExtension({ + icons: { + 16: "favicon.ico", + }, + }); }); add_task(async function GetState() { @@ -460,7 +468,7 @@ var currentStateObj = async function (isPrivateWindowValue, hiddenEngine = "") { ), }; for (let engine of await Services.search.getVisibleEngines()) { - let uri = engine.getIconURL(16); + let uri = await engine.getIconURL(16); state.engines.push({ name: engine.name, iconData: await iconDataFromURI(uri), @@ -476,7 +484,7 @@ var currentStateObj = async function (isPrivateWindowValue, hiddenEngine = "") { }; async function constructEngineObj(engine) { - let uriFavicon = engine.getIconURL(16); + let uriFavicon = await engine.getIconURL(16); return { name: engine.name, iconData: await iconDataFromURI(uriFavicon), @@ -491,7 +499,7 @@ function iconDataFromURI(uri) { ); } - if (!uri.startsWith("data:")) { + if (!uri.startsWith("data:") && !uri.startsWith("blob:")) { plainURIIconTested = true; return Promise.resolve(uri); } diff --git a/browser/components/search/test/browser/browser_contentSearchUI.js b/browser/components/search/test/browser/browser_contentSearchUI.js index 9196b1355c..da6044f35f 100644 --- a/browser/components/search/test/browser/browser_contentSearchUI.js +++ b/browser/components/search/test/browser/browser_contentSearchUI.js @@ -24,17 +24,6 @@ ChromeUtils.defineESModuleGetters(this, { "resource://gre/modules/SearchSuggestionController.sys.mjs", }); -const pageURL = getRootDirectory(gTestPath) + TEST_PAGE_BASENAME; -BrowserTestUtils.registerAboutPage( - registerCleanupFunction, - "test-about-content-search-ui", - pageURL, - Ci.nsIAboutModule.URI_SAFE_FOR_UNTRUSTED_CONTENT | - Ci.nsIAboutModule.URI_MUST_LOAD_IN_CHILD | - Ci.nsIAboutModule.ALLOW_SCRIPT | - Ci.nsIAboutModule.URI_CAN_LOAD_IN_PRIVILEGEDABOUT_PROCESS -); - requestLongerTimeout(2); function waitForSuggestions() { @@ -261,6 +250,19 @@ let extension1; let extension2; add_setup(async function () { + const pageURL = getRootDirectory(gTestPath) + TEST_PAGE_BASENAME; + + let cleanupAboutPage; + await BrowserTestUtils.registerAboutPage( + callback => (cleanupAboutPage = callback), + "test-about-content-search-ui", + pageURL, + Ci.nsIAboutModule.URI_SAFE_FOR_UNTRUSTED_CONTENT | + Ci.nsIAboutModule.URI_MUST_LOAD_IN_CHILD | + Ci.nsIAboutModule.ALLOW_SCRIPT | + Ci.nsIAboutModule.URI_CAN_LOAD_IN_PRIVILEGEDABOUT_PROCESS + ); + let originalOnMessageSearch = ContentSearch._onMessageSearch; let originalOnMessageManageEngines = ContentSearch._onMessageManageEngines; @@ -290,8 +292,20 @@ add_setup(async function () { } registerCleanupFunction(async () => { + // Ensure tabs are closed before we continue on with the cleanup. + for (let tab of tabs) { + BrowserTestUtils.removeTab(tab); + } + Services.search.restoreDefaultEngines(); + + await TestUtils.waitForTick(); + ContentSearch._onMessageSearch = originalOnMessageSearch; ContentSearch._onMessageManageEngines = originalOnMessageManageEngines; + + if (cleanupAboutPage) { + await cleanupAboutPage(); + } }); await promiseTab(); @@ -1096,10 +1110,6 @@ add_task(async function settings() { await msg("reset"); }); -add_task(async function cleanup() { - Services.search.restoreDefaultEngines(); -}); - function checkState( actualState, expectedInputVal, @@ -1147,10 +1157,10 @@ function checkState( } var gMsgMan; - +var tabs = []; async function promiseTab() { let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser); - registerCleanupFunction(() => BrowserTestUtils.removeTab(tab)); + tabs.push(tab); let loadedPromise = BrowserTestUtils.firstBrowserLoaded(window); openTrustedLinkIn("about:test-about-content-search-ui", "current"); diff --git a/browser/components/search/test/browser/browser_contentSearchUI_default.js b/browser/components/search/test/browser/browser_contentSearchUI_default.js index 47114fa6da..5410cfc826 100644 --- a/browser/components/search/test/browser/browser_contentSearchUI_default.js +++ b/browser/components/search/test/browser/browser_contentSearchUI_default.js @@ -58,11 +58,22 @@ async function ensureIcon(tab, expectedIcon) { "Search Icon not set." ); - Assert.equal( - computedStyle.getPropertyValue("--newtab-search-icon"), - `url(${icon})`, - "Should have the expected icon" - ); + if (icon.startsWith("blob:")) { + // We don't check the data here as `browser_contentSearch.js` performs + // those checks. + Assert.ok( + computedStyle + .getPropertyValue("--newtab-search-icon") + .startsWith("url(blob:"), + "Should have a blob URL" + ); + } else { + Assert.equal( + computedStyle.getPropertyValue("--newtab-search-icon"), + `url(${icon})`, + "Should have the expected icon" + ); + } } ); } @@ -96,7 +107,7 @@ async function runNewTabTest(isHandoff) { waitForLoad: false, }); - let engineIcon = defaultEngine.getIconURL(16); + let engineIcon = await defaultEngine.getIconURL(16); await ensureIcon(tab, engineIcon); if (isHandoff) { @@ -162,7 +173,7 @@ add_task(async function test_content_search_attributes_in_private_window() { }); let tab = win.gBrowser.selectedTab; - let engineIcon = defaultEngine.getIconURL(16); + let engineIcon = await defaultEngine.getIconURL(16); await ensureIcon(tab, engineIcon); await ensurePlaceholder( diff --git a/browser/components/search/test/browser/browser_defaultPrivate_nimbus.js b/browser/components/search/test/browser/browser_defaultPrivate_nimbus.js index ce5acc91a0..41b9ce7576 100644 --- a/browser/components/search/test/browser/browser_defaultPrivate_nimbus.js +++ b/browser/components/search/test/browser/browser_defaultPrivate_nimbus.js @@ -31,6 +31,47 @@ const CONFIG_DEFAULT = [ }, ]; +const CONFIG_V2 = [ + { + recordType: "engine", + identifier: "basic", + base: { + name: "basic", + urls: { + search: { + base: "https://example.com", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + recordType: "engine", + identifier: "private", + base: { + name: "private", + urls: { + search: { + base: "https://example.com", + searchTermParamName: "q", + }, + }, + }, + variants: [{ environment: { allRegionsAndLocales: true } }], + }, + { + recordType: "defaultEngines", + globalDefault: "basic", + globalDefaultPrivate: "private", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + SearchTestUtils.init(this); add_setup(async () => { @@ -50,7 +91,9 @@ add_setup(async () => { }); SearchTestUtils.useMockIdleService(); - await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_DEFAULT); + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled ? CONFIG_V2 : CONFIG_DEFAULT + ); registerCleanupFunction(async () => { let settingsWritten = SearchTestUtils.promiseSearchNotification( diff --git a/browser/components/search/test/browser/browser_google_behavior.js b/browser/components/search/test/browser/browser_google_behavior.js index cce3b3ce1f..ccc84e8bba 100644 --- a/browser/components/search/test/browser/browser_google_behavior.js +++ b/browser/components/search/test/browser/browser_google_behavior.js @@ -55,7 +55,7 @@ if (code) { } function promiseContentSearchReady(browser) { - return SpecialPowers.spawn(browser, [], async function (args) { + return SpecialPowers.spawn(browser, [], async function () { return new Promise(resolve => { SpecialPowers.pushPrefEnv({ set: [ @@ -175,7 +175,7 @@ async function testSearchEngine(engineDetails) { await promiseContentSearchReady(browser); }, async run(tab) { - await SpecialPowers.spawn(tab.linkedBrowser, [], async function (args) { + await SpecialPowers.spawn(tab.linkedBrowser, [], async function () { let input = content.document.querySelector("input[id*=search-]"); input.focus(); input.value = "foo"; diff --git a/browser/components/search/test/browser/browser_oneOffContextMenu_setDefault.js b/browser/components/search/test/browser/browser_oneOffContextMenu_setDefault.js index 9f05e948ed..174a86ba3f 100644 --- a/browser/components/search/test/browser/browser_oneOffContextMenu_setDefault.js +++ b/browser/components/search/test/browser/browser_oneOffContextMenu_setDefault.js @@ -71,10 +71,10 @@ async function testSearchBarChangeEngine(win, testPrivate, isPrivateWindow) { if (testPrivate == isPrivateWindow) { let expectedName = originalEngine.name; - let expectedImage = originalEngine.getIconURL(); + let expectedImage = await originalEngine.getIconURL(); if (isPrivateWindow) { expectedName = originalPrivateEngine.name; - expectedImage = originalPrivateEngine.getIconURL(); + expectedImage = await originalPrivateEngine.getIconURL(); } Assert.equal( diff --git a/browser/components/search/test/browser/browser_rich_suggestions.js b/browser/components/search/test/browser/browser_rich_suggestions.js index 98adedcee5..3daefc12d9 100644 --- a/browser/components/search/test/browser/browser_rich_suggestions.js +++ b/browser/components/search/test/browser/browser_rich_suggestions.js @@ -17,6 +17,58 @@ const CONFIG_DEFAULT = [ }, ]; +const CONFIG_V2 = [ + { + recordType: "engine", + identifier: "basic", + base: { + name: "basic", + urls: { + search: { + base: "https://example.com", + searchTermParamName: "q", + }, + trending: { + base: "https://example.com/browser/browser/components/search/test/browser/trendingSuggestionEngine.sjs", + method: "GET", + params: [ + { + name: "richsuggestions", + value: "true", + }, + ], + }, + suggestions: { + base: "https://example.com/browser/browser/components/search/test/browser/trendingSuggestionEngine.sjs", + method: "GET", + params: [ + { + name: "richsuggestions", + value: "true", + }, + ], + searchTermParamName: "query", + }, + }, + aliases: ["basic"], + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "basic", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + SearchTestUtils.init(this); add_setup(async () => { @@ -37,7 +89,9 @@ add_setup(async () => { }); SearchTestUtils.useMockIdleService(); - await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_DEFAULT); + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled ? CONFIG_V2 : CONFIG_DEFAULT + ); registerCleanupFunction(async () => { let settingsWritten = SearchTestUtils.promiseSearchNotification( diff --git a/browser/components/search/test/browser/browser_searchEngine_behaviors.js b/browser/components/search/test/browser/browser_searchEngine_behaviors.js index 15a30583bf..295e069857 100644 --- a/browser/components/search/test/browser/browser_searchEngine_behaviors.js +++ b/browser/components/search/test/browser/browser_searchEngine_behaviors.js @@ -22,9 +22,13 @@ const SEARCH_ENGINE_DETAILS = [ }, { alias: "b", - baseURL: `https://www.bing.com/search?{code}pc=${ - SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "MOZR" : "MOZI" - }&q=foo`, + baseURL: SearchUtils.newSearchConfigEnabled + ? `https://www.bing.com/search?pc=${ + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "MOZR" : "MOZI" + }&{code}q=foo` + : `https://www.bing.com/search?{code}pc=${ + SearchUtils.MODIFIED_APP_CHANNEL == "esr" ? "MOZR" : "MOZI" + }&q=foo`, codes: { context: "form=MOZCON&", keyword: "form=MOZLBR&", @@ -74,7 +78,7 @@ const SEARCH_ENGINE_DETAILS = [ ]; function promiseContentSearchReady(browser) { - return SpecialPowers.spawn(browser, [], async function (args) { + return SpecialPowers.spawn(browser, [], async function () { SpecialPowers.pushPrefEnv({ set: [ [ diff --git a/browser/components/search/test/browser/browser_searchbar_keyboard_navigation.js b/browser/components/search/test/browser/browser_searchbar_keyboard_navigation.js index ee292db1b5..7e2be41993 100644 --- a/browser/components/search/test/browser/browser_searchbar_keyboard_navigation.js +++ b/browser/components/search/test/browser/browser_searchbar_keyboard_navigation.js @@ -29,18 +29,10 @@ async function checkHeader(engine) { // The header can be updated after getting the engine, so we may have to // wait for it. let header = searchPopup.searchbarEngineName; - if (!header.getAttribute("value").includes(engine.name)) { - await new Promise(resolve => { - let observer = new MutationObserver(() => { - observer.disconnect(); - resolve(); - }); - observer.observe(searchPopup.searchbarEngineName, { - attributes: true, - attributeFilter: ["value"], - }); - }); - } + await TestUtils.waitForCondition( + () => header.getAttribute("value").includes(engine.name), + "Should have the correct engine name displayed in the header" + ); Assert.ok( header.getAttribute("value").includes(engine.name), "Should have the correct engine name displayed in the header" diff --git a/browser/components/search/test/browser/browser_searchbar_openpopup.js b/browser/components/search/test/browser/browser_searchbar_openpopup.js index 2653e65e8d..32c6995f69 100644 --- a/browser/components/search/test/browser/browser_searchbar_openpopup.js +++ b/browser/components/search/test/browser/browser_searchbar_openpopup.js @@ -126,7 +126,7 @@ add_task(async function open_empty() { let image = searchPopup.querySelector(".searchbar-engine-image"); Assert.equal( image.src, - engine.getIconURL(16), + await engine.getIconURL(16), "Should have the correct icon" ); @@ -267,6 +267,13 @@ add_no_popup_task(async function right_click_doesnt_open_popup() { context_click(textbox); let contextPopup = await promise; + // Assert that the context menu click inside the popup does nothing. If it + // opens something, assert_no_popup_task will make us fail. On macOS this + // doesn't work because of native context menus. + if (!navigator.platform.includes("Mac")) { + context_click(contextPopup); + } + is( Services.focus.focusedElement, textbox, diff --git a/browser/components/search/test/browser/browser_trending_suggestions.js b/browser/components/search/test/browser/browser_trending_suggestions.js index 74d0b944d5..efe54d2da5 100644 --- a/browser/components/search/test/browser/browser_trending_suggestions.js +++ b/browser/components/search/test/browser/browser_trending_suggestions.js @@ -22,6 +22,65 @@ const CONFIG_DEFAULT = [ }, ]; +const CONFIG_V2 = [ + { + recordType: "engine", + identifier: "basic", + base: { + name: "basic", + urls: { + search: { + base: "https://example.com", + searchTermParamName: "q", + }, + trending: { + base: "https://example.com/browser/browser/components/search/test/browser/trendingSuggestionEngine.sjs", + method: "GET", + }, + }, + aliases: ["basic"], + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "engine", + identifier: "private", + base: { + name: "private", + urls: { + search: { + base: "https://example.com", + searchTermParamName: "q", + }, + suggestions: { + base: "https://example.com", + method: "GET", + searchTermParamName: "search", + }, + }, + aliases: ["private"], + }, + variants: [ + { + environment: { allRegionsAndLocales: true }, + }, + ], + }, + { + recordType: "defaultEngines", + globalDefault: "basic", + specificDefaults: [], + }, + { + recordType: "engineOrders", + orders: [], + }, +]; + SearchTestUtils.init(this); add_setup(async () => { @@ -38,7 +97,9 @@ add_setup(async () => { }); SearchTestUtils.useMockIdleService(); - await SearchTestUtils.updateRemoteSettingsConfig(CONFIG_DEFAULT); + await SearchTestUtils.updateRemoteSettingsConfig( + SearchUtils.newSearchConfigEnabled ? CONFIG_V2 : CONFIG_DEFAULT + ); Services.telemetry.clearScalars(); registerCleanupFunction(async () => { diff --git a/browser/components/search/test/browser/contentSearchUI.html b/browser/components/search/test/browser/contentSearchUI.html index 09abe822b2..7fa41b9d86 100644 --- a/browser/components/search/test/browser/contentSearchUI.html +++ b/browser/components/search/test/browser/contentSearchUI.html @@ -13,6 +13,9 @@ </head> <body> +<!-- Dummy Button is used to ensure pressing Shift+Tab on <input> will make the new focus + - remains in the same document, rather than the Chrome UI. --> +<button>Dummy Button</button> <div id="container"><input type="text" value=""/></div> <script src="chrome://mochitests/content/browser/browser/components/search/test/browser/contentSearchUI.js"> diff --git a/browser/components/search/test/browser/head.js b/browser/components/search/test/browser/head.js index 7a45a9f4f5..6051ef1caa 100644 --- a/browser/components/search/test/browser/head.js +++ b/browser/components/search/test/browser/head.js @@ -123,7 +123,7 @@ async function searchInSearchbar(inputText, win = window) { return sb.textbox.popup; } -function clearSearchbarHistory(win = window) { +function clearSearchbarHistory() { info("cleanup the search history"); return FormHistory.update({ op: "remove", fieldname: "searchbar-history" }); } diff --git a/browser/components/search/test/browser/telemetry/browser.toml b/browser/components/search/test/browser/telemetry/browser.toml index 49d8f256aa..660fc4eae2 100644 --- a/browser/components/search/test/browser/telemetry/browser.toml +++ b/browser/components/search/test/browser/telemetry/browser.toml @@ -4,10 +4,7 @@ support-files = ["head.js", "head-spa.js"] prefs = ["browser.search.log=true"] ["browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js"] -support-files = [ - "domain_category_mappings.json", - "searchTelemetryDomainCategorizationReporting.html", -] +support-files = ["searchTelemetryDomainCategorizationReporting.html"] ["browser_search_glean_serp_event_telemetry_enabled_by_nimbus_variable.js"] support-files = ["searchTelemetryAd.html"] @@ -31,9 +28,16 @@ support-files = [ "searchTelemetryAd_components_carousel_outer_container.html", "searchTelemetryAd_components_text.html", "searchTelemetryAd_components_visibility.html", + "searchTelemetryAd_components_cookie_banner.html", "serp.css", ] +["browser_search_telemetry_adImpression_component_skipCount_children.js"] +support-files = ["searchTelemetryAd_searchbox_with_content.html", "serp.css"] + +["browser_search_telemetry_adImpression_component_skipCount_parent.js"] +support-files = ["searchTelemetryAd_searchbox_with_content.html", "serp.css"] + ["browser_search_telemetry_categorization_timing.js"] ["browser_search_telemetry_content.js"] @@ -42,7 +46,6 @@ support-files = [ support-files = ["searchTelemetryDomainCategorizationReporting.html"] ["browser_search_telemetry_domain_categorization_download_timer.js"] -support-files = ["domain_category_mappings.json"] ["browser_search_telemetry_domain_categorization_extraction.js"] support-files = ["searchTelemetryDomainExtraction.html"] @@ -85,6 +88,15 @@ support-files = [ "serp.css", ] +["browser_search_telemetry_engagement_eventListeners_children.js"] +support-files = ["searchTelemetryAd_searchbox_with_content.html", "serp.css"] + +["browser_search_telemetry_engagement_eventListeners_parent.js"] +support-files = ["searchTelemetryAd_searchbox_with_content.html", "serp.css"] + +["browser_search_telemetry_engagement_ignoreLinkRegexps.js"] +support-files = ["searchTelemetryAd_searchbox_with_content.html", "serp.css"] + ["browser_search_telemetry_engagement_multiple_tabs.js"] support-files = [ "searchTelemetryAd_searchbox_with_content.html", @@ -98,6 +110,14 @@ support-files = [ "serp.css", ] +["browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js"] +support-files = [ + "searchTelemetryAd_searchbox_with_redirecting_links.html", + "searchTelemetryAd_shopping.html", + "searchTelemetry_redirect_with_js.html", + "serp.css", +] + ["browser_search_telemetry_engagement_query_params.js"] support-files = [ "searchTelemetryAd_components_query_parameters.html", diff --git a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js index ed71a7c5ed..e73a9601d4 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js +++ b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js @@ -38,7 +38,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component.js index 8049406d40..5a09353ed6 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component.js @@ -78,6 +78,15 @@ const TEST_PROVIDER_INFO = [ type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, default: true, }, + { + type: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + included: { + parent: { + selector: "#banner", + }, + }, + topDown: true, + }, ], }, ]; @@ -500,3 +509,35 @@ add_task(async function test_impressions_without_ads() { BrowserTestUtils.removeTab(tab); }); + +add_task(async function test_ad_impressions_with_cookie_banner() { + resetTelemetry(); + let url = getSERPUrl("searchTelemetryAd_components_cookie_banner.html"); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + + await waitForPageWithAdImpressions(); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", + }, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_children.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_children.js new file mode 100644 index 0000000000..65cd612a49 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_children.js @@ -0,0 +1,149 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests skipCount property on elements in the children. + */ + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + }, +]; + +const IMPRESSION = { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", +}; + +const SERP_URL = getSERPUrl("searchTelemetryAd_searchbox_with_content.html"); + +async function replaceIncludedProperty(included) { + let components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + included, + topDown: true, + }, + ]; + TEST_PROVIDER_INFO[0].components = components; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +// For older clients, skipCount won't be available. +add_task(async function test_skip_count_not_provided() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + }, + children: [ + { + selector: "a", + }, + ], + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_skip_count_is_false() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + }, + children: [ + { + selector: "a", + skipCount: false, + }, + ], + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_skip_count_is_true() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + }, + children: [ + { + selector: "a", + skipCount: true, + }, + ], + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [], + }, + ]); + + await cleanup(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_parent.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_parent.js new file mode 100644 index 0000000000..8471215840 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_adImpression_component_skipCount_parent.js @@ -0,0 +1,134 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests skipCount property on parent elements. + */ + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + }, +]; + +const IMPRESSION = { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", +}; + +const SERP_URL = getSERPUrl("searchTelemetryAd_searchbox_with_content.html"); + +async function replaceIncludedProperty(included) { + let components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + included, + topDown: true, + }, + ]; + TEST_PROVIDER_INFO[0].components = components; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +// For older clients, skipCount won't be available. +add_task(async function test_skip_count_not_provided() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + }, + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_skip_count_is_false() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: false, + }, + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_skip_count_is_true() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + }); + + let { cleanup } = await openSerpInNewTab(SERP_URL); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + adImpressions: [], + }, + ]); + + await cleanup(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js index ce18f64e9f..246caf6f47 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js @@ -30,7 +30,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js index d01141d826..b8dd85da97 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js @@ -33,7 +33,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, @@ -119,6 +119,7 @@ add_task(async function test_download_after_failure() { id: "example_id", version: 1, filename: "domain_category_mappings.json", + mapping: CONVERTED_ATTACHMENT_VALUES, }); await db.create(record); await db.importChanges({}, Date.now()); @@ -173,6 +174,7 @@ add_task(async function test_download_after_multiple_failures() { id: "example_id", version: 1, filename: "domain_category_mappings.json", + mapping: CONVERTED_ATTACHMENT_VALUES, }); await db.create(record); await db.importChanges({}, Date.now()); @@ -220,6 +222,7 @@ add_task(async function test_cancel_download_timer() { id: "example_id", version: 1, filename: "domain_category_mappings.json", + mapping: CONVERTED_ATTACHMENT_VALUES, }); await db.create(record); await db.importChanges({}, Date.now()); @@ -277,6 +280,7 @@ add_task(async function test_download_adjust() { id: "example_id", version: 1, filename: "domain_category_mappings.json", + mapping: CONVERTED_ATTACHMENT_VALUES, }); await db.create(record); await db.importChanges({}, Date.now()); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js index 03ddb75481..e653be6c48 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js @@ -11,6 +11,10 @@ ChromeUtils.defineESModuleGetters(this, { SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", }); +// The search provider's name is provided to ensure we can extract domains +// from relative links, e.g. /url?=https://www.foobar.com +const SEARCH_PROVIDER_NAME = "example"; + const TESTS = [ { title: "Extract domain from href (absolute URL) - one link.", @@ -35,7 +39,7 @@ const TESTS = [ expectedDomains: ["foo.com", "bar.com", "baz.com", "qux.com"], }, { - title: "Extract domain from href (relative URL).", + title: "Extract domain from href (relative URL / URL matching provider)", extractorInfos: [ { selectors: @@ -43,38 +47,33 @@ const TESTS = [ method: "href", }, ], - expectedDomains: ["example.org"], + expectedDomains: [], }, { title: "Extract domain from data attribute - one link.", extractorInfos: [ { selectors: "#test4 [data-dtld]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "dtld", }, }, ], - expectedDomains: ["www.abc.com"], + expectedDomains: ["abc.com"], }, { title: "Extract domain from data attribute - multiple links.", extractorInfos: [ { selectors: "#test5 [data-dtld]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "dtld", }, }, ], - expectedDomains: [ - "www.foo.com", - "www.bar.com", - "www.baz.com", - "www.qux.com", - ], + expectedDomains: ["foo.com", "bar.com", "baz.com", "qux.com"], }, { title: "Extract domain from an href's query param value.", @@ -88,7 +87,7 @@ const TESTS = [ }, }, ], - expectedDomains: ["def.com"], + expectedDomains: ["def.com", "bar.com", "baz.com"], }, { title: @@ -144,7 +143,7 @@ const TESTS = [ }, { selectors: "#test10 [data-dtld]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "dtld", }, @@ -158,7 +157,7 @@ const TESTS = [ }, }, ], - expectedDomains: ["foobar.com", "www.abc.com", "def.com"], + expectedDomains: ["foobar.com", "abc.com", "def.com"], }, { title: "No elements match the selectors.", @@ -176,7 +175,7 @@ const TESTS = [ extractorInfos: [ { selectors: "#test12 [data-dtld]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "dtld", }, @@ -208,6 +207,161 @@ const TESTS = [ ], expectedDomains: [], }, + { + title: "Second-level domains to a top-level domain.", + extractorInfos: [ + { + selectors: "#test15 a", + method: "href", + }, + ], + expectedDomains: [ + "foobar.gc.ca", + "foobar.gov.uk", + "foobar.co.uk", + "foobar.co.il", + ], + }, + { + title: "URL with a long subdomain.", + extractorInfos: [ + { + selectors: "#test16 a", + method: "href", + }, + ], + expectedDomains: ["foobar.com"], + }, + { + title: "URLs with the same top level domain.", + extractorInfos: [ + { + selectors: "#test17 a", + method: "href", + }, + ], + expectedDomains: ["foobar.com"], + }, + { + title: "Maximum domains extracted from a single selector.", + extractorInfos: [ + { + selectors: "#test18 a", + method: "href", + }, + ], + expectedDomains: [ + "foobar1.com", + "foobar2.com", + "foobar3.com", + "foobar4.com", + "foobar5.com", + "foobar6.com", + "foobar7.com", + "foobar8.com", + "foobar9.com", + "foobar10.com", + ], + }, + { + // This is just in case we use multiple selectors meant for separate SERPs + // and the provider switches to re-using their markup. + title: "Maximum domains extracted from multiple matching selectors.", + extractorInfos: [ + { + selectors: "#test19 a.foo", + method: "href", + }, + { + selectors: "#test19 a.baz", + method: "href", + }, + ], + expectedDomains: [ + "foobar1.com", + "foobar2.com", + "foobar3.com", + "foobar4.com", + "foobar5.com", + "foobar6.com", + "foobar7.com", + "foobar8.com", + "foobar9.com", + // This is from the second selector. + "foobaz1.com", + ], + }, + { + title: "Bing organic result.", + extractorInfos: [ + { + selectors: "#test20 #b_results .b_algo .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["organic.com"], + }, + { + title: "Bing sponsored result.", + extractorInfos: [ + { + selectors: "#test21 #b_results .b_ad .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["sponsored.com"], + }, + { + title: "Bing carousel result.", + extractorInfos: [ + { + selectors: "#test22 .adsMvCarousel cite", + method: "textContent", + }, + ], + expectedDomains: ["fixedupfromthecarousel.com"], + }, + { + title: "Bing sidebar result.", + extractorInfos: [ + { + selectors: "#test23 aside cite", + method: "textContent", + }, + ], + expectedDomains: ["fixedupfromthesidebar.com"], + }, + { + title: "Extraction threshold respected using text content method.", + extractorInfos: [ + { + selectors: "#test24 #b_results .b_ad .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: [ + "sponsored1.com", + "sponsored2.com", + "sponsored3.com", + "sponsored4.com", + "sponsored5.com", + "sponsored6.com", + "sponsored7.com", + "sponsored8.com", + "sponsored9.com", + "sponsored10.com", + ], + }, + { + title: "Bing organic result with no protocol.", + extractorInfos: [ + { + selectors: "#test25 #b_results .b_algo .b_attribution cite", + method: "textContent", + }, + ], + expectedDomains: ["organic.com"], + }, ]; add_setup(async function () { @@ -240,14 +394,15 @@ add_task(async function test_domain_extraction_heuristics() { let expectedDomains = new Set(currentTest.expectedDomains); let actualDomains = await SpecialPowers.spawn( gBrowser.selectedBrowser, - [currentTest.extractorInfos], - extractorInfos => { + [currentTest.extractorInfos, SEARCH_PROVIDER_NAME], + (extractorInfos, searchProviderName) => { const { domainExtractor } = ChromeUtils.importESModule( "resource:///actors/SearchSERPTelemetryChild.sys.mjs" ); return domainExtractor.extractDomainsFromDocument( content.document, - extractorInfos + extractorInfos, + searchProviderName ); } ); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js index f328bb4f79..4c47b0b14a 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js @@ -30,7 +30,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js index b7edb8763f..973f17b760 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js @@ -30,7 +30,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js index cfb8590960..9d3ac2c931 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js @@ -31,7 +31,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js index cb95164221..c73e224eae 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js @@ -32,7 +32,7 @@ const TEST_PROVIDER_INFO = [ ads: [ { selectors: "[data-ad-domain]", - method: "data-attribute", + method: "dataAttribute", options: { dataAttributeKey: "adDomain", }, diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_content.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_content.js index a7ea62ebd5..f94e6b0bd8 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_content.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_content.js @@ -138,8 +138,8 @@ add_task(async function test_click_tab() { { impression: { provider: "example", - tagged: "false", - partner_code: "", + tagged: "true", + partner_code: "ff", source: "unknown", is_shopping_page: "false", is_private: "false", @@ -217,8 +217,8 @@ add_task(async function test_click_shopping() { { impression: { provider: "example", - tagged: "false", - partner_code: "", + tagged: "true", + partner_code: "ff", source: "unknown", is_shopping_page: "true", is_private: "false", diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_children.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_children.js new file mode 100644 index 0000000000..4f5aaf9378 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_children.js @@ -0,0 +1,480 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests eventListeners property on children elements in topDown searches. + */ + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + }, +]; + +const IMPRESSION = { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", +}; + +const SELECTOR = ".arrow"; +const SERP_URL = getSERPUrl("searchTelemetryAd_searchbox_with_content.html"); + +async function replaceIncludedProperty(included) { + TEST_PROVIDER_INFO[0].components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + included, + topDown: true, + }, + ]; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +add_task(async function test_listeners_not_provided() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_no_listeners() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_click_listener() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [{ eventType: "click" }], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: ".arrow-next", + tab, + }); + await synthesizePageAction({ + selector: ".arrow-prev", + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +/** + * The click event is by far our most used event so by default, we translate + * a "click" eventType to a "clicked" action. If no action is provided for + * another type of event, nothing should be reported. + */ +add_task(async function test_event_with_no_default_action() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [{ eventType: "mousedown" }], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_event_no_default_action_with_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "mousedown", + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + }, + ], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_target_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [{ eventType: "click", target: "custom_target" }], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: "custom_target", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_target_and_action_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + action: "custom_action", + target: "custom_target", + }, + ], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "custom_action", + target: "custom_target", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_multiple_listeners() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: " .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + }, + { + eventType: "mouseover", + action: "mouseovered", + }, + ], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + event: { + type: "mouseover", + }, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + { + action: "mouseovered", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_condition() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: ".arrow", + skipCount: true, + eventListeners: [ + { + eventType: "keydown", + action: "keydowned", + condition: "keydownEnter", + }, + ], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await SpecialPowers.spawn(tab.linkedBrowser, [SELECTOR], async function (s) { + let el = content.document.querySelector(s); + el.focus(); + }); + + await EventUtils.synthesizeKey("A"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + let pageActionPromise = waitForPageWithAction(); + await EventUtils.synthesizeKey("KEY_Enter"); + await pageActionPromise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "keydowned", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_condition_invalid() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons", + skipCount: true, + }, + children: [ + { + selector: ".arrow", + skipCount: true, + eventListeners: [ + { + eventType: "keydown", + action: "keydowned", + condition: "noConditionExistsWithThisName", + }, + ], + }, + ], + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await SpecialPowers.spawn(tab.linkedBrowser, [SELECTOR], async function (s) { + let el = content.document.querySelector(s); + el.focus(); + }); + + await EventUtils.synthesizeKey("A"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + await EventUtils.synthesizeKey("KEY_Enter"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_parent.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_parent.js new file mode 100644 index 0000000000..4e3c635b4c --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_eventListeners_parent.js @@ -0,0 +1,434 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests eventListeners property on parent elements in topDown searches. + */ + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + }, +]; + +// The impression doesn't change in these tests. +const IMPRESSION = { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", +}; + +const SELECTOR = ".arrow"; +const SERP_URL = getSERPUrl("searchTelemetryAd_searchbox_with_content.html"); + +async function replaceIncludedProperty(included) { + TEST_PROVIDER_INFO[0].components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + included, + topDown: true, + }, + ]; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +add_task(async function test_listeners_not_provided() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_no_listeners() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_click_listener() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: ".arrow-next", + tab, + }); + await synthesizePageAction({ + selector: ".arrow-prev", + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +/** + * The click event is by far our most used event so by default, we translate + * a "click" eventType to a "clicked" action. If no action is provided for + * another type of event, nothing should be reported. + */ +add_task(async function test_event_with_no_default_action_parent() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "mousedown", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + expectEngagement: false, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_event_no_default_action_with_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "mousedown", + action: "clicked", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_target_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + target: "custom_target", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: "custom_target", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_target_and_action_override() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + target: "custom_target", + action: "custom_action", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "custom_action", + target: "custom_target", + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_multiple_listeners() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "click", + action: "clicked", + }, + { + eventType: "mouseover", + action: "mouseovered", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await synthesizePageAction({ + selector: SELECTOR, + tab, + }); + await synthesizePageAction({ + selector: SELECTOR, + tab, + event: { + type: "mouseover", + }, + }); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + { + action: "mouseovered", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_condition() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "keydown", + action: "keydowned", + condition: "keydownEnter", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await SpecialPowers.spawn(tab.linkedBrowser, [SELECTOR], async function (s) { + let el = content.document.querySelector(s); + el.focus(); + }); + + await EventUtils.synthesizeKey("A"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + let pageActionPromise = waitForPageWithAction(); + await EventUtils.synthesizeKey("KEY_Enter"); + await pageActionPromise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "keydowned", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + ]); + + await cleanup(); +}); + +add_task(async function test_condition_invalid() { + await replaceIncludedProperty({ + parent: { + selector: ".refined-search-buttons .arrow", + skipCount: true, + eventListeners: [ + { + eventType: "keydown", + action: "keydowned", + condition: "noConditionExistsWithThisName", + }, + ], + }, + }); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + await SpecialPowers.spawn(tab.linkedBrowser, [SELECTOR], async function (s) { + let el = content.document.querySelector(s); + el.focus(); + }); + + await EventUtils.synthesizeKey("A"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + await EventUtils.synthesizeKey("KEY_Enter"); + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(resolve => setTimeout(resolve, 10)); + + assertSERPTelemetry([ + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_ignoreLinkRegexps.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_ignoreLinkRegexps.js new file mode 100644 index 0000000000..10f2a2d836 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_ignoreLinkRegexps.js @@ -0,0 +1,223 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +/** + * Tests ignoreLinkRegexps property in search telemetry that explicitly results + * in our network code ignoring the link. The main reason for doing so is for + * rare situations where we need to find a components from a topDown approach + * but it loads a page in the network process. + */ + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + ignoreLinkRegexps: [ + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd_searchbox_with_content.html\?s=test&page=images/, + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd_searchbox_with_content.html\?s=test&page=shopping/, + ], + components: [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, + default: true, + }, + ], + hrefToComponentMapAugmentation: [ + { + action: "clicked_something", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + url: "https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox.html", + }, + ], + }, +]; + +// The impression doesn't change in these tests. +const IMPRESSION = { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", +}; + +const SERP_URL = getSERPUrl("searchTelemetryAd_searchbox_with_content.html"); + +async function replaceIncludedProperty(included) { + TEST_PROVIDER_INFO[0].components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + included, + topDown: true, + }, + ]; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +} + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + // Enable local telemetry recording for the duration of the tests. + let oldCanRecord = Services.telemetry.canRecordExtended; + Services.telemetry.canRecordExtended = true; + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + Services.telemetry.canRecordExtended = oldCanRecord; + resetTelemetry(); + }); +}); + +add_task(async function test_click_link_1_matching_ignore_link_regexps() { + resetTelemetry(); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + let promise = BrowserTestUtils.browserLoaded(tab.linkedBrowser); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#images", + {}, + tab.linkedBrowser + ); + await promise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + abandonment: { + reason: SearchSERPTelemetryUtils.ABANDONMENTS.NAVIGATION, + }, + }, + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_click_link_2_matching_ignore_link_regexps() { + resetTelemetry(); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + let promise = BrowserTestUtils.browserLoaded(tab.linkedBrowser); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#shopping", + {}, + tab.linkedBrowser + ); + await promise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + abandonment: { + reason: SearchSERPTelemetryUtils.ABANDONMENTS.NAVIGATION, + }, + }, + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_click_link_3_not_matching_ignore_link_regexps() { + resetTelemetry(); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + let promise = BrowserTestUtils.browserLoaded(tab.linkedBrowser); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#extra", + {}, + tab.linkedBrowser + ); + await promise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: SearchSERPTelemetryUtils.COMPONENTS.NON_ADS_LINK, + }, + ], + }, + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); + +add_task(async function test_click_listener_with_ignore_link_regexps() { + resetTelemetry(); + + TEST_PROVIDER_INFO[0].components = [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + topDown: true, + included: { + parent: { + selector: "nav a", + skipCount: true, + eventListeners: [ + { + eventType: "click", + action: "clicked", + }, + ], + }, + }, + }, + { + type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, + default: true, + }, + ]; + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + + let { tab, cleanup } = await openSerpInNewTab(SERP_URL); + + let promise = BrowserTestUtils.browserLoaded(tab.linkedBrowser); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#images", + {}, + tab.linkedBrowser + ); + await promise; + + assertSERPTelemetry([ + { + impression: IMPRESSION, + engagements: [ + { + action: "clicked", + target: SearchSERPTelemetryUtils.COMPONENTS.REFINED_SEARCH_BUTTONS, + }, + ], + }, + { + impression: IMPRESSION, + }, + ]); + + await cleanup(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js new file mode 100644 index 0000000000..93a6b7993e --- /dev/null +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js @@ -0,0 +1,252 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * These tests load SERPs and click on links. + */ + +"use strict"; + +const TEST_PROVIDER_INFO = [ + { + telemetryId: "example", + searchPageRegexp: + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry/, + queryParamNames: ["s"], + codeParamName: "abc", + taggedCodes: ["ff"], + adServerAttributes: ["mozAttr"], + nonAdsLinkRegexps: [ + /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry_redirect_with_js/, + ], + nonAdsLinkQueryParamNames: ["url"], + extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + shoppingTab: { + regexp: "&page=shopping", + selector: "nav a", + inspectRegexpInSERP: true, + }, + components: [ + { + type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, + default: true, + }, + ], + }, +]; + +add_setup(async function () { + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); + + registerCleanupFunction(async () => { + SearchSERPTelemetry.overrideSearchTelemetryForTests(); + resetTelemetry(); + }); +}); + +add_task(async function test_click_absolute_url_in_query_param() { + resetTelemetry(); + + let url = getSERPUrl( + "searchTelemetryAd_searchbox_with_redirecting_links.html" + ); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + let browserLoadedPromise = BrowserTestUtils.browserLoaded( + tab.linkedBrowser, + true + ); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#shopping-redirect-absolute-link", + {}, + tab.linkedBrowser + ); + await browserLoadedPromise; + await waitForPageWithAdImpressions(); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "true", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "true", + is_private: "false", + shopping_tab_displayed: "true", + }, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); + + // Reset state for other tests. + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +}); + +add_task(async function test_click_relative_href_in_query_param() { + resetTelemetry(); + + let url = getSERPUrl( + "searchTelemetryAd_searchbox_with_redirecting_links.html" + ); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + let browserLoadedPromise = BrowserTestUtils.browserLoaded( + tab.linkedBrowser, + true + ); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#shopping-redirect-relative-link", + {}, + tab.linkedBrowser + ); + await browserLoadedPromise; + await waitForPageWithAdImpressions(); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "true", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "true", + is_private: "false", + shopping_tab_displayed: "true", + }, + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); + + // Reset state for other tests. + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +}); + +add_task(async function test_click_irrelevant_href_in_query_param() { + resetTelemetry(); + + let url = getSERPUrl( + "searchTelemetryAd_searchbox_with_redirecting_links.html" + ); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + let browserLoadedPromise = BrowserTestUtils.browserLoaded( + tab.linkedBrowser, + true, + "https://example.org/foo/bar" + ); + await BrowserTestUtils.synthesizeMouseAtCenter( + "#organic-redirect", + {}, + tab.linkedBrowser + ); + await browserLoadedPromise; + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "true", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED, + target: SearchSERPTelemetryUtils.COMPONENTS.NON_ADS_LINK, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); + + // Reset state for other tests. + SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); + await waitForIdle(); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_target.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_target.js index b30a7bc0c1..8f7f7f4e05 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_target.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_engagement_target.js @@ -22,6 +22,7 @@ const TEST_PROVIDER_INFO = [ /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetryAd_nonAdsLink_redirect.html/, ], extraAdServersRegexps: [/^https:\/\/example\.com\/ad/], + ignoreLinkRegexps: [/^https:\/\/example\.org\/consent\?data=/], components: [ { type: SearchSERPTelemetryUtils.COMPONENTS.AD_CAROUSEL, @@ -90,6 +91,44 @@ const TEST_PROVIDER_INFO = [ type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK, default: true, }, + { + type: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + topDown: true, + included: { + parent: { + selector: "#banner", + }, + children: [ + { + selector: "#cookie_accept", + eventListeners: [ + { + eventType: "click", + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_ACCEPT, + }, + ], + }, + { + selector: "#cookie_reject", + eventListeners: [ + { + eventType: "click", + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_REJECT, + }, + ], + }, + { + selector: "#cookie_more_options", + eventListeners: [ + { + eventType: "click", + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_MORE_OPTIONS, + }, + ], + }, + ], + }, + }, ], }, ]; @@ -455,3 +494,138 @@ add_task(async function test_click_link_with_special_characters_in_path() { SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO); await waitForIdle(); }); + +// Test that clicking the accept button on the cookie banner is correctly +// tracked as an engagement event. +add_task(async function test_click_cookie_banner_accept() { + resetTelemetry(); + let url = getSERPUrl("searchTelemetryAd_components_cookie_banner.html"); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + await BrowserTestUtils.synthesizeMouseAtCenter( + "#cookie_accept", + {}, + tab.linkedBrowser + ); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_ACCEPT, + target: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); +}); + +// Test that clicking the reject button on the cookie banner is accurately +// recorded as an engagement event. +add_task(async function test_click_cookie_banner_reject() { + resetTelemetry(); + let url = getSERPUrl("searchTelemetryAd_components_cookie_banner.html"); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + await BrowserTestUtils.synthesizeMouseAtCenter( + "#cookie_reject", + {}, + tab.linkedBrowser + ); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_REJECT, + target: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); +}); + +// Test that clicking the more options button on the cookie banner is accurately +// recorded as an engagement event. +add_task(async function test_click_cookie_banner_more_options() { + resetTelemetry(); + let url = getSERPUrl("searchTelemetryAd_components_cookie_banner.html"); + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await waitForPageWithAdImpressions(); + + await BrowserTestUtils.synthesizeMouseAtCenter( + "#cookie_more_options", + {}, + tab.linkedBrowser + ); + + assertSERPTelemetry([ + { + impression: { + provider: "example", + tagged: "true", + partner_code: "ff", + source: "unknown", + is_shopping_page: "false", + is_private: "false", + shopping_tab_displayed: "false", + }, + engagements: [ + { + action: SearchSERPTelemetryUtils.ACTIONS.CLICKED_MORE_OPTIONS, + target: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + }, + ], + adImpressions: [ + { + component: SearchSERPTelemetryUtils.COMPONENTS.COOKIE_BANNER, + ads_loaded: "1", + ads_visible: "1", + ads_hidden: "0", + }, + ], + }, + ]); + + BrowserTestUtils.removeTab(tab); +}); diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_sources_webextension.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_sources_webextension.js index f7b22f004b..cb9e123622 100644 --- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_sources_webextension.js +++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_sources_webextension.js @@ -152,7 +152,7 @@ async function track_ad_click( add_task(async function test_source_webextension_search() { /* global browser */ - async function background(SEARCH_TERM) { + async function background() { // Search with no tabId browser.search.search({ query: "searchSuggestion", engine: "Example" }); } @@ -184,7 +184,7 @@ add_task(async function test_source_webextension_search() { }); add_task(async function test_source_webextension_query() { - async function background(SEARCH_TERM) { + async function background() { // Search with no tabId browser.search.query({ text: "searchSuggestion", diff --git a/browser/components/search/test/browser/telemetry/domain_category_mappings.json b/browser/components/search/test/browser/telemetry/domain_category_mappings.json deleted file mode 100644 index 2f8d0d2af2..0000000000 --- a/browser/components/search/test/browser/telemetry/domain_category_mappings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "DqNorjpE3CBY9OZh0wf1uA==": [2, 90], - "kpuib0kvhtSp1moICEmGWg==": [2, 95], - "+5WbbjV3Nmxp0mBZODcJWg==": [2, 78, 4, 10], - "OIHlWZ/yMyTHHuY78AV9VQ==": [3, 90], - "r1hDZinn+oNrQjabn8IB9w==": [4, 90], - "AtlIam7nqWvzFzTGkYI01w==": [4, 90] -} diff --git a/browser/components/search/test/browser/telemetry/head.js b/browser/components/search/test/browser/telemetry/head.js index 416451e400..b798099bdd 100644 --- a/browser/components/search/test/browser/telemetry/head.js +++ b/browser/components/search/test/browser/telemetry/head.js @@ -45,6 +45,10 @@ ChromeUtils.defineLazyGetter(this, "SEARCH_AD_CLICK_SCALARS", () => { ]; }); +ChromeUtils.defineLazyGetter(this, "gCryptoHash", () => { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); + // For use with categorization. const APP_MAJOR_VERSION = parseInt(Services.appinfo.version).toString(); const CHANNEL = SearchUtils.MODIFIED_APP_CHANNEL; @@ -207,6 +211,11 @@ function resetTelemetry() { * values we use to validate the recorded Glean impression events. */ function assertSERPTelemetry(expectedEvents) { + // Do a deep copy of impressions in case the input is using constants, as + // we insert impression id into the expected events to make it easier to + // run Assert.deepEqual() on the expected and actual result. + expectedEvents = JSON.parse(JSON.stringify(expectedEvents)); + // A single test might run assertImpressionEvents more than once // so the Set needs to be cleared or else the impression event // check will throw. @@ -385,6 +394,46 @@ add_setup(function () { }); }); +async function openSerpInNewTab(url, expectedAds = true) { + let promise; + if (expectedAds) { + promise = waitForPageWithAdImpressions(); + } + let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url); + await promise; + + let cleanup = async () => { + await BrowserTestUtils.removeTab(tab); + resetTelemetry(); + }; + + return { tab, cleanup }; +} + +async function synthesizePageAction({ + selector, + event = {}, + tab, + expectEngagement = true, +} = {}) { + let promise; + if (expectEngagement) { + promise = waitForPageWithAction(); + } else { + // Wait roughly around how much it might take for a possible page action + // to be registered in telemetry. + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + promise = new Promise(resolve => setTimeout(resolve, 50)); + } + await BrowserTestUtils.synthesizeMouseAtCenter( + selector, + event, + tab.linkedBrowser + ); + + await promise; +} + function assertCategorizationValues(expectedResults) { // TODO Bug 1868476: Replace with calls to Glean telemetry. let actualResults = [...fakeTelemetryStorage]; @@ -435,6 +484,10 @@ function assertCategorizationValues(expectedResults) { } } +function waitForPageWithAction() { + return TestUtils.topicObserved("reported-page-with-action"); +} + function waitForPageWithAdImpressions() { return TestUtils.topicObserved("reported-page-with-ad-impressions"); } @@ -459,10 +512,9 @@ registerCleanupFunction(async () => { await PlacesUtils.history.clear(); }); -async function mockRecordWithAttachment({ id, version, filename }) { +async function mockRecordWithAttachment({ id, version, filename, mapping }) { // Get the bytes of the file for the hash and size for attachment metadata. - let data = await IOUtils.readUTF8(getTestFilePath(filename)); - let buffer = new TextEncoder().encode(data).buffer; + let buffer = new TextEncoder().encode(JSON.stringify(mapping)).buffer; let stream = Cc["@mozilla.org/io/arraybuffer-input-stream;1"].createInstance( Ci.nsIArrayBufferInputStream ); @@ -506,6 +558,30 @@ async function resetCategorizationCollection(record) { await client.db.importChanges({}, Date.now()); } +const MOCK_ATTACHMENT_VALUES = { + "abc.com": [2, 95], + "abc.org": [4, 90], + "def.com": [2, 78, 4, 10], + "def.org": [4, 90], + "foobar.org": [3, 90], +}; + +const CONVERTED_ATTACHMENT_VALUES = convertDomainsToHashes( + MOCK_ATTACHMENT_VALUES +); + +function convertDomainsToHashes(domainsToCategories) { + let newObj = {}; + for (let [key, value] of Object.entries(domainsToCategories)) { + gCryptoHash.init(gCryptoHash.SHA256); + let bytes = new TextEncoder().encode(key); + gCryptoHash.update(bytes, key.length); + let hash = gCryptoHash.finish(true); + newObj[hash] = value; + } + return newObj; +} + async function insertRecordIntoCollection() { const client = RemoteSettings(TELEMETRY_CATEGORIZATION_KEY); const db = client.db; @@ -515,6 +591,7 @@ async function insertRecordIntoCollection() { id: "example_id", version: 1, filename: "domain_category_mappings.json", + mapping: CONVERTED_ATTACHMENT_VALUES, }); await db.create(record); await client.attachments.cacheImpl.set(record.id, attachment); diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryAd_components_cookie_banner.html b/browser/components/search/test/browser/telemetry/searchTelemetryAd_components_cookie_banner.html new file mode 100644 index 0000000000..e33afb2672 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/searchTelemetryAd_components_cookie_banner.html @@ -0,0 +1,16 @@ +<html> +<head> + <title>A top-level page with cookie banner</title> +</head> +<body> + <h1>This is the top-level page</h1> + <div id="banner"> + <button id="cookie_accept">Accept</button> + <button id="cookie_reject">Reject</button> + <button id="cookie_more_options" + onclick="location.href='https:example.org/consent?data='"> + More Options + </button> + </div> +</body> +</html> diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html b/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html index 9c4d371691..d23255984f 100644 --- a/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html +++ b/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html @@ -11,13 +11,16 @@ </form> </section> <nav> - <a id="images" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test&page=images">Images</a> - <a id="shopping" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test&page=shopping">Shopping</a> - <a id="extra" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox.html?s=test">Extra Page</a> + <a id="images" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test&page=images&abc=ff">Images</a> + <a id="shopping" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test&page=shopping&abc=ff">Shopping</a> + <a id="extra" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox.html?s=test&abc=ff">Extra Page</a> </nav> <section class="refined-search-buttons"> + <button class="arrow arrow-prev">← Prev</button> <a id="refined-search-button" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test's">Test's</a> <a id="refined-search-button-with-partner-code" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_content.html?s=test2&abc=ff">Test 2</a> + <a href="javascript:void(0)">Element relying on Javascript</a> + <button class="arrow arrow-next">Next →</button> </section> <section id="searchresults"> <div class="lhs"> diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_redirecting_links.html b/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_redirecting_links.html new file mode 100644 index 0000000000..2b09409126 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/searchTelemetryAd_searchbox_with_redirecting_links.html @@ -0,0 +1,40 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <link rel="stylesheet" type="text/css" href="./serp.css" /> +</head> +<body> + <nav> + <a href="/">All Results</a> + <a id="shopping-redirect-relative-link" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_shopping.html?s=test&page=shopping&abc=ff">Shopping Relative</a> + <a id="shopping-redirect-absolute-link" href="https://example.org/browser/browser/components/search/test/browser/telemetry/searchTelemetryAd_shopping.html?s=test&page=shopping&abc=ff">Shopping Absolute</a> + </nav> + <section> + <form role="search"> + <input type="text" value="test" /> + </form> + </section> + <section id="searchresults"> + <a id="organic-redirect" href="https://example.org/foo/bar">Organic Redirect Result</a> + </section> +</body> +<script type="text/javascript"> + const ORIGIN = "https://example.org"; + const PATH = "/browser/browser/components/search/test/browser/telemetry/" + const REDIRECT_URL = `${ORIGIN + PATH}searchTelemetry_redirect_with_js.html`; + const SHOPPING_PAGE = "searchTelemetryAd_shopping.html?s=test&page=shopping&abc=ff"; + document.getElementById("shopping-redirect-relative-link").addEventListener("click", event => { + event.preventDefault(); + window.location.href = `${REDIRECT_URL}?url=${encodeURIComponent(PATH + SHOPPING_PAGE)}`; + }); + document.getElementById("shopping-redirect-absolute-link").addEventListener("click", event => { + event.preventDefault(); + window.location.href = `${REDIRECT_URL}?url=${encodeURIComponent(ORIGIN + PATH + SHOPPING_PAGE)}`; + }); + document.getElementById("organic-redirect").addEventListener("click", event => { + event.preventDefault(); + window.location.href = `${REDIRECT_URL}?url=${encodeURIComponent("https://example.org/foo/bar")}`; + }); +</script> +</html> diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html index b49e5610ae..28c31af959 100644 --- a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html +++ b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html @@ -25,6 +25,8 @@ <div id="test3"> <div data-layout="organic"> <a href="/dummy-page" data-testid="result-title-a">Extract domain from href (relative URL).</a> + <a href="https://example.org/dummy-page" data-testid="result-title-a">Extract domain from href.</a> + <a href="https://www.example.org/dummy-page" data-testid="result-title-a">Extract domain from href.</a> </div> </div> @@ -40,7 +42,9 @@ </div> <div id="test6"> - <a href="example.com/testing?ad_domain=def.com" class="js-carousel-item-title">Extract domain from an href's query param value.</a> + <a href="https://www.example.org/testing?ad_domain=def.com" class="js-carousel-item-title">Extract domain from an href's query param value.</a> + <a href="https://example.org/testing?ad_domain=bar.com" class="js-carousel-item-title">Extract domain from an href's query param value.</a> + <a href="/testing?ad_domain=baz.com" class="js-carousel-item-title">Extract domain from a relative href containing a relevant query param value.</a> </div> <div id="test7"> @@ -79,6 +83,179 @@ <div id="test14"> <a href="git://testing.com/testrepo">Non-standard URL scheme.</a> </div> + + <div id="test15"> + <h5>Second-level domains to a top-level domain.</h5> + <a href="https://www.foobar.gc.ca/">Link</a> + <a href="https://www.foobar.gov.uk/">Link</a> + <a href="https://foobar.co.uk">Link</a> + <a href="https://www.foobar.co.il">Link</a> + </div> + + <div id="test16"> + <a href="https://ab.cd.ef.gh.foobar.com/">URL with a long subdomain</a> + </div> + + <div id="test17"> + <h5>URL with the same top level domain.</h5> + <a href="https://foobar.com/">Link</a> + <a href="https://www.foobar.com/">Link</a> + <a href="https://abc.def.foobar.com/">Link</a> + </div> + + <div id="test18"> + <h5>More than the threshold of links.</h5> + <a href="https://foobar1.com/">Link</a> + <a href="https://foobar1.com/">Duplicate Link</a> + <a href="https://foobar2.com/">Link</a> + <a href="https://foobar3.com/">Link</a> + <a href="https://foobar4.com/">Link</a> + <a href="https://foobar5.com/">Link</a> + <a href="https://foobar6.com/">Link</a> + <a href="https://foobar7.com/">Link</a> + <a href="https://foobar8.com/">Link</a> + <a href="https://foobar9.com/">Link</a> + <a href="https://foobar10.com/">Link</a> + <a href="https://foobar11.com/">Link Outside Threshold</a> + </div> + + <div id="test19"> + <h5>More than the threshold of links using multiple matching selectors.</h5> + <a class="foo" href="https://foobar1.com/">Link</a> + <a class="foo" href="https://foobar2.com/">Link</a> + <a class="foo" href="https://foobar3.com/">Link</a> + <a class="foo" href="https://foobar4.com/">Link</a> + <a class="foo" href="https://foobar5.com/">Link</a> + <a class="foo" href="https://foobar6.com/">Link</a> + <a class="foo" href="https://foobar7.com/">Link</a> + <a class="foo" href="https://foobar8.com/">Link</a> + <a class="foo" href="https://foobar9.com/">Link</a> + <a class="baz" href="https://foobaz1.com/">Link</a> + <a class="baz" href="https://foobaz2.com/">Link Outside Threshold</a> + </div> + + <div id="test20"> + <div id="b_results"> + <div class="b_algo"> + <div class="b_attribution"> + <cite>https://organic.com</cite> + </div> + </div> + </div> + </div> + + <div id="test21"> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored.com</cite> + </div> + </div> + </div> + </div> + + <div id="test22"> + <div class="adsMvCarousel"> + <cite>Fixed up from the carousel</cite> + </div> + </div> + + <div id="test23"> + <aside> + <cite>Fixed up from the sidebar</cite> + </aside> + </div> + + <div id="test24"> + <h5>More than the threshold of links using the text content selection method.</h5> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored1.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored2.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored3.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored4.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored5.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored6.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored7.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored8.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored9.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored10.com</cite> + </div> + </div> + </div> + <div id="b_results"> + <div class="b_ad"> + <div class="b_attribution"> + <cite>https://sponsored11.com</cite> + </div> + </div> + </div> + </div> + + <div id="test25"> + <div id="b_results"> + <div class="b_algo"> + <div class="b_attribution"> + <cite>organic.com</cite> + </div> + </div> + </div> + </div> </div> </body> </html> diff --git a/browser/components/search/test/browser/telemetry/searchTelemetrySinglePageApp.html b/browser/components/search/test/browser/telemetry/searchTelemetrySinglePageApp.html index 7598da694e..f52088206f 100644 --- a/browser/components/search/test/browser/telemetry/searchTelemetrySinglePageApp.html +++ b/browser/components/search/test/browser/telemetry/searchTelemetrySinglePageApp.html @@ -204,7 +204,7 @@ } }) - window.addEventListener("DOMContentLoaded", (event) => { + window.addEventListener("DOMContentLoaded", () => { let url = new URL(window.location.href); searchKey = url.searchParams.has("r") ? "r": "s"; @@ -219,7 +219,7 @@ updateSuggestions(); }); - window.addEventListener("popstate", (event) => { + window.addEventListener("popstate", () => { let baseUrl = new URL(window.location.href); let page = baseUrl.searchParams.get("page"); switch (page) { diff --git a/browser/components/search/test/browser/telemetry/searchTelemetry_redirect_with_js.html b/browser/components/search/test/browser/telemetry/searchTelemetry_redirect_with_js.html new file mode 100644 index 0000000000..744fcd2906 --- /dev/null +++ b/browser/components/search/test/browser/telemetry/searchTelemetry_redirect_with_js.html @@ -0,0 +1,25 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="UTF-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Page will redirect using the url param</title> + <script> + let parentWindow = window.parent; + let params = new URLSearchParams(window.location.search); + let paramValue = params.get("url"); + if (paramValue) { + // Replicate how some SERPs load pages by encoding the true destination + // in the query param value. + let url = paramValue.startsWith("https://") ? + new URL(paramValue).href : + new URL(paramValue, "https://example.org/").href; + window.location.href = url; + } + </script> + </head> + <body> + <h1>Redirecting...</h1> + </body> +</html> diff --git a/browser/components/search/test/unit/domain_category_mappings_1a.json b/browser/components/search/test/unit/domain_category_mappings_1a.json deleted file mode 100644 index 51b18e12a7..0000000000 --- a/browser/components/search/test/unit/domain_category_mappings_1a.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "Wrq9YDsieAMC3Y2DSY5Rcg==": [1, 100] -} diff --git a/browser/components/search/test/unit/domain_category_mappings_1b.json b/browser/components/search/test/unit/domain_category_mappings_1b.json deleted file mode 100644 index 698ef45f1a..0000000000 --- a/browser/components/search/test/unit/domain_category_mappings_1b.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "G99y4E1rUMgqSMfk3TjMaQ==": [2, 90] -} diff --git a/browser/components/search/test/unit/domain_category_mappings_2a.json b/browser/components/search/test/unit/domain_category_mappings_2a.json deleted file mode 100644 index 08db2fa8c2..0000000000 --- a/browser/components/search/test/unit/domain_category_mappings_2a.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "Wrq9YDsieAMC3Y2DSY5Rcg==": [1, 80] -} diff --git a/browser/components/search/test/unit/domain_category_mappings_2b.json b/browser/components/search/test/unit/domain_category_mappings_2b.json deleted file mode 100644 index dec2d130c1..0000000000 --- a/browser/components/search/test/unit/domain_category_mappings_2b.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "G99y4E1rUMgqSMfk3TjMaQ==": [2, 50, 4, 80] -} diff --git a/browser/components/search/test/unit/test_search_telemetry_categorization_logic.js b/browser/components/search/test/unit/test_search_telemetry_categorization_logic.js index 947a7aae46..44b9147c50 100644 --- a/browser/components/search/test/unit/test_search_telemetry_categorization_logic.js +++ b/browser/components/search/test/unit/test_search_telemetry_categorization_logic.js @@ -15,90 +15,111 @@ ChromeUtils.defineESModuleGetters(this, { SearchSERPTelemetryUtils: "resource:///modules/SearchSERPTelemetry.sys.mjs", }); -const TEST_DOMAIN_TO_CATEGORIES_MAP_SIMPLE = { - "byVQ4ej7T7s2xf/cPqgMyw==": [2, 90], - "1TEnSjgNCuobI6olZinMiQ==": [2, 95], - "/Bnju09b9iBPjg7K+5ENIw==": [2, 78, 4, 10], - "Ja6RJq5LQftdl7NQrX1avQ==": [2, 56, 4, 24], - "Jy26Qt99JrUderAcURtQ5A==": [2, 89], - "sZnJyyzY9QcN810Q6jfbvw==": [2, 43], - "QhmteGKeYk0okuB/bXzwRw==": [2, 65], - "CKQZZ1IJjzjjE4LUV8vUSg==": [2, 67], - "FK7mL5E1JaE6VzOiGMmlZg==": [2, 89], - "mzcR/nhDcrs0ed4kTf+ZFg==": [2, 99], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_INCONCLUSIVE = { - "IkOfhoSlHTMIZzWXkYf7fg==": [0, 0], - "PIAHxeaBOeDNY2tvZKqQuw==": [0, 0], - "DKx2mqmFtEvxrHAqpwSevA==": [0, 0], - "DlZKnz9ryYqbxJq9wodzlA==": [0, 0], - "n3NWT4N9JlKX0I7MUtAsYg==": [0, 0], - "A6KyupOlu5zXt8loti90qw==": [0, 0], - "gf5rpseruOaq8nXOSJPG3Q==": [0, 0], - "vlQYOvbcbAp6sMx54OwqCQ==": [0, 0], - "8PcaPATLgmHD9SR0/961Sw==": [0, 0], - "l+hLycEAW2v/OPE/XFpNwQ==": [0, 0], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_UNKNOWN_AND_INCONCLUSIVE = { - "CEA642T3hV+Fdi2PaRH9BQ==": [0, 0], - "cVqopYLASYxcWdDW4F+w2w==": [0, 0], - "X61OdTU20n8pxZ76K2eAHg==": [0, 0], - "/srrOggOAwgaBGCsPdC4bA==": [0, 0], - "onnMGn+MmaCQx3RNLBzGOQ==": [0, 0], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_ALL_TYPES = { - "VSXaqgDKYWrJ/yjsFomUdg==": [3, 90], - "6re74Kk34n2V6VCdLmCD5w==": [3, 88], - "s8gOGIaFnly5hHX7nPncnw==": [3, 90, 6, 2], - "zfRJyKV+2jd1RKNsSHm9pw==": [3, 78, 6, 7], - "zcW+KbRfLRO6Dljf5qnuwQ==": [3, 97], - "Rau9mfbBcIRiRQIliUxkow==": [0, 0], - "4AFhUOmLQ8804doOsI4jBA==": [0, 0], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_TIE = { - "fmEqRSc+pBr9noi0l99nGw==": [1, 50, 2, 50], - "cms8ipz0JQ3WS9o48RtvnQ==": [1, 50, 2, 50], - "y8Haj7Qdmx+k762RaxCPvA==": [1, 50, 2, 50], - "tCbLmi5xJ/OrF8tbRm8PrA==": [1, 50, 2, 50], - "uYNQECmDShqI409HrSTdLQ==": [1, 50, 2, 50], - "D88hdsmzLWIXYhkrDal33w==": [3, 50, 4, 50], - "1mhx0I0B4cEaI91x8zor7Q==": [5, 50, 6, 50], - "dVZYATQixuBHmalCFR9+Lw==": [7, 50, 8, 50], - "pdOFJG49D7hE/+FtsWDihQ==": [9, 50, 10, 50], - "+gl+dBhWE0nx0AM69m2g5w==": [11, 50, 12, 50], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_1 = { - "VSXaqgDKYWrJ/yjsFomUdg==": [1, 45], - "6re74Kk34n2V6VCdLmCD5w==": [2, 45], - "s8gOGIaFnly5hHX7nPncnw==": [3, 45], - "zfRJyKV+2jd1RKNsSHm9pw==": [4, 45], - "zcW+KbRfLRO6Dljf5qnuwQ==": [5, 45], - "Rau9mfbBcIRiRQIliUxkow==": [6, 45], - "4AFhUOmLQ8804doOsI4jBA==": [7, 45], - "YZ3aEL73MR+Cjog0D7A24w==": [8, 45], - "crMclD9rwInEQ30DpZLg+g==": [9, 45], - "/r7oPRoE6LJAE95nuwmu7w==": [10, 45], -}; - -const TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_2 = { - "sHWSmFwSYL3snycBZCY8Kg==": [1, 35, 2, 4], - "FZ5zPYh6ByI0KGWKkmpDoA==": [1, 5, 2, 94], -}; +ChromeUtils.defineLazyGetter(this, "gCryptoHash", () => { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); + +function convertDomainsToHashes(domainsToCategories) { + let newObj = {}; + for (let [key, value] of Object.entries(domainsToCategories)) { + gCryptoHash.init(gCryptoHash.SHA256); + let bytes = new TextEncoder().encode(key); + gCryptoHash.update(bytes, key.length); + let hash = gCryptoHash.finish(true); + newObj[hash] = value; + } + return newObj; +} + +const TEST_DOMAIN_TO_CATEGORIES_MAP_SIMPLE = convertDomainsToHashes({ + "test1.com": [2, 90], + "test2.com": [2, 95], + "test3.com": [2, 78, 4, 10], + "test4.com": [2, 56, 4, 24], + "test5.com": [2, 89], + "test6.com": [2, 43], + "test7.com": [2, 65], + "test8.com": [2, 67], + "test9.com": [2, 89], + "test10.com": [2, 99], +}); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_INCONCLUSIVE = convertDomainsToHashes({ + "test11.com": [0, 0], + "test12.com": [0, 0], + "test13.com": [0, 0], + "test14.com": [0, 0], + "test15.com": [0, 0], + "test16.com": [0, 0], + "test17.com": [0, 0], + "test18.com": [0, 0], + "test19.com": [0, 0], + "test20.com": [0, 0], +}); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_UNKNOWN_AND_INCONCLUSIVE = + convertDomainsToHashes({ + "test31.com": [0, 0], + "test32.com": [0, 0], + "test33.com": [0, 0], + "test34.com": [0, 0], + "test35.com": [0, 0], + }); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_ALL_TYPES = convertDomainsToHashes({ + "test51.com": [3, 90], + "test52.com": [3, 88], + "test53.com": [3, 90, 6, 2], + "test54.com": [3, 78, 6, 7], + "test55.com": [3, 97], + "test56.com": [0, 0], + "test57.com": [0, 0], +}); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_TIE = convertDomainsToHashes({ + "test41.com": [1, 50, 2, 50], + "test42.com": [1, 50, 2, 50], + "test43.com": [1, 50, 2, 50], + "test44.com": [1, 50, 2, 50], + "test45.com": [1, 50, 2, 50], + "test46.com": [3, 50, 4, 50], + "test47.com": [5, 50, 6, 50], + "test48.com": [7, 50, 8, 50], + "test49.com": [9, 50, 10, 50], + "test50.com": [11, 50, 12, 50], +}); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_1 = + convertDomainsToHashes({ + "test51.com": [1, 45], + "test52.com": [2, 45], + "test53.com": [3, 45], + "test54.com": [4, 45], + "test55.com": [5, 45], + "test56.com": [6, 45], + "test57.com": [7, 45], + "test58.com": [8, 45], + "test59.com": [9, 45], + "test60.com": [10, 45], + }); + +const TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_2 = + convertDomainsToHashes({ + "test61.com": [1, 35, 2, 4], + "test62.com": [1, 5, 2, 94], + }); add_setup(async () => { + do_get_profile(); Services.prefs.setBoolPref( "browser.search.serpEventTelemetryCategorization.enabled", true ); + await SearchSERPDomainToCategoriesMap.init(); }); add_task(async function test_categorization_simple() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_SIMPLE ); @@ -115,8 +136,9 @@ add_task(async function test_categorization_simple() { "test10.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -126,7 +148,7 @@ add_task(async function test_categorization_simple() { }); add_task(async function test_categorization_inconclusive() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_INCONCLUSIVE ); @@ -143,8 +165,9 @@ add_task(async function test_categorization_inconclusive() { "test20.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -161,7 +184,7 @@ add_task(async function test_categorization_inconclusive() { add_task(async function test_categorization_unknown() { // Reusing TEST_DOMAIN_TO_CATEGORIES_MAP_SIMPLE since none of this task's // domains will be keys within it. - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_SIMPLE ); @@ -178,8 +201,9 @@ add_task(async function test_categorization_unknown() { "test30.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -194,7 +218,7 @@ add_task(async function test_categorization_unknown() { }); add_task(async function test_categorization_unknown_and_inconclusive() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_UNKNOWN_AND_INCONCLUSIVE ); @@ -211,8 +235,9 @@ add_task(async function test_categorization_unknown_and_inconclusive() { "test40.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -228,7 +253,7 @@ add_task(async function test_categorization_unknown_and_inconclusive() { // Tests a mixture of categorized, inconclusive and unknown domains. add_task(async function test_categorization_all_types() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_ALL_TYPES ); @@ -247,8 +272,9 @@ add_task(async function test_categorization_all_types() { "test60.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -263,7 +289,7 @@ add_task(async function test_categorization_all_types() { }); add_task(async function test_categorization_tie() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_TIE ); @@ -280,8 +306,9 @@ add_task(async function test_categorization_tie() { "test50.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.equal( [1, 2].includes(resultsToReport.category), @@ -301,7 +328,7 @@ add_task(async function test_categorization_tie() { }); add_task(async function test_rank_penalization_equal_scores() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_1 ); @@ -318,8 +345,9 @@ add_task(async function test_rank_penalization_equal_scores() { "test60.com", ]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, @@ -329,14 +357,15 @@ add_task(async function test_rank_penalization_equal_scores() { }); add_task(async function test_rank_penalization_highest_score_lower_on_page() { - SearchSERPDomainToCategoriesMap.overrideMapForTests( + await SearchSERPDomainToCategoriesMap.overrideMapForTests( TEST_DOMAIN_TO_CATEGORIES_MAP_RANK_PENALIZATION_2 ); let domains = new Set(["test61.com", "test62.com"]); - let resultsToReport = - SearchSERPCategorization.applyCategorizationLogic(domains); + let resultsToReport = await SearchSERPCategorization.applyCategorizationLogic( + domains + ); Assert.deepEqual( resultsToReport, diff --git a/browser/components/search/test/unit/test_search_telemetry_categorization_process_domains.js b/browser/components/search/test/unit/test_search_telemetry_categorization_process_domains.js deleted file mode 100644 index 84acedaa7a..0000000000 --- a/browser/components/search/test/unit/test_search_telemetry_categorization_process_domains.js +++ /dev/null @@ -1,89 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - http://creativecommons.org/publicdomain/zero/1.0/ */ - -/* - * This test ensures we are correctly processing the domains that have been - * extracted from a SERP. - */ - -ChromeUtils.defineESModuleGetters(this, { - BrowserSearchTelemetry: "resource:///modules/BrowserSearchTelemetry.sys.mjs", - SearchSERPCategorization: "resource:///modules/SearchSERPTelemetry.sys.mjs", - SearchSERPTelemetry: "resource:///modules/SearchSERPTelemetry.sys.mjs", - SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs", - sinon: "resource://testing-common/Sinon.sys.mjs", -}); - -// Links including the provider name are not extracted. -const PROVIDER = "example"; - -const TESTS = [ - { - title: "Domains matching the provider.", - domains: ["example.com", "www.example.com", "www.foobar.com"], - expected: ["foobar.com"], - }, - { - title: "Second-level domains to a top-level domain.", - domains: [ - "www.foobar.gc.ca", - "www.foobar.gov.uk", - "foobar.co.uk", - "www.foobar.co.il", - ], - expected: ["foobar.gc.ca", "foobar.gov.uk", "foobar.co.uk", "foobar.co.il"], - }, - { - title: "Long subdomain.", - domains: ["ab.cd.ef.gh.foobar.com"], - expected: ["foobar.com"], - }, - { - title: "Same top-level domain.", - domains: ["foobar.com", "www.foobar.com", "abc.def.foobar.com"], - expected: ["foobar.com"], - }, - { - title: "Empty input.", - domains: [""], - expected: [], - }, -]; - -add_setup(async function () { - Services.prefs.setBoolPref( - SearchUtils.BROWSER_SEARCH_PREF + "serpEventTelemetry.enabled", - true - ); - Services.prefs.setBoolPref( - SearchUtils.BROWSER_SEARCH_PREF + - "serpEventTelemetryCategorization.enabled", - true - ); - - // Required or else BrowserSearchTelemetry will throw. - sinon.stub(BrowserSearchTelemetry, "shouldRecordSearchCount").returns(true); - await SearchSERPTelemetry.init(); -}); - -add_task(async function test_parsing_extracted_urls() { - for (let i = 0; i < TESTS.length; i++) { - let currentTest = TESTS[i]; - let domains = new Set(currentTest.domains); - - if (currentTest.title) { - info(currentTest.title); - } - let expectedDomains = new Set(currentTest.expected); - let actualDomains = SearchSERPCategorization.processDomains( - domains, - PROVIDER - ); - - Assert.deepEqual( - Array.from(actualDomains), - Array.from(expectedDomains), - "Domains should have been parsed correctly." - ); - } -}); diff --git a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js index 423ee0a81d..40d38efbba 100644 --- a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js +++ b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js @@ -16,16 +16,34 @@ ChromeUtils.defineESModuleGetters(this, { TestUtils: "resource://testing-common/TestUtils.sys.mjs", }); +ChromeUtils.defineLazyGetter(this, "gCryptoHash", () => { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); + +function convertDomainsToHashes(domainsToCategories) { + let newObj = {}; + for (let [key, value] of Object.entries(domainsToCategories)) { + gCryptoHash.init(gCryptoHash.SHA256); + let bytes = new TextEncoder().encode(key); + gCryptoHash.update(bytes, key.length); + let hash = gCryptoHash.finish(true); + newObj[hash] = value; + } + return newObj; +} + async function waitForDomainToCategoriesUpdate() { return TestUtils.topicObserved("domain-to-categories-map-update-complete"); } -async function mockRecordWithCachedAttachment({ id, version, filename }) { +async function mockRecordWithCachedAttachment({ + id, + version, + filename, + mapping, +}) { // Get the bytes of the file for the hash and size for attachment metadata. - let data = await IOUtils.readUTF8( - PathUtils.join(do_get_cwd().path, filename) - ); - let buffer = new TextEncoder().encode(data).buffer; + let buffer = new TextEncoder().encode(JSON.stringify(mapping)).buffer; let stream = Cc["@mozilla.org/io/arraybuffer-input-stream;1"].createInstance( Ci.nsIArrayBufferInputStream ); @@ -73,21 +91,33 @@ const RECORDS = { id: RECORD_A_ID, version: 1, filename: "domain_category_mappings_1a.json", + mapping: convertDomainsToHashes({ + "example.com": [1, 100], + }), }, record1b: { id: RECORD_B_ID, version: 1, filename: "domain_category_mappings_1b.json", + mapping: convertDomainsToHashes({ + "example.org": [2, 90], + }), }, record2a: { id: RECORD_A_ID, version: 2, filename: "domain_category_mappings_2a.json", + mapping: convertDomainsToHashes({ + "example.com": [1, 80], + }), }, record2b: { id: RECORD_B_ID, version: 2, filename: "domain_category_mappings_2b.json", + mapping: convertDomainsToHashes({ + "example.org": [2, 50, 4, 80], + }), }, }; @@ -115,13 +145,13 @@ add_task(async function test_initial_import() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [{ category: 1, score: 100 }], "Return value from lookup of example.com should be the same." ); Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.org"), + await SearchSERPDomainToCategoriesMap.get("example.org"), [{ category: 2, score: 90 }], "Return value from lookup of example.org should be the same." ); @@ -167,13 +197,13 @@ add_task(async function test_update_records() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [{ category: 1, score: 80 }], "Return value from lookup of example.com should have changed." ); Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.org"), + await SearchSERPDomainToCategoriesMap.get("example.org"), [ { category: 2, score: 50 }, { category: 4, score: 80 }, @@ -224,13 +254,13 @@ add_task(async function test_delayed_initial_import() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [{ category: 1, score: 100 }], "Return value from lookup of example.com should be the same." ); Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.org"), + await SearchSERPDomainToCategoriesMap.get("example.org"), [{ category: 2, score: 90 }], "Return value from lookup of example.org should be the same." ); @@ -264,7 +294,7 @@ add_task(async function test_remove_record() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [{ category: 1, score: 80 }], "Initialized properly." ); @@ -283,13 +313,13 @@ add_task(async function test_remove_record() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [{ category: 1, score: 80 }], "Return value from lookup of example.com should remain unchanged." ); Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.org"), + await SearchSERPDomainToCategoriesMap.get("example.org"), [], "Return value from lookup of example.org should be empty." ); @@ -323,7 +353,7 @@ add_task(async function test_different_versions_coexisting() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [ { category: 1, @@ -334,7 +364,7 @@ add_task(async function test_different_versions_coexisting() { ); Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.org"), + await SearchSERPDomainToCategoriesMap.get("example.org"), [ { category: 2, score: 50 }, { category: 4, score: 80 }, @@ -367,7 +397,7 @@ add_task(async function test_download_error() { await promise; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [ { category: 1, @@ -406,7 +436,7 @@ add_task(async function test_download_error() { await observeDownloadError; Assert.deepEqual( - SearchSERPDomainToCategoriesMap.get("example.com"), + await SearchSERPDomainToCategoriesMap.get("example.com"), [], "Domain should not exist in store." ); diff --git a/browser/components/search/test/unit/xpcshell.toml b/browser/components/search/test/unit/xpcshell.toml index 61cdb83378..423d218d19 100644 --- a/browser/components/search/test/unit/xpcshell.toml +++ b/browser/components/search/test/unit/xpcshell.toml @@ -8,16 +8,8 @@ firefox-appdir = "browser" ["test_search_telemetry_categorization_logic.js"] -["test_search_telemetry_categorization_process_domains.js"] - ["test_search_telemetry_categorization_sync.js"] prefs = ["browser.search.serpEventTelemetryCategorization.enabled=true"] -support-files = [ - "domain_category_mappings_1a.json", - "domain_category_mappings_1b.json", - "domain_category_mappings_2a.json", - "domain_category_mappings_2b.json", -] ["test_search_telemetry_compare_urls.js"] |