summaryrefslogtreecommitdiffstats
path: root/browser/components/search
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-15 03:35:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-15 03:35:49 +0000
commitd8bbc7858622b6d9c278469aab701ca0b609cddf (patch)
treeeff41dc61d9f714852212739e6b3738b82a2af87 /browser/components/search
parentReleasing progress-linux version 125.0.3-1~progress7.99u1. (diff)
downloadfirefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.tar.xz
firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.zip
Merging upstream version 126.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'browser/components/search')
-rw-r--r--browser/components/search/.eslintrc.js2
-rw-r--r--browser/components/search/DomainToCategoriesMap.worker.mjs101
-rw-r--r--browser/components/search/SearchSERPTelemetry.sys.mjs860
-rw-r--r--browser/components/search/metrics.yaml104
-rw-r--r--browser/components/search/moz.build6
-rw-r--r--browser/components/search/schema/search-telemetry-v2-schema.json (renamed from browser/components/search/schema/search-telemetry-schema.json)0
-rw-r--r--browser/components/search/schema/search-telemetry-v2-ui-schema.json (renamed from browser/components/search/schema/search-telemetry-ui-schema.json)2
-rw-r--r--browser/components/search/test/browser/telemetry/browser.toml23
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js15
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js13
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js35
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js46
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js141
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js302
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js12
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js60
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js21
-rw-r--r--browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js19
-rw-r--r--browser/components/search/test/browser/telemetry/head.js59
-rw-r--r--browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html18
-rw-r--r--browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html31
-rw-r--r--browser/components/search/test/marionette/manifest.toml2
-rw-r--r--browser/components/search/test/marionette/telemetry/manifest.toml4
-rw-r--r--browser/components/search/test/marionette/telemetry/test_ping_submitted.py89
-rw-r--r--browser/components/search/test/unit/corruptDB.sqlitebin0 -> 32772 bytes
-rw-r--r--browser/components/search/test/unit/test_domain_to_categories_store.js361
-rw-r--r--browser/components/search/test/unit/test_search_telemetry_categorization_sync.js75
-rw-r--r--browser/components/search/test/unit/test_search_telemetry_config_validation.js2
-rw-r--r--browser/components/search/test/unit/test_ui_schemas_valid.js31
-rw-r--r--browser/components/search/test/unit/xpcshell.toml11
30 files changed, 2186 insertions, 259 deletions
diff --git a/browser/components/search/.eslintrc.js b/browser/components/search/.eslintrc.js
index 39079432e7..7224dc6eb7 100644
--- a/browser/components/search/.eslintrc.js
+++ b/browser/components/search/.eslintrc.js
@@ -5,8 +5,6 @@
"use strict";
module.exports = {
- extends: ["plugin:mozilla/require-jsdoc"],
-
rules: {
"mozilla/var-only-at-top-level": "error",
},
diff --git a/browser/components/search/DomainToCategoriesMap.worker.mjs b/browser/components/search/DomainToCategoriesMap.worker.mjs
deleted file mode 100644
index 07dc52cfb8..0000000000
--- a/browser/components/search/DomainToCategoriesMap.worker.mjs
+++ /dev/null
@@ -1,101 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { PromiseWorker } from "resource://gre/modules/workers/PromiseWorker.mjs";
-
-/**
- * Boilerplate to connect with the main thread PromiseWorker.
- */
-const worker = new PromiseWorker.AbstractWorker();
-worker.dispatch = function (method, args = []) {
- return agent[method](...args);
-};
-worker.postMessage = function (message, ...transfers) {
- self.postMessage(message, ...transfers);
-};
-worker.close = function () {
- self.close();
-};
-
-self.addEventListener("message", msg => worker.handleMessage(msg));
-self.addEventListener("unhandledrejection", function (error) {
- throw error.reason;
-});
-
-/**
- * Stores and manages the Domain-to-Categories Map.
- */
-class Agent {
- /**
- * @type {Map<string, Array<number>>} Hashes mapped to categories and values.
- */
- #map = new Map();
-
- /**
- * Converts data from the array directly into a Map.
- *
- * @param {Array<ArrayBuffer>} fileContents Files
- * @returns {boolean} Returns whether the Map contains results.
- */
- populateMap(fileContents) {
- this.#map.clear();
-
- for (let fileContent of fileContents) {
- let obj;
- try {
- obj = JSON.parse(new TextDecoder().decode(fileContent));
- } catch (ex) {
- return false;
- }
- for (let objKey in obj) {
- if (Object.hasOwn(obj, objKey)) {
- this.#map.set(objKey, obj[objKey]);
- }
- }
- }
- return this.#map.size > 0;
- }
-
- /**
- * Retrieves scores for the hash from the map.
- *
- * @param {string} hash Key to look up in the map.
- * @returns {Array<number>}
- */
- getScores(hash) {
- if (this.#map.has(hash)) {
- return this.#map.get(hash);
- }
- return [];
- }
-
- /**
- * Empties the internal map.
- *
- * @returns {boolean}
- */
- emptyMap() {
- this.#map.clear();
- return true;
- }
-
- /**
- * Test only function to allow the map to contain information without
- * having to go through Remote Settings.
- *
- * @param {object} obj The data to directly import into the Map.
- * @returns {boolean} Whether the map contains values.
- */
- overrideMapForTests(obj) {
- this.#map.clear();
- for (let objKey in obj) {
- if (Object.hasOwn(obj, objKey)) {
- this.#map.set(objKey, obj[objKey]);
- }
- }
- return this.#map.size > 0;
- }
-}
-
-const agent = new Agent();
diff --git a/browser/components/search/SearchSERPTelemetry.sys.mjs b/browser/components/search/SearchSERPTelemetry.sys.mjs
index fa593be08c..2a9ed88db1 100644
--- a/browser/components/search/SearchSERPTelemetry.sys.mjs
+++ b/browser/components/search/SearchSERPTelemetry.sys.mjs
@@ -7,12 +7,12 @@ import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
- BasePromiseWorker: "resource://gre/modules/PromiseWorker.sys.mjs",
BrowserSearchTelemetry: "resource:///modules/BrowserSearchTelemetry.sys.mjs",
PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
Region: "resource://gre/modules/Region.sys.mjs",
RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs",
+ Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
});
ChromeUtils.defineLazyGetter(lazy, "gCryptoHash", () => {
@@ -52,11 +52,15 @@ export const SEARCH_TELEMETRY_SHARED = {
const impressionIdsWithoutEngagementsSet = new Set();
export const CATEGORIZATION_SETTINGS = {
+ STORE_SCHEMA: 1,
+ STORE_FILE: "domain_to_categories.sqlite",
+ STORE_NAME: "domain_to_categories",
MAX_DOMAINS_TO_CATEGORIZE: 10,
MINIMUM_SCORE: 0,
STARTING_RANK: 2,
IDLE_TIMEOUT_SECONDS: 60 * 60,
WAKE_TIMEOUT_MS: 60 * 60 * 1000,
+ PING_SUBMISSION_THRESHOLD: 10,
};
ChromeUtils.defineLazyGetter(lazy, "logConsole", () => {
@@ -83,15 +87,20 @@ XPCOMUtils.defineLazyPreferenceGetter(
false,
(aPreference, previousValue, newValue) => {
if (newValue) {
- SearchSERPDomainToCategoriesMap.init();
- SearchSERPCategorizationEventScheduler.init();
+ SearchSERPCategorization.init();
} else {
- SearchSERPDomainToCategoriesMap.uninit();
- SearchSERPCategorizationEventScheduler.uninit();
+ SearchSERPCategorization.uninit({ deleteMap: true });
}
}
);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "activityLimit",
+ "telemetry.fog.test.activity_limit",
+ 120
+);
+
export const SearchSERPTelemetryUtils = {
ACTIONS: {
CLICKED: "clicked",
@@ -380,7 +389,7 @@ class TelemetryHandler {
* unit tests can set it to easy to test values.
*
* @param {Array} providerInfo
- * See {@link https://searchfox.org/mozilla-central/search?q=search-telemetry-schema.json}
+ * See {@link https://searchfox.org/mozilla-central/search?q=search-telemetry-v2-schema.json}
* for type information.
*/
overrideSearchTelemetryForTests(providerInfo) {
@@ -1641,7 +1650,10 @@ class ContentHandler {
!telemetryState.adImpressionsReported
) {
for (let [componentType, data] of info.adImpressions.entries()) {
- telemetryState.adsVisible += data.adsVisible;
+ // Not all ad impressions are sponsored.
+ if (AD_COMPONENTS.includes(componentType)) {
+ telemetryState.adsVisible += data.adsVisible;
+ }
lazy.logConsole.debug("Counting ad:", { type: componentType, ...data });
Glean.serp.adImpression.record({
@@ -1772,6 +1784,8 @@ class ContentHandler {
let item = this._findItemForBrowser(browser);
let telemetryState = item.browserTelemetryStateMap.get(browser);
if (lazy.serpEventTelemetryCategorization && telemetryState) {
+ lazy.logConsole.debug("Ad domains:", Array.from(info.adDomains));
+ lazy.logConsole.debug("Non ad domains:", Array.from(info.nonAdDomains));
let result = await SearchSERPCategorization.maybeCategorizeSERP(
info.nonAdDomains,
info.adDomains,
@@ -1789,6 +1803,7 @@ class ContentHandler {
partner_code: impressionInfo.partnerCode,
provider: impressionInfo.provider,
tagged: impressionInfo.tagged,
+ is_shopping_page: impressionInfo.isShoppingPage,
num_ads_clicked: telemetryState.adsClicked,
num_ads_visible: telemetryState.adsVisible,
});
@@ -1843,6 +1858,22 @@ class ContentHandler {
* Categorizes SERPs.
*/
class SERPCategorizer {
+ async init() {
+ if (lazy.serpEventTelemetryCategorization) {
+ lazy.logConsole.debug("Initialize SERP categorizer.");
+ await SearchSERPDomainToCategoriesMap.init();
+ SearchSERPCategorizationEventScheduler.init();
+ SERPCategorizationRecorder.init();
+ }
+ }
+
+ async uninit({ deleteMap = false } = {}) {
+ lazy.logConsole.debug("Uninit SERP categorizer.");
+ await SearchSERPDomainToCategoriesMap.uninit(deleteMap);
+ SearchSERPCategorizationEventScheduler.uninit();
+ SERPCategorizationRecorder.uninit();
+ }
+
/**
* Categorizes domains extracted from SERPs. Note that we don't process
* domains if the domain-to-categories map is empty (if the client couldn't
@@ -1999,12 +2030,8 @@ class CategorizationEventScheduler {
*/
#mostRecentMs = null;
- constructor() {
- this.init();
- }
-
init() {
- if (!lazy.serpEventTelemetryCategorization || this.#init) {
+ if (this.#init) {
return;
}
@@ -2114,6 +2141,61 @@ class CategorizationEventScheduler {
* Handles reporting SERP categorization telemetry to Glean.
*/
class CategorizationRecorder {
+ #init = false;
+
+ // The number of SERP categorizations that have been recorded but not yet
+ // reported in a Glean ping.
+ #serpCategorizationsCount = 0;
+
+ // When the user started interacting with the SERP.
+ #userInteractionStartTime = null;
+
+ async init() {
+ if (this.#init) {
+ return;
+ }
+
+ Services.obs.addObserver(this, "user-interaction-active");
+ Services.obs.addObserver(this, "user-interaction-inactive");
+ this.#init = true;
+ this.submitPing("startup");
+ Services.obs.notifyObservers(null, "categorization-recorder-init");
+ }
+
+ uninit() {
+ if (this.#init) {
+ Services.obs.removeObserver(this, "user-interaction-active");
+ Services.obs.removeObserver(this, "user-interaction-inactive");
+ this.#resetCategorizationRecorderData();
+ this.#init = false;
+ }
+ }
+
+ observe(subject, topic, _data) {
+ switch (topic) {
+ case "user-interaction-active": {
+ // If the user is already active, we don't want to overwrite the start
+ // time.
+ if (this.#userInteractionStartTime == null) {
+ this.#userInteractionStartTime = Date.now();
+ }
+ break;
+ }
+ case "user-interaction-inactive": {
+ let currentTime = Date.now();
+ let activityLimitInMs = lazy.activityLimit * 1000;
+ if (
+ this.#userInteractionStartTime &&
+ currentTime - this.#userInteractionStartTime >= activityLimitInMs
+ ) {
+ this.submitPing("inactivity");
+ }
+ this.#userInteractionStartTime = null;
+ break;
+ }
+ }
+ }
+
/**
* Helper function for recording the SERP categorization event.
*
@@ -2125,7 +2207,37 @@ class CategorizationRecorder {
"Reporting the following categorization result:",
resultToReport
);
- // TODO: Bug 1868476 - Report result to Glean.
+ Glean.serp.categorization.record(resultToReport);
+
+ this.#serpCategorizationsCount++;
+ if (
+ this.#serpCategorizationsCount >=
+ CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD
+ ) {
+ this.submitPing("threshold_reached");
+ this.#serpCategorizationsCount = 0;
+ }
+ }
+
+ submitPing(reason) {
+ lazy.logConsole.debug("Submitting SERP categorization ping:", reason);
+ GleanPings.serpCategorization.submit(reason);
+ }
+
+ /**
+ * Tests are able to clear telemetry on demand. When that happens, we need to
+ * ensure we're doing to the same here or else the internal count in tests
+ * will be inaccurate.
+ */
+ testReset() {
+ if (Cu.isInAutomation) {
+ this.#resetCategorizationRecorderData();
+ }
+ }
+
+ #resetCategorizationRecorderData() {
+ this.#serpCategorizationsCount = 0;
+ this.#userInteractionStartTime = null;
}
}
@@ -2144,10 +2256,8 @@ class CategorizationRecorder {
*/
/**
- * Maps domain to categories, with its data synced using Remote Settings. The
- * data is downloaded from Remote Settings and stored in a map in a worker
- * thread to avoid processing the data from the attachments from occupying
- * the main thread.
+ * Maps domain to categories. Data is downloaded from Remote Settings and
+ * stored inside DomainToCategoriesStore.
*/
class DomainToCategoriesMap {
/**
@@ -2195,40 +2305,63 @@ class DomainToCategoriesMap {
#downloadRetries = 0;
/**
- * Whether the mappings are empty.
- */
- #empty = true;
-
- /**
- * @type {BasePromiseWorker|null} Worker used to access the raw domain
- * to categories map data.
+ * A reference to the data store.
+ *
+ * @type {DomainToCategoriesStore | null}
*/
- #worker = null;
+ #store = null;
/**
* Runs at application startup with startup idle tasks. If the SERP
* categorization preference is enabled, it creates a Remote Settings
- * client to listen to updates, and populates the map.
+ * client to listen to updates, and populates the store.
*/
async init() {
- if (!lazy.serpEventTelemetryCategorization || this.#init) {
+ if (this.#init) {
return;
}
lazy.logConsole.debug("Initializing domain-to-categories map.");
- this.#worker = new lazy.BasePromiseWorker(
- "resource:///modules/DomainToCategoriesMap.worker.mjs",
- { type: "module" }
- );
- await this.#setupClientAndMap();
+
+ // Set early to allow un-init from an initialization.
this.#init = true;
+
+ try {
+ await this.#setupClientAndStore();
+ } catch (ex) {
+ lazy.logConsole.error(ex);
+ await this.uninit();
+ return;
+ }
+
+ // If we don't have a client and store, it likely means an un-init process
+ // started during the initialization process.
+ if (this.#client && this.#store) {
+ lazy.logConsole.debug("Initialized domain-to-categories map.");
+ Services.obs.notifyObservers(null, "domain-to-categories-map-init");
+ }
}
- uninit() {
+ async uninit(shouldDeleteStore) {
if (this.#init) {
lazy.logConsole.debug("Un-initializing domain-to-categories map.");
- this.#clearClientAndWorker();
+ this.#clearClient();
this.#cancelAndNullifyTimer();
+
+ if (this.#store) {
+ if (shouldDeleteStore) {
+ try {
+ await this.#store.dropData();
+ } catch (ex) {
+ lazy.logConsole.error(ex);
+ }
+ }
+ await this.#store.uninit();
+ this.#store = null;
+ }
+
+ lazy.logConsole.debug("Un-initialized domain-to-categories map.");
this.#init = false;
+ Services.obs.notifyObservers(null, "domain-to-categories-map-uninit");
}
}
@@ -2241,14 +2374,14 @@ class DomainToCategoriesMap {
* for the domain is available, return an empty array.
*/
async get(domain) {
- if (this.empty) {
+ if (!this.#store || this.#store.empty || !this.#store.ready) {
return [];
}
lazy.gCryptoHash.init(lazy.gCryptoHash.SHA256);
let bytes = new TextEncoder().encode(domain);
lazy.gCryptoHash.update(bytes, domain.length);
let hash = lazy.gCryptoHash.finish(true);
- let rawValues = await this.#worker.post("getScores", [hash]);
+ let rawValues = await this.#store.getCategories(hash);
if (rawValues?.length) {
let output = [];
// Transform data into a more readable format.
@@ -2275,12 +2408,15 @@ class DomainToCategoriesMap {
}
/**
- * Whether the map is empty of data.
+ * Whether the store is empty of data.
*
* @returns {boolean}
*/
get empty() {
- return this.#empty;
+ if (!this.#store) {
+ return true;
+ }
+ return this.#store.empty;
}
/**
@@ -2290,15 +2426,26 @@ class DomainToCategoriesMap {
* @param {object} domainToCategoriesMap
* An object where the key is a hashed domain and the value is an array
* containing an arbitrary number of DomainCategoryScores.
+ * @param {number} version
+ * The version number for the store.
*/
- async overrideMapForTests(domainToCategoriesMap) {
- let hasResults = await this.#worker.post("overrideMapForTests", [
- domainToCategoriesMap,
- ]);
- this.#empty = !hasResults;
+ async overrideMapForTests(domainToCategoriesMap, version = 1) {
+ if (Cu.isInAutomation || Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) {
+ await this.#store.init();
+ await this.#store.dropData();
+ await this.#store.insertObject(domainToCategoriesMap, version);
+ }
}
- async #setupClientAndMap() {
+ /**
+ * Connect with Remote Settings and retrieve the records associated with
+ * categorization. Then, check if the records match the store version. If
+ * no records exist, return early. If records exist but the version stored
+ * on the records differ from the store version, then attempt to
+ * empty the store and fill it with data from downloaded attachments. Only
+ * reuse the store if the version in each record matches the store.
+ */
+ async #setupClientAndStore() {
if (this.#client && !this.empty) {
return;
}
@@ -2308,11 +2455,33 @@ class DomainToCategoriesMap {
this.#onSettingsSync = event => this.#sync(event.data);
this.#client.on("sync", this.#onSettingsSync);
+ this.#store = new DomainToCategoriesStore();
+ await this.#store.init();
+
let records = await this.#client.get();
- await this.#clearAndPopulateMap(records);
+ // Even though records don't exist, this is still technically initialized
+ // since the next sync from Remote Settings will populate the store with
+ // records.
+ if (!records.length) {
+ lazy.logConsole.debug("No records found for domain-to-categories map.");
+ return;
+ }
+
+ this.#version = this.#retrieveLatestVersion(records);
+ let storeVersion = await this.#store.getVersion();
+ if (storeVersion == this.#version && !this.#store.empty) {
+ lazy.logConsole.debug("Reuse existing domain-to-categories map.");
+ Services.obs.notifyObservers(
+ null,
+ "domain-to-categories-map-update-complete"
+ );
+ return;
+ }
+
+ await this.#clearAndPopulateStore(records);
}
- #clearClientAndWorker() {
+ #clearClient() {
if (this.#client) {
lazy.logConsole.debug("Removing Remote Settings client.");
this.#client.off("sync", this.#onSettingsSync);
@@ -2320,17 +2489,6 @@ class DomainToCategoriesMap {
this.#onSettingsSync = null;
this.#downloadRetries = 0;
}
-
- if (!this.#empty) {
- lazy.logConsole.debug("Clearing domain-to-categories map.");
- this.#empty = true;
- this.#version = null;
- }
-
- if (this.#worker) {
- this.#worker.terminate();
- this.#worker = null;
- }
}
/**
@@ -2377,27 +2535,50 @@ class DomainToCategoriesMap {
// again in case there's a new download error.
this.#downloadRetries = 0;
- this.#clearAndPopulateMap(data?.current);
+ try {
+ await this.#clearAndPopulateStore(data?.current);
+ } catch (ex) {
+ lazy.logConsole.error("Error populating map: ", ex);
+ await this.uninit();
+ }
}
/**
- * Clear the existing map and populate it with attachments found in the
+ * Clear the existing store and populate it with attachments found in the
* records. If no attachments are found, or no record containing an
* attachment contained the latest version, then nothing will change.
*
* @param {Array<DomainToCategoriesRecord>} records
* The records containing attachments.
- *
+ * @throws {Error}
+ * Will throw if it was not able to drop the store data, or it was unable
+ * to insert data into the store.
*/
- async #clearAndPopulateMap(records) {
- // Empty map so that if there are errors in the download process, callers
- // querying the map won't use information we know is already outdated.
- await this.#worker.post("emptyMap");
+ async #clearAndPopulateStore(records) {
+ // If we don't have a handle to a store, it would mean that it was removed
+ // during an uninitialization process.
+ if (!this.#store) {
+ lazy.logConsole.debug(
+ "Could not populate store because no store was available."
+ );
+ return;
+ }
+
+ if (!this.#store.ready) {
+ lazy.logConsole.debug(
+ "Could not populate store because it was not ready."
+ );
+ return;
+ }
+
+ // Empty table so that if there are errors in the download process, callers
+ // querying the map won't use information we know is probably outdated.
+ await this.#store.dropData();
- this.#empty = true;
this.#version = null;
this.#cancelAndNullifyTimer();
+ // A collection with no records is still a valid init state.
if (!records?.length) {
lazy.logConsole.debug("No records found for domain-to-categories map.");
return;
@@ -2418,41 +2599,24 @@ class DomainToCategoriesMap {
fileContents.push(result.buffer);
}
ChromeUtils.addProfilerMarker(
- "SearchSERPTelemetry.#clearAndPopulateMap",
+ "SearchSERPTelemetry.#clearAndPopulateStore",
start,
"Download attachments."
);
- // Attachments should have a version number.
this.#version = this.#retrieveLatestVersion(records);
-
if (!this.#version) {
lazy.logConsole.debug("Could not find a version number for any record.");
return;
}
- Services.tm.idleDispatchToMainThread(async () => {
- start = Cu.now();
- let hasResults;
- try {
- hasResults = await this.#worker.post("populateMap", [fileContents]);
- } catch (ex) {
- console.error(ex);
- }
+ await this.#store.insertFileContents(fileContents, this.#version);
- this.#empty = !hasResults;
-
- ChromeUtils.addProfilerMarker(
- "SearchSERPTelemetry.#clearAndPopulateMap",
- start,
- "Convert contents to JSON."
- );
- lazy.logConsole.debug("Updated domain-to-categories map.");
- Services.obs.notifyObservers(
- null,
- "domain-to-categories-map-update-complete"
- );
- });
+ lazy.logConsole.debug("Finished updating domain-to-categories store.");
+ Services.obs.notifyObservers(
+ null,
+ "domain-to-categories-map-update-complete"
+ );
}
#cancelAndNullifyTimer() {
@@ -2466,7 +2630,8 @@ class DomainToCategoriesMap {
#createTimerToPopulateMap() {
if (
this.#downloadRetries >=
- TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.maxTriesPerSession
+ TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.maxTriesPerSession ||
+ !this.#client
) {
return;
}
@@ -2486,7 +2651,12 @@ class DomainToCategoriesMap {
async () => {
this.#downloadRetries += 1;
let records = await this.#client.get();
- this.#clearAndPopulateMap(records);
+ try {
+ await this.#clearAndPopulateStore(records);
+ } catch (ex) {
+ lazy.logConsole.error("Error populating store: ", ex);
+ await this.uninit();
+ }
},
delay,
Ci.nsITimer.TYPE_ONE_SHOT
@@ -2494,6 +2664,514 @@ class DomainToCategoriesMap {
}
}
+/**
+ * Handles the storage of data containing domains to categories.
+ */
+export class DomainToCategoriesStore {
+ #init = false;
+
+ /**
+ * The connection to the store.
+ *
+ * @type {object | null}
+ */
+ #connection = null;
+
+ /**
+ * Reference for the shutdown blocker in case we need to remove it before
+ * shutdown.
+ *
+ * @type {Function | null}
+ */
+ #asyncShutdownBlocker = null;
+
+ /**
+ * Whether the store is empty of data.
+ *
+ * @type {boolean}
+ */
+ #empty = true;
+
+ /**
+ * For a particular subset of errors, we'll attempt to rebuild the database
+ * from scratch.
+ */
+ #rebuildableErrors = ["NS_ERROR_FILE_CORRUPTED"];
+
+ /**
+ * Initializes the store. If the store is initialized it should have cached
+ * a connection to the store and ensured the store exists.
+ */
+ async init() {
+ if (this.#init) {
+ return;
+ }
+ lazy.logConsole.debug("Initializing domain-to-categories store.");
+
+ // Attempts to cache a connection to the store.
+ // If a failure occured, try to re-build the store.
+ let rebuiltStore = false;
+ try {
+ await this.#initConnection();
+ } catch (ex1) {
+ lazy.logConsole.error(`Error initializing a connection: ${ex1}`);
+ if (this.#rebuildableErrors.includes(ex1.name)) {
+ try {
+ await this.#rebuildStore();
+ } catch (ex2) {
+ await this.#closeConnection();
+ lazy.logConsole.error(`Could not rebuild store: ${ex2}`);
+ return;
+ }
+ rebuiltStore = true;
+ }
+ }
+
+ // If we don't have a connection, bail because the browser could be
+ // shutting down ASAP, or re-creating the store is impossible.
+ if (!this.#connection) {
+ lazy.logConsole.debug(
+ "Bailing from DomainToCategoriesStore.init because connection doesn't exist."
+ );
+ return;
+ }
+
+ // If we weren't forced to re-build the store, we only have the connection.
+ // We want to ensure the store exists so calls to public methods can pass
+ // without throwing errors due to the absence of the store.
+ if (!rebuiltStore) {
+ try {
+ await this.#initSchema();
+ } catch (ex) {
+ lazy.logConsole.error(`Error trying to create store: ${ex}`);
+ await this.#closeConnection();
+ return;
+ }
+ }
+
+ lazy.logConsole.debug("Initialized domain-to-categories store.");
+ this.#init = true;
+ }
+
+ async uninit() {
+ if (this.#init) {
+ lazy.logConsole.debug("Un-initializing domain-to-categories store.");
+ await this.#closeConnection();
+ this.#asyncShutdownBlocker = null;
+ lazy.logConsole.debug("Un-initialized domain-to-categories store.");
+ }
+ }
+
+ /**
+ * Whether the store has an open connection to the physical store.
+ *
+ * @returns {boolean}
+ */
+ get ready() {
+ return this.#init;
+ }
+
+ /**
+ * Whether the store is devoid of data.
+ *
+ * @returns {boolean}
+ */
+ get empty() {
+ return this.#empty;
+ }
+
+ /**
+ * Clears information in the store. If dropping data encountered a failure,
+ * try to delete the file containing the store and re-create it.
+ *
+ * @throws {Error} Will throw if it was unable to clear information from the
+ * store.
+ */
+ async dropData() {
+ if (!this.#connection) {
+ return;
+ }
+ let tableExists = await this.#connection.tableExists(
+ CATEGORIZATION_SETTINGS.STORE_NAME
+ );
+ if (tableExists) {
+ lazy.logConsole.debug("Drop domain_to_categories.");
+ // This can fail if the permissions of the store are read-only.
+ await this.#connection.executeTransaction(async () => {
+ await this.#connection.execute(`DROP TABLE domain_to_categories`);
+ const createDomainToCategoriesTable = `
+ CREATE TABLE IF NOT EXISTS
+ domain_to_categories (
+ string_id
+ TEXT PRIMARY KEY NOT NULL,
+ categories
+ TEXT
+ );
+ `;
+ await this.#connection.execute(createDomainToCategoriesTable);
+ await this.#connection.execute(`DELETE FROM moz_meta`);
+ await this.#connection.executeCached(
+ `
+ INSERT INTO
+ moz_meta (key, value)
+ VALUES
+ (:key, :value)
+ ON CONFLICT DO UPDATE SET
+ value = :value
+ `,
+ { key: "version", value: 0 }
+ );
+ });
+
+ this.#empty = true;
+ }
+ }
+
+ /**
+ * Given file contents, try moving them into the store. If a failure occurs,
+ * it will attempt to drop existing data to ensure callers aren't accessing
+ * a partially filled store.
+ *
+ * @param {Array<ArrayBuffer>} fileContents
+ * Contents to convert.
+ * @param {number} version
+ * The version for the store.
+ * @throws {Error}
+ * Will throw if the insertion failed and dropData was unable to run
+ * successfully.
+ */
+ async insertFileContents(fileContents, version) {
+ if (!this.#init || !fileContents?.length || !version) {
+ return;
+ }
+
+ try {
+ await this.#insert(fileContents, version);
+ } catch (ex) {
+ lazy.logConsole.error(`Could not insert file contents: ${ex}`);
+ await this.dropData();
+ }
+ }
+
+ /**
+ * Convenience function to make it trivial to insert Javascript objects into
+ * the store. This avoids having to set up the collection in Remote Settings.
+ *
+ * @param {object} domainToCategoriesMap
+ * An object whose keys should be hashed domains with values containing
+ * an array of integers.
+ * @param {number} version
+ * The version for the store.
+ * @returns {boolean}
+ * Whether the operation was successful.
+ */
+ async insertObject(domainToCategoriesMap, version) {
+ if (!Cu.isInAutomation || !this.#init) {
+ return false;
+ }
+ let buffer = new TextEncoder().encode(
+ JSON.stringify(domainToCategoriesMap)
+ ).buffer;
+ await this.insertFileContents([buffer], version);
+ return true;
+ }
+
+ /**
+ * Retrieves domains mapped to the key.
+ *
+ * @param {string} key
+ * The value to lookup in the store.
+ * @returns {Array<number>}
+ * An array of numbers corresponding to the category and score. If the key
+ * does not exist in the store or the store is having issues retrieving the
+ * value, returns an empty array.
+ */
+ async getCategories(key) {
+ if (!this.#init) {
+ return [];
+ }
+
+ let rows;
+ try {
+ rows = await this.#connection.executeCached(
+ `
+ SELECT
+ categories
+ FROM
+ domain_to_categories
+ WHERE
+ string_id = :key
+ `,
+ {
+ key,
+ }
+ );
+ } catch (ex) {
+ lazy.logConsole.error(`Could not retrieve from the store: ${ex}`);
+ return [];
+ }
+
+ if (!rows.length) {
+ return [];
+ }
+ return JSON.parse(rows[0].getResultByName("categories")) ?? [];
+ }
+
+ /**
+ * Retrieves the version number of the store.
+ *
+ * @returns {number}
+ * The version number. Returns 0 if the version was never set or if there
+ * was an issue accessing the version number.
+ */
+ async getVersion() {
+ if (this.#connection) {
+ let rows;
+ try {
+ rows = await this.#connection.executeCached(
+ `
+ SELECT
+ value
+ FROM
+ moz_meta
+ WHERE
+ key = "version"
+ `
+ );
+ } catch (ex) {
+ lazy.logConsole.error(`Could not retrieve version of the store: ${ex}`);
+ return 0;
+ }
+ if (rows.length) {
+ return parseInt(rows[0].getResultByName("value")) ?? 0;
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * Test only function allowing tests to delete the store.
+ */
+ async testDelete() {
+ if (Cu.isInAutomation) {
+ await this.#closeConnection();
+ await this.#delete();
+ }
+ }
+
+ /**
+ * If a connection is available, close it and remove shutdown blockers.
+ */
+ async #closeConnection() {
+ this.#init = false;
+ this.#empty = true;
+ if (this.#asyncShutdownBlocker) {
+ lazy.Sqlite.shutdown.removeBlocker(this.#asyncShutdownBlocker);
+ this.#asyncShutdownBlocker = null;
+ }
+
+ if (this.#connection) {
+ lazy.logConsole.debug("Closing connection.");
+ // An error could occur while closing the connection. We suppress the
+ // error since it is not a critical part of the browser.
+ try {
+ await this.#connection.close();
+ } catch (ex) {
+ lazy.logConsole.error(ex);
+ }
+ this.#connection = null;
+ }
+ }
+
+ /**
+ * Initialize the schema for the store.
+ *
+ * @throws {Error}
+ * Will throw if a permissions error prevents creating the store.
+ */
+ async #initSchema() {
+ if (!this.#connection) {
+ return;
+ }
+ lazy.logConsole.debug("Create store.");
+ // Creation can fail if the store is read only.
+ await this.#connection.executeTransaction(async () => {
+ // Let outer try block handle the exception.
+ const createDomainToCategoriesTable = `
+ CREATE TABLE IF NOT EXISTS
+ domain_to_categories (
+ string_id
+ TEXT PRIMARY KEY NOT NULL,
+ categories
+ TEXT
+ ) WITHOUT ROWID;
+ `;
+ await this.#connection.execute(createDomainToCategoriesTable);
+ const createMetaTable = `
+ CREATE TABLE IF NOT EXISTS
+ moz_meta (
+ key
+ TEXT PRIMARY KEY NOT NULL,
+ value
+ INTEGER
+ ) WITHOUT ROWID;
+ `;
+ await this.#connection.execute(createMetaTable);
+ await this.#connection.setSchemaVersion(
+ CATEGORIZATION_SETTINGS.STORE_SCHEMA
+ );
+ });
+
+ let rows = await this.#connection.executeCached(
+ "SELECT count(*) = 0 FROM domain_to_categories"
+ );
+ this.#empty = !!rows[0].getResultByIndex(0);
+ }
+
+ /**
+ * Attempt to delete the store.
+ *
+ * @throws {Error}
+ * Will throw if the permissions for the file prevent its deletion.
+ */
+ async #delete() {
+ lazy.logConsole.debug("Attempt to delete the store.");
+ try {
+ await IOUtils.remove(
+ PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ ),
+ { ignoreAbsent: true }
+ );
+ } catch (ex) {
+ lazy.logConsole.error(ex);
+ }
+ this.#empty = true;
+ lazy.logConsole.debug("Store was deleted.");
+ }
+
+ /**
+ * Tries to establish a connection to the store.
+ *
+ * @throws {Error}
+ * Will throw if there was an issue establishing a connection or adding
+ * adding a shutdown blocker.
+ */
+ async #initConnection() {
+ if (this.#connection) {
+ return;
+ }
+
+ // This could fail if the store is corrupted.
+ this.#connection = await lazy.Sqlite.openConnection({
+ path: PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ ),
+ });
+
+ await this.#connection.execute("PRAGMA journal_mode = TRUNCATE");
+
+ this.#asyncShutdownBlocker = async () => {
+ await this.#connection.close();
+ this.#connection = null;
+ };
+
+ // This could fail if we're adding it during shutdown. In this case,
+ // don't throw but close the connection.
+ try {
+ lazy.Sqlite.shutdown.addBlocker(
+ "SearchSERPTelemetry:DomainToCategoriesSqlite closing",
+ this.#asyncShutdownBlocker
+ );
+ } catch (ex) {
+ lazy.logConsole.error(ex);
+ await this.#closeConnection();
+ }
+ }
+
+ /**
+ * Inserts into the store.
+ *
+ * @param {Array<ArrayBuffer>} fileContents
+ * The data that should be converted and inserted into the store.
+ * @param {number} version
+ * The version number that should be inserted into the store.
+ * @throws {Error}
+ * Will throw if a connection is not present, if the store is not
+ * able to be updated (permissions error, corrupted file), or there is
+ * something wrong with the file contents.
+ */
+ async #insert(fileContents, version) {
+ let start = Cu.now();
+ await this.#connection.executeTransaction(async () => {
+ lazy.logConsole.debug("Insert into domain_to_categories table.");
+ for (let fileContent of fileContents) {
+ await this.#connection.executeCached(
+ `
+ INSERT INTO
+ domain_to_categories (string_id, categories)
+ SELECT
+ json_each.key AS string_id,
+ json_each.value AS categories
+ FROM
+ json_each(json(:obj))
+ `,
+ {
+ obj: new TextDecoder().decode(fileContent),
+ }
+ );
+ }
+ // Once the insertions have successfully completed, update the version.
+ await this.#connection.executeCached(
+ `
+ INSERT INTO
+ moz_meta (key, value)
+ VALUES
+ (:key, :value)
+ ON CONFLICT DO UPDATE SET
+ value = :value
+ `,
+ { key: "version", value: version }
+ );
+ });
+ ChromeUtils.addProfilerMarker(
+ "DomainToCategoriesSqlite.#insert",
+ start,
+ "Move file contents into table."
+ );
+
+ if (fileContents?.length) {
+ this.#empty = false;
+ }
+ }
+
+ /**
+ * Deletes and re-build's the store. Used in cases where we encounter a
+ * failure and we want to try fixing the error by starting with an
+ * entirely fresh store.
+ *
+ * @throws {Error}
+ * Will throw if a connection could not be established, if it was
+ * unable to delete the store, or it was unable to build a new store.
+ */
+ async #rebuildStore() {
+ lazy.logConsole.debug("Try rebuilding store.");
+ // Step 1. Close all connections.
+ await this.#closeConnection();
+
+ // Step 2. Delete the existing store.
+ await this.#delete();
+
+ // Step 3. Re-establish the connection.
+ await this.#initConnection();
+
+ // Step 4. If a connection exists, try creating the store.
+ await this.#initSchema();
+ }
+}
+
function randomInteger(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
diff --git a/browser/components/search/metrics.yaml b/browser/components/search/metrics.yaml
index 12fd44a0e2..c7636b9d04 100644
--- a/browser/components/search/metrics.yaml
+++ b/browser/components/search/metrics.yaml
@@ -331,6 +331,110 @@ serp:
- fx-search-telemetry@mozilla.com
expires: never
+ categorization:
+ type: event
+ description: >
+ A high-level categorization of a SERP (a best guess as to its topic),
+ using buckets such as "sports" or "travel".
+ bugs:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1868476
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1869064
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1887686
+ data_reviews:
+ - https://bugzilla.mozilla.org/show_bug.cgi?id=1868476
+ data_sensitivity:
+ - stored_content
+ notification_emails:
+ - fx-search-telemetry@mozilla.com
+ - rev-data@mozilla.com
+ expires: never
+ extra_keys:
+ sponsored_category:
+ description: >
+ An index corresponding to a broad category for the SERP, derived from
+ sponsored domains.
+ type: quantity
+ sponsored_num_domains:
+ description: >
+ The total number of sponsored domains used in the categorization
+ process for the SERP.
+ type: quantity
+ sponsored_num_unknown:
+ description: >
+ The count of sponsored domains extracted from the SERP that are not
+ found in the domain-to-categories mapping.
+ type: quantity
+ sponsored_num_inconclusive:
+ description: >
+ The count of sponsored domains extracted from the SERP that are found
+ in the domain-to-categories mapping but are deemed inconclusive.
+ type: quantity
+ organic_category:
+ description: >
+ An index corresponding to a broad category for the SERP, derived from
+ organic domains.
+ type: quantity
+ organic_num_domains:
+ description: >
+ The total number of organic domains used in the categorization
+ process for the SERP.
+ type: quantity
+ organic_num_unknown:
+ description: >
+ The count of organic domains extracted from the SERP that are not
+ found in the domain-to-categories mapping.
+ type: quantity
+ organic_num_inconclusive:
+ description: >
+ The count of organic domains extracted from the SERP that are found
+ in the domain-to-categories mapping but are deemed inconclusive.
+ type: quantity
+ region:
+ description: >
+ A two-letter country code indicating where the SERP was loaded.
+ type: string
+ channel:
+ description: >
+ The type of update channel, for example: “nightly”, “beta”, “release”.
+ type: string
+ provider:
+ description: >
+ The name of the provider.
+ type: string
+ tagged:
+ description: >
+ Whether the search is tagged (true) or organic (false).
+ type: boolean
+ partner_code:
+ description: >
+ Any partner_code parsing in the URL or an empty string if not
+ available.
+ type: string
+ app_version:
+ description: >
+ The Firefox major version used, for example: 126.
+ type: quantity
+ mappings_version:
+ description: >
+ Version number for the Remote Settings attachments used to generate
+ the domain-to-categories map used in the SERP categorization process.
+ type: quantity
+ is_shopping_page:
+ description: >
+ Indicates if the page is a shopping page.
+ type: boolean
+ num_ads_visible:
+ description: >
+ Number of ads visible on the page at the time of categorizing the
+ page.
+ type: quantity
+ num_ads_clicked:
+ description: >
+ Number of ads clicked on the page.
+ type: quantity
+ send_in_pings:
+ - serp-categorization
+
search_with:
reporting_url:
type: url
diff --git a/browser/components/search/moz.build b/browser/components/search/moz.build
index 0289f32979..ff49a259ed 100644
--- a/browser/components/search/moz.build
+++ b/browser/components/search/moz.build
@@ -6,7 +6,6 @@
EXTRA_JS_MODULES += [
"BrowserSearchTelemetry.sys.mjs",
- "DomainToCategoriesMap.worker.mjs",
"SearchOneOffs.sys.mjs",
"SearchSERPTelemetry.sys.mjs",
"SearchUIUtils.sys.mjs",
@@ -18,7 +17,10 @@ BROWSER_CHROME_MANIFESTS += [
"test/browser/telemetry/browser.toml",
]
-MARIONETTE_MANIFESTS += ["test/marionette/manifest.toml"]
+MARIONETTE_MANIFESTS += [
+ "test/marionette/manifest.toml",
+ "test/marionette/telemetry/manifest.toml",
+]
XPCSHELL_TESTS_MANIFESTS += ["test/unit/xpcshell.toml"]
diff --git a/browser/components/search/schema/search-telemetry-schema.json b/browser/components/search/schema/search-telemetry-v2-schema.json
index 50b6e124fc..50b6e124fc 100644
--- a/browser/components/search/schema/search-telemetry-schema.json
+++ b/browser/components/search/schema/search-telemetry-v2-schema.json
diff --git a/browser/components/search/schema/search-telemetry-ui-schema.json b/browser/components/search/schema/search-telemetry-v2-ui-schema.json
index 781da5a626..749063db72 100644
--- a/browser/components/search/schema/search-telemetry-ui-schema.json
+++ b/browser/components/search/schema/search-telemetry-v2-ui-schema.json
@@ -11,10 +11,12 @@
"organicCodes",
"followOnParamNames",
"followOnCookies",
+ "ignoreLinkRegexps",
"extraAdServersRegexps",
"adServerAttributes",
"components",
"nonAdsLinkRegexps",
+ "nonAdsLinkQueryParamNames",
"shoppingTab",
"domainExtraction",
"isSPA",
diff --git a/browser/components/search/test/browser/telemetry/browser.toml b/browser/components/search/test/browser/telemetry/browser.toml
index 660fc4eae2..5e42a9187d 100644
--- a/browser/components/search/test/browser/telemetry/browser.toml
+++ b/browser/components/search/test/browser/telemetry/browser.toml
@@ -50,6 +50,15 @@ support-files = ["searchTelemetryDomainCategorizationReporting.html"]
["browser_search_telemetry_domain_categorization_extraction.js"]
support-files = ["searchTelemetryDomainExtraction.html"]
+["browser_search_telemetry_domain_categorization_no_sponsored_values.js"]
+support-files = ["searchTelemetryDomainCategorizationReportingWithoutAds.html"]
+
+["browser_search_telemetry_domain_categorization_ping_submission.js"]
+support-files = [
+ "searchTelemetryDomainCategorizationReporting.html",
+ "searchTelemetryDomainExtraction.html",
+]
+
["browser_search_telemetry_domain_categorization_region.js"]
support-files = ["searchTelemetryDomainCategorizationReporting.html"]
@@ -103,13 +112,6 @@ support-files = [
"searchTelemetryAd_searchbox_with_content.html^headers^",
]
-["browser_search_telemetry_engagement_non_ad.js"]
-support-files = [
- "searchTelemetryAd_searchbox_with_content.html",
- "searchTelemetryAd_searchbox_with_content.html^headers^",
- "serp.css",
-]
-
["browser_search_telemetry_engagement_nonAdsLinkQueryParamNames.js"]
support-files = [
"searchTelemetryAd_searchbox_with_redirecting_links.html",
@@ -118,6 +120,13 @@ support-files = [
"serp.css",
]
+["browser_search_telemetry_engagement_non_ad.js"]
+support-files = [
+ "searchTelemetryAd_searchbox_with_content.html",
+ "searchTelemetryAd_searchbox_with_content.html^headers^",
+ "serp.css",
+]
+
["browser_search_telemetry_engagement_query_params.js"]
support-files = [
"searchTelemetryAd_components_query_parameters.html",
diff --git a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js
index e73a9601d4..8e9db64fae 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_glean_serp_event_telemetry_categorization_enabled_by_nimbus_variable.js
@@ -74,11 +74,14 @@ add_setup(async function () {
let oldCanRecord = Services.telemetry.canRecordExtended;
Services.telemetry.canRecordExtended = true;
- await insertRecordIntoCollectionAndSync();
// If the categorization preference is enabled, we should also wait for the
// sync event to update the domain to categories map.
if (lazy.serpEventsCategorizationEnabled) {
- await waitForDomainToCategoriesUpdate();
+ let promise = waitForDomainToCategoriesUpdate();
+ await insertRecordIntoCollectionAndSync();
+ await promise;
+ } else {
+ await insertRecordIntoCollectionAndSync();
}
registerCleanupFunction(async () => {
@@ -99,6 +102,11 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() {
// the default branch, and not overwrite the user branch.
prefBranch.setBoolPref(TELEMETRY_PREF, false);
+ // If it was true, we should wait until the map is fully un-inited.
+ if (originalPrefValue) {
+ await waitForDomainToCategoriesUninit();
+ }
+
Assert.equal(
lazy.serpEventsCategorizationEnabled,
false,
@@ -152,6 +160,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -160,6 +169,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() {
info("End experiment.");
await doExperimentCleanup();
+ await waitForDomainToCategoriesUninit();
Assert.equal(
lazy.serpEventsCategorizationEnabled,
@@ -179,6 +189,7 @@ add_task(async function test_enable_experiment_when_pref_is_not_enabled() {
await new Promise(resolve => setTimeout(resolve, 1500));
BrowserTestUtils.removeTab(tab);
+ // We should not record telemetry if the experiment is un-enrolled.
assertCategorizationValues([]);
// Clean up.
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js
index 246caf6f47..daccbf0c93 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ad_values.js
@@ -71,6 +71,16 @@ add_setup(async function () {
await promise;
registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
SearchSERPTelemetry.overrideSearchTelemetryForTests();
resetTelemetry();
});
@@ -103,6 +113,7 @@ add_task(async function test_load_serp_and_categorize() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -143,6 +154,7 @@ add_task(async function test_load_serp_and_categorize_and_click_organic() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -181,6 +193,7 @@ add_task(async function test_load_serp_and_categorize_and_click_sponsored() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "1",
num_ads_visible: "2",
},
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js
index b8dd85da97..9bd215f697 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_download_timer.js
@@ -82,11 +82,20 @@ add_setup(async function () {
await db.clear();
- // Set the state of the pref to false so that tests toggle the preference,
- // triggering the map to be updated.
- await SpecialPowers.pushPrefEnv({
- set: [["browser.search.serpEventTelemetryCategorization.enabled", false]],
- });
+ // If the pref is by default on, disable it as the following tests toggle
+ // the preference to check what happens when the preference is off and the
+ // preference is turned on.
+ if (
+ Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ let promise = waitForDomainToCategoriesUninit();
+ await SpecialPowers.pushPrefEnv({
+ set: [["browser.search.serpEventTelemetryCategorization.enabled", false]],
+ });
+ await promise;
+ }
let defaultDownloadSettings = {
...TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS,
@@ -104,6 +113,16 @@ add_setup(async function () {
TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.maxAdjust = 0;
registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesInit();
+ }
SearchSERPTelemetry.overrideSearchTelemetryForTests();
resetTelemetry();
TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS = {
@@ -159,6 +178,7 @@ add_task(async function test_download_after_failure() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_visible: "2",
num_ads_clicked: "0",
},
@@ -166,6 +186,7 @@ add_task(async function test_download_after_failure() {
// Clean up.
await SpecialPowers.popPrefEnv();
+ await waitForDomainToCategoriesUninit();
await resetCategorizationCollection(record);
});
@@ -214,6 +235,7 @@ add_task(async function test_download_after_multiple_failures() {
// Clean up.
await SpecialPowers.popPrefEnv();
+ await waitForDomainToCategoriesUninit();
await resetCategorizationCollection(record);
});
@@ -245,6 +267,7 @@ add_task(async function test_cancel_download_timer() {
});
await SpecialPowers.popPrefEnv();
await observeCancel;
+ await waitForDomainToCategoriesUninit();
// To ensure we don't attempt another download, wait a bit over how long the
// the download error should take.
@@ -263,7 +286,6 @@ add_task(async function test_cancel_download_timer() {
Assert.ok(SearchSERPDomainToCategoriesMap.empty, "Map is empty");
// Clean up.
- await SpecialPowers.popPrefEnv();
await resetCategorizationCollection(record);
});
@@ -310,6 +332,7 @@ add_task(async function test_download_adjust() {
// Clean up.
await SpecialPowers.popPrefEnv();
+ await waitForDomainToCategoriesUninit();
await resetCategorizationCollection(record);
TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.base = TIMEOUT_IN_MS;
TELEMETRY_CATEGORIZATION_DOWNLOAD_SETTINGS.minAdjust = 0;
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js
index e653be6c48..2d13b147a2 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_extraction.js
@@ -362,19 +362,57 @@ const TESTS = [
],
expectedDomains: ["organic.com"],
},
+ {
+ title: "Bing organic result with a path in the URL.",
+ extractorInfos: [
+ {
+ selectors: "#test26 #b_results .b_algo .b_attribution cite",
+ method: "textContent",
+ },
+ ],
+ expectedDomains: ["organic.com"],
+ },
+ {
+ title: "Bing organic result with a path and query param in the URL.",
+ extractorInfos: [
+ {
+ selectors: "#test27 #b_results .b_algo .b_attribution cite",
+ method: "textContent",
+ },
+ ],
+ expectedDomains: ["organic.com"],
+ },
+ {
+ title:
+ "Bing organic result with a path in the URL, but protocol appears in separate HTML element.",
+ extractorInfos: [
+ {
+ selectors: "#test28 #b_results .b_algo .b_attribution cite",
+ method: "textContent",
+ },
+ ],
+ expectedDomains: ["wikipedia.org"],
+ },
];
add_setup(async function () {
await SpecialPowers.pushPrefEnv({
- set: [
- ["browser.search.serpEventTelemetry.enabled", true],
- ["browser.search.serpEventTelemetryCategorization.enabled", true],
- ],
+ set: [["browser.search.serpEventTelemetryCategorization.enabled", true]],
});
await SearchSERPTelemetry.init();
registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
resetTelemetry();
});
});
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js
new file mode 100644
index 0000000000..2375cad82a
--- /dev/null
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_no_sponsored_values.js
@@ -0,0 +1,141 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/*
+ * Checks reporting of pages without ads is accurate.
+ */
+
+ChromeUtils.defineESModuleGetters(this, {
+ SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs",
+});
+
+const TEST_PROVIDER_INFO = [
+ {
+ telemetryId: "example",
+ searchPageRegexp:
+ /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry/,
+ queryParamNames: ["s"],
+ codeParamName: "abc",
+ taggedCodes: ["ff"],
+ adServerAttributes: ["mozAttr"],
+ nonAdsLinkRegexps: [],
+ extraAdServersRegexps: [/^https:\/\/example\.com\/ad/],
+ shoppingTab: {
+ selector: "#shopping",
+ },
+ // The search telemetry entry responsible for targeting the specific results.
+ domainExtraction: {
+ ads: [],
+ nonAds: [
+ {
+ selectors: "#results .organic a",
+ method: "href",
+ },
+ ],
+ },
+ components: [
+ {
+ type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK,
+ default: true,
+ },
+ ],
+ },
+];
+
+add_setup(async function () {
+ SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO);
+ await waitForIdle();
+
+ let promise = waitForDomainToCategoriesUpdate();
+ await insertRecordIntoCollectionAndSync();
+ await SpecialPowers.pushPrefEnv({
+ set: [["browser.search.serpEventTelemetryCategorization.enabled", true]],
+ });
+ await promise;
+
+ registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
+ SearchSERPTelemetry.overrideSearchTelemetryForTests();
+ resetTelemetry();
+ });
+});
+
+add_task(
+ async function test_load_serp_without_sponsored_links_and_categorize() {
+ resetTelemetry();
+
+ let url = getSERPUrl(
+ "searchTelemetryDomainCategorizationReportingWithoutAds.html"
+ );
+ info("Load a SERP with organic and ad components that are non-sponsored.");
+ let promise = waitForPageWithCategorizedDomains();
+ let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url);
+ await promise;
+
+ info("Assert there is a non-sponsored component on the page.");
+ assertSERPTelemetry([
+ {
+ impression: {
+ shopping_tab_displayed: "true",
+ provider: "example",
+ source: "unknown",
+ tagged: "true",
+ is_private: "false",
+ is_shopping_page: "false",
+ partner_code: "ff",
+ },
+ adImpressions: [
+ {
+ component: SearchSERPTelemetryUtils.COMPONENTS.SHOPPING_TAB,
+ ads_loaded: "1",
+ ads_visible: "1",
+ ads_hidden: "0",
+ },
+ ],
+ },
+ ]);
+
+ info("Click on the non-sponsored component.");
+ await BrowserTestUtils.synthesizeMouseAtCenter(
+ "#shopping",
+ {},
+ tab.linkedBrowser
+ );
+
+ await BrowserTestUtils.removeTab(tab);
+ info("Assert no ads were visible or clicked on.");
+ assertCategorizationValues([
+ {
+ organic_category: "3",
+ organic_num_domains: "1",
+ organic_num_inconclusive: "0",
+ organic_num_unknown: "0",
+ sponsored_category: "0",
+ sponsored_num_domains: "0",
+ sponsored_num_inconclusive: "0",
+ sponsored_num_unknown: "0",
+ mappings_version: "1",
+ app_version: APP_MAJOR_VERSION,
+ channel: CHANNEL,
+ region: REGION,
+ partner_code: "ff",
+ provider: "example",
+ tagged: "true",
+ is_shopping_page: "false",
+ num_ads_clicked: "0",
+ num_ads_visible: "0",
+ },
+ ]);
+ }
+);
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js
new file mode 100644
index 0000000000..0196483b8c
--- /dev/null
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_ping_submission.js
@@ -0,0 +1,302 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/*
+ * This test ensures we are correctly submitting the custom ping for SERP
+ * categorization. (Please see the search component's Marionette tests for
+ * a test of the ping's submission upon startup.)
+ */
+
+ChromeUtils.defineESModuleGetters(this, {
+ CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+ SearchSERPTelemetry: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ SERPCategorizationRecorder: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ TELEMETRY_CATEGORIZATION_KEY:
+ "resource:///modules/SearchSERPTelemetry.sys.mjs",
+});
+
+const TEST_PROVIDER_INFO = [
+ {
+ telemetryId: "example",
+ searchPageRegexp:
+ /^https:\/\/example.org\/browser\/browser\/components\/search\/test\/browser\/telemetry\/searchTelemetry/,
+ queryParamNames: ["s"],
+ codeParamName: "abc",
+ taggedCodes: ["ff"],
+ adServerAttributes: ["mozAttr"],
+ nonAdsLinkRegexps: [/^https:\/\/example.com/],
+ extraAdServersRegexps: [/^https:\/\/example\.com\/ad/],
+ // The search telemetry entry responsible for targeting the specific results.
+ domainExtraction: {
+ ads: [
+ {
+ selectors: "[data-ad-domain]",
+ method: "data-attribute",
+ options: {
+ dataAttributeKey: "adDomain",
+ },
+ },
+ {
+ selectors: ".ad",
+ method: "href",
+ options: {
+ queryParamKey: "ad_domain",
+ },
+ },
+ ],
+ nonAds: [
+ {
+ selectors: "#results .organic a",
+ method: "href",
+ },
+ ],
+ },
+ components: [
+ {
+ type: SearchSERPTelemetryUtils.COMPONENTS.AD_LINK,
+ default: true,
+ },
+ ],
+ },
+];
+
+const client = RemoteSettings(TELEMETRY_CATEGORIZATION_KEY);
+const db = client.db;
+
+function sleep(ms) {
+ // eslint-disable-next-line mozilla/no-arbitrary-setTimeout
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+add_setup(async function () {
+ SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO);
+ await waitForIdle();
+ // Enable local telemetry recording for the duration of the tests.
+ let oldCanRecord = Services.telemetry.canRecordExtended;
+ Services.telemetry.canRecordExtended = true;
+ await db.clear();
+
+ let promise = waitForDomainToCategoriesUpdate();
+ await insertRecordIntoCollectionAndSync();
+ await SpecialPowers.pushPrefEnv({
+ set: [["browser.search.serpEventTelemetryCategorization.enabled", true]],
+ });
+ await promise;
+
+ registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
+
+ SearchSERPTelemetry.overrideSearchTelemetryForTests();
+ Services.telemetry.canRecordExtended = oldCanRecord;
+ resetTelemetry();
+ });
+});
+
+add_task(async function test_threshold_reached() {
+ resetTelemetry();
+
+ let oldThreshold = CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD;
+ // For testing, it's fine to categorize fewer SERPs before sending the ping.
+ CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD = 2;
+ SERPCategorizationRecorder.uninit();
+ SERPCategorizationRecorder.init();
+
+ Assert.equal(
+ null,
+ Glean.serp.categorization.testGetValue(),
+ "Should not have recorded any metrics yet."
+ );
+
+ let submitted = false;
+ GleanPings.serpCategorization.testBeforeNextSubmit(reason => {
+ submitted = true;
+ Assert.equal(
+ "threshold_reached",
+ reason,
+ "Ping submission reason should be 'threshold_reached'."
+ );
+ });
+
+ // Categorize first SERP, which results in one organic and one sponsored
+ // reporting.
+ let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html");
+ info("Load a sample SERP with organic and sponsored results.");
+ let promise = waitForPageWithCategorizedDomains();
+ let tab1 = await BrowserTestUtils.openNewForegroundTab(gBrowser, url);
+ await promise;
+
+ Assert.equal(
+ false,
+ submitted,
+ "Ping should not be submitted before threshold is reached."
+ );
+
+ // Categorize second SERP, which results in one organic and one sponsored
+ // reporting.
+ url = getSERPUrl("searchTelemetryDomainExtraction.html");
+ info("Load a sample SERP with organic and sponsored results.");
+ promise = waitForPageWithCategorizedDomains();
+ let tab2 = await BrowserTestUtils.openNewForegroundTab(gBrowser, url);
+ await promise;
+
+ BrowserTestUtils.removeTab(tab1);
+ BrowserTestUtils.removeTab(tab2);
+
+ Assert.equal(
+ true,
+ submitted,
+ "Ping should be submitted once threshold is reached."
+ );
+
+ CATEGORIZATION_SETTINGS.PING_SUBMISSION_THRESHOLD = oldThreshold;
+});
+
+add_task(async function test_quick_activity_to_inactivity_alternation() {
+ resetTelemetry();
+
+ Assert.equal(
+ null,
+ Glean.serp.categorization.testGetValue(),
+ "Should not have recorded any metrics yet."
+ );
+
+ let submitted = false;
+ GleanPings.serpCategorization.testBeforeNextSubmit(() => {
+ submitted = true;
+ });
+
+ let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html");
+ info("Load a sample SERP with organic and sponsored results.");
+ let promise = waitForPageWithCategorizedDomains();
+ let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url);
+ await promise;
+
+ let activityDetectedPromise = TestUtils.topicObserved(
+ "user-interaction-active"
+ );
+ // Simulate ~2.5 seconds of activity.
+ for (let i = 0; i < 25; i++) {
+ EventUtils.synthesizeKey("KEY_Enter");
+ await sleep(100);
+ }
+ await activityDetectedPromise;
+
+ let inactivityDetectedPromise = TestUtils.topicObserved(
+ "user-interaction-inactive"
+ );
+ await inactivityDetectedPromise;
+
+ Assert.equal(
+ false,
+ submitted,
+ "Ping should not be submitted after a quick alternation from activity to inactivity."
+ );
+
+ BrowserTestUtils.removeTab(tab);
+});
+
+add_task(async function test_submit_after_activity_then_inactivity() {
+ resetTelemetry();
+ let oldActivityLimit = Services.prefs.getIntPref(
+ "telemetry.fog.test.activity_limit"
+ );
+ Services.prefs.setIntPref("telemetry.fog.test.activity_limit", 2);
+
+ Assert.equal(
+ null,
+ Glean.serp.categorization.testGetValue(),
+ "Should not have recorded any metrics yet."
+ );
+
+ let submitted = false;
+ GleanPings.serpCategorization.testBeforeNextSubmit(reason => {
+ submitted = true;
+ Assert.equal(
+ "inactivity",
+ reason,
+ "Ping submission reason should be 'inactivity'."
+ );
+ });
+
+ let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html");
+ info("Load a sample SERP with organic and sponsored results.");
+ let promise = waitForPageWithCategorizedDomains();
+ let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, url);
+ await promise;
+
+ let activityDetectedPromise = TestUtils.topicObserved(
+ "user-interaction-active"
+ );
+ // Simulate ~2.5 seconds of activity.
+ for (let i = 0; i < 25; i++) {
+ EventUtils.synthesizeKey("KEY_Enter");
+ await sleep(100);
+ }
+ await activityDetectedPromise;
+
+ let inactivityDetectedPromise = TestUtils.topicObserved(
+ "user-interaction-inactive"
+ );
+ await inactivityDetectedPromise;
+
+ Assert.equal(
+ true,
+ submitted,
+ "Ping should be submitted after 2+ seconds of activity, followed by inactivity."
+ );
+
+ BrowserTestUtils.removeTab(tab);
+ Services.prefs.setIntPref(
+ "telemetry.fog.test.activity_limit",
+ oldActivityLimit
+ );
+});
+
+add_task(async function test_no_observers_added_if_pref_is_off() {
+ resetTelemetry();
+
+ let prefOnActiveObserverCount = Array.from(
+ Services.obs.enumerateObservers("user-interaction-active")
+ ).length;
+ let prefOnInactiveObserverCount = Array.from(
+ Services.obs.enumerateObservers("user-interaction-inactive")
+ ).length;
+
+ await SpecialPowers.pushPrefEnv({
+ set: [["browser.search.serpEventTelemetryCategorization.enabled", false]],
+ });
+ await waitForDomainToCategoriesUninit();
+
+ let prefOffActiveObserverCount = Array.from(
+ Services.obs.enumerateObservers("user-interaction-active")
+ ).length;
+ let prefOffInactiveObserverCount = Array.from(
+ Services.obs.enumerateObservers("user-interaction-inactive")
+ ).length;
+
+ Assert.equal(
+ prefOnActiveObserverCount - prefOffActiveObserverCount,
+ 1,
+ "There should be one fewer active observer when the pref is off."
+ );
+ Assert.equal(
+ prefOnInactiveObserverCount - prefOffInactiveObserverCount,
+ 1,
+ "There should be one fewer inactive observer when the pref is off."
+ );
+
+ await SpecialPowers.popPrefEnv();
+ await waitForDomainToCategoriesInit();
+});
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js
index 4c47b0b14a..7dbf605396 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_region.js
@@ -78,6 +78,17 @@ add_setup(async function () {
Assert.equal(Region.home, "DE", "Region");
registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
+
Region._setHomeRegion(originalHomeRegion);
Region._setCurrentRegion(originalCurrentRegion);
@@ -113,6 +124,7 @@ add_task(async function test_categorize_page_with_different_region() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js
index 973f17b760..3c439844d7 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting.js
@@ -19,6 +19,7 @@ const TEST_PROVIDER_INFO = [
queryParamNames: ["s"],
codeParamName: "abc",
taggedCodes: ["ff"],
+ organicCodes: [],
adServerAttributes: ["mozAttr"],
nonAdsLinkRegexps: [],
extraAdServersRegexps: [
@@ -56,6 +57,9 @@ const TEST_PROVIDER_INFO = [
default: true,
},
],
+ shoppingTab: {
+ regexp: "&page=shop",
+ },
},
];
@@ -69,6 +73,10 @@ add_setup(async function () {
SearchSERPTelemetry.overrideSearchTelemetryForTests(TEST_PROVIDER_INFO);
await waitForIdle();
+ // Enable local telemetry recording for the duration of the tests.
+ let oldCanRecord = Services.telemetry.canRecordExtended;
+ Services.telemetry.canRecordExtended = true;
+
let { record, attachment } = await insertRecordIntoCollection();
categorizationRecord = record;
categorizationAttachment = attachment;
@@ -82,7 +90,18 @@ add_setup(async function () {
await promise;
registerCleanupFunction(async () => {
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ }
SearchSERPTelemetry.overrideSearchTelemetryForTests();
+ Services.telemetry.canRecordExtended = oldCanRecord;
resetTelemetry();
await db.clear();
});
@@ -115,6 +134,7 @@ add_task(async function test_categorization_reporting() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -147,6 +167,7 @@ add_task(async function test_no_reporting_if_download_failure() {
await promise;
await BrowserTestUtils.removeTab(tab);
+ // We should not record telemetry if attachments weren't downloaded.
assertCategorizationValues([]);
// Re-insert the attachment for other tests.
@@ -177,6 +198,7 @@ add_task(async function test_no_reporting_if_no_records() {
await promise;
await BrowserTestUtils.removeTab(tab);
+ // We should not record telemetry if there are no records.
assertCategorizationValues([]);
});
@@ -218,8 +240,46 @@ add_task(async function test_reporting_limited_to_10_domains_of_each_kind() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "12",
},
]);
});
+
+add_task(async function test_categorization_reporting_for_shopping_page() {
+ resetTelemetry();
+
+ let url = getSERPUrl("searchTelemetryDomainCategorizationReporting.html");
+ let shoppingUrl = new URL(url);
+ shoppingUrl.searchParams.set("page", "shop");
+ shoppingUrl = shoppingUrl.toString();
+ info("Load a sample shopping page SERP with organic and sponsored results.");
+ let promise = waitForPageWithCategorizedDomains();
+ let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, shoppingUrl);
+ await promise;
+
+ await BrowserTestUtils.removeTab(tab);
+ assertCategorizationValues([
+ {
+ organic_category: "3",
+ organic_num_domains: "1",
+ organic_num_inconclusive: "0",
+ organic_num_unknown: "0",
+ sponsored_category: "4",
+ sponsored_num_domains: "2",
+ sponsored_num_inconclusive: "0",
+ sponsored_num_unknown: "0",
+ mappings_version: "1",
+ app_version: APP_MAJOR_VERSION,
+ channel: CHANNEL,
+ region: REGION,
+ partner_code: "ff",
+ provider: "example",
+ tagged: "true",
+ is_shopping_page: "true",
+ num_ads_clicked: "0",
+ num_ads_visible: "2",
+ },
+ ]);
+});
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js
index 9d3ac2c931..0e2d1c07fd 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer.js
@@ -87,9 +87,20 @@ add_setup(async function () {
await promise;
registerCleanupFunction(async () => {
- // The scheduler uses the mock idle service.
- SearchSERPCategorizationEventScheduler.uninit();
- SearchSERPCategorizationEventScheduler.init();
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ } else {
+ // The scheduler uses the mock idle service.
+ SearchSERPCategorizationEventScheduler.uninit();
+ SearchSERPCategorizationEventScheduler.init();
+ }
SearchSERPTelemetry.overrideSearchTelemetryForTests();
resetTelemetry();
});
@@ -126,6 +137,7 @@ add_task(async function test_categorize_serp_and_wait() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -170,6 +182,7 @@ add_task(async function test_categorize_serp_open_multiple_tabs() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
});
@@ -223,6 +236,7 @@ add_task(async function test_categorize_serp_close_tab_and_wait() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -276,6 +290,7 @@ add_task(async function test_categorize_serp_open_ad_and_wait() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "1",
num_ads_visible: "2",
},
diff --git a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js
index c73e224eae..43c520a8d0 100644
--- a/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js
+++ b/browser/components/search/test/browser/telemetry/browser_search_telemetry_domain_categorization_reporting_timer_wakeup.js
@@ -92,9 +92,20 @@ add_setup(async function () {
await promise;
registerCleanupFunction(async () => {
- // The scheduler uses the mock idle service.
- SearchSERPCategorizationEventScheduler.uninit();
- SearchSERPCategorizationEventScheduler.init();
+ // Manually unload the pref so that we can check if we should wait for the
+ // the categories map to be un-initialized.
+ await SpecialPowers.popPrefEnv();
+ if (
+ !Services.prefs.getBoolPref(
+ "browser.search.serpEventTelemetryCategorization.enabled"
+ )
+ ) {
+ await waitForDomainToCategoriesUninit();
+ } else {
+ // The scheduler uses the mock idle service.
+ SearchSERPCategorizationEventScheduler.uninit();
+ SearchSERPCategorizationEventScheduler.init();
+ }
CATEGORIZATION_SETTINGS.WAKE_TIMEOUT_MS = oldWakeTimeout;
SearchSERPTelemetry.overrideSearchTelemetryForTests();
resetTelemetry();
@@ -138,6 +149,7 @@ add_task(async function test_categorize_serp_and_sleep() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
@@ -195,6 +207,7 @@ add_task(async function test_categorize_serp_and_sleep_not_long_enough() {
partner_code: "ff",
provider: "example",
tagged: "true",
+ is_shopping_page: "false",
num_ads_clicked: "0",
num_ads_visible: "2",
},
diff --git a/browser/components/search/test/browser/telemetry/head.js b/browser/components/search/test/browser/telemetry/head.js
index b798099bdd..ecc6e38fa9 100644
--- a/browser/components/search/test/browser/telemetry/head.js
+++ b/browser/components/search/test/browser/telemetry/head.js
@@ -4,11 +4,14 @@
ChromeUtils.defineESModuleGetters(this, {
ADLINK_CHECK_TIMEOUT_MS:
"resource:///actors/SearchSERPTelemetryChild.sys.mjs",
+ CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs",
CustomizableUITestUtils:
"resource://testing-common/CustomizableUITestUtils.sys.mjs",
Region: "resource://gre/modules/Region.sys.mjs",
RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
SEARCH_TELEMETRY_SHARED: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ SearchSERPDomainToCategoriesMap:
+ "resource:///modules/SearchSERPTelemetry.sys.mjs",
SearchSERPTelemetry: "resource:///modules/SearchSERPTelemetry.sys.mjs",
SearchSERPTelemetryUtils: "resource:///modules/SearchSERPTelemetry.sys.mjs",
SearchTestUtils: "resource://testing-common/SearchTestUtils.sys.mjs",
@@ -193,11 +196,10 @@ async function assertSearchSourcesTelemetry(
}
function resetTelemetry() {
- // TODO Bug 1868476: Replace when we're using Glean telemetry.
- fakeTelemetryStorage = [];
searchCounts.clear();
Services.telemetry.clearScalars();
Services.fog.testResetFOG();
+ SERPCategorizationRecorder.testReset();
}
/**
@@ -377,23 +379,6 @@ function assertSERPTelemetry(expectedEvents) {
);
}
-// TODO Bug 1868476: Replace when we're using Glean telemetry.
-let categorizationSandbox;
-let fakeTelemetryStorage = [];
-add_setup(function () {
- categorizationSandbox = sinon.createSandbox();
- categorizationSandbox
- .stub(SERPCategorizationRecorder, "recordCategorizationTelemetry")
- .callsFake(input => {
- fakeTelemetryStorage.push(input);
- });
-
- registerCleanupFunction(() => {
- categorizationSandbox.restore();
- fakeTelemetryStorage = [];
- });
-});
-
async function openSerpInNewTab(url, expectedAds = true) {
let promise;
if (expectedAds) {
@@ -435,12 +420,11 @@ async function synthesizePageAction({
}
function assertCategorizationValues(expectedResults) {
- // TODO Bug 1868476: Replace with calls to Glean telemetry.
- let actualResults = [...fakeTelemetryStorage];
+ let actualResults = Glean.serp.categorization.testGetValue() ?? [];
Assert.equal(
- expectedResults.length,
actualResults.length,
+ expectedResults.length,
"Should have the correct number of categorization impressions."
);
@@ -458,7 +442,7 @@ function assertCategorizationValues(expectedResults) {
}
}
for (let actual of actualResults) {
- for (let key in actual) {
+ for (let key in actual.extra) {
keys.add(key);
}
}
@@ -467,14 +451,21 @@ function assertCategorizationValues(expectedResults) {
for (let index = 0; index < expectedResults.length; ++index) {
info(`Checking categorization at index: ${index}`);
let expected = expectedResults[index];
- let actual = actualResults[index];
+ let actual = actualResults[index].extra;
+
+ Assert.ok(
+ Number(actual?.organic_num_domains) <=
+ CATEGORIZATION_SETTINGS.MAX_DOMAINS_TO_CATEGORIZE,
+ "Number of organic domains categorized should not exceed threshold."
+ );
+
+ Assert.ok(
+ Number(actual?.sponsored_num_domains) <=
+ CATEGORIZATION_SETTINGS.MAX_DOMAINS_TO_CATEGORIZE,
+ "Number of sponsored domains categorized should not exceed threshold."
+ );
+
for (let key of keys) {
- // TODO Bug 1868476: This conversion to strings is to mimic Glean
- // converting all values into strings. Once we receive real values from
- // Glean, it can be removed.
- if (actual[key] != null && typeof actual[key] !== "string") {
- actual[key] = actual[key].toString();
- }
Assert.equal(
actual[key],
expected[key],
@@ -508,6 +499,14 @@ function waitForDomainToCategoriesUpdate() {
return TestUtils.topicObserved("domain-to-categories-map-update-complete");
}
+function waitForDomainToCategoriesInit() {
+ return TestUtils.topicObserved("domain-to-categories-map-init");
+}
+
+function waitForDomainToCategoriesUninit() {
+ return TestUtils.topicObserved("domain-to-categories-map-uninit");
+}
+
registerCleanupFunction(async () => {
await PlacesUtils.history.clear();
});
diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html b/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html
new file mode 100644
index 0000000000..13d023e45d
--- /dev/null
+++ b/browser/components/search/test/browser/telemetry/searchTelemetryDomainCategorizationReportingWithoutAds.html
@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Document</title>
+</head>
+<body>
+ <div>
+ <a id="shopping" href="https://www.example.org/shopping">Shopping</a>
+ </div>
+ <div id="results">
+ <div class="organic">
+ <a href="https://www.foobar.org">Link</a>
+ </div>
+ </div>
+</body>
+</html>
diff --git a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html
index 28c31af959..fe52bb8b48 100644
--- a/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html
+++ b/browser/components/search/test/browser/telemetry/searchTelemetryDomainExtraction.html
@@ -256,6 +256,37 @@
</div>
</div>
</div>
+
+ <div id="test26">
+ <div id="b_results">
+ <div class="b_algo">
+ <div class="b_attribution">
+ <cite>https://organic.com/cats</cite>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div id="test27">
+ <div id="b_results">
+ <div class="b_algo">
+ <div class="b_attribution">
+ <cite>https://organic.com/testing?q=cats</cite>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div id="test28">
+ <div id="b_results">
+ <div class="b_algo">
+ <div class="b_attribution">
+ <span>HTTPS</span>
+ <cite>en.wikipedia.org/wiki/Cat</cite>
+ </div>
+ </div>
+ </div>
+ </div>
</div>
</body>
</html>
diff --git a/browser/components/search/test/marionette/manifest.toml b/browser/components/search/test/marionette/manifest.toml
index 152442bc5b..9cc88e9f84 100644
--- a/browser/components/search/test/marionette/manifest.toml
+++ b/browser/components/search/test/marionette/manifest.toml
@@ -1,4 +1,6 @@
[DEFAULT]
run-if = ["buildapp == 'browser'"]
+["include:telemetry/manifest.toml"]
+
["test_engines_on_restart.py"]
diff --git a/browser/components/search/test/marionette/telemetry/manifest.toml b/browser/components/search/test/marionette/telemetry/manifest.toml
new file mode 100644
index 0000000000..1fe35945c9
--- /dev/null
+++ b/browser/components/search/test/marionette/telemetry/manifest.toml
@@ -0,0 +1,4 @@
+[DEFAULT]
+run-if = ["buildapp == 'browser'"]
+
+["test_ping_submitted.py"]
diff --git a/browser/components/search/test/marionette/telemetry/test_ping_submitted.py b/browser/components/search/test/marionette/telemetry/test_ping_submitted.py
new file mode 100644
index 0000000000..cefe2d72d1
--- /dev/null
+++ b/browser/components/search/test/marionette/telemetry/test_ping_submitted.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from marionette_driver import Wait
+from marionette_harness.marionette_test import MarionetteTestCase
+
+
+class TestPingSubmitted(MarionetteTestCase):
+ def setUp(self):
+ super(TestPingSubmitted, self).setUp()
+
+ self.marionette.set_context(self.marionette.CONTEXT_CHROME)
+
+ self.marionette.enforce_gecko_prefs(
+ {
+ "datareporting.healthreport.uploadEnabled": True,
+ "telemetry.fog.test.localhost_port": 3000,
+ "browser.search.log": True,
+ }
+ )
+ # The categorization ping is submitted on startup. If anything delays
+ # its initialization, turning the preference on and immediately
+ # attaching a categorization event could result in the ping being
+ # submitted after the test event is reported but before the browser
+ # restarts.
+ script = """
+ let [outerResolve] = arguments;
+ (async () => {
+ if (!Services.prefs.getBoolPref("browser.search.serpEventTelemetryCategorization.enabled")) {
+ let inited = new Promise(innerResolve => {
+ Services.obs.addObserver(function callback() {
+ Services.obs.removeObserver(callback, "categorization-recorder-init");
+ innerResolve();
+ }, "categorization-recorder-init");
+ });
+ Services.prefs.setBoolPref("browser.search.serpEventTelemetryCategorization.enabled", true);
+ await inited;
+ }
+ })().then(outerResolve);
+ """
+ self.marionette.execute_async_script(script)
+
+ def test_ping_submit_on_start(self):
+ # Record an event for the ping to eventually submit.
+ self.marionette.execute_script(
+ """
+ Glean.serp.categorization.record({
+ organic_category: "3",
+ organic_num_domains: "1",
+ organic_num_inconclusive: "0",
+ organic_num_unknown: "0",
+ sponsored_category: "4",
+ sponsored_num_domains: "2",
+ sponsored_num_inconclusive: "0",
+ sponsored_num_unknown: "0",
+ mappings_version: "1",
+ app_version: "124",
+ channel: "nightly",
+ region: "US",
+ partner_code: "ff",
+ provider: "example",
+ tagged: "true",
+ num_ads_clicked: "0",
+ num_ads_visible: "2",
+ });
+ """
+ )
+
+ Wait(self.marionette, timeout=60).until(
+ lambda _: self.marionette.execute_script(
+ """
+ return (Glean.serp.categorization.testGetValue()?.length ?? 0) == 1;
+ """
+ ),
+ message="Should have recorded a SERP categorization event before restart.",
+ )
+
+ self.marionette.restart(clean=False, in_app=True)
+
+ Wait(self.marionette, timeout=60).until(
+ lambda _: self.marionette.execute_script(
+ """
+ return (Glean.serp.categorization.testGetValue()?.length ?? 0) == 0;
+ """
+ ),
+ message="SERP categorization should have been sent some time after restart.",
+ )
diff --git a/browser/components/search/test/unit/corruptDB.sqlite b/browser/components/search/test/unit/corruptDB.sqlite
new file mode 100644
index 0000000000..b234246cac
--- /dev/null
+++ b/browser/components/search/test/unit/corruptDB.sqlite
Binary files differ
diff --git a/browser/components/search/test/unit/test_domain_to_categories_store.js b/browser/components/search/test/unit/test_domain_to_categories_store.js
new file mode 100644
index 0000000000..e3af0c8de5
--- /dev/null
+++ b/browser/components/search/test/unit/test_domain_to_categories_store.js
@@ -0,0 +1,361 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Ensure that the domain to categories store public methods work as expected
+ * and it handles all error cases as expected.
+ */
+
+ChromeUtils.defineESModuleGetters(this, {
+ CATEGORIZATION_SETTINGS: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ DomainToCategoriesStore: "resource:///modules/SearchSERPTelemetry.sys.mjs",
+ sinon: "resource://testing-common/Sinon.sys.mjs",
+ Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
+});
+
+let store = new DomainToCategoriesStore();
+let defaultStorePath;
+let fileContents = [convertToBuffer({ foo: [0, 1] })];
+
+async function createCorruptedStore() {
+ info("Create a corrupted store.");
+ let storePath = PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ );
+ let src = PathUtils.join(do_get_cwd().path, "corruptDB.sqlite");
+ await IOUtils.copy(src, storePath);
+ Assert.ok(await IOUtils.exists(storePath), "Store exists.");
+ return storePath;
+}
+
+function convertToBuffer(obj) {
+ return new TextEncoder().encode(JSON.stringify(obj)).buffer;
+}
+
+/**
+ * Deletes data from the store and removes any files that were generated due
+ * to them.
+ */
+async function cleanup() {
+ info("Clean up store.");
+
+ // In these tests, we sometimes use read-only files to test permission error
+ // handling. On Windows, we have to change it to writable to allow for their
+ // deletion so that subsequent tests aren't affected.
+ if (
+ (await IOUtils.exists(defaultStorePath)) &&
+ Services.appinfo.OS == "WINNT"
+ ) {
+ await IOUtils.setPermissions(defaultStorePath, 0o600);
+ }
+
+ await store.testDelete();
+ Assert.equal(store.empty, true, "Store should be empty.");
+ Assert.equal(await IOUtils.exists(defaultStorePath), false, "Store exists.");
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should be 0 when store is empty."
+ );
+
+ await store.uninit();
+}
+
+async function createReadOnlyStore() {
+ info("Create a store that can't be read.");
+ let storePath = PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ );
+
+ let conn = await Sqlite.openConnection({ path: storePath });
+ await conn.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)");
+ await conn.close();
+
+ await changeStoreToReadOnly();
+}
+
+async function changeStoreToReadOnly() {
+ info("Change store to read only.");
+ let storePath = PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ );
+ let stat = await IOUtils.stat(storePath);
+ await IOUtils.setPermissions(storePath, 0o444);
+ stat = await IOUtils.stat(storePath);
+ Assert.equal(stat.permissions, 0o444, "Permissions should be read only.");
+ Assert.ok(await IOUtils.exists(storePath), "Store exists.");
+}
+
+add_setup(async function () {
+ // We need a profile directory to create the store and open a connection.
+ do_get_profile();
+ defaultStorePath = PathUtils.join(
+ PathUtils.profileDir,
+ CATEGORIZATION_SETTINGS.STORE_FILE
+ );
+ registerCleanupFunction(async () => {
+ await cleanup();
+ });
+});
+
+// Ensure the test only function deletes the store.
+add_task(async function delete_store() {
+ let storePath = await createCorruptedStore();
+ await store.testDelete();
+ Assert.ok(!(await IOUtils.exists(storePath)), "Store doesn't exist.");
+});
+
+/**
+ * These tests check common no fail scenarios.
+ */
+
+add_task(async function init_insert_uninit() {
+ await store.init();
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should not be set."
+ );
+ Assert.equal(store.empty, true, "Store should be empty.");
+
+ info("Try inserting after init.");
+ await store.insertFileContents(fileContents, 1);
+
+ result = await store.getCategories("foo");
+ Assert.deepEqual(result, [0, 1], "foo should have a matching result.");
+ Assert.equal(await store.getVersion(), 1, "Version number should be set.");
+ Assert.equal(store.empty, false, "Store should not be empty.");
+
+ info("Un-init store.");
+ await store.uninit();
+
+ result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should be removed from store.");
+ Assert.equal(store.empty, true, "Store should be empty.");
+ Assert.equal(await store.getVersion(), 0, "Version should be reset.");
+
+ await cleanup();
+});
+
+add_task(async function insert_and_re_init() {
+ await store.init();
+ await store.insertFileContents(fileContents, 20240202);
+
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [0, 1], "foo should have a matching result.");
+ Assert.equal(
+ await store.getVersion(),
+ 20240202,
+ "Version number should be set."
+ );
+ Assert.equal(store.empty, false, "Is store empty.");
+
+ info("Simulate a restart.");
+ await store.uninit();
+ await store.init();
+
+ result = await store.getCategories("foo");
+ Assert.deepEqual(
+ result,
+ [0, 1],
+ "After restart, foo should still be in the store."
+ );
+ Assert.equal(
+ await store.getVersion(),
+ 20240202,
+ "Version number should still be in the store."
+ );
+ Assert.equal(store.empty, false, "Is store empty.");
+
+ await cleanup();
+});
+
+// Simulate consecutive updates.
+add_task(async function insert_multiple_times() {
+ await store.init();
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should not be set."
+ );
+ Assert.equal(store.empty, true, "Is store empty.");
+
+ for (let i = 0; i < 3; ++i) {
+ info("Try inserting after init.");
+ await store.insertFileContents(fileContents, 1);
+
+ result = await store.getCategories("foo");
+ Assert.deepEqual(result, [0, 1], "foo should have a matching result.");
+ Assert.equal(store.empty, false, "Is store empty.");
+ Assert.equal(await store.getVersion(), 1, "Version number is set.");
+
+ await store.dropData();
+ result = await store.getCategories("foo");
+ Assert.deepEqual(
+ result,
+ [],
+ "After dropping data, foo should no longer have a matching result."
+ );
+ Assert.equal(await store.getVersion(), 0, "Version should be reset.");
+ Assert.equal(store.empty, true, "Is store empty.");
+ }
+
+ await cleanup();
+});
+
+/**
+ * The following tests check failures on store initialization.
+ */
+
+add_task(async function init_with_corrupted_store() {
+ await createCorruptedStore();
+
+ info("Initialize the store.");
+ await store.init();
+
+ info("Try inserting after the corrupted store was replaced.");
+ await store.insertFileContents(fileContents, 1);
+
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [0, 1], "foo should have a matching result.");
+ Assert.equal(await store.getVersion(), 1, "Version number is set.");
+ Assert.equal(store.empty, false, "Is store empty.");
+
+ await cleanup();
+});
+
+add_task(async function init_with_unfixable_store() {
+ let sandbox = sinon.createSandbox();
+ sandbox.stub(Sqlite, "openConnection").throws();
+
+ info("Initialize the store.");
+ await store.init();
+
+ info("Try inserting content even if the connection is impossible to fix.");
+ await store.dropData();
+ await store.insertFileContents(fileContents, 20240202);
+
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(await store.getVersion(), 0, "Version should be reset.");
+ Assert.equal(store.empty, true, "Store should be empty.");
+
+ sandbox.restore();
+ await cleanup();
+});
+
+add_task(async function init_read_only_store() {
+ await createReadOnlyStore();
+ await store.init();
+
+ info("Insert contents into the store.");
+ await store.insertFileContents(fileContents, 20240202);
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should not be set."
+ );
+ Assert.equal(store.empty, true, "Store should be empty.");
+
+ await cleanup();
+});
+
+add_task(async function init_close_to_shutdown() {
+ let sandbox = sinon.createSandbox();
+ sandbox.stub(Sqlite.shutdown, "addBlocker").throws(new Error());
+ await store.init();
+
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should not be set."
+ );
+ Assert.equal(store.empty, true, "Store should be empty.");
+
+ sandbox.restore();
+ await cleanup();
+});
+
+/**
+ * The following tests check error handling when inserting data into the store.
+ */
+
+add_task(async function insert_broken_file() {
+ await store.init();
+
+ Assert.equal(
+ await store.getVersion(),
+ 0,
+ "Version number should not be set."
+ );
+
+ info("Try inserting one valid file and an invalid file.");
+ let contents = [...fileContents, new ArrayBuffer(0).buffer];
+ await store.insertFileContents(contents, 20240202);
+
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(await store.getVersion(), 0, "Version should remain unset.");
+ Assert.equal(store.empty, true, "Store should remain empty.");
+
+ await cleanup();
+});
+
+add_task(async function insert_into_read_only_store() {
+ await createReadOnlyStore();
+ await store.init();
+
+ await store.dropData();
+ await store.insertFileContents(fileContents, 20240202);
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [], "foo should not have a result.");
+ Assert.equal(await store.getVersion(), 0, "Version should remain unset.");
+ Assert.equal(store.empty, true, "Store should remain empty.");
+
+ await cleanup();
+});
+
+// If the store becomes read only with content already inside of it,
+// the next time we try opening it, we'll encounter an error trying to write to
+// it. Since we are no longer able to manipulate it, the results should always
+// be empty.
+add_task(async function restart_with_read_only_store() {
+ await store.init();
+ await store.insertFileContents(fileContents, 20240202);
+
+ info("Check store has content.");
+ let result = await store.getCategories("foo");
+ Assert.deepEqual(result, [0, 1], "foo should have a matching result.");
+ Assert.equal(
+ await store.getVersion(),
+ 20240202,
+ "Version number should be set."
+ );
+ Assert.equal(store.empty, false, "Store should not be empty.");
+
+ await changeStoreToReadOnly();
+ await store.uninit();
+ await store.init();
+
+ result = await store.getCategories("foo");
+ Assert.deepEqual(
+ result,
+ [],
+ "foo should no longer have a matching value from the store."
+ );
+ Assert.equal(await store.getVersion(), 0, "Version number should be unset.");
+ Assert.equal(store.empty, true, "Store should be empty.");
+
+ await cleanup();
+});
diff --git a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js
index 40d38efbba..2351347d77 100644
--- a/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js
+++ b/browser/components/search/test/unit/test_search_telemetry_categorization_sync.js
@@ -9,6 +9,7 @@
ChromeUtils.defineESModuleGetters(this, {
RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+ SearchSERPCategorization: "resource:///modules/SearchSERPTelemetry.sys.mjs",
SearchSERPDomainToCategoriesMap:
"resource:///modules/SearchSERPTelemetry.sys.mjs",
TELEMETRY_CATEGORIZATION_KEY:
@@ -158,7 +159,7 @@ add_task(async function test_initial_import() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
add_task(async function test_update_records() {
@@ -219,7 +220,7 @@ add_task(async function test_update_records() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
add_task(async function test_delayed_initial_import() {
@@ -273,7 +274,7 @@ add_task(async function test_delayed_initial_import() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
add_task(async function test_remove_record() {
@@ -332,7 +333,7 @@ add_task(async function test_remove_record() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
add_task(async function test_different_versions_coexisting() {
@@ -380,7 +381,7 @@ add_task(async function test_different_versions_coexisting() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
add_task(async function test_download_error() {
@@ -449,5 +450,67 @@ add_task(async function test_download_error() {
// Clean up.
await db.clear();
- SearchSERPDomainToCategoriesMap.uninit();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
+});
+
+add_task(async function test_mock_restart() {
+ info("Create record containing domain_category_mappings_2a.json attachment.");
+ let record2a = await mockRecordWithCachedAttachment(RECORDS.record2a);
+ await db.create(record2a);
+
+ info("Create record containing domain_category_mappings_2b.json attachment.");
+ let record2b = await mockRecordWithCachedAttachment(RECORDS.record2b);
+ await db.create(record2b);
+
+ info("Add data to Remote Settings DB.");
+ await db.importChanges({}, Date.now());
+
+ info("Initialize search categorization mappings.");
+ let promise = waitForDomainToCategoriesUpdate();
+ await SearchSERPCategorization.init();
+ await promise;
+
+ Assert.deepEqual(
+ await SearchSERPDomainToCategoriesMap.get("example.com"),
+ [
+ {
+ category: 1,
+ score: 80,
+ },
+ ],
+ "Should have a record."
+ );
+
+ Assert.equal(
+ SearchSERPDomainToCategoriesMap.version,
+ 2,
+ "Version should be the latest."
+ );
+
+ info("Mock a restart by un-initializing the map.");
+ await SearchSERPCategorization.uninit();
+ promise = waitForDomainToCategoriesUpdate();
+ await SearchSERPCategorization.init();
+ await promise;
+
+ Assert.deepEqual(
+ await SearchSERPDomainToCategoriesMap.get("example.com"),
+ [
+ {
+ category: 1,
+ score: 80,
+ },
+ ],
+ "Should have a record."
+ );
+
+ Assert.equal(
+ SearchSERPDomainToCategoriesMap.version,
+ 2,
+ "Version should be the latest."
+ );
+
+ // Clean up.
+ await db.clear();
+ await SearchSERPDomainToCategoriesMap.uninit(true);
});
diff --git a/browser/components/search/test/unit/test_search_telemetry_config_validation.js b/browser/components/search/test/unit/test_search_telemetry_config_validation.js
index 8897b1e7c7..d14f7a3918 100644
--- a/browser/components/search/test/unit/test_search_telemetry_config_validation.js
+++ b/browser/components/search/test/unit/test_search_telemetry_config_validation.js
@@ -57,7 +57,7 @@ function disallowAdditionalProperties(section) {
add_task(async function test_search_telemetry_validates_to_schema() {
let schema = await IOUtils.readJSON(
- PathUtils.join(do_get_cwd().path, "search-telemetry-schema.json")
+ PathUtils.join(do_get_cwd().path, "search-telemetry-v2-schema.json")
);
disallowAdditionalProperties(schema);
diff --git a/browser/components/search/test/unit/test_ui_schemas_valid.js b/browser/components/search/test/unit/test_ui_schemas_valid.js
new file mode 100644
index 0000000000..3396f38238
--- /dev/null
+++ b/browser/components/search/test/unit/test_ui_schemas_valid.js
@@ -0,0 +1,31 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+let schemas = [
+ ["search-telemetry-v2-schema.json", "search-telemetry-v2-ui-schema.json"],
+];
+
+async function checkUISchemaValid(configSchema, uiSchema) {
+ for (let key of Object.keys(configSchema.properties)) {
+ Assert.ok(
+ uiSchema["ui:order"].includes(key),
+ `Should have ${key} listed at the top-level of the ui schema`
+ );
+ }
+}
+
+add_task(async function test_ui_schemas_valid() {
+ for (let [schema, uiSchema] of schemas) {
+ info(`Validating ${uiSchema} has every top-level from ${schema}`);
+ let schemaData = await IOUtils.readJSON(
+ PathUtils.join(do_get_cwd().path, schema)
+ );
+ let uiSchemaData = await IOUtils.readJSON(
+ PathUtils.join(do_get_cwd().path, uiSchema)
+ );
+
+ await checkUISchemaValid(schemaData, uiSchemaData);
+ }
+});
diff --git a/browser/components/search/test/unit/xpcshell.toml b/browser/components/search/test/unit/xpcshell.toml
index 423d218d19..24e1d78eb5 100644
--- a/browser/components/search/test/unit/xpcshell.toml
+++ b/browser/components/search/test/unit/xpcshell.toml
@@ -6,6 +6,9 @@ prefs = ["browser.search.log=true"]
skip-if = ["os == 'android'"] # bug 1730213
firefox-appdir = "browser"
+["test_domain_to_categories_store.js"]
+support-files = ["corruptDB.sqlite"]
+
["test_search_telemetry_categorization_logic.js"]
["test_search_telemetry_categorization_sync.js"]
@@ -14,7 +17,13 @@ prefs = ["browser.search.serpEventTelemetryCategorization.enabled=true"]
["test_search_telemetry_compare_urls.js"]
["test_search_telemetry_config_validation.js"]
-support-files = ["../../schema/search-telemetry-schema.json"]
+support-files = ["../../schema/search-telemetry-v2-schema.json"]
+
+["test_ui_schemas_valid.js"]
+support-files = [
+ "../../schema/search-telemetry-v2-schema.json",
+ "../../schema/search-telemetry-v2-ui-schema.json",
+]
["test_urlTelemetry.js"]