diff options
Diffstat (limited to '')
5 files changed, 1490 insertions, 0 deletions
diff --git a/browser/components/urlbar/private/BaseFeature.sys.mjs b/browser/components/urlbar/private/BaseFeature.sys.mjs new file mode 100644 index 0000000000..b703f239f4 --- /dev/null +++ b/browser/components/urlbar/private/BaseFeature.sys.mjs @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs", + UrlbarUtils: "resource:///modules/UrlbarUtils.sys.mjs", +}); + +/** + * Base class for quick suggest features. It can be extended to implement a + * feature that is part of the larger quick suggest feature and that should be + * enabled only when quick suggest is enabled. + * + * You can extend this class as an alternative to implementing your feature + * directly in `QuickSuggest`. Doing so has the following advantages: + * + * - If your feature is gated on a Nimbus variable or preference, `QuickSuggest` + * will manage its lifetime automatically. This is really only useful if the + * feature has state that must be initialized when the feature is enabled and + * uninitialized when it's disabled. + * + * - Encapsulation. You can keep all the code related to your feature in one + * place, without mixing it with unrelated code and cluttering up + * `QuickSuggest`. You can also test it in isolation from `QuickSuggest`. + * + * - Your feature will automatically get its own logger. + * + * If your feature can't benefit from these advantages, especially the first, + * feel free to implement it directly in `QuickSuggest`. + * + * To register your subclass with `QuickSuggest`, add it to the `FEATURES` const + * in QuickSuggest.sys.mjs. + */ +export class BaseFeature { + /** + * {boolean} + * Whether the feature should be enabled. Typically the subclass will check + * the values of one or more Nimbus variables or preferences. `QuickSuggest` + * will access this getter only when the quick suggest feature as a whole is + * enabled. Otherwise the subclass feature will be disabled automatically. + */ + get shouldEnable() { + throw new Error("`shouldEnable` must be overridden"); + } + + /** + * @returns {Array} + * If the subclass's `shouldEnable` implementation depends on preferences + * instead of Nimbus variables, the subclass should override this getter and + * return their names in this array so that `enable()` can be called when + * they change. Names should be in the same format that `UrlbarPrefs.get()` + * expects. + */ + get enablingPreferences() { + return null; + } + + /** + * This method should initialize or uninitialize any state related to the + * feature. + * + * @param {boolean} enabled + * Whether the feature should be enabled or not. + */ + enable(enabled) { + throw new Error("`enable()` must be overridden"); + } + + /** + * @returns {Logger} + * The feature's logger. + */ + get logger() { + if (!this._logger) { + this._logger = lazy.UrlbarUtils.getLogger({ + prefix: `QuickSuggest.${this.constructor.name}`, + }); + } + return this._logger; + } + + /** + * @returns {boolean} + * Whether the feature is enabled. The enabled status is automatically + * managed by `QuickSuggest` and subclasses should not override this. + */ + get isEnabled() { + return this.#isEnabled; + } + + /** + * Enables or disables the feature according to `shouldEnable` and whether + * quick suggest is enabled. If the feature is already enabled appropriately, + * does nothing. + */ + update() { + let enable = + lazy.UrlbarPrefs.get("quickSuggestEnabled") && this.shouldEnable; + if (enable != this.isEnabled) { + this.logger.info(`Setting enabled = ${enable}`); + this.enable(enable); + this.#isEnabled = enable; + } + } + + #isEnabled = false; +} diff --git a/browser/components/urlbar/private/BlockedSuggestions.sys.mjs b/browser/components/urlbar/private/BlockedSuggestions.sys.mjs new file mode 100644 index 0000000000..d74a0979d1 --- /dev/null +++ b/browser/components/urlbar/private/BlockedSuggestions.sys.mjs @@ -0,0 +1,187 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + TaskQueue: "resource:///modules/UrlbarUtils.sys.mjs", + UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs", +}); + +/** + * A set of blocked suggestions for quick suggest. + */ +export class BlockedSuggestions extends BaseFeature { + constructor() { + super(); + this.#taskQueue = new lazy.TaskQueue(); + lazy.UrlbarPrefs.addObserver(this); + } + + get shouldEnable() { + // Return true so that we'll always load blocked digests when quick suggest + // is enabled, even if blocking new suggestions is currently disabled. + // Blocking may have been enabled previously, and blocked suggestions should + // remain blocked as long as quick suggest as a whole remains enabled. + return true; + } + + enable(enabled) { + if (enabled) { + this.#loadDigests(); + } + } + + /** + * Blocks a suggestion. + * + * @param {string} originalUrl + * The suggestion's original URL with its unreplaced timestamp template. + */ + async add(originalUrl) { + this.logger.debug(`Queueing add: ${originalUrl}`); + await this.#taskQueue.queue(async () => { + this.logger.info(`Blocking suggestion: ${originalUrl}`); + let digest = await this.#getDigest(originalUrl); + this.logger.debug(`Got digest for '${originalUrl}': ${digest}`); + this.#digests.add(digest); + let json = JSON.stringify([...this.#digests]); + this.#updatingDigests = true; + try { + lazy.UrlbarPrefs.set("quicksuggest.blockedDigests", json); + } finally { + this.#updatingDigests = false; + } + this.logger.debug(`All blocked suggestions: ${json}`); + }); + } + + /** + * Gets whether a suggestion is blocked. + * + * @param {string} originalUrl + * The suggestion's original URL with its unreplaced timestamp template. + * @returns {boolean} + * Whether the suggestion is blocked. + */ + async has(originalUrl) { + this.logger.debug(`Queueing has: ${originalUrl}`); + return this.#taskQueue.queue(async () => { + this.logger.info(`Getting blocked status: ${originalUrl}`); + let digest = await this.#getDigest(originalUrl); + this.logger.debug(`Got digest for '${originalUrl}': ${digest}`); + let isBlocked = this.#digests.has(digest); + this.logger.info(`Blocked status for '${originalUrl}': ${isBlocked}`); + return isBlocked; + }); + } + + /** + * Unblocks all suggestions. + */ + async clear() { + this.logger.debug(`Queueing clearBlockedSuggestions`); + await this.#taskQueue.queue(() => { + this.logger.info(`Clearing all blocked suggestions`); + this.#digests.clear(); + lazy.UrlbarPrefs.clear("quicksuggest.blockedDigests"); + }); + } + + /** + * Called when a urlbar pref changes. + * + * @param {string} pref + * The name of the pref relative to `browser.urlbar`. + */ + onPrefChanged(pref) { + switch (pref) { + case "quicksuggest.blockedDigests": + if (!this.#updatingDigests) { + this.logger.info( + "browser.urlbar.quicksuggest.blockedDigests changed" + ); + this.#loadDigests(); + } + break; + } + } + + /** + * Loads blocked suggestion digests from the pref into `#digests`. + */ + async #loadDigests() { + this.logger.debug(`Queueing #loadDigests`); + await this.#taskQueue.queue(() => { + this.logger.info(`Loading blocked suggestion digests`); + let json = lazy.UrlbarPrefs.get("quicksuggest.blockedDigests"); + this.logger.debug( + `browser.urlbar.quicksuggest.blockedDigests value: ${json}` + ); + if (!json) { + this.logger.info(`There are no blocked suggestion digests`); + this.#digests.clear(); + } else { + try { + this.#digests = new Set(JSON.parse(json)); + this.logger.info(`Successfully loaded blocked suggestion digests`); + } catch (error) { + this.logger.error( + `Error loading blocked suggestion digests: ${error}` + ); + } + } + }); + } + + /** + * Returns the SHA-1 digest of a string as a 40-character hex-encoded string. + * + * @param {string} string + * The string to convert to SHA-1 + * @returns {string} + * The hex-encoded digest of the given string. + */ + async #getDigest(string) { + let stringArray = new TextEncoder().encode(string); + let hashBuffer = await crypto.subtle.digest("SHA-1", stringArray); + let hashArray = new Uint8Array(hashBuffer); + return Array.from(hashArray, b => b.toString(16).padStart(2, "0")).join(""); + } + + get _test_readyPromise() { + return this.#taskQueue.emptyPromise; + } + + get _test_digests() { + return this.#digests; + } + + _test_getDigest(string) { + return this.#getDigest(string); + } + + // Set of digests of the original URLs of blocked suggestions. A suggestion's + // "original URL" is its URL straight from the source with an unreplaced + // timestamp template. For details on the digests, see `#getDigest()`. + // + // The only reason we use URL digests is that suggestions currently do not + // have persistent IDs. We could use the URLs themselves but SHA-1 digests are + // only 40 chars long, so they save a little space. This is also consistent + // with how blocked tiles on the newtab page are stored, but they use MD5. We + // do *not* store digests for any security or obfuscation reason. + // + // This value is serialized as a JSON'ed array to the + // `browser.urlbar.quicksuggest.blockedDigests` pref. + #digests = new Set(); + + // Used to serialize access to blocked suggestions. This is only necessary + // because getting a suggestion's URL digest is async. + #taskQueue = null; + + // Whether blocked digests are currently being updated. + #updatingDigests = false; +} diff --git a/browser/components/urlbar/private/ImpressionCaps.sys.mjs b/browser/components/urlbar/private/ImpressionCaps.sys.mjs new file mode 100644 index 0000000000..711ae9c338 --- /dev/null +++ b/browser/components/urlbar/private/ImpressionCaps.sys.mjs @@ -0,0 +1,569 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs", + QuickSuggest: "resource:///modules/QuickSuggest.sys.mjs", + UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs", + clearInterval: "resource://gre/modules/Timer.sys.mjs", + setInterval: "resource://gre/modules/Timer.sys.mjs", +}); + +const IMPRESSION_COUNTERS_RESET_INTERVAL_MS = 60 * 60 * 1000; // 1 hour + +// This object maps impression stats object keys to their corresponding keys in +// the `extra` object of impression cap telemetry events. The main reason this +// is necessary is because the keys of the `extra` object are limited to 15 +// characters in length, which some stats object keys exceed. It also forces us +// to be deliberate about keys we add to the `extra` object, since the `extra` +// object is limited to 10 keys. +const TELEMETRY_IMPRESSION_CAP_EXTRA_KEYS = { + // stats object key -> `extra` telemetry event object key + intervalSeconds: "intervalSeconds", + startDateMs: "startDate", + count: "count", + maxCount: "maxCount", + impressionDateMs: "impressionDate", +}; + +/** + * Impression caps and stats for quick suggest suggestions. + */ +export class ImpressionCaps extends BaseFeature { + constructor() { + super(); + lazy.UrlbarPrefs.addObserver(this); + } + + get shouldEnable() { + return ( + lazy.UrlbarPrefs.get("quickSuggestImpressionCapsSponsoredEnabled") || + lazy.UrlbarPrefs.get("quickSuggestImpressionCapsNonSponsoredEnabled") + ); + } + + enable(enabled) { + if (enabled) { + this.#init(); + } else { + this.#uninit(); + } + } + + /** + * Increments the user's impression stats counters for the given type of + * suggestion. This should be called only when a suggestion impression is + * recorded. + * + * @param {string} type + * The suggestion type, one of: "sponsored", "nonsponsored" + */ + updateStats(type) { + this.logger.info("Starting impression stats update"); + this.logger.debug( + JSON.stringify({ + type, + currentStats: this.#stats, + impression_caps: + lazy.QuickSuggest.remoteSettings.config.impression_caps, + }) + ); + + // Don't bother recording anything if caps are disabled. + let isSponsored = type == "sponsored"; + if ( + (isSponsored && + !lazy.UrlbarPrefs.get("quickSuggestImpressionCapsSponsoredEnabled")) || + (!isSponsored && + !lazy.UrlbarPrefs.get("quickSuggestImpressionCapsNonSponsoredEnabled")) + ) { + this.logger.info("Impression caps disabled, skipping update"); + return; + } + + // Get the user's impression stats. Since stats are synced from caps, if the + // stats don't exist then the caps don't exist, and don't bother recording + // anything in that case. + let stats = this.#stats[type]; + if (!stats) { + this.logger.info("Impression caps undefined, skipping update"); + return; + } + + // Increment counters. + for (let stat of stats) { + stat.count++; + stat.impressionDateMs = Date.now(); + + // Record a telemetry event for each newly hit cap. + if (stat.count == stat.maxCount) { + this.logger.info(`'${type}' impression cap hit`); + this.logger.debug(JSON.stringify({ type, hitStat: stat })); + this.#recordCapEvent({ + stat, + eventType: "hit", + suggestionType: type, + }); + } + } + + // Save the stats. + this.#updatingStats = true; + try { + lazy.UrlbarPrefs.set( + "quicksuggest.impressionCaps.stats", + JSON.stringify(this.#stats) + ); + } finally { + this.#updatingStats = false; + } + + this.logger.info("Finished impression stats update"); + this.logger.debug(JSON.stringify({ newStats: this.#stats })); + } + + /** + * Returns a non-null value if an impression cap has been reached for the + * given suggestion type and null otherwise. This method can therefore be used + * to tell whether a cap has been reached for a given type. The actual return + * value an object describing the impression stats that caused the cap to be + * reached. + * + * @param {string} type + * The suggestion type, one of: "sponsored", "nonsponsored" + * @returns {object} + * An impression stats object or null. + */ + getHitStats(type) { + this.#resetElapsedCounters(); + let stats = this.#stats[type]; + if (stats) { + let hitStats = stats.filter(s => s.maxCount <= s.count); + if (hitStats.length) { + return hitStats; + } + } + return null; + } + + /** + * Called when a urlbar pref changes. + * + * @param {string} pref + * The name of the pref relative to `browser.urlbar`. + */ + onPrefChanged(pref) { + switch (pref) { + case "quicksuggest.impressionCaps.stats": + if (!this.#updatingStats) { + this.logger.info( + "browser.urlbar.quicksuggest.impressionCaps.stats changed" + ); + this.#loadStats(); + } + break; + } + } + + #init() { + this.#loadStats(); + + // Validate stats against any changes to the impression caps in the config. + this._onConfigSet = () => this.#validateStats(); + lazy.QuickSuggest.remoteSettings.emitter.on( + "config-set", + this._onConfigSet + ); + + // Periodically record impression counters reset telemetry. + this.#setCountersResetInterval(); + + // On shutdown, record any final impression counters reset telemetry. + this._shutdownBlocker = () => this.#resetElapsedCounters(); + lazy.AsyncShutdown.profileChangeTeardown.addBlocker( + "QuickSuggest: Record impression counters reset telemetry", + this._shutdownBlocker + ); + } + + #uninit() { + lazy.QuickSuggest.remoteSettings.emitter.off( + "config-set", + this._onConfigSet + ); + this._onConfigSet = null; + + lazy.clearInterval(this._impressionCountersResetInterval); + this._impressionCountersResetInterval = 0; + + lazy.AsyncShutdown.profileChangeTeardown.removeBlocker( + this._shutdownBlocker + ); + this._shutdownBlocker = null; + } + + /** + * Loads and validates impression stats. + */ + #loadStats() { + let json = lazy.UrlbarPrefs.get("quicksuggest.impressionCaps.stats"); + if (!json) { + this.#stats = {}; + } else { + try { + this.#stats = JSON.parse( + json, + // Infinity, which is the `intervalSeconds` for the lifetime cap, is + // stringified as `null` in the JSON, so convert it back to Infinity. + (key, value) => + key == "intervalSeconds" && value === null ? Infinity : value + ); + } catch (error) {} + } + this.#validateStats(); + } + + /** + * Validates impression stats, which includes two things: + * + * - Type checks stats and discards any that are invalid. We do this because + * stats are stored in prefs where anyone can modify them. + * - Syncs stats with impression caps so that there is one stats object + * corresponding to each impression cap. See the `#stats` comment for info. + */ + #validateStats() { + let { impression_caps } = lazy.QuickSuggest.remoteSettings.config; + + this.logger.info("Validating impression stats"); + this.logger.debug( + JSON.stringify({ + impression_caps, + currentStats: this.#stats, + }) + ); + + if (!this.#stats || typeof this.#stats != "object") { + this.#stats = {}; + } + + for (let [type, cap] of Object.entries(impression_caps || {})) { + // Build a map from interval seconds to max counts in the caps. + let maxCapCounts = (cap.custom || []).reduce( + (map, { interval_s, max_count }) => { + map.set(interval_s, max_count); + return map; + }, + new Map() + ); + if (typeof cap.lifetime == "number") { + maxCapCounts.set(Infinity, cap.lifetime); + } + + let stats = this.#stats[type]; + if (!Array.isArray(stats)) { + stats = []; + this.#stats[type] = stats; + } + + // Validate existing stats: + // + // * Discard stats with invalid properties. + // * Collect and remove stats with intervals that aren't in the caps. This + // should only happen when caps are changed or removed. + // * For stats with intervals that are in the caps: + // * Keep track of the max `stat.count` across all stats so we can + // update the lifetime stat below. + // * Set `stat.maxCount` to the max count in the corresponding cap. + let orphanStats = []; + let maxCountInStats = 0; + for (let i = 0; i < stats.length; ) { + let stat = stats[i]; + if ( + typeof stat.intervalSeconds != "number" || + typeof stat.startDateMs != "number" || + typeof stat.count != "number" || + typeof stat.maxCount != "number" || + typeof stat.impressionDateMs != "number" + ) { + stats.splice(i, 1); + } else { + maxCountInStats = Math.max(maxCountInStats, stat.count); + let maxCount = maxCapCounts.get(stat.intervalSeconds); + if (maxCount === undefined) { + stats.splice(i, 1); + orphanStats.push(stat); + } else { + stat.maxCount = maxCount; + i++; + } + } + } + + // Create stats for caps that don't already have corresponding stats. + for (let [intervalSeconds, maxCount] of maxCapCounts.entries()) { + if (!stats.some(s => s.intervalSeconds == intervalSeconds)) { + stats.push({ + maxCount, + intervalSeconds, + startDateMs: Date.now(), + count: 0, + impressionDateMs: 0, + }); + } + } + + // Merge orphaned stats into other ones if possible. For each orphan, if + // its interval is no bigger than an existing stat's interval, then the + // orphan's count can contribute to the existing stat's count, so merge + // the two. + for (let orphan of orphanStats) { + for (let stat of stats) { + if (orphan.intervalSeconds <= stat.intervalSeconds) { + stat.count = Math.max(stat.count, orphan.count); + stat.startDateMs = Math.min(stat.startDateMs, orphan.startDateMs); + stat.impressionDateMs = Math.max( + stat.impressionDateMs, + orphan.impressionDateMs + ); + } + } + } + + // If the lifetime stat exists, make its count the max count found above. + // This is only necessary when the lifetime cap wasn't present before, but + // it doesn't hurt to always do it. + let lifetimeStat = stats.find(s => s.intervalSeconds == Infinity); + if (lifetimeStat) { + lifetimeStat.count = maxCountInStats; + } + + // Sort the stats by interval ascending. This isn't necessary except that + // it guarantees an ordering for tests. + stats.sort((a, b) => a.intervalSeconds - b.intervalSeconds); + } + + this.logger.debug(JSON.stringify({ newStats: this.#stats })); + } + + /** + * Resets the counters of impression stats whose intervals have elapased. + */ + #resetElapsedCounters() { + this.logger.info("Checking for elapsed impression cap intervals"); + this.logger.debug( + JSON.stringify({ + currentStats: this.#stats, + impression_caps: + lazy.QuickSuggest.remoteSettings.config.impression_caps, + }) + ); + + let now = Date.now(); + for (let [type, stats] of Object.entries(this.#stats)) { + for (let stat of stats) { + let elapsedMs = now - stat.startDateMs; + let intervalMs = 1000 * stat.intervalSeconds; + let elapsedIntervalCount = Math.floor(elapsedMs / intervalMs); + if (elapsedIntervalCount) { + // At least one interval period elapsed for the stat, so reset it. We + // may also need to record a telemetry event for the reset. + this.logger.info( + `Resetting impression counter for interval ${stat.intervalSeconds}s` + ); + this.logger.debug( + JSON.stringify({ type, stat, elapsedMs, elapsedIntervalCount }) + ); + + let newStartDateMs = + stat.startDateMs + elapsedIntervalCount * intervalMs; + + // Compute the portion of `elapsedIntervalCount` that happened after + // startup. This will be the interval count we report in the telemetry + // event. By design we don't report intervals that elapsed while the + // app wasn't running. For example, if the user stopped using Firefox + // for a year, we don't want to report a year's worth of intervals. + // + // First, compute the count of intervals that elapsed before startup. + // This is the same arithmetic used above except here it's based on + // the startup date instead of `now`. Keep in mind that startup may be + // before the stat's start date. Then subtract that count from + // `elapsedIntervalCount` to get the portion after startup. + let startupDateMs = this._getStartupDateMs(); + let elapsedIntervalCountBeforeStartup = Math.floor( + Math.max(0, startupDateMs - stat.startDateMs) / intervalMs + ); + let elapsedIntervalCountAfterStartup = + elapsedIntervalCount - elapsedIntervalCountBeforeStartup; + + if (elapsedIntervalCountAfterStartup) { + this.#recordCapEvent({ + eventType: "reset", + suggestionType: type, + eventDateMs: newStartDateMs, + eventCount: elapsedIntervalCountAfterStartup, + stat: { + ...stat, + startDateMs: + stat.startDateMs + + elapsedIntervalCountBeforeStartup * intervalMs, + }, + }); + } + + // Reset the stat. + stat.startDateMs = newStartDateMs; + stat.count = 0; + } + } + } + + this.logger.debug(JSON.stringify({ newStats: this.#stats })); + } + + /** + * Records an impression cap telemetry event. + * + * @param {object} options + * Options object + * @param {"hit" | "reset"} options.eventType + * One of: "hit", "reset" + * @param {string} options.suggestionType + * One of: "sponsored", "nonsponsored" + * @param {object} options.stat + * The stats object whose max count was hit or whose counter was reset. + * @param {number} options.eventCount + * The number of intervals that elapsed since the last event. + * @param {number} options.eventDateMs + * The `eventDate` that should be recorded in the event's `extra` object. + * We include this in `extra` even though events are timestamped because + * "reset" events are batched during periods where the user doesn't perform + * any searches and therefore impression counters are not reset. + */ + #recordCapEvent({ + eventType, + suggestionType, + stat, + eventCount = 1, + eventDateMs = Date.now(), + }) { + // All `extra` object values must be strings. + let extra = { + type: suggestionType, + eventDate: String(eventDateMs), + eventCount: String(eventCount), + }; + for (let [statKey, value] of Object.entries(stat)) { + let extraKey = TELEMETRY_IMPRESSION_CAP_EXTRA_KEYS[statKey]; + if (!extraKey) { + throw new Error("Unrecognized stats object key: " + statKey); + } + extra[extraKey] = String(value); + } + Services.telemetry.recordEvent( + lazy.QuickSuggest.TELEMETRY_EVENT_CATEGORY, + "impression_cap", + eventType, + "", + extra + ); + } + + /** + * Creates a repeating timer that resets impression counters and records + * related telemetry. Since counters are also reset when suggestions are + * triggered, the only point of this is to make sure we record reset telemetry + * events in a timely manner during periods when suggestions aren't triggered. + * + * @param {number} ms + * The number of milliseconds in the interval. + */ + #setCountersResetInterval(ms = IMPRESSION_COUNTERS_RESET_INTERVAL_MS) { + if (this._impressionCountersResetInterval) { + lazy.clearInterval(this._impressionCountersResetInterval); + } + this._impressionCountersResetInterval = lazy.setInterval( + () => this.#resetElapsedCounters(), + ms + ); + } + + /** + * Gets the timestamp of app startup in ms since Unix epoch. This is only + * defined as its own method so tests can override it to simulate arbitrary + * startups. + * + * @returns {number} + * Startup timestamp in ms since Unix epoch. + */ + _getStartupDateMs() { + return Services.startup.getStartupInfo().process.getTime(); + } + + get _test_stats() { + return this.#stats; + } + + _test_reloadStats() { + this.#stats = null; + this.#loadStats(); + } + + _test_resetElapsedCounters() { + this.#resetElapsedCounters(); + } + + _test_setCountersResetInterval(ms) { + this.#setCountersResetInterval(ms); + } + + // An object that keeps track of impression stats per sponsored and + // non-sponsored suggestion types. It looks like this: + // + // { sponsored: statsArray, nonsponsored: statsArray } + // + // The `statsArray` values are arrays of stats objects, one per impression + // cap, which look like this: + // + // { intervalSeconds, startDateMs, count, maxCount, impressionDateMs } + // + // {number} intervalSeconds + // The number of seconds in the corresponding cap's time interval. + // {number} startDateMs + // The timestamp at which the current interval period started and the + // object's `count` was reset to zero. This is a value returned from + // `Date.now()`. When the current date/time advances past `startDateMs + + // 1000 * intervalSeconds`, a new interval period will start and `count` + // will be reset to zero. + // {number} count + // The number of impressions during the current interval period. + // {number} maxCount + // The maximum number of impressions allowed during an interval period. + // This value is the same as the `max_count` value in the corresponding + // cap. It's stored in the stats object for convenience. + // {number} impressionDateMs + // The timestamp of the most recent impression, i.e., when `count` was + // last incremented. + // + // There are two types of impression caps: interval and lifetime. Interval + // caps are periodically reset, and lifetime caps are never reset. For stats + // objects corresponding to interval caps, `intervalSeconds` will be the + // `interval_s` value of the cap. For stats objects corresponding to lifetime + // caps, `intervalSeconds` will be `Infinity`. + // + // `#stats` is kept in sync with impression caps, and there is a one-to-one + // relationship between stats objects and caps. A stats object's corresponding + // cap is the one with the same suggestion type (sponsored or non-sponsored) + // and interval. See `#validateStats()` for more. + // + // Impression caps are stored in the remote settings config. See + // `QuickSuggestRemoteSettingsClient.confg.impression_caps`. + #stats = {}; + + // Whether impression stats are currently being updated. + #updatingStats = false; +} diff --git a/browser/components/urlbar/private/RemoteSettingsClient.sys.mjs b/browser/components/urlbar/private/RemoteSettingsClient.sys.mjs new file mode 100644 index 0000000000..a2543d34c6 --- /dev/null +++ b/browser/components/urlbar/private/RemoteSettingsClient.sys.mjs @@ -0,0 +1,453 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + EventEmitter: "resource://gre/modules/EventEmitter.sys.mjs", + TaskQueue: "resource:///modules/UrlbarUtils.sys.mjs", + UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + RemoteSettings: "resource://services-settings/remote-settings.js", +}); + +const RS_COLLECTION = "quicksuggest"; + +// Categories that should show "Firefox Suggest" instead of "Sponsored" +const NONSPONSORED_IAB_CATEGORIES = new Set(["5 - Education"]); + +// Default score for remote settings suggestions. +const DEFAULT_SUGGESTION_SCORE = 0.2; + +// Entries are added to the `#resultsByKeyword` map in chunks, and each chunk +// will add at most this many entries. +const ADD_RESULTS_CHUNK_SIZE = 1000; + +const TELEMETRY_LATENCY = "FX_URLBAR_QUICK_SUGGEST_REMOTE_SETTINGS_LATENCY_MS"; + +/** + * Fetches the suggestions data from RemoteSettings and builds the structures + * to provide suggestions for UrlbarProviderQuickSuggest. + */ +export class RemoteSettingsClient extends BaseFeature { + /** + * @returns {number} + * The default score for remote settings suggestions, a value in the range + * [0, 1]. All suggestions require a score that can be used for comparison, + * so if a remote settings suggestion does not have one, it's assigned this + * value. + */ + static get DEFAULT_SUGGESTION_SCORE() { + return DEFAULT_SUGGESTION_SCORE; + } + + constructor() { + super(); + this.#taskQueue = new lazy.TaskQueue(); + this.#emitter = new lazy.EventEmitter(); + } + + get shouldEnable() { + return ( + lazy.UrlbarPrefs.get("suggest.quicksuggest.nonsponsored") || + lazy.UrlbarPrefs.get("suggest.quicksuggest.sponsored") + ); + } + + get enablingPreferences() { + return [ + "suggest.quicksuggest.nonsponsored", + "suggest.quicksuggest.sponsored", + ]; + } + + /** + * @returns {EventEmitter} + * The client will emit events on this object. + */ + get emitter() { + return this.#emitter; + } + + /** + * @returns {Promise} + * Resolves when any ongoing updates to the suggestions data are done. + */ + get readyPromise() { + return this.#taskQueue.emptyPromise; + } + + /** + * @returns {object} + * Global quick suggest configuration stored in remote settings. When the + * config changes the `emitter` property will emit a "config-set" event. The + * config is an object that looks like this: + * + * { + * best_match: { + * min_search_string_length, + * blocked_suggestion_ids, + * }, + * impression_caps: { + * nonsponsored: { + * lifetime, + * custom: [ + * { interval_s, max_count }, + * ], + * }, + * sponsored: { + * lifetime, + * custom: [ + * { interval_s, max_count }, + * ], + * }, + * }, + * } + */ + get config() { + return this.#config; + } + + enable(enabled) { + this.#queueSettingsSetup(enabled); + } + + /** + * Fetches remote settings suggestions. + * + * @param {string} searchString + * The search string. + * @returns {Array} + * The remote settings suggestions. If there are no matches, an empty array + * is returned. + */ + async fetch(searchString) { + let suggestions; + let stopwatchInstance = {}; + TelemetryStopwatch.start(TELEMETRY_LATENCY, stopwatchInstance); + try { + suggestions = await this.#fetchHelper(searchString); + TelemetryStopwatch.finish(TELEMETRY_LATENCY, stopwatchInstance); + } catch (error) { + TelemetryStopwatch.cancel(TELEMETRY_LATENCY, stopwatchInstance); + this.logger.error("Error fetching suggestions: " + error); + } + + return suggestions || []; + } + + /** + * Helper for `fetch()` that actually looks up the matching suggestions. + * + * @param {string} phrase + * The search string. + * @returns {Array} + * The matched suggestion objects. If there are no matches, an empty array + * is returned. + */ + async #fetchHelper(phrase) { + this.logger.info("Handling query: " + JSON.stringify(phrase)); + + phrase = phrase.toLowerCase(); + let object = this.#resultsByKeyword.get(phrase); + if (!object) { + return []; + } + + // `object` will be a single result object if there's only one match or an + // array of result objects if there's more than one match. + let results = [object].flat(); + + // Start each icon fetch at the same time and wait for them all to finish. + let icons = await Promise.all( + results.map(({ icon }) => this.#fetchIcon(icon)) + ); + + return results.map(result => ({ + full_keyword: this.getFullKeyword(phrase, result.keywords), + title: result.title, + url: result.url, + click_url: result.click_url, + impression_url: result.impression_url, + block_id: result.id, + advertiser: result.advertiser, + iab_category: result.iab_category, + is_sponsored: !NONSPONSORED_IAB_CATEGORIES.has(result.iab_category), + score: + typeof result.score == "number" + ? result.score + : DEFAULT_SUGGESTION_SCORE, + source: "remote-settings", + icon: icons.shift(), + position: result.position, + })); + } + + /** + * Gets the full keyword (i.e., suggestion) for a result and query. The data + * doesn't include full keywords, so we make our own based on the result's + * keyword phrases and a particular query. We use two heuristics: + * + * (1) Find the first keyword phrase that has more words than the query. Use + * its first `queryWords.length` words as the full keyword. e.g., if the + * query is "moz" and `result.keywords` is ["moz", "mozi", "mozil", + * "mozill", "mozilla", "mozilla firefox"], pick "mozilla firefox", pop + * off the "firefox" and use "mozilla" as the full keyword. + * (2) If there isn't any keyword phrase with more words, then pick the + * longest phrase. e.g., pick "mozilla" in the previous example (assuming + * the "mozilla firefox" phrase isn't there). That might be the query + * itself. + * + * @param {string} query + * The query string that matched `result`. + * @param {Array} keywords + * An array of result keywords. + * @returns {string} + * The full keyword. + */ + getFullKeyword(query, keywords) { + let longerPhrase; + let trimmedQuery = query.trim(); + let queryWords = trimmedQuery.split(" "); + + for (let phrase of keywords) { + if (phrase.startsWith(query)) { + let trimmedPhrase = phrase.trim(); + let phraseWords = trimmedPhrase.split(" "); + // As an exception to (1), if the query ends with a space, then look for + // phrases with one more word so that the suggestion includes a word + // following the space. + let extra = query.endsWith(" ") ? 1 : 0; + let len = queryWords.length + extra; + if (len < phraseWords.length) { + // We found a phrase with more words. + return phraseWords.slice(0, len).join(" "); + } + if ( + query.length < phrase.length && + (!longerPhrase || longerPhrase.length < trimmedPhrase.length) + ) { + // We found a longer phrase with the same number of words. + longerPhrase = trimmedPhrase; + } + } + } + return longerPhrase || trimmedQuery; + } + + /** + * Queues a task to ensure our remote settings client is initialized or torn + * down as appropriate. + * + * @param {boolean} enabled + * Whether the feature should be enabled. + */ + #queueSettingsSetup(enabled) { + this.#taskQueue.queue(() => { + if (enabled && !this.#rs) { + this.#onSettingsSync = (...args) => this.#queueSettingsSync(...args); + this.#rs = lazy.RemoteSettings(RS_COLLECTION); + this.#rs.on("sync", this.#onSettingsSync); + this.#queueSettingsSync(); + } else if (!enabled && this.#rs) { + this.#rs.off("sync", this.#onSettingsSync); + this.#rs = null; + this.#onSettingsSync = null; + } + }); + } + + /** + * Queues a task to populate the results map from the remote settings data + * plus any other work that needs to be done on sync. + * + * @param {object} [event] + * The event object passed to the "sync" event listener if you're calling + * this from the listener. + */ + async #queueSettingsSync(event = null) { + await this.#taskQueue.queue(async () => { + if (!this.#rs || this._test_ignoreSettingsSync) { + return; + } + + // Remove local files of deleted records + if (event?.data?.deleted) { + await Promise.all( + event.data.deleted + .filter(d => d.attachment) + .map(entry => + Promise.all([ + this.#rs.attachments.deleteDownloaded(entry), // type: data + this.#rs.attachments.deleteFromDisk(entry), // type: icon + ]) + ) + ); + } + + let dataType = lazy.UrlbarPrefs.get("quickSuggestRemoteSettingsDataType"); + this.logger.debug("Loading data with type: " + dataType); + + let [configArray, data] = await Promise.all([ + this.#rs.get({ filters: { type: "configuration" } }), + this.#rs.get({ filters: { type: dataType } }), + this.#rs + .get({ filters: { type: "icon" } }) + .then(icons => + Promise.all(icons.map(i => this.#rs.attachments.downloadToDisk(i))) + ), + ]); + + this.logger.debug("Got configuration: " + JSON.stringify(configArray)); + this.#setConfig(configArray?.[0]?.configuration || {}); + + this.#resultsByKeyword.clear(); + + this.logger.debug(`Got data with ${data.length} records`); + for (let record of data) { + let { buffer } = await this.#rs.attachments.download(record); + let results = JSON.parse(new TextDecoder("utf-8").decode(buffer)); + this.logger.debug(`Adding ${results.length} results`); + await this.#addResults(results); + } + }); + } + + /** + * Sets the quick suggest config and emits a "config-set" event. + * + * @param {object} config + * The config object. + */ + #setConfig(config) { + this.#config = config || {}; + this.#emitter.emit("config-set"); + } + + /** + * Adds a list of result objects to the results map. This method is also used + * by tests to set up mock suggestions. + * + * @param {Array} results + * Array of result objects. + */ + async #addResults(results) { + // There can be many results, and each result can have many keywords. To + // avoid blocking the main thread for too long, update the map in chunks, + // and to avoid blocking the UI and other higher priority work, do each + // chunk only when the main thread is idle. During each chunk, we'll add at + // most `_addResultsChunkSize` entries to the map. + let resultIndex = 0; + let keywordIndex = 0; + + // Keep adding chunks until all results have been fully added. + while (resultIndex < results.length) { + await new Promise(resolve => { + Services.tm.idleDispatchToMainThread(() => { + // Keep updating the map until the current chunk is done. + let indexInChunk = 0; + while ( + indexInChunk < this._addResultsChunkSize && + resultIndex < results.length + ) { + let result = results[resultIndex]; + if (keywordIndex == result.keywords.length) { + resultIndex++; + keywordIndex = 0; + continue; + } + // If the keyword's only result is `result`, store it directly as + // the value. Otherwise store an array of results. For details, see + // the `#resultsByKeyword` comment. + let keyword = result.keywords[keywordIndex]; + let object = this.#resultsByKeyword.get(keyword); + if (!object) { + this.#resultsByKeyword.set(keyword, result); + } else if (!Array.isArray(object)) { + this.#resultsByKeyword.set(keyword, [object, result]); + } else { + object.push(result); + } + keywordIndex++; + indexInChunk++; + } + + // The current chunk is done. + resolve(); + }); + }); + } + } + + /** + * Fetch the icon from RemoteSettings attachments. + * + * @param {string} path + * The icon's remote settings path. + */ + async #fetchIcon(path) { + if (!path || !this.#rs) { + return null; + } + let record = ( + await this.#rs.get({ + filters: { id: `icon-${path}` }, + }) + ).pop(); + if (!record) { + return null; + } + return this.#rs.attachments.downloadToDisk(record); + } + + get _test_rs() { + return this.#rs; + } + + get _test_resultsByKeyword() { + return this.#resultsByKeyword; + } + + _test_setConfig(config) { + this.#setConfig(config); + } + + async _test_addResults(results) { + await this.#addResults(results); + } + + // The RemoteSettings client. + #rs = null; + + // Task queue for serializing access to remote settings and related data. + // Methods in this class should use this when they need to to modify or access + // the settings client. It ensures settings accesses are serialized, do not + // overlap, and happen only one at a time. It also lets clients, especially + // tests, use this class without having to worry about whether a settings sync + // or initialization is ongoing; see `readyPromise`. + #taskQueue = null; + + // Configuration data synced from remote settings. See the `config` getter. + #config = {}; + + // Maps each keyword in the dataset to one or more results for the keyword. If + // only one result uses a keyword, the keyword's value in the map will be the + // result object. If more than one result uses the keyword, the value will be + // an array of the results. The reason for not always using an array is that + // we expect the vast majority of keywords to be used by only one result, and + // since there are potentially very many keywords and results and we keep them + // in memory all the time, we want to save as much memory as possible. + #resultsByKeyword = new Map(); + + // This is only defined as a property so that tests can override it. + _addResultsChunkSize = ADD_RESULTS_CHUNK_SIZE; + + #onSettingsSync = null; + #emitter = null; +} diff --git a/browser/components/urlbar/private/Weather.sys.mjs b/browser/components/urlbar/private/Weather.sys.mjs new file mode 100644 index 0000000000..611994c8f8 --- /dev/null +++ b/browser/components/urlbar/private/Weather.sys.mjs @@ -0,0 +1,171 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + clearInterval: "resource://gre/modules/Timer.sys.mjs", + MerinoClient: "resource:///modules/MerinoClient.sys.mjs", + PromiseUtils: "resource://gre/modules/PromiseUtils.sys.mjs", + setInterval: "resource://gre/modules/Timer.sys.mjs", + UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs", +}); + +const FETCH_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes +const MERINO_PROVIDER = "accuweather"; +const MERINO_TIMEOUT_MS = 5000; // 5s + +const HISTOGRAM_LATENCY = "FX_URLBAR_MERINO_LATENCY_WEATHER_MS"; +const HISTOGRAM_RESPONSE = "FX_URLBAR_MERINO_RESPONSE_WEATHER"; + +/** + * A feature that periodically fetches weather suggestions from Merino. + */ +export class Weather extends BaseFeature { + get shouldEnable() { + return ( + lazy.UrlbarPrefs.get("weatherFeatureGate") && + lazy.UrlbarPrefs.get("suggest.weather") && + lazy.UrlbarPrefs.get("merinoEnabled") + ); + } + + get enablingPreferences() { + return ["suggest.weather"]; + } + + /** + * @returns {object} + * The last weather suggestion fetched from Merino or null if none. + */ + get suggestion() { + return this.#suggestion; + } + + enable(enabled) { + if (enabled) { + this.#init(); + } else { + this.#uninit(); + } + } + + /** + * Returns a promise that resolves when all pending fetches finish, if there + * are pending fetches. If there aren't, the promise resolves when all pending + * fetches starting with the next fetch finish. + * + * @returns {Promise} + */ + waitForFetches() { + if (!this.#waitForFetchesDeferred) { + this.#waitForFetchesDeferred = lazy.PromiseUtils.defer(); + } + return this.#waitForFetchesDeferred.promise; + } + + #init() { + this.#merino = new lazy.MerinoClient(this.constructor.name); + + this.#fetchInterval = lazy.setInterval( + () => this.#fetch(), + this.#fetchIntervalMs + ); + + // `#fetch()` is async but there's no need to await it here. + this.#fetch(); + } + + #uninit() { + this.#merino = null; + this.#suggestion = null; + lazy.clearInterval(this.#fetchInterval); + this.#fetchInterval = 0; + } + + async #fetch() { + this.logger.info("Fetching suggestion"); + + // This `Weather` instance may be uninitialized while awaiting the fetch or + // even uninitialized and re-initialized a number of times. Discard the + // fetched suggestion if the `#merino` after the fetch isn't the same as the + // one before. + let merino = this.#merino; + + let suggestions; + this.#pendingFetchCount++; + try { + suggestions = await merino.fetch({ + query: "", + providers: [MERINO_PROVIDER], + timeoutMs: this.#timeoutMs, + extraLatencyHistogram: HISTOGRAM_LATENCY, + extraResponseHistogram: HISTOGRAM_RESPONSE, + }); + } finally { + this.#pendingFetchCount--; + } + + // Reset the Merino client's session so different fetches use different + // sessions. A single session is intended to represent a single user + // engagement in the urlbar, which this is not. Practically this isn't + // necessary since the client automatically resets the session on a timer + // whose period is much shorter than our fetch period, but there's no reason + // to keep it ticking in the meantime. + merino.resetSession(); + + if (merino != this.#merino) { + this.logger.info("Fetch canceled, discarding fetched suggestion, if any"); + } else { + let suggestion = suggestions?.[0]; + if (!suggestion) { + this.logger.info("No suggestion received"); + this.#suggestion = null; + } else { + this.logger.info("Got suggestion"); + this.logger.debug(JSON.stringify({ suggestion })); + this.#suggestion = { ...suggestion, source: "merino" }; + } + } + + if (!this.#pendingFetchCount) { + this.#waitForFetchesDeferred?.resolve(); + this.#waitForFetchesDeferred = null; + } + } + + get _test_merino() { + return this.#merino; + } + + get _test_fetchInterval() { + return this.#fetchInterval; + } + + get _test_pendingFetchCount() { + return this.#pendingFetchCount; + } + + async _test_fetch() { + await this.#fetch(); + } + + _test_setFetchIntervalMs(ms) { + this.#fetchIntervalMs = ms < 0 ? FETCH_INTERVAL_MS : ms; + } + + _test_setTimeoutMs(ms) { + this.#timeoutMs = ms < 0 ? MERINO_TIMEOUT_MS : ms; + } + + #merino = null; + #suggestion = null; + #fetchInterval = 0; + #fetchIntervalMs = FETCH_INTERVAL_MS; + #timeoutMs = MERINO_TIMEOUT_MS; + #waitForFetchesDeferred = null; + #pendingFetchCount = 0; +} |