diff options
Diffstat (limited to 'browser/components/newtab/lib')
59 files changed, 28709 insertions, 0 deletions
diff --git a/browser/components/newtab/lib/ASRouter.jsm b/browser/components/newtab/lib/ASRouter.jsm new file mode 100644 index 0000000000..231c1dfbb9 --- /dev/null +++ b/browser/components/newtab/lib/ASRouter.jsm @@ -0,0 +1,2101 @@ +/* vim: set ts=2 sw=2 sts=2 et tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Downloader: "resource://services-settings/Attachments.sys.mjs", + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + MacAttribution: "resource:///modules/MacAttribution.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PanelTestProvider: "resource://activity-stream/lib/PanelTestProvider.sys.mjs", + RemoteL10n: "resource://activity-stream/lib/RemoteL10n.sys.mjs", + SnippetsTestMessageProvider: + "resource://activity-stream/lib/SnippetsTestMessageProvider.sys.mjs", + SpecialMessageActions: + "resource://messaging-system/lib/SpecialMessageActions.sys.mjs", + TargetingContext: "resource://messaging-system/targeting/Targeting.sys.mjs", + Utils: "resource://services-settings/Utils.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + Spotlight: "resource://activity-stream/lib/Spotlight.jsm", + ToastNotification: "resource://activity-stream/lib/ToastNotification.jsm", + ToolbarBadgeHub: "resource://activity-stream/lib/ToolbarBadgeHub.jsm", + ToolbarPanelHub: "resource://activity-stream/lib/ToolbarPanelHub.jsm", + MomentsPageHub: "resource://activity-stream/lib/MomentsPageHub.jsm", + InfoBar: "resource://activity-stream/lib/InfoBar.jsm", + ASRouterTargeting: "resource://activity-stream/lib/ASRouterTargeting.jsm", + ASRouterPreferences: "resource://activity-stream/lib/ASRouterPreferences.jsm", + TARGETING_PREFERENCES: + "resource://activity-stream/lib/ASRouterPreferences.jsm", + ASRouterTriggerListeners: + "resource://activity-stream/lib/ASRouterTriggerListeners.jsm", + KintoHttpClient: "resource://services-common/kinto-http-client.js", +}); + +XPCOMUtils.defineLazyServiceGetters(lazy, { + BrowserHandler: ["@mozilla.org/browser/clh;1", "nsIBrowserHandler"], +}); +const { actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const { CFRMessageProvider } = ChromeUtils.importESModule( + "resource://activity-stream/lib/CFRMessageProvider.sys.mjs" +); +const { OnboardingMessageProvider } = ChromeUtils.import( + "resource://activity-stream/lib/OnboardingMessageProvider.jsm" +); +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); +const { CFRPageActions } = ChromeUtils.import( + "resource://activity-stream/lib/CFRPageActions.jsm" +); +const { AttributionCode } = ChromeUtils.importESModule( + "resource:///modules/AttributionCode.sys.mjs" +); + +// List of hosts for endpoints that serve router messages. +// Key is allowed host, value is a name for the endpoint host. +const DEFAULT_ALLOWLIST_HOSTS = { + "activity-stream-icons.services.mozilla.com": "production", + "snippets-admin.mozilla.org": "preview", +}; +const SNIPPETS_ENDPOINT_ALLOWLIST = + "browser.newtab.activity-stream.asrouter.allowHosts"; +// Max possible impressions cap for any message +const MAX_MESSAGE_LIFETIME_CAP = 100; + +const LOCAL_MESSAGE_PROVIDERS = { + OnboardingMessageProvider, + CFRMessageProvider, +}; +const STARTPAGE_VERSION = "6"; + +// Remote Settings +const RS_MAIN_BUCKET = "main"; +const RS_COLLECTION_L10N = "ms-language-packs"; // "ms" stands for Messaging System +const RS_PROVIDERS_WITH_L10N = ["cfr"]; +const RS_FLUENT_VERSION = "v1"; +const RS_FLUENT_RECORD_PREFIX = `cfr-${RS_FLUENT_VERSION}`; +const RS_DOWNLOAD_MAX_RETRIES = 2; +// This is the list of providers for which we want to cache the targeting +// expression result and reuse between calls. Cache duration is defined in +// ASRouterTargeting where evaluation takes place. +const JEXL_PROVIDER_CACHE = new Set(["snippets"]); + +// To observe the app locale change notification. +const TOPIC_INTL_LOCALE_CHANGED = "intl:app-locales-changed"; +const TOPIC_EXPERIMENT_ENROLLMENT_CHANGED = "nimbus:enrollments-updated"; +// To observe the pref that controls if ASRouter should use the remote Fluent files for l10n. +const USE_REMOTE_L10N_PREF = + "browser.newtabpage.activity-stream.asrouter.useRemoteL10n"; + +const MESSAGING_EXPERIMENTS_DEFAULT_FEATURES = [ + "cfr", + "fxms-message-1", + "fxms-message-2", + "fxms-message-3", + "fxms-message-4", + "fxms-message-5", + "fxms-message-6", + "fxms-message-7", + "fxms-message-8", + "fxms-message-9", + "fxms-message-10", + "fxms-message-11", + "infobar", + "moments-page", + "pbNewtab", + "spotlight", +]; + +// Experiment groups that need to report the reach event in Messaging-Experiments. +// If you're adding new groups to it, make sure they're also added in the +// `messaging_experiments.reach.objects` defined in "toolkit/components/telemetry/Events.yaml" +const REACH_EVENT_GROUPS = ["cfr", "moments-page", "infobar", "spotlight"]; +const REACH_EVENT_CATEGORY = "messaging_experiments"; +const REACH_EVENT_METHOD = "reach"; + +const MessageLoaderUtils = { + STARTPAGE_VERSION, + REMOTE_LOADER_CACHE_KEY: "RemoteLoaderCache", + _errors: [], + + reportError(e) { + console.error(e); + this._errors.push({ + timestamp: new Date(), + error: { message: e.toString(), stack: e.stack }, + }); + }, + + get errors() { + const errors = this._errors; + this._errors = []; + return errors; + }, + + /** + * _localLoader - Loads messages for a local provider (i.e. one that lives in mozilla central) + * + * @param {obj} provider An AS router provider + * @param {Array} provider.messages An array of messages + * @returns {Array} the array of messages + */ + _localLoader(provider) { + return provider.messages; + }, + + async _remoteLoaderCache(storage) { + let allCached; + try { + allCached = + (await storage.get(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY)) || {}; + } catch (e) { + // istanbul ignore next + MessageLoaderUtils.reportError(e); + // istanbul ignore next + allCached = {}; + } + return allCached; + }, + + /** + * _remoteLoader - Loads messages for a remote provider + * + * @param {obj} provider An AS router provider + * @param {string} provider.url An endpoint that returns an array of messages as JSON + * @param {obj} options.storage A storage object with get() and set() methods for caching. + * @returns {Promise} resolves with an array of messages, or an empty array if none could be fetched + */ + async _remoteLoader(provider, options) { + let remoteMessages = []; + if (provider.url) { + const allCached = await MessageLoaderUtils._remoteLoaderCache( + options.storage + ); + const cached = allCached[provider.id]; + let etag; + + if ( + cached && + cached.url === provider.url && + cached.version === STARTPAGE_VERSION + ) { + const { lastFetched, messages } = cached; + if ( + !MessageLoaderUtils.shouldProviderUpdate({ + ...provider, + lastUpdated: lastFetched, + }) + ) { + // Cached messages haven't expired, return early. + return messages; + } + etag = cached.etag; + remoteMessages = messages; + } + + let headers = new Headers(); + if (etag) { + headers.set("If-None-Match", etag); + } + + let response; + try { + response = await fetch(provider.url, { + headers, + credentials: "omit", + }); + } catch (e) { + MessageLoaderUtils.reportError(e); + } + if ( + response && + response.ok && + response.status >= 200 && + response.status < 400 + ) { + let jsonResponse; + try { + jsonResponse = await response.json(); + } catch (e) { + MessageLoaderUtils.reportError(e); + return remoteMessages; + } + if (jsonResponse && jsonResponse.messages) { + remoteMessages = jsonResponse.messages.map(msg => ({ + ...msg, + provider_url: provider.url, + })); + + // Cache the results if this isn't a preview URL. + if (provider.updateCycleInMs > 0) { + etag = response.headers.get("ETag"); + const cacheInfo = { + messages: remoteMessages, + etag, + lastFetched: Date.now(), + version: STARTPAGE_VERSION, + }; + + options.storage.set(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY, { + ...allCached, + [provider.id]: cacheInfo, + }); + } + } else { + MessageLoaderUtils.reportError( + `No messages returned from ${provider.url}.` + ); + } + } else if (response) { + MessageLoaderUtils.reportError( + `Invalid response status ${response.status} from ${provider.url}.` + ); + } + } + return remoteMessages; + }, + + /** + * _remoteSettingsLoader - Loads messages for a RemoteSettings provider + * + * Note: + * 1). The "cfr" provider requires the Fluent file for l10n, so there is + * another file downloading phase for those two providers after their messages + * are successfully fetched from Remote Settings. Currently, they share the same + * attachment of the record "${RS_FLUENT_RECORD_PREFIX}-${locale}" in the + * "ms-language-packs" collection. E.g. for "en-US" with version "v1", + * the Fluent file is attched to the record with ID "cfr-v1-en-US". + * + * 2). The Remote Settings downloader is able to detect the duplicate download + * requests for the same attachment and ignore the redundent requests automatically. + * + * @param {object} provider An AS router provider + * @param {string} provider.id The id of the provider + * @param {string} provider.collection Remote Settings collection name + * @param {object} options + * @param {function} options.dispatchCFRAction Action handler function + * @returns {Promise<object[]>} Resolves with an array of messages, or an + * empty array if none could be fetched + */ + async _remoteSettingsLoader(provider, options) { + let messages = []; + if (provider.collection) { + try { + messages = await MessageLoaderUtils._getRemoteSettingsMessages( + provider.collection + ); + if (!messages.length) { + MessageLoaderUtils._handleRemoteSettingsUndesiredEvent( + "ASR_RS_NO_MESSAGES", + provider.id, + options.dispatchCFRAction + ); + } else if ( + RS_PROVIDERS_WITH_L10N.includes(provider.id) && + lazy.RemoteL10n.isLocaleSupported(MessageLoaderUtils.locale) + ) { + const recordId = `${RS_FLUENT_RECORD_PREFIX}-${MessageLoaderUtils.locale}`; + const kinto = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL); + const record = await kinto + .bucket(RS_MAIN_BUCKET) + .collection(RS_COLLECTION_L10N) + .getRecord(recordId); + if (record && record.data) { + const downloader = new lazy.Downloader( + RS_MAIN_BUCKET, + RS_COLLECTION_L10N, + "browser", + "newtab" + ); + // Await here in order to capture the exceptions for reporting. + await downloader.downloadToDisk(record.data, { + retries: RS_DOWNLOAD_MAX_RETRIES, + }); + lazy.RemoteL10n.reloadL10n(); + } else { + MessageLoaderUtils._handleRemoteSettingsUndesiredEvent( + "ASR_RS_NO_MESSAGES", + RS_COLLECTION_L10N, + options.dispatchCFRAction + ); + } + } + } catch (e) { + MessageLoaderUtils._handleRemoteSettingsUndesiredEvent( + "ASR_RS_ERROR", + provider.id, + options.dispatchCFRAction + ); + MessageLoaderUtils.reportError(e); + } + } + return messages; + }, + + /** + * Fetch messages from a given collection in Remote Settings. + * + * @param {string} collection The remote settings collection identifier + * @returns {Promise<object[]>} Resolves with an array of messages + */ + _getRemoteSettingsMessages(collection) { + return RemoteSettings(collection).get(); + }, + + /** + * Return messages from active Nimbus experiments and rollouts. + * + * @param {object} provider A messaging experiments provider. + * @param {string[]?} provider.featureIds + * An optional array of Nimbus feature IDs to check for + * enrollments. If not provided, we will fall back to the + * set of default features. Otherwise, if provided and + * empty, we will not ingest messages from any features. + * + * @return {object[]} The list of messages from active enrollments, as well as + * the messages defined in unenrolled branches so that they + * reach events can be recorded (if we record reach events + * for that feature). + */ + async _experimentsAPILoader(provider) { + // Allow tests to override the set of featureIds + const featureIds = Array.isArray(provider.featureIds) + ? provider.featureIds + : MESSAGING_EXPERIMENTS_DEFAULT_FEATURES; + let experiments = []; + for (const featureId of featureIds) { + const featureAPI = lazy.NimbusFeatures[featureId]; + const experimentData = lazy.ExperimentAPI.getExperimentMetaData({ + featureId, + }); + + // We are not enrolled in any experiment or rollout for this feature, so + // we can skip the feature. + if ( + !experimentData && + !lazy.ExperimentAPI.getRolloutMetaData({ featureId }) + ) { + continue; + } + + const featureValue = featureAPI.getAllVariables(); + + // If the value is a multi-message config, add each message in the + // messages array. Cache the Nimbus feature ID on each message, because + // there is not a 1-1 correspondance between templates and features. + // This is used when recording expose events (see |sendTriggerMessage|). + const messages = + featureValue?.template === "multi" && + Array.isArray(featureValue.messages) + ? featureValue.messages + : [featureValue]; + for (const message of messages) { + if (message?.id) { + message._nimbusFeature = featureId; + experiments.push(message); + } + } + + // Add Reach messages from unenrolled sibling branches, provided we are + // recording Reach events for this feature. If we are in a rollout, we do + // not have sibling branches. + if (!REACH_EVENT_GROUPS.includes(featureId) || !experimentData) { + continue; + } + + // Check other sibling branches for triggers, add them to the return array + // if found any. The `forReachEvent` label is used to identify those + // branches so that they would only be used to record the Reach event. + const branches = + (await lazy.ExperimentAPI.getAllBranches(experimentData.slug)) || []; + for (const branch of branches) { + let branchValue = branch[featureId].value; + if (!branchValue || branch.slug === experimentData.branch.slug) { + continue; + } + const branchMessages = + branchValue?.template === "multi" && + Array.isArray(branchValue.messages) + ? branchValue.messages + : [branchValue]; + for (const message of branchMessages) { + if (!message?.trigger) { + continue; + } + experiments.push({ + forReachEvent: { sent: false, group: featureId }, + experimentSlug: experimentData.slug, + branchSlug: branch.slug, + ...message, + }); + } + } + } + + return experiments; + }, + + _handleRemoteSettingsUndesiredEvent(event, providerId, dispatchCFRAction) { + if (dispatchCFRAction) { + dispatchCFRAction( + ac.ASRouterUserEvent({ + action: "asrouter_undesired_event", + event, + message_id: "n/a", + event_context: providerId, + }) + ); + } + }, + + /** + * _getMessageLoader - return the right loading function given the provider's type + * + * @param {obj} provider An AS Router provider + * @returns {func} A loading function + */ + _getMessageLoader(provider) { + switch (provider.type) { + case "remote": + return this._remoteLoader; + case "remote-settings": + return this._remoteSettingsLoader; + case "remote-experiments": + return this._experimentsAPILoader; + case "local": + default: + return this._localLoader; + } + }, + + /** + * shouldProviderUpdate - Given the current time, should a provider update its messages? + * + * @param {any} provider An AS Router provider + * @param {int} provider.updateCycleInMs The number of milliseconds we should wait between updates + * @param {Date} provider.lastUpdated If the provider has been updated, the time the last update occurred + * @param {Date} currentTime The time we should check against. (defaults to Date.now()) + * @returns {bool} Should an update happen? + */ + shouldProviderUpdate(provider, currentTime = Date.now()) { + return ( + !(provider.lastUpdated >= 0) || + currentTime - provider.lastUpdated > provider.updateCycleInMs + ); + }, + + async _loadDataForProvider(provider, options) { + const loader = this._getMessageLoader(provider); + let messages = await loader(provider, options); + // istanbul ignore if + if (!messages) { + messages = []; + MessageLoaderUtils.reportError( + new Error( + `Tried to load messages for ${provider.id} but the result was not an Array.` + ) + ); + } + + return { messages }; + }, + + /** + * loadMessagesForProvider - Load messages for a provider, given the provider's type. + * + * @param {obj} provider An AS Router provider + * @param {string} provider.type An AS Router provider type (defaults to "local") + * @param {obj} options.storage A storage object with get() and set() methods for caching. + * @param {func} options.dispatchCFRAction dispatch an action the main AS Store + * @returns {obj} Returns an object with .messages (an array of messages) and .lastUpdated (the time the messages were updated) + */ + async loadMessagesForProvider(provider, options) { + let { messages } = await this._loadDataForProvider(provider, options); + // Filter out messages we temporarily want to exclude + if (provider.exclude && provider.exclude.length) { + messages = messages.filter( + message => !provider.exclude.includes(message.id) + ); + } + const lastUpdated = Date.now(); + return { + messages: messages + .map(messageData => { + const message = { + weight: 100, + ...messageData, + groups: messageData.groups || [], + provider: provider.id, + }; + + return message; + }) + .filter(message => message.weight > 0), + lastUpdated, + errors: MessageLoaderUtils.errors, + }; + }, + + /** + * cleanupCache - Removes cached data of removed providers. + * + * @param {Array} providers A list of activer AS Router providers + */ + async cleanupCache(providers, storage) { + const ids = providers.filter(p => p.type === "remote").map(p => p.id); + const cache = await MessageLoaderUtils._remoteLoaderCache(storage); + let dirty = false; + for (let id in cache) { + if (!ids.includes(id)) { + delete cache[id]; + dirty = true; + } + } + if (dirty) { + await storage.set(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY, cache); + } + }, + + /** + * The locale to use for RemoteL10n. + * + * This may map the app's actual locale into something that RemoteL10n + * supports. + */ + get locale() { + const localeMap = { + "ja-JP-macos": "ja-JP-mac", + + // While it's not a valid locale, "und" is commonly observed on + // Linux platforms. Per l10n team, it's reasonable to fallback to + // "en-US", therefore, we should allow the fetch for it. + und: "en-US", + }; + + const locale = Services.locale.appLocaleAsBCP47; + return localeMap[locale] ?? locale; + }, +}; + +/** + * @class _ASRouter - Keeps track of all messages, UI surfaces, and + * handles blocking, rotation, etc. Inspecting ASRouter.state will + * tell you what the current displayed message is in all UI surfaces. + * + * Note: This is written as a constructor rather than just a plain object + * so that it can be more easily unit tested. + */ +class _ASRouter { + constructor(localProviders = LOCAL_MESSAGE_PROVIDERS) { + this.initialized = false; + this.clearChildMessages = null; + this.clearChildProviders = null; + this.updateAdminState = null; + this.sendTelemetry = null; + this.dispatchCFRAction = null; + this._storage = null; + this._resetInitialization(); + this._state = { + providers: [], + messageBlockList: [], + messageImpressions: {}, + screenImpressions: {}, + messages: [], + groups: [], + errors: [], + localeInUse: Services.locale.appLocaleAsBCP47, + }; + this._experimentChangedListeners = new Map(); + this._triggerHandler = this._triggerHandler.bind(this); + this._localProviders = localProviders; + this.blockMessageById = this.blockMessageById.bind(this); + this.unblockMessageById = this.unblockMessageById.bind(this); + this.handleMessageRequest = this.handleMessageRequest.bind(this); + this.addImpression = this.addImpression.bind(this); + this.addScreenImpression = this.addScreenImpression.bind(this); + this._handleTargetingError = this._handleTargetingError.bind(this); + this.onPrefChange = this.onPrefChange.bind(this); + this._onLocaleChanged = this._onLocaleChanged.bind(this); + this.isUnblockedMessage = this.isUnblockedMessage.bind(this); + this.unblockAll = this.unblockAll.bind(this); + this.forceWNPanel = this.forceWNPanel.bind(this); + this._onExperimentEnrollmentsUpdated = + this._onExperimentEnrollmentsUpdated.bind(this); + this.forcePBWindow = this.forcePBWindow.bind(this); + Services.telemetry.setEventRecordingEnabled(REACH_EVENT_CATEGORY, true); + } + + async onPrefChange(prefName) { + if (lazy.TARGETING_PREFERENCES.includes(prefName)) { + let invalidMessages = []; + // Notify all tabs of messages that have become invalid after pref change + const context = this._getMessagesContext(); + const targetingContext = new lazy.TargetingContext(context); + + for (const msg of this.state.messages.filter(this.isUnblockedMessage)) { + if (!msg.targeting) { + continue; + } + const isMatch = await targetingContext.evalWithDefault(msg.targeting); + if (!isMatch) { + invalidMessages.push(msg.id); + } + } + this.clearChildMessages(invalidMessages); + } else { + // Update message providers and fetch new messages on pref change + this._loadLocalProviders(); + let invalidProviders = await this._updateMessageProviders(); + if (invalidProviders.length) { + this.clearChildProviders(invalidProviders); + } + await this.loadMessagesFromAllProviders(); + // Any change in user prefs can disable or enable groups + await this.setState(state => ({ + groups: state.groups.map(this._checkGroupEnabled), + })); + } + } + + // Fetch and decode the message provider pref JSON, and update the message providers + async _updateMessageProviders() { + lazy.ASRouterPreferences.console.debug("entering updateMessageProviders"); + + const previousProviders = this.state.providers; + const providers = await Promise.all( + [ + // If we have added a `preview` provider, hold onto it + ...previousProviders.filter(p => p.id === "preview"), + // The provider should be enabled and not have a user preference set to false + ...lazy.ASRouterPreferences.providers.filter( + p => + p.enabled && + lazy.ASRouterPreferences.getUserPreference(p.id) !== false + ), + ].map(async _provider => { + // make a copy so we don't modify the source of the pref + const provider = { ..._provider }; + + if (provider.type === "local" && !provider.messages) { + // Get the messages from the local message provider + const localProvider = this._localProviders[provider.localProvider]; + provider.messages = []; + if (localProvider) { + provider.messages = await localProvider.getMessages(); + } + } + if (provider.type === "remote" && provider.url) { + provider.url = provider.url.replace( + /%STARTPAGE_VERSION%/g, + STARTPAGE_VERSION + ); + provider.url = Services.urlFormatter.formatURL(provider.url); + } + if (provider.id === "messaging-experiments") { + // By default, the messaging-experiments provider lacks a featureIds + // property, so fall back to the list of default features. + if (!provider.featureIds) { + provider.featureIds = MESSAGING_EXPERIMENTS_DEFAULT_FEATURES; + } + } + // Reset provider update timestamp to force message refresh + provider.lastUpdated = undefined; + return provider; + }) + ); + + const providerIDs = providers.map(p => p.id); + let invalidProviders = []; + + // Clear old messages for providers that are no longer enabled + for (const prevProvider of previousProviders) { + if (!providerIDs.includes(prevProvider.id)) { + invalidProviders.push(prevProvider.id); + } + } + + return this.setState(prevState => ({ + providers, + // Clear any messages from removed providers + messages: [ + ...prevState.messages.filter(message => + providerIDs.includes(message.provider) + ), + ], + })).then(() => invalidProviders); + } + + get state() { + return this._state; + } + + set state(value) { + throw new Error( + "Do not modify this.state directy. Instead, call this.setState(newState)" + ); + } + + /** + * _resetInitialization - adds the following to the instance: + * .initialized {bool} Has AS Router been initialized? + * .waitForInitialized {Promise} A promise that resolves when initializion is complete + * ._finishInitializing {func} A function that, when called, resolves the .waitForInitialized + * promise and sets .initialized to true. + * @memberof _ASRouter + */ + _resetInitialization() { + this.initialized = false; + this.initializing = false; + this.waitForInitialized = new Promise(resolve => { + this._finishInitializing = () => { + this.initialized = true; + this.initializing = false; + resolve(); + }; + }); + } + + /** + * Check all provided groups are enabled. + * @param groups Set of groups to verify + * @returns bool + */ + hasGroupsEnabled(groups = []) { + return this.state.groups + .filter(({ id }) => groups.includes(id)) + .every(({ enabled }) => enabled); + } + + /** + * Verify that the provider block the message through the `exclude` field + * @param message Message to verify + * @returns bool + */ + isExcludedByProvider(message) { + // preview snippets are never excluded + if (message.provider === "preview") { + return false; + } + const provider = this.state.providers.find(p => p.id === message.provider); + if (!provider) { + return true; + } + if (provider.exclude) { + return provider.exclude.includes(message.id); + } + return false; + } + + /** + * Takes a group and sets the correct `enabled` state based on message config + * and user preferences + * + * @param {GroupConfig} group + * @returns {GroupConfig} + */ + _checkGroupEnabled(group) { + return { + ...group, + enabled: + group.enabled && + // And if defined user preferences are true. If multiple prefs are + // defined then at least one has to be enabled. + (Array.isArray(group.userPreferences) + ? group.userPreferences.some(pref => + lazy.ASRouterPreferences.getUserPreference(pref) + ) + : true), + }; + } + + /** + * Fetch all message groups and update Router.state.groups. + * There are two cases to consider: + * 1. The provider needs to update as determined by the update cycle + * 2. Some pref change occured which could invalidate one of the existing + * groups. + */ + async loadAllMessageGroups() { + const provider = this.state.providers.find( + p => + p.id === "message-groups" && MessageLoaderUtils.shouldProviderUpdate(p) + ); + let remoteMessages = null; + if (provider) { + const { messages } = await MessageLoaderUtils._loadDataForProvider( + provider, + { + storage: this._storage, + dispatchCFRAction: this.dispatchCFRAction, + } + ); + remoteMessages = messages; + } + await this.setState(state => ({ + // If fetching remote messages fails we default to existing state.groups. + groups: (remoteMessages || state.groups).map(this._checkGroupEnabled), + })); + } + + /** + * loadMessagesFromAllProviders - Loads messages from all providers if they require updates. + * Checks the .lastUpdated field on each provider to see if updates are needed + * @param toUpdate An optional list of providers to update. This overrides + * the checks to determine which providers to update. + * @memberof _ASRouter + */ + async loadMessagesFromAllProviders(toUpdate = undefined) { + const needsUpdate = Array.isArray(toUpdate) + ? toUpdate + : this.state.providers.filter(provider => + MessageLoaderUtils.shouldProviderUpdate(provider) + ); + lazy.ASRouterPreferences.console.debug( + "entering loadMessagesFromAllProviders" + ); + + await this.loadAllMessageGroups(); + // Don't do extra work if we don't need any updates + if (needsUpdate.length) { + let newState = { messages: [], providers: [] }; + for (const provider of this.state.providers) { + if (needsUpdate.includes(provider)) { + const { messages, lastUpdated, errors } = + await MessageLoaderUtils.loadMessagesForProvider(provider, { + storage: this._storage, + dispatchCFRAction: this.dispatchCFRAction, + }); + newState.providers.push({ ...provider, lastUpdated, errors }); + newState.messages = [...newState.messages, ...messages]; + } else { + // Skip updating this provider's messages if no update is required + let messages = this.state.messages.filter( + msg => msg.provider === provider.id + ); + newState.providers.push(provider); + newState.messages = [...newState.messages, ...messages]; + } + } + + // Some messages have triggers that require us to initalise trigger listeners + const unseenListeners = new Set(lazy.ASRouterTriggerListeners.keys()); + for (const { trigger } of newState.messages) { + if (trigger && lazy.ASRouterTriggerListeners.has(trigger.id)) { + lazy.ASRouterTriggerListeners.get(trigger.id).init( + this._triggerHandler, + trigger.params, + trigger.patterns + ); + unseenListeners.delete(trigger.id); + } + } + // We don't need these listeners, but they may have previously been + // initialised, so uninitialise them + for (const triggerID of unseenListeners) { + lazy.ASRouterTriggerListeners.get(triggerID).uninit(); + } + + // We don't want to cache preview endpoints, remove them after messages are fetched + await this.setState(this._removePreviewEndpoint(newState)); + await this.cleanupImpressions(); + } + + await this._fireMessagesLoadedTrigger(); + + return this.state; + } + + async _fireMessagesLoadedTrigger() { + const win = Services.wm.getMostRecentBrowserWindow() ?? null; + const browser = win?.gBrowser?.selectedBrowser ?? null; + // pass skipLoadingMessages to avoid infinite recursion. pass browser and + // window into context so messages that may need a window or browser can + // target accordingly. + await this.sendTriggerMessage( + { + id: "messagesLoaded", + browser, + context: { browser, browserWindow: win }, + }, + true + ); + } + + async _maybeUpdateL10nAttachment() { + const { localeInUse } = this.state.localeInUse; + const newLocale = Services.locale.appLocaleAsBCP47; + if (newLocale !== localeInUse) { + const providers = [...this.state.providers]; + let needsUpdate = false; + providers.forEach(provider => { + if (RS_PROVIDERS_WITH_L10N.includes(provider.id)) { + // Force to refresh the messages as well as the attachment. + provider.lastUpdated = undefined; + needsUpdate = true; + } + }); + if (needsUpdate) { + await this.setState({ + localeInUse: newLocale, + providers, + }); + await this.loadMessagesFromAllProviders(); + } + } + return this.state; + } + + async _onLocaleChanged(subject, topic, data) { + await this._maybeUpdateL10nAttachment(); + } + + observe(aSubject, aTopic, aPrefName) { + switch (aPrefName) { + case USE_REMOTE_L10N_PREF: + CFRPageActions.reloadL10n(); + break; + } + } + + toWaitForInitFunc(func) { + return (...args) => this.waitForInitialized.then(() => func(...args)); + } + + /** + * init - Initializes the MessageRouter. + * + * @param {obj} parameters parameters to initialize ASRouter + * @memberof _ASRouter + */ + async init({ + storage, + sendTelemetry, + clearChildMessages, + clearChildProviders, + updateAdminState, + dispatchCFRAction, + }) { + if (this.initializing || this.initialized) { + return null; + } + this.initializing = true; + this._storage = storage; + this.ALLOWLIST_HOSTS = this._loadSnippetsAllowHosts(); + this.clearChildMessages = this.toWaitForInitFunc(clearChildMessages); + this.clearChildProviders = this.toWaitForInitFunc(clearChildProviders); + // NOTE: This is only necessary to sync devtools and snippets when devtools is active. + this.updateAdminState = this.toWaitForInitFunc(updateAdminState); + this.sendTelemetry = sendTelemetry; + this.dispatchCFRAction = this.toWaitForInitFunc(dispatchCFRAction); + + lazy.ASRouterPreferences.init(); + lazy.ASRouterPreferences.addListener(this.onPrefChange); + lazy.ToolbarBadgeHub.init(this.waitForInitialized, { + handleMessageRequest: this.handleMessageRequest, + addImpression: this.addImpression, + blockMessageById: this.blockMessageById, + unblockMessageById: this.unblockMessageById, + sendTelemetry: this.sendTelemetry, + }); + lazy.ToolbarPanelHub.init(this.waitForInitialized, { + getMessages: this.handleMessageRequest, + sendTelemetry: this.sendTelemetry, + }); + lazy.MomentsPageHub.init(this.waitForInitialized, { + handleMessageRequest: this.handleMessageRequest, + addImpression: this.addImpression, + blockMessageById: this.blockMessageById, + sendTelemetry: this.sendTelemetry, + }); + + this._loadLocalProviders(); + + const messageBlockList = + (await this._storage.get("messageBlockList")) || []; + const messageImpressions = + (await this._storage.get("messageImpressions")) || {}; + const groupImpressions = + (await this._storage.get("groupImpressions")) || {}; + const screenImpressions = + (await this._storage.get("screenImpressions")) || {}; + const previousSessionEnd = + (await this._storage.get("previousSessionEnd")) || 0; + + await this.setState({ + messageBlockList, + groupImpressions, + messageImpressions, + screenImpressions, + previousSessionEnd, + ...(lazy.ASRouterPreferences.specialConditions || {}), + initialized: false, + }); + await this._updateMessageProviders(); + await this.loadMessagesFromAllProviders(); + await MessageLoaderUtils.cleanupCache(this.state.providers, storage); + + lazy.SpecialMessageActions.blockMessageById = this.blockMessageById; + Services.obs.addObserver(this._onLocaleChanged, TOPIC_INTL_LOCALE_CHANGED); + Services.obs.addObserver( + this._onExperimentEnrollmentsUpdated, + TOPIC_EXPERIMENT_ENROLLMENT_CHANGED + ); + Services.prefs.addObserver(USE_REMOTE_L10N_PREF, this); + // sets .initialized to true and resolves .waitForInitialized promise + this._finishInitializing(); + return this.state; + } + + uninit() { + this._storage.set("previousSessionEnd", Date.now()); + + this.clearChildMessages = null; + this.clearChildProviders = null; + this.updateAdminState = null; + this.sendTelemetry = null; + this.dispatchCFRAction = null; + + lazy.ASRouterPreferences.removeListener(this.onPrefChange); + lazy.ASRouterPreferences.uninit(); + lazy.ToolbarPanelHub.uninit(); + lazy.ToolbarBadgeHub.uninit(); + lazy.MomentsPageHub.uninit(); + + // Uninitialise all trigger listeners + for (const listener of lazy.ASRouterTriggerListeners.values()) { + listener.uninit(); + } + Services.obs.removeObserver( + this._onLocaleChanged, + TOPIC_INTL_LOCALE_CHANGED + ); + Services.obs.removeObserver( + this._onExperimentEnrollmentsUpdated, + TOPIC_EXPERIMENT_ENROLLMENT_CHANGED + ); + Services.prefs.removeObserver(USE_REMOTE_L10N_PREF, this); + // If we added any CFR recommendations, they need to be removed + CFRPageActions.clearRecommendations(); + this._resetInitialization(); + } + + setState(callbackOrObj) { + lazy.ASRouterPreferences.console.debug( + "in setState, callbackOrObj = ", + callbackOrObj + ); + lazy.ASRouterPreferences.console.trace(); + const newState = + typeof callbackOrObj === "function" + ? callbackOrObj(this.state) + : callbackOrObj; + this._state = { + ...this.state, + ...newState, + }; + if (lazy.ASRouterPreferences.devtoolsEnabled) { + return this.updateTargetingParameters().then(state => { + this.updateAdminState(state); + return state; + }); + } + return Promise.resolve(this.state); + } + + updateTargetingParameters() { + return this.getTargetingParameters( + lazy.ASRouterTargeting.Environment, + this._getMessagesContext() + ).then(targetingParameters => ({ + ...this.state, + providerPrefs: lazy.ASRouterPreferences.providers, + userPrefs: lazy.ASRouterPreferences.getAllUserPreferences(), + targetingParameters, + errors: this.errors, + })); + } + + getMessageById(id) { + return this.state.messages.find(message => message.id === id); + } + + _loadLocalProviders() { + // If we're in ASR debug mode add the local test providers + if (lazy.ASRouterPreferences.devtoolsEnabled) { + this._localProviders = { + ...this._localProviders, + SnippetsTestMessageProvider: lazy.SnippetsTestMessageProvider, + PanelTestProvider: lazy.PanelTestProvider, + }; + } + } + + /** + * Used by ASRouter Admin returns all ASRouterTargeting.Environment + * and ASRouter._getMessagesContext parameters and values + */ + async getTargetingParameters(environment, localContext) { + // Resolve objects that may contain promises. + async function resolve(object) { + if (typeof object === "object" && object !== null) { + if (Array.isArray(object)) { + return Promise.all(object.map(async item => resolve(await item))); + } + + if (object instanceof Date) { + return object; + } + + const target = {}; + const promises = Object.entries(object).map(async ([key, value]) => [ + key, + await resolve(await value), + ]); + for (const [key, value] of await Promise.all(promises)) { + target[key] = value; + } + return target; + } + + return object; + } + + const targetingParameters = { + ...(await resolve(environment)), + ...(await resolve(localContext)), + }; + + return targetingParameters; + } + + _handleTargetingError(error, message) { + console.error(error); + this.dispatchCFRAction( + ac.ASRouterUserEvent({ + message_id: message.id, + action: "asrouter_undesired_event", + event: "TARGETING_EXPRESSION_ERROR", + event_context: {}, + }) + ); + } + + // Return an object containing targeting parameters used to select messages + _getMessagesContext() { + const { messageImpressions, previousSessionEnd, screenImpressions } = + this.state; + + return { + get messageImpressions() { + return messageImpressions; + }, + get previousSessionEnd() { + return previousSessionEnd; + }, + get screenImpressions() { + return screenImpressions; + }, + }; + } + + async evaluateExpression({ expression, context }) { + const targetingContext = new lazy.TargetingContext(context); + let evaluationStatus; + try { + evaluationStatus = { + result: await targetingContext.evalWithDefault(expression), + success: true, + }; + } catch (e) { + evaluationStatus = { result: e.message, success: false }; + } + return Promise.resolve({ evaluationStatus }); + } + + unblockAll() { + return this.setState({ messageBlockList: [] }); + } + + isUnblockedMessage(message) { + const { state } = this; + return ( + !state.messageBlockList.includes(message.id) && + (!message.campaign || + !state.messageBlockList.includes(message.campaign)) && + this.hasGroupsEnabled(message.groups) && + !this.isExcludedByProvider(message) + ); + } + + // Work out if a message can be shown based on its and its provider's frequency caps. + isBelowFrequencyCaps(message) { + const { messageImpressions, groupImpressions } = this.state; + const impressionsForMessage = messageImpressions[message.id]; + + const _belowItemFrequencyCap = this._isBelowItemFrequencyCap( + message, + impressionsForMessage, + MAX_MESSAGE_LIFETIME_CAP + ); + if (!_belowItemFrequencyCap) { + lazy.ASRouterPreferences.console.debug( + `isBelowFrequencyCaps: capped by item: `, + message, + "impressions =", + impressionsForMessage + ); + } + + const _belowGroupFrequencyCaps = message.groups.every(messageGroup => { + const belowThisGroupCap = this._isBelowItemFrequencyCap( + this.state.groups.find(({ id }) => id === messageGroup), + groupImpressions[messageGroup] + ); + + if (!belowThisGroupCap) { + lazy.ASRouterPreferences.console.debug( + `isBelowFrequencyCaps: ${message.id} capped by group ${messageGroup}` + ); + } else { + lazy.ASRouterPreferences.console.debug( + `isBelowFrequencyCaps: ${message.id} allowed by group ${messageGroup}, groupImpressions = `, + groupImpressions + ); + } + + return belowThisGroupCap; + }); + + return _belowItemFrequencyCap && _belowGroupFrequencyCaps; + } + + // Helper for isBelowFrecencyCaps - work out if the frequency cap for the given + // item has been exceeded or not + _isBelowItemFrequencyCap(item, impressions, maxLifetimeCap = Infinity) { + if (item && item.frequency && impressions && impressions.length) { + if ( + item.frequency.lifetime && + impressions.length >= Math.min(item.frequency.lifetime, maxLifetimeCap) + ) { + lazy.ASRouterPreferences.console.debug( + `${item.id} capped by lifetime (${item.frequency.lifetime})` + ); + + return false; + } + if (item.frequency.custom) { + const now = Date.now(); + for (const setting of item.frequency.custom) { + let { period } = setting; + const impressionsInPeriod = impressions.filter(t => now - t < period); + if (impressionsInPeriod.length >= setting.cap) { + lazy.ASRouterPreferences.console.debug( + `${item.id} capped by impressions (${impressionsInPeriod.length}) in period (${period}) >= ${setting.cap}` + ); + return false; + } + } + } + } + return true; + } + + async _extraTemplateStrings(originalMessage) { + let extraTemplateStrings; + let localProvider = this._findProvider(originalMessage.provider); + if (localProvider && localProvider.getExtraAttributes) { + extraTemplateStrings = await localProvider.getExtraAttributes(); + } + + return extraTemplateStrings; + } + + _findProvider(providerID) { + return this._localProviders[ + this.state.providers.find(i => i.id === providerID).localProvider + ]; + } + + routeCFRMessage(message, browser, trigger, force = false) { + if (!message) { + return { message: {} }; + } + + switch (message.template) { + case "whatsnew_panel_message": + if (force) { + lazy.ToolbarPanelHub.forceShowMessage(browser, message); + } + break; + case "cfr_doorhanger": + case "milestone_message": + if (force) { + CFRPageActions.forceRecommendation( + browser, + message, + this.dispatchCFRAction + ); + } else { + CFRPageActions.addRecommendation( + browser, + trigger.param && trigger.param.host, + message, + this.dispatchCFRAction + ); + } + break; + case "cfr_urlbar_chiclet": + if (force) { + CFRPageActions.forceRecommendation( + browser, + message, + this.dispatchCFRAction + ); + } else { + CFRPageActions.addRecommendation( + browser, + null, + message, + this.dispatchCFRAction + ); + } + break; + case "toolbar_badge": + lazy.ToolbarBadgeHub.registerBadgeNotificationListener(message, { + force, + }); + break; + case "update_action": + lazy.MomentsPageHub.executeAction(message); + break; + case "infobar": + lazy.InfoBar.showInfoBarMessage( + browser, + message, + this.dispatchCFRAction + ); + break; + case "spotlight": + lazy.Spotlight.showSpotlightDialog( + browser, + message, + this.dispatchCFRAction + ); + break; + case "toast_notification": + lazy.ToastNotification.showToastNotification( + message, + this.dispatchCFRAction + ); + break; + } + + return { message }; + } + + addScreenImpression(screen) { + lazy.ASRouterPreferences.console.debug( + `entering addScreenImpression for ${screen.id}` + ); + + const time = Date.now(); + + let screenImpressions = { ...this.state.screenImpressions }; + screenImpressions[screen.id] = time; + + this.setState({ screenImpressions }); + lazy.ASRouterPreferences.console.debug( + screen.id, + `screen impression added, screenImpressions[screen.id]: `, + screenImpressions[screen.id] + ); + this._storage.set("screenImpressions", screenImpressions); + } + + addImpression(message) { + lazy.ASRouterPreferences.console.debug( + `entering addImpression for ${message.id}` + ); + + const groupsWithFrequency = this.state.groups.filter( + ({ frequency, id }) => frequency && message.groups.includes(id) + ); + // We only need to store impressions for messages that have frequency, or + // that have providers that have frequency + if (message.frequency || groupsWithFrequency.length) { + const time = Date.now(); + return this.setState(state => { + const messageImpressions = this._addImpressionForItem( + state.messageImpressions, + message, + "messageImpressions", + time + ); + // Initialize this with state.groupImpressions, and then assign the + // newly-updated copy to it during each iteration so that + // all the changes get captured and either returned or passed into the + // _addImpressionsForItem call on the next iteration. + let { groupImpressions } = state; + for (const group of groupsWithFrequency) { + groupImpressions = this._addImpressionForItem( + groupImpressions, + group, + "groupImpressions", + time + ); + } + + return { messageImpressions, groupImpressions }; + }); + } + return Promise.resolve(); + } + + // Helper for addImpression - calculate the updated impressions object for the given + // item, then store it and return it + _addImpressionForItem(currentImpressions, item, impressionsString, time) { + // The destructuring here is to avoid mutating passed parameters + // (see https://redux.js.org/recipes/structuring-reducers/prerequisite-concepts#immutable-data-management) + const impressions = { ...currentImpressions }; + if (item.frequency) { + impressions[item.id] = [...(impressions[item.id] ?? []), time]; + + lazy.ASRouterPreferences.console.debug( + item.id, + "impression added, impressions[item.id]: ", + impressions[item.id] + ); + + this._storage.set(impressionsString, impressions); + } + return impressions; + } + + /** + * getLongestPeriod + * + * @param {obj} item Either an ASRouter message or an ASRouter provider + * @returns {int|null} if the item has custom frequency caps, the longest period found in the list of caps. + if the item has no custom frequency caps, null + * @memberof _ASRouter + */ + getLongestPeriod(item) { + if (!item.frequency || !item.frequency.custom) { + return null; + } + return item.frequency.custom.sort((a, b) => b.period - a.period)[0].period; + } + + /** + * cleanupImpressions - this function cleans up obsolete impressions whenever + * messages are refreshed or fetched. It will likely need to be more sophisticated in the future, + * but the current behaviour for when both message impressions and provider impressions are + * cleared is as follows (where `item` is either `message` or `provider`): + * + * 1. If the item id for a list of item impressions no longer exists in the ASRouter state, it + * will be cleared. + * 2. If the item has time-bound frequency caps but no lifetime cap, any item impressions older + * than the longest time period will be cleared. + */ + cleanupImpressions() { + return this.setState(state => { + const messageImpressions = this._cleanupImpressionsForItems( + state, + state.messages, + "messageImpressions" + ); + const groupImpressions = this._cleanupImpressionsForItems( + state, + state.groups, + "groupImpressions" + ); + return { messageImpressions, groupImpressions }; + }); + } + + /** _cleanupImpressionsForItems - Helper for cleanupImpressions - calculate the updated + /* impressions object for the given items, then store it and return it + * + * @param {obj} state Reference to ASRouter internal state + * @param {array} items Can be messages, providers or groups that we count impressions for + * @param {string} impressionsString Key name for entry in state where impressions are stored + */ + _cleanupImpressionsForItems(state, items, impressionsString) { + const impressions = { ...state[impressionsString] }; + let needsUpdate = false; + Object.keys(impressions).forEach(id => { + const [item] = items.filter(x => x.id === id); + // Don't keep impressions for items that no longer exist + if (!item || !item.frequency || !Array.isArray(impressions[id])) { + lazy.ASRouterPreferences.console.debug( + "_cleanupImpressionsForItem: removing impressions for deleted or changed item: ", + item + ); + lazy.ASRouterPreferences.console.trace(); + delete impressions[id]; + needsUpdate = true; + return; + } + if (!impressions[id].length) { + return; + } + // If we don't want to store impressions older than the longest period + if (item.frequency.custom && !item.frequency.lifetime) { + lazy.ASRouterPreferences.console.debug( + "_cleanupImpressionsForItem: removing impressions older than longest period for item: ", + item + ); + const now = Date.now(); + impressions[id] = impressions[id].filter( + t => now - t < this.getLongestPeriod(item) + ); + needsUpdate = true; + } + }); + if (needsUpdate) { + this._storage.set(impressionsString, impressions); + } + return impressions; + } + + handleMessageRequest({ + messages: candidates, + triggerId, + triggerParam, + triggerContext, + template, + provider, + ordered = false, + returnAll = false, + }) { + let shouldCache; + lazy.ASRouterPreferences.console.debug( + "in handleMessageRequest, arguments = ", + Array.from(arguments) // eslint-disable-line prefer-rest-params + ); + lazy.ASRouterPreferences.console.trace(); + const messages = + candidates || + this.state.messages.filter(m => { + if (provider && m.provider !== provider) { + lazy.ASRouterPreferences.console.debug(m.id, " filtered by provider"); + return false; + } + if (template && m.template !== template) { + lazy.ASRouterPreferences.console.debug(m.id, " filtered by template"); + return false; + } + if (triggerId && !m.trigger) { + lazy.ASRouterPreferences.console.debug(m.id, " filtered by trigger"); + return false; + } + if (triggerId && m.trigger.id !== triggerId) { + lazy.ASRouterPreferences.console.debug( + m.id, + " filtered by triggerId" + ); + return false; + } + if (!this.isUnblockedMessage(m)) { + lazy.ASRouterPreferences.console.debug( + m.id, + " filtered because blocked" + ); + return false; + } + if (!this.isBelowFrequencyCaps(m)) { + lazy.ASRouterPreferences.console.debug( + m.id, + " filtered because capped" + ); + return false; + } + + if (shouldCache !== false) { + shouldCache = JEXL_PROVIDER_CACHE.has(m.provider); + } + + return true; + }); + + if (!messages.length) { + return returnAll ? messages : null; + } + + const context = this._getMessagesContext(); + + // Find a message that matches the targeting context as well as the trigger context (if one is provided) + // If no trigger is provided, we should find a message WITHOUT a trigger property defined. + return lazy.ASRouterTargeting.findMatchingMessage({ + messages, + trigger: triggerId && { + id: triggerId, + param: triggerParam, + context: triggerContext, + }, + context, + onError: this._handleTargetingError, + ordered, + shouldCache, + returnAll, + }); + } + + setMessageById({ id, ...data }, force, browser) { + return this.routeCFRMessage(this.getMessageById(id), browser, data, force); + } + + blockMessageById(idOrIds) { + lazy.ASRouterPreferences.console.debug( + "blockMessageById called, idOrIds = ", + idOrIds + ); + lazy.ASRouterPreferences.console.trace(); + + const idsToBlock = Array.isArray(idOrIds) ? idOrIds : [idOrIds]; + + return this.setState(state => { + const messageBlockList = [...state.messageBlockList]; + const messageImpressions = { ...state.messageImpressions }; + + idsToBlock.forEach(id => { + const message = state.messages.find(m => m.id === id); + const idToBlock = message && message.campaign ? message.campaign : id; + if (!messageBlockList.includes(idToBlock)) { + messageBlockList.push(idToBlock); + } + + // When a message is blocked, its impressions should be cleared as well + delete messageImpressions[id]; + }); + + this._storage.set("messageBlockList", messageBlockList); + this._storage.set("messageImpressions", messageImpressions); + return { messageBlockList, messageImpressions }; + }); + } + + unblockMessageById(idOrIds) { + const idsToUnblock = Array.isArray(idOrIds) ? idOrIds : [idOrIds]; + + return this.setState(state => { + const messageBlockList = [...state.messageBlockList]; + idsToUnblock + .map(id => state.messages.find(m => m.id === id)) + // Remove all `id`s (or `campaign`s for snippets) from the message + // block list + .forEach(message => { + const idToUnblock = + message && message.campaign ? message.campaign : message.id; + messageBlockList.splice(messageBlockList.indexOf(idToUnblock), 1); + }); + + this._storage.set("messageBlockList", messageBlockList); + return { messageBlockList }; + }); + } + + resetGroupsState() { + const newGroupImpressions = {}; + for (let { id } of this.state.groups) { + newGroupImpressions[id] = []; + } + // Update storage + this._storage.set("groupImpressions", newGroupImpressions); + return this.setState(({ groups }) => ({ + groupImpressions: newGroupImpressions, + })); + } + + resetMessageState() { + const newMessageImpressions = {}; + for (let { id } of this.state.messages) { + newMessageImpressions[id] = []; + } + // Update storage + this._storage.set("messageImpressions", newMessageImpressions); + return this.setState(() => ({ + messageImpressions: newMessageImpressions, + })); + } + + _validPreviewEndpoint(url) { + try { + const endpoint = new URL(url); + if (!this.ALLOWLIST_HOSTS[endpoint.host]) { + console.error( + `The preview URL host ${endpoint.host} is not in the list of allowed hosts.` + ); + } + if (endpoint.protocol !== "https:") { + console.error("The URL protocol is not https."); + } + return ( + endpoint.protocol === "https:" && this.ALLOWLIST_HOSTS[endpoint.host] + ); + } catch (e) { + return false; + } + } + + _loadSnippetsAllowHosts() { + let additionalHosts = []; + const allowPrefValue = Services.prefs.getStringPref( + SNIPPETS_ENDPOINT_ALLOWLIST, + "" + ); + try { + additionalHosts = JSON.parse(allowPrefValue); + } catch (e) { + if (allowPrefValue) { + console.error( + `Pref ${SNIPPETS_ENDPOINT_ALLOWLIST} value is not valid JSON` + ); + } + } + + if (!additionalHosts.length) { + return DEFAULT_ALLOWLIST_HOSTS; + } + + // If there are additional hosts we want to allow, add them as + // `preview` so that the updateCycle is 0 + return additionalHosts.reduce( + (allow_hosts, host) => { + allow_hosts[host] = "preview"; + Services.console.logStringMessage( + `Adding ${host} to list of allowed hosts.` + ); + return allow_hosts; + }, + { ...DEFAULT_ALLOWLIST_HOSTS } + ); + } + + // To be passed to ASRouterTriggerListeners + _triggerHandler(browser, trigger) { + // Disable ASRouterTriggerListeners in kiosk mode. + if (lazy.BrowserHandler.kiosk) { + return Promise.resolve(); + } + return this.sendTriggerMessage({ ...trigger, browser }); + } + + _removePreviewEndpoint(state) { + state.providers = state.providers.filter(p => p.id !== "preview"); + return state; + } + + addPreviewEndpoint(url, browser) { + const providers = [...this.state.providers]; + if ( + this._validPreviewEndpoint(url) && + !providers.find(p => p.url === url) + ) { + // When you view a preview snippet we want to hide all real content - + // sending EnterSnippetsPreviewMode puts this browser tab in that state. + browser.sendMessageToActor("EnterSnippetsPreviewMode", {}, "ASRouter"); + providers.push({ + id: "preview", + type: "remote", + enabled: true, + url, + updateCycleInMs: 0, + }); + return this.setState({ providers }); + } + return Promise.resolve(); + } + + /** + * forceAttribution - this function should only be called from within about:newtab#asrouter. + * It forces the browser attribution to be set to something specified in asrouter admin + * tools, and reloads the providers in order to get messages that are dependant on this + * attribution data (see Return to AMO flow in bug 1475354 for example). Note - OSX and Windows only + * @param {data} Object an object containing the attribtion data that came from asrouter admin page + */ + async forceAttribution(data) { + // Extract the parameters from data that will make up the referrer url + const attributionData = AttributionCode.allowedCodeKeys + .map(key => `${key}=${encodeURIComponent(data[key] || "")}`) + .join("&"); + if (AppConstants.platform === "win") { + // The whole attribution data is encoded (again) for windows + await AttributionCode.writeAttributionFile( + encodeURIComponent(attributionData) + ); + } else if (AppConstants.platform === "macosx") { + let appPath = lazy.MacAttribution.applicationPath; + let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService( + Ci.nsIMacAttributionService + ); + + // The attribution data is treated as a url query for mac + let referrer = `https://www.mozilla.org/anything/?${attributionData}`; + + // This sets the Attribution to be the referrer + attributionSvc.setReferrerUrl(appPath, referrer, true); + + // Delete attribution data file + await AttributionCode.deleteFileAsync(); + } + + // Clear cache call is only possible in a testing environment + Services.env.set("XPCSHELL_TEST_PROFILE_DIR", "testing"); + + // Clear and refresh Attribution, and then fetch the messages again to update + AttributionCode._clearCache(); + await AttributionCode.getAttrDataAsync(); + await this._updateMessageProviders(); + return this.loadMessagesFromAllProviders(); + } + + async sendPBNewTabMessage({ tabId, hideDefault }) { + let message = null; + const PromoInfo = { + FOCUS: { enabledPref: "browser.promo.focus.enabled" }, + VPN: { enabledPref: "browser.vpn_promo.enabled" }, + PIN: { enabledPref: "browser.promo.pin.enabled" }, + COOKIE_BANNERS: { enabledPref: "browser.promo.cookiebanners.enabled" }, + }; + await this.loadMessagesFromAllProviders(); + + // If message has hideDefault property set to true + // remove from state all pb_newtab messages with type default + if (hideDefault) { + await this.setState(state => ({ + messages: state.messages.filter( + m => !(m.template === "pb_newtab" && m.type === "default") + ), + })); + } + + // Remove from state pb_newtab messages with PromoType disabled + await this.setState(state => ({ + messages: state.messages.filter( + m => + !( + m.template === "pb_newtab" && + !Services.prefs.getBoolPref( + PromoInfo[m.content?.promoType]?.enabledPref, + true + ) + ) + ), + })); + + const telemetryObject = { tabId }; + TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + message = await this.handleMessageRequest({ + template: "pb_newtab", + }); + TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + + // Format urls if any are defined + ["infoLinkUrl"].forEach(key => { + if (message?.content?.[key]) { + message.content[key] = Services.urlFormatter.formatURL( + message.content[key] + ); + } + }); + + return { message }; + } + + async sendNewTabMessage({ endpoint, tabId, browser }) { + let message; + + // Load preview endpoint for snippets if one is sent + if (endpoint) { + await this.addPreviewEndpoint(endpoint.url, browser); + } + + // Load all messages + await this.loadMessagesFromAllProviders(); + + if (endpoint) { + message = await this.handleMessageRequest({ provider: "preview" }); + + // We don't want to cache preview messages, remove them after we selected the message to show + if (message) { + await this.setState(state => ({ + messages: state.messages.filter(m => m.id !== message.id), + })); + } + } else { + const telemetryObject = { tabId }; + TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + message = await this.handleMessageRequest({ provider: "snippets" }); + TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + } + + return this.routeCFRMessage(message, browser, undefined, false); + } + + _recordReachEvent(message) { + const messageGroup = message.forReachEvent.group; + // Events telemetry only accepts understores for the event `object` + const underscored = messageGroup.split("-").join("_"); + const extra = { branches: message.branchSlug }; + Services.telemetry.recordEvent( + REACH_EVENT_CATEGORY, + REACH_EVENT_METHOD, + underscored, + message.experimentSlug, + extra + ); + } + + /** + * Fire a trigger, look for a matching message, and route it to the + * appropriate message handler/messaging surface. + * @param {object} trigger + * @param {string} trigger.id the name of the trigger, e.g. "openURL" + * @param {object} [trigger.param] an object with host, url, type, etc. keys + * whose values are used to match against the message's trigger params + * @param {object} [trigger.context] an object with data about the source of + * the trigger, matched against the message's targeting expression + * @param {MozBrowser} trigger.browser the browser to route messages to + * @param {number} [trigger.tabId] identifier used only for exposure testing + * @param {boolean} [skipLoadingMessages=false] pass true to skip looking for + * new messages. use when calling from loadMessagesFromAllProviders to avoid + * recursion. we call this from loadMessagesFromAllProviders in order to + * fire the messagesLoaded trigger. + * @returns {Promise<object>} + * @resolves {message} an object with the routed message + */ + async sendTriggerMessage( + { tabId, browser, ...trigger }, + skipLoadingMessages = false + ) { + if (!skipLoadingMessages) { + await this.loadMessagesFromAllProviders(); + } + const telemetryObject = { tabId }; + TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + // Return all the messages so that it can record the Reach event + const messages = + (await this.handleMessageRequest({ + triggerId: trigger.id, + triggerParam: trigger.param, + triggerContext: trigger.context, + returnAll: true, + })) || []; + TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + + // Record the Reach event for all the messages with `forReachEvent`, + // only send the first message without forReachEvent to the target + const nonReachMessages = []; + for (const message of messages) { + if (message.forReachEvent) { + if (!message.forReachEvent.sent) { + this._recordReachEvent(message); + message.forReachEvent.sent = true; + } + } else { + nonReachMessages.push(message); + } + } + + if (nonReachMessages.length) { + let featureId = nonReachMessages[0]._nimbusFeature; + if (featureId) { + lazy.NimbusFeatures[featureId].recordExposureEvent({ once: true }); + } + } + + return this.routeCFRMessage( + nonReachMessages[0] || null, + browser, + trigger, + false + ); + } + + async forceWNPanel(browser) { + let win = browser.ownerGlobal; + await lazy.ToolbarPanelHub.enableToolbarButton(); + + win.PanelUI.showSubView( + "PanelUI-whatsNew", + win.document.getElementById("whats-new-menu-button") + ); + + let panel = win.document.getElementById("customizationui-widget-panel"); + // Set the attribute to keep the panel open + panel.setAttribute("noautohide", true); + } + + async closeWNPanel(browser) { + let win = browser.ownerGlobal; + let panel = win.document.getElementById("customizationui-widget-panel"); + // Set the attribute to allow the panel to close + panel.setAttribute("noautohide", false); + // Removing the button is enough to close the panel. + await lazy.ToolbarPanelHub._hideToolbarButton(win); + } + + async _onExperimentEnrollmentsUpdated() { + const experimentProvider = this.state.providers.find( + p => p.id === "messaging-experiments" + ); + if (!experimentProvider?.enabled) { + return; + } + await this.loadMessagesFromAllProviders([experimentProvider]); + } + + async forcePBWindow(browser, msg) { + const privateBrowserOpener = await new Promise( + ( + resolveOnContentBrowserCreated // wrap this in a promise to give back the right browser + ) => + browser.ownerGlobal.openTrustedLinkIn( + "about:privatebrowsing?debug", + "window", + { + private: true, + triggeringPrincipal: + Services.scriptSecurityManager.getSystemPrincipal({}), + csp: null, + resolveOnContentBrowserCreated, + opener: "devtools", + } + ) + ); + + lazy.setTimeout(() => { + // setTimeout is necessary to make sure the private browsing window has a chance to open before the message is sent + privateBrowserOpener.browsingContext.currentWindowGlobal + .getActor("AboutPrivateBrowsing") + .sendAsyncMessage("ShowDevToolsMessage", msg); + }, 100); + + return privateBrowserOpener; + } +} + +/** + * ASRouter - singleton instance of _ASRouter that controls all messages + * in the new tab page. + */ +const ASRouter = new _ASRouter(); + +const EXPORTED_SYMBOLS = ["_ASRouter", "ASRouter", "MessageLoaderUtils"]; diff --git a/browser/components/newtab/lib/ASRouterDefaultConfig.jsm b/browser/components/newtab/lib/ASRouterDefaultConfig.jsm new file mode 100644 index 0000000000..84ccad5200 --- /dev/null +++ b/browser/components/newtab/lib/ASRouterDefaultConfig.jsm @@ -0,0 +1,65 @@ +/* vim: set ts=2 sw=2 sts=2 et tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["ASRouterDefaultConfig"]; + +const { ASRouter } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouter.jsm" +); +const { TelemetryFeed } = ChromeUtils.import( + "resource://activity-stream/lib/TelemetryFeed.jsm" +); +const { ASRouterParentProcessMessageHandler } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouterParentProcessMessageHandler.jsm" +); +const { SpecialMessageActions } = ChromeUtils.importESModule( + "resource://messaging-system/lib/SpecialMessageActions.sys.mjs" +); +const { ASRouterPreferences } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouterPreferences.jsm" +); +const { QueryCache } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouterTargeting.jsm" +); +const { ActivityStreamStorage } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamStorage.jsm" +); + +const createStorage = async telemetryFeed => { + const dbStore = new ActivityStreamStorage({ + storeNames: ["sectionPrefs", "snippets"], + telemetry: { + handleUndesiredEvent: e => telemetryFeed.SendASRouterUndesiredEvent(e), + }, + }); + // Accessing the db causes the object stores to be created / migrated. + // This needs to happen before other instances try to access the db, which + // would update only a subset of the stores to the latest version. + try { + await dbStore.db; // eslint-disable-line no-unused-expressions + } catch (e) { + return Promise.reject(e); + } + return dbStore.getDbTable("snippets"); +}; + +const ASRouterDefaultConfig = () => { + const router = ASRouter; + const telemetry = new TelemetryFeed(); + const messageHandler = new ASRouterParentProcessMessageHandler({ + router, + preferences: ASRouterPreferences, + specialMessageActions: SpecialMessageActions, + queryCache: QueryCache, + sendTelemetry: telemetry.onAction.bind(telemetry), + }); + return { + router, + messageHandler, + createStorage: createStorage.bind(null, telemetry), + }; +}; diff --git a/browser/components/newtab/lib/ASRouterNewTabHook.sys.mjs b/browser/components/newtab/lib/ASRouterNewTabHook.sys.mjs new file mode 100644 index 0000000000..d0fdbfdae4 --- /dev/null +++ b/browser/components/newtab/lib/ASRouterNewTabHook.sys.mjs @@ -0,0 +1,117 @@ +/* vim: set ts=2 sw=2 sts=2 et tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +class ASRouterNewTabHookInstance { + constructor() { + this._newTabMessageHandler = null; + this._parentProcessMessageHandler = null; + this._router = null; + this._clearChildMessages = (...params) => + this._newTabMessageHandler === null + ? Promise.resolve() + : this._newTabMessageHandler.clearChildMessages(...params); + this._clearChildProviders = (...params) => + this._newTabMessageHandler === null + ? Promise.resolve() + : this._newTabMessageHandler.clearChildProviders(...params); + this._updateAdminState = (...params) => + this._newTabMessageHandler === null + ? Promise.resolve() + : this._newTabMessageHandler.updateAdminState(...params); + } + + /** + * Params: + * object - { + * messageHandler: message handler for parent process messages + * { + * handleCFRAction: Responds to CFR action and returns a Promise + * handleTelemetry: Logs telemetry events and returns nothing + * }, + * router: ASRouter instance + * createStorage: function to create DB storage for ASRouter + * } + */ + async initialize({ messageHandler, router, createStorage }) { + this._parentProcessMessageHandler = messageHandler; + this._router = router; + if (!this._router.initialized) { + const storage = await createStorage(); + await this._router.init({ + storage, + sendTelemetry: this._parentProcessMessageHandler.handleTelemetry, + dispatchCFRAction: this._parentProcessMessageHandler.handleCFRAction, + clearChildMessages: this._clearChildMessages, + clearChildProviders: this._clearChildProviders, + updateAdminState: this._updateAdminState, + }); + } + } + + destroy() { + if (this._router?.initialized) { + this.disconnect(); + this._router.uninit(); + } + } + + /** + * Connects new tab message handler to hook. + * Note: Should only ever be called on an initialized instance + * Params: + * newTabMessageHandler - { + * clearChildMessages: clears child messages and returns Promise + * clearChildProviders: clears child providers and returns Promise. + * updateAdminState: updates admin state and returns Promise + * } + * Returns: parentProcessMessageHandler + */ + connect(newTabMessageHandler) { + this._newTabMessageHandler = newTabMessageHandler; + return this._parentProcessMessageHandler; + } + + /** + * Disconnects new tab message handler from hook. + */ + disconnect() { + this._newTabMessageHandler = null; + } +} + +class AwaitSingleton { + constructor() { + this.instance = null; + const initialized = new Promise(resolve => { + this.setInstance = instance => { + this.setInstance = () => {}; + this.instance = instance; + resolve(instance); + }; + }); + this.getInstance = () => initialized; + } +} + +export const ASRouterNewTabHook = (() => { + const singleton = new AwaitSingleton(); + const instance = new ASRouterNewTabHookInstance(); + return { + getInstance: singleton.getInstance, + + /** + * Param: + * params - see ASRouterNewTabHookInstance.init + */ + createInstance: async params => { + await instance.initialize(params); + singleton.setInstance(instance); + }, + + destroy: () => { + instance.destroy(); + }, + }; +})(); diff --git a/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm b/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm new file mode 100644 index 0000000000..dca79b29d0 --- /dev/null +++ b/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm @@ -0,0 +1,179 @@ +/* vim: set ts=2 sw=2 sts=2 et tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["ASRouterParentProcessMessageHandler"]; + +const { ASRouterPreferences } = ChromeUtils.import( + "resource://activity-stream/lib/ASRouterPreferences.jsm" +); + +const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule( + "resource://activity-stream/common/ActorConstants.sys.mjs" +); + +class ASRouterParentProcessMessageHandler { + constructor({ + router, + preferences, + specialMessageActions, + queryCache, + sendTelemetry, + }) { + this._router = router; + this._preferences = preferences; + this._specialMessageActions = specialMessageActions; + this._queryCache = queryCache; + this.handleTelemetry = sendTelemetry; + this.handleMessage = this.handleMessage.bind(this); + this.handleCFRAction = this.handleCFRAction.bind(this); + } + + handleCFRAction({ type, data }, browser) { + switch (type) { + case msg.INFOBAR_TELEMETRY: + case msg.TOOLBAR_BADGE_TELEMETRY: + case msg.TOOLBAR_PANEL_TELEMETRY: + case msg.MOMENTS_PAGE_TELEMETRY: + case msg.DOORHANGER_TELEMETRY: + case msg.SPOTLIGHT_TELEMETRY: + case msg.TOAST_NOTIFICATION_TELEMETRY: { + return this.handleTelemetry({ type, data }); + } + default: { + return this.handleMessage(type, data, { browser }); + } + } + } + + handleMessage(name, data, { id: tabId, browser } = { browser: null }) { + switch (name) { + case msg.AS_ROUTER_TELEMETRY_USER_EVENT: + return this.handleTelemetry({ + type: msg.AS_ROUTER_TELEMETRY_USER_EVENT, + data, + }); + case msg.BLOCK_MESSAGE_BY_ID: { + ASRouterPreferences.console.debug( + "handleMesssage(): about to block, data = ", + data + ); + ASRouterPreferences.console.trace(); + + // Block the message but don't dismiss it in case the action taken has + // another state that needs to be visible + return this._router + .blockMessageById(data.id) + .then(() => !data.preventDismiss); + } + case msg.USER_ACTION: { + return this._specialMessageActions.handleAction(data, browser); + } + case msg.IMPRESSION: { + return this._router.addImpression(data); + } + case msg.TRIGGER: { + return this._router.sendTriggerMessage({ + ...(data && data.trigger), + tabId, + browser, + }); + } + case msg.PBNEWTAB_MESSAGE_REQUEST: { + return this._router.sendPBNewTabMessage({ + ...data, + tabId, + browser, + }); + } + case msg.NEWTAB_MESSAGE_REQUEST: { + return this._router.sendNewTabMessage({ + ...data, + tabId, + browser, + }); + } + + // ADMIN Messages + case msg.ADMIN_CONNECT_STATE: { + if (data && data.endpoint) { + return this._router + .addPreviewEndpoint(data.endpoint.url) + .then(() => this._router.loadMessagesFromAllProviders()); + } + return this._router.updateTargetingParameters(); + } + case msg.UNBLOCK_MESSAGE_BY_ID: { + return this._router.unblockMessageById(data.id); + } + case msg.UNBLOCK_ALL: { + return this._router.unblockAll(); + } + case msg.BLOCK_BUNDLE: { + return this._router.blockMessageById(data.bundle.map(b => b.id)); + } + case msg.UNBLOCK_BUNDLE: { + return this._router.setState(state => { + const messageBlockList = [...state.messageBlockList]; + for (let message of data.bundle) { + messageBlockList.splice(messageBlockList.indexOf(message.id), 1); + } + this._router._storage.set("messageBlockList", messageBlockList); + return { messageBlockList }; + }); + } + case msg.DISABLE_PROVIDER: { + this._preferences.enableOrDisableProvider(data, false); + return Promise.resolve(); + } + case msg.ENABLE_PROVIDER: { + this._preferences.enableOrDisableProvider(data, true); + return Promise.resolve(); + } + case msg.EVALUATE_JEXL_EXPRESSION: { + return this._router.evaluateExpression(data); + } + case msg.EXPIRE_QUERY_CACHE: { + this._queryCache.expireAll(); + return Promise.resolve(); + } + case msg.FORCE_ATTRIBUTION: { + return this._router.forceAttribution(data); + } + case msg.FORCE_PRIVATE_BROWSING_WINDOW: { + return this._router.forcePBWindow(browser, data.message); + } + case msg.FORCE_WHATSNEW_PANEL: { + return this._router.forceWNPanel(browser); + } + case msg.CLOSE_WHATSNEW_PANEL: { + return this._router.closeWNPanel(browser); + } + case msg.MODIFY_MESSAGE_JSON: { + return this._router.routeCFRMessage(data.content, browser, data, true); + } + case msg.OVERRIDE_MESSAGE: { + return this._router.setMessageById(data, true, browser); + } + case msg.RESET_PROVIDER_PREF: { + this._preferences.resetProviderPref(); + return Promise.resolve(); + } + case msg.SET_PROVIDER_USER_PREF: { + this._preferences.setUserPreference(data.id, data.value); + return Promise.resolve(); + } + case msg.RESET_GROUPS_STATE: { + return this._router + .resetGroupsState(data) + .then(() => this._router.loadMessagesFromAllProviders()); + } + default: { + return Promise.reject(new Error(`Unknown message received: ${name}`)); + } + } + } +} diff --git a/browser/components/newtab/lib/ASRouterPreferences.jsm b/browser/components/newtab/lib/ASRouterPreferences.jsm new file mode 100644 index 0000000000..c2fc071fbd --- /dev/null +++ b/browser/components/newtab/lib/ASRouterPreferences.jsm @@ -0,0 +1,259 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const PROVIDER_PREF_BRANCH = + "browser.newtabpage.activity-stream.asrouter.providers."; +const DEVTOOLS_PREF = + "browser.newtabpage.activity-stream.asrouter.devtoolsEnabled"; + +/** + * Use `ASRouterPreferences.console.debug()` and friends from ASRouter files to + * log messages during development. See LOG_LEVELS in ConsoleAPI.jsm for the + * available methods as well as the available values for this pref. + */ +const DEBUG_PREF = "browser.newtabpage.activity-stream.asrouter.debugLogLevel"; + +const FXA_USERNAME_PREF = "services.sync.username"; + +const DEFAULT_STATE = { + _initialized: false, + _providers: null, + _providerPrefBranch: PROVIDER_PREF_BRANCH, + _devtoolsEnabled: null, + _devtoolsPref: DEVTOOLS_PREF, +}; + +const USER_PREFERENCES = { + snippets: "browser.newtabpage.activity-stream.feeds.snippets", + cfrAddons: "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons", + cfrFeatures: + "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features", +}; + +// Preferences that influence targeting attributes. When these change we need +// to re-evaluate if the message targeting still matches +const TARGETING_PREFERENCES = [FXA_USERNAME_PREF]; + +const TEST_PROVIDERS = [ + { + id: "snippets_local_testing", + type: "local", + localProvider: "SnippetsTestMessageProvider", + enabled: true, + }, + { + id: "panel_local_testing", + type: "local", + localProvider: "PanelTestProvider", + enabled: true, + }, +]; + +class _ASRouterPreferences { + constructor() { + Object.assign(this, DEFAULT_STATE); + this._callbacks = new Set(); + + XPCOMUtils.defineLazyGetter(this, "console", () => { + let { ConsoleAPI } = ChromeUtils.importESModule( + "resource://gre/modules/Console.sys.mjs" + ); + let consoleOptions = { + maxLogLevel: "error", + maxLogLevelPref: DEBUG_PREF, + prefix: "ASRouter", + }; + return new ConsoleAPI(consoleOptions); + }); + } + + _transformPersonalizedCfrScores(value) { + let result = {}; + try { + result = JSON.parse(value); + } catch (e) { + console.error(e); + } + return result; + } + + _getProviderConfig() { + const prefList = Services.prefs.getChildList(this._providerPrefBranch); + return prefList.reduce((filtered, pref) => { + let value; + try { + value = JSON.parse(Services.prefs.getStringPref(pref, "")); + } catch (e) { + console.error( + `Could not parse ASRouter preference. Try resetting ${pref} in about:config.` + ); + } + if (value) { + filtered.push(value); + } + return filtered; + }, []); + } + + get providers() { + if (!this._initialized || this._providers === null) { + const config = this._getProviderConfig(); + const providers = config.map(provider => Object.freeze(provider)); + if (this.devtoolsEnabled) { + providers.unshift(...TEST_PROVIDERS); + } + this._providers = Object.freeze(providers); + } + + return this._providers; + } + + enableOrDisableProvider(id, value) { + const providers = this._getProviderConfig(); + const config = providers.find(p => p.id === id); + if (!config) { + console.error( + `Cannot set enabled state for '${id}' because the pref ${this._providerPrefBranch}${id} does not exist or is not correctly formatted.` + ); + return; + } + + Services.prefs.setStringPref( + this._providerPrefBranch + id, + JSON.stringify({ ...config, enabled: value }) + ); + } + + resetProviderPref() { + for (const pref of Services.prefs.getChildList(this._providerPrefBranch)) { + Services.prefs.clearUserPref(pref); + } + for (const id of Object.keys(USER_PREFERENCES)) { + Services.prefs.clearUserPref(USER_PREFERENCES[id]); + } + } + + /** + * Bug 1800087 - Migrate the ASRouter message provider prefs' values to the + * current format (provider.bucket -> provider.collection). + * + * TODO (Bug 1800937): Remove migration code after the next watershed release. + */ + _migrateProviderPrefs() { + const prefList = Services.prefs.getChildList(this._providerPrefBranch); + for (const pref of prefList) { + if (!Services.prefs.prefHasUserValue(pref)) { + continue; + } + try { + let value = JSON.parse(Services.prefs.getStringPref(pref, "")); + if (value && "bucket" in value && !("collection" in value)) { + const { bucket, ...rest } = value; + Services.prefs.setStringPref( + pref, + JSON.stringify({ + ...rest, + collection: bucket, + }) + ); + } + } catch (e) { + Services.prefs.clearUserPref(pref); + } + } + } + + get devtoolsEnabled() { + if (!this._initialized || this._devtoolsEnabled === null) { + this._devtoolsEnabled = Services.prefs.getBoolPref( + this._devtoolsPref, + false + ); + } + return this._devtoolsEnabled; + } + + observe(aSubject, aTopic, aPrefName) { + if (aPrefName && aPrefName.startsWith(this._providerPrefBranch)) { + this._providers = null; + } else if (aPrefName === this._devtoolsPref) { + this._providers = null; + this._devtoolsEnabled = null; + } + this._callbacks.forEach(cb => cb(aPrefName)); + } + + getUserPreference(name) { + const prefName = USER_PREFERENCES[name] || name; + return Services.prefs.getBoolPref(prefName, true); + } + + getAllUserPreferences() { + const values = {}; + for (const id of Object.keys(USER_PREFERENCES)) { + values[id] = this.getUserPreference(id); + } + return values; + } + + setUserPreference(providerId, value) { + if (!USER_PREFERENCES[providerId]) { + return; + } + Services.prefs.setBoolPref(USER_PREFERENCES[providerId], value); + } + + addListener(callback) { + this._callbacks.add(callback); + } + + removeListener(callback) { + this._callbacks.delete(callback); + } + + init() { + if (this._initialized) { + return; + } + this._migrateProviderPrefs(); + Services.prefs.addObserver(this._providerPrefBranch, this); + Services.prefs.addObserver(this._devtoolsPref, this); + for (const id of Object.keys(USER_PREFERENCES)) { + Services.prefs.addObserver(USER_PREFERENCES[id], this); + } + for (const targetingPref of TARGETING_PREFERENCES) { + Services.prefs.addObserver(targetingPref, this); + } + this._initialized = true; + } + + uninit() { + if (this._initialized) { + Services.prefs.removeObserver(this._providerPrefBranch, this); + Services.prefs.removeObserver(this._devtoolsPref, this); + for (const id of Object.keys(USER_PREFERENCES)) { + Services.prefs.removeObserver(USER_PREFERENCES[id], this); + } + for (const targetingPref of TARGETING_PREFERENCES) { + Services.prefs.removeObserver(targetingPref, this); + } + } + Object.assign(this, DEFAULT_STATE); + this._callbacks.clear(); + } +} + +const ASRouterPreferences = new _ASRouterPreferences(); + +const EXPORTED_SYMBOLS = [ + "_ASRouterPreferences", + "ASRouterPreferences", + "TEST_PROVIDERS", + "TARGETING_PREFERENCES", +]; diff --git a/browser/components/newtab/lib/ASRouterTargeting.jsm b/browser/components/newtab/lib/ASRouterTargeting.jsm new file mode 100644 index 0000000000..3e5d969f32 --- /dev/null +++ b/browser/components/newtab/lib/ASRouterTargeting.jsm @@ -0,0 +1,1225 @@ +/* This Source Code Form is subject to the terms of the Mozilla PublicddonMa + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const FXA_ENABLED_PREF = "identity.fxaccounts.enabled"; +const DISTRIBUTION_ID_PREF = "distribution.id"; +const DISTRIBUTION_ID_CHINA_REPACK = "MozillaOnline"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { NewTabUtils } = ChromeUtils.importESModule( + "resource://gre/modules/NewTabUtils.sys.mjs" +); +const { ShellService } = ChromeUtils.importESModule( + "resource:///modules/ShellService.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AttributionCode: "resource:///modules/AttributionCode.sys.mjs", + ClientEnvironment: "resource://normandy/lib/ClientEnvironment.sys.mjs", + CustomizableUI: "resource:///modules/CustomizableUI.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + ProfileAge: "resource://gre/modules/ProfileAge.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + TargetingContext: "resource://messaging-system/targeting/Targeting.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + ASRouterPreferences: "resource://activity-stream/lib/ASRouterPreferences.jsm", + HomePage: "resource:///modules/HomePage.jsm", + AboutNewTab: "resource:///modules/AboutNewTab.jsm", + BrowserWindowTracker: "resource:///modules/BrowserWindowTracker.jsm", +}); + +XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "cfrFeaturesUserPref", + "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features", + true +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "cfrAddonsUserPref", + "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons", + true +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "isWhatsNewPanelEnabled", + "browser.messaging-system.whatsNewPanel.enabled", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "hasAccessedFxAPanel", + "identity.fxaccounts.toolbar.accessed", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "clientsDevicesDesktop", + "services.sync.clients.devices.desktop", + 0 +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "clientsDevicesMobile", + "services.sync.clients.devices.mobile", + 0 +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "syncNumClients", + "services.sync.numClients", + 0 +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "devtoolsSelfXSSCount", + "devtools.selfxss.count", + 0 +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "isFxAEnabled", + FXA_ENABLED_PREF, + true +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "isXPIInstallEnabled", + "xpinstall.enabled", + true +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "snippetsUserPref", + "browser.newtabpage.activity-stream.feeds.snippets", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "hasMigratedBookmarks", + "browser.migrate.interactions.bookmarks", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "hasMigratedHistory", + "browser.migrate.interactions.history", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "hasMigratedPasswords", + "browser.migrate.interactions.passwords", + false +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "useEmbeddedMigrationWizard", + "browser.migrate.content-modal.about-welcome-behavior", + "default", + null, + behaviorString => { + return behaviorString === "embedded"; + } +); + +XPCOMUtils.defineLazyServiceGetters(lazy, { + AUS: ["@mozilla.org/updates/update-service;1", "nsIApplicationUpdateService"], + BrowserHandler: ["@mozilla.org/browser/clh;1", "nsIBrowserHandler"], + TrackingDBService: [ + "@mozilla.org/tracking-db-service;1", + "nsITrackingDBService", + ], + UpdateCheckSvc: ["@mozilla.org/updates/update-checker;1", "nsIUpdateChecker"], +}); + +const FXA_USERNAME_PREF = "services.sync.username"; + +const { activityStreamProvider: asProvider } = NewTabUtils; + +const FXA_ATTACHED_CLIENTS_UPDATE_INTERVAL = 4 * 60 * 60 * 1000; // Four hours +const FRECENT_SITES_UPDATE_INTERVAL = 6 * 60 * 60 * 1000; // Six hours +const FRECENT_SITES_IGNORE_BLOCKED = false; +const FRECENT_SITES_NUM_ITEMS = 25; +const FRECENT_SITES_MIN_FRECENCY = 100; + +const CACHE_EXPIRATION = 5 * 60 * 1000; +const jexlEvaluationCache = new Map(); + +/** + * CachedTargetingGetter + * @param property {string} Name of the method + * @param options {any=} Options passed to the method + * @param updateInterval {number?} Update interval for query. Defaults to FRECENT_SITES_UPDATE_INTERVAL + */ +function CachedTargetingGetter( + property, + options = null, + updateInterval = FRECENT_SITES_UPDATE_INTERVAL, + getter = asProvider +) { + return { + _lastUpdated: 0, + _value: null, + // For testing + expire() { + this._lastUpdated = 0; + this._value = null; + }, + async get() { + const now = Date.now(); + if (now - this._lastUpdated >= updateInterval) { + this._value = await getter[property](options); + this._lastUpdated = now; + } + return this._value; + }, + }; +} + +function CacheListAttachedOAuthClients() { + return { + _lastUpdated: 0, + _value: null, + expire() { + this._lastUpdated = 0; + this._value = null; + }, + get() { + const now = Date.now(); + if (now - this._lastUpdated >= FXA_ATTACHED_CLIENTS_UPDATE_INTERVAL) { + this._value = new Promise(resolve => { + lazy.fxAccounts + .listAttachedOAuthClients() + .then(clients => { + resolve(clients); + }) + .catch(() => resolve([])); + }); + this._lastUpdated = now; + } + return this._value; + }, + }; +} + +function CheckBrowserNeedsUpdate( + updateInterval = FRECENT_SITES_UPDATE_INTERVAL +) { + const checker = { + _lastUpdated: 0, + _value: null, + // For testing. Avoid update check network call. + setUp(value) { + this._lastUpdated = Date.now(); + this._value = value; + }, + expire() { + this._lastUpdated = 0; + this._value = null; + }, + async get() { + const now = Date.now(); + if ( + !AppConstants.MOZ_UPDATER || + now - this._lastUpdated < updateInterval + ) { + return this._value; + } + if (!lazy.AUS.canCheckForUpdates) { + return false; + } + this._lastUpdated = now; + let check = lazy.UpdateCheckSvc.checkForUpdates( + lazy.UpdateCheckSvc.FOREGROUND_CHECK + ); + let result = await check.result; + if (!result.succeeded) { + throw result.request; + } + checker._value = !!result.updates.length; + return checker._value; + }, + }; + + return checker; +} + +const QueryCache = { + expireAll() { + Object.keys(this.queries).forEach(query => { + this.queries[query].expire(); + }); + Object.keys(this.getters).forEach(key => { + this.getters[key].expire(); + }); + }, + queries: { + TopFrecentSites: new CachedTargetingGetter("getTopFrecentSites", { + ignoreBlocked: FRECENT_SITES_IGNORE_BLOCKED, + numItems: FRECENT_SITES_NUM_ITEMS, + topsiteFrecency: FRECENT_SITES_MIN_FRECENCY, + onePerDomain: true, + includeFavicon: false, + }), + TotalBookmarksCount: new CachedTargetingGetter("getTotalBookmarksCount"), + CheckBrowserNeedsUpdate: new CheckBrowserNeedsUpdate(), + RecentBookmarks: new CachedTargetingGetter("getRecentBookmarks"), + ListAttachedOAuthClients: new CacheListAttachedOAuthClients(), + UserMonthlyActivity: new CachedTargetingGetter("getUserMonthlyActivity"), + }, + getters: { + doesAppNeedPin: new CachedTargetingGetter( + "doesAppNeedPin", + null, + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + doesAppNeedPrivatePin: new CachedTargetingGetter( + "doesAppNeedPin", + true, + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + isDefaultBrowser: new CachedTargetingGetter( + "isDefaultBrowser", + null, + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + currentThemes: new CachedTargetingGetter( + "getAddonsByTypes", + ["theme"], + FRECENT_SITES_UPDATE_INTERVAL, + lazy.AddonManager // eslint-disable-line mozilla/valid-lazy + ), + isDefaultHTMLHandler: new CachedTargetingGetter( + "isDefaultHandlerFor", + [".html"], + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + isDefaultPDFHandler: new CachedTargetingGetter( + "isDefaultHandlerFor", + [".pdf"], + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + defaultPDFHandler: new CachedTargetingGetter( + "getDefaultPDFHandler", + null, + FRECENT_SITES_UPDATE_INTERVAL, + ShellService + ), + }, +}; + +/** + * sortMessagesByWeightedRank + * + * Each message has an associated weight, which is guaranteed to be strictly + * positive. Sort the messages so that higher weighted messages are more likely + * to come first. + * + * Specifically, sort them so that the probability of message x_1 with weight + * w_1 appearing before message x_2 with weight w_2 is (w_1 / (w_1 + w_2)). + * + * This is equivalent to requiring that x_1 appearing before x_2 is (w_1 / w_2) + * "times" as likely as x_2 appearing before x_1. + * + * See Bug 1484996, Comment 2 for a justification of the method. + * + * @param {Array} messages - A non-empty array of messages to sort, all with + * strictly positive weights + * @returns the sorted array + */ +function sortMessagesByWeightedRank(messages) { + return messages + .map(message => ({ + message, + rank: Math.pow(Math.random(), 1 / message.weight), + })) + .sort((a, b) => b.rank - a.rank) + .map(({ message }) => message); +} + +/** + * getSortedMessages - Given an array of Messages, applies sorting and filtering rules + * in expected order. + * + * @param {Array<Message>} messages + * @param {{}} options + * @param {boolean} options.ordered - Should .order be used instead of random weighted sorting? + * @returns {Array<Message>} + */ +function getSortedMessages(messages, options = {}) { + let { ordered } = { ordered: false, ...options }; + let result = messages; + + if (!ordered) { + result = sortMessagesByWeightedRank(result); + } + + result.sort((a, b) => { + // Next, sort by priority + if (a.priority > b.priority || (!isNaN(a.priority) && isNaN(b.priority))) { + return -1; + } + if (a.priority < b.priority || (isNaN(a.priority) && !isNaN(b.priority))) { + return 1; + } + + // Sort messages with targeting expressions higher than those with none + if (a.targeting && !b.targeting) { + return -1; + } + if (!a.targeting && b.targeting) { + return 1; + } + + // Next, sort by order *ascending* if ordered = true + if (ordered) { + if (a.order > b.order || (!isNaN(a.order) && isNaN(b.order))) { + return 1; + } + if (a.order < b.order || (isNaN(a.order) && !isNaN(b.order))) { + return -1; + } + } + + return 0; + }); + + return result; +} + +/** + * parseAboutPageURL - Parse a URL string retrieved from about:home and about:new, returns + * its type (web extenstion or custom url) and the parsed url(s) + * + * @param {string} url - A URL string for home page or newtab page + * @returns {Object} { + * isWebExt: boolean, + * isCustomUrl: boolean, + * urls: Array<{url: string, host: string}> + * } + */ +function parseAboutPageURL(url) { + let ret = { + isWebExt: false, + isCustomUrl: false, + urls: [], + }; + if (url.startsWith("moz-extension://")) { + ret.isWebExt = true; + ret.urls.push({ url, host: "" }); + } else { + // The home page URL could be either a single URL or a list of "|" separated URLs. + // Note that it should work with "about:home" and "about:blank", in which case the + // "host" is set as an empty string. + for (const _url of url.split("|")) { + if (!["about:home", "about:newtab", "about:blank"].includes(_url)) { + ret.isCustomUrl = true; + } + try { + const parsedURL = new URL(_url); + const host = parsedURL.hostname.replace(/^www\./i, ""); + ret.urls.push({ url: _url, host }); + } catch (e) {} + } + // If URL parsing failed, just return the given url with an empty host + if (!ret.urls.length) { + ret.urls.push({ url, host: "" }); + } + } + + return ret; +} + +/** + * Get the number of records in autofill storage, e.g. credit cards/addresses. + * + * @param {Object} [data] + * @param {string} [data.collectionName] + * The name used to specify which collection to retrieve records. + * @param {string} [data.searchString] + * The typed string for filtering out the matched records. + * @param {string} [data.info] + * The input autocomplete property's information. + * @returns {Promise<number>} The number of matched records. + * @see FormAutofillParent._getRecords + */ +async function getAutofillRecords(data) { + let actor; + try { + const win = Services.wm.getMostRecentBrowserWindow(); + actor = + win.gBrowser.selectedBrowser.browsingContext.currentWindowGlobal.getActor( + "FormAutofill" + ); + } catch (error) { + // If the actor is not available, we can't get the records. We could import + // the records directly from FormAutofillStorage to avoid the messiness of + // JSActors, but that would import a lot of code for a targeting attribute. + return 0; + } + let records = await actor?.receiveMessage({ + name: "FormAutofill:GetRecords", + data, + }); + return records?.length ?? 0; +} + +// Attribution data can be encoded multiple times so we need this function to +// get a cleartext value. +function decodeAttributionValue(value) { + if (!value) { + return null; + } + + let decodedValue = value; + + while (decodedValue.includes("%")) { + try { + const result = decodeURIComponent(decodedValue); + if (result === decodedValue) { + break; + } + decodedValue = result; + } catch (e) { + break; + } + } + + return decodedValue; +} + +const TargetingGetters = { + get locale() { + return Services.locale.appLocaleAsBCP47; + }, + get localeLanguageCode() { + return ( + Services.locale.appLocaleAsBCP47 && + Services.locale.appLocaleAsBCP47.substr(0, 2) + ); + }, + get browserSettings() { + const { settings } = lazy.TelemetryEnvironment.currentEnvironment; + return { + update: settings.update, + }; + }, + get attributionData() { + // Attribution is determined at startup - so we can use the cached attribution at this point + return lazy.AttributionCode.getCachedAttributionData(); + }, + get currentDate() { + return new Date(); + }, + get profileAgeCreated() { + return lazy.ProfileAge().then(times => times.created); + }, + get profileAgeReset() { + return lazy.ProfileAge().then(times => times.reset); + }, + get usesFirefoxSync() { + return Services.prefs.prefHasUserValue(FXA_USERNAME_PREF); + }, + get isFxAEnabled() { + return lazy.isFxAEnabled; + }, + get isFxASignedIn() { + return new Promise(resolve => { + if (!lazy.isFxAEnabled) { + resolve(false); + } + if (Services.prefs.getStringPref(FXA_USERNAME_PREF, "")) { + resolve(true); + } + lazy.fxAccounts + .getSignedInUser() + .then(data => resolve(!!data)) + .catch(e => resolve(false)); + }); + }, + get sync() { + return { + desktopDevices: lazy.clientsDevicesDesktop, + mobileDevices: lazy.clientsDevicesMobile, + totalDevices: lazy.syncNumClients, + }; + }, + get xpinstallEnabled() { + // This is needed for all add-on recommendations, to know if we allow xpi installs in the first place + return lazy.isXPIInstallEnabled; + }, + get addonsInfo() { + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + if (bts?.isBackgroundTaskMode) { + return { addons: {}, isFullData: true }; + } + + return lazy.AddonManager.getActiveAddons(["extension", "service"]).then( + ({ addons, fullData }) => { + const info = {}; + for (const addon of addons) { + info[addon.id] = { + version: addon.version, + type: addon.type, + isSystem: addon.isSystem, + isWebExtension: addon.isWebExtension, + }; + if (fullData) { + Object.assign(info[addon.id], { + name: addon.name, + userDisabled: addon.userDisabled, + installDate: addon.installDate, + }); + } + } + return { addons: info, isFullData: fullData }; + } + ); + }, + get searchEngines() { + const NONE = { installed: [], current: "" }; + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + if (bts?.isBackgroundTaskMode) { + return Promise.resolve(NONE); + } + return new Promise(resolve => { + // Note: calling init ensures this code is only executed after Search has been initialized + Services.search + .getAppProvidedEngines() + .then(engines => { + resolve({ + current: Services.search.defaultEngine.identifier, + installed: engines.map(engine => engine.identifier), + }); + }) + .catch(() => resolve(NONE)); + }); + }, + get isDefaultBrowser() { + return QueryCache.getters.isDefaultBrowser.get().catch(() => null); + }, + get devToolsOpenedCount() { + return lazy.devtoolsSelfXSSCount; + }, + get topFrecentSites() { + return QueryCache.queries.TopFrecentSites.get().then(sites => + sites.map(site => ({ + url: site.url, + host: new URL(site.url).hostname, + frecency: site.frecency, + lastVisitDate: site.lastVisitDate, + })) + ); + }, + get recentBookmarks() { + return QueryCache.queries.RecentBookmarks.get(); + }, + get pinnedSites() { + return NewTabUtils.pinnedLinks.links.map(site => + site + ? { + url: site.url, + host: new URL(site.url).hostname, + searchTopSite: site.searchTopSite, + } + : {} + ); + }, + get providerCohorts() { + return lazy.ASRouterPreferences.providers.reduce((prev, current) => { + prev[current.id] = current.cohort || ""; + return prev; + }, {}); + }, + get totalBookmarksCount() { + return QueryCache.queries.TotalBookmarksCount.get(); + }, + get firefoxVersion() { + return parseInt(AppConstants.MOZ_APP_VERSION.match(/\d+/), 10); + }, + get region() { + return lazy.Region.home || ""; + }, + get needsUpdate() { + return QueryCache.queries.CheckBrowserNeedsUpdate.get(); + }, + get hasPinnedTabs() { + for (let win of Services.wm.getEnumerator("navigator:browser")) { + if (win.closed || !win.ownerGlobal.gBrowser) { + continue; + } + if (win.ownerGlobal.gBrowser.visibleTabs.filter(t => t.pinned).length) { + return true; + } + } + + return false; + }, + get hasAccessedFxAPanel() { + return lazy.hasAccessedFxAPanel; + }, + get isWhatsNewPanelEnabled() { + return lazy.isWhatsNewPanelEnabled; + }, + get userPrefs() { + return { + cfrFeatures: lazy.cfrFeaturesUserPref, + cfrAddons: lazy.cfrAddonsUserPref, + snippets: lazy.snippetsUserPref, + }; + }, + get totalBlockedCount() { + return lazy.TrackingDBService.sumAllEvents(); + }, + get blockedCountByType() { + const idToTextMap = new Map([ + [Ci.nsITrackingDBService.TRACKERS_ID, "trackerCount"], + [Ci.nsITrackingDBService.TRACKING_COOKIES_ID, "cookieCount"], + [Ci.nsITrackingDBService.CRYPTOMINERS_ID, "cryptominerCount"], + [Ci.nsITrackingDBService.FINGERPRINTERS_ID, "fingerprinterCount"], + [Ci.nsITrackingDBService.SOCIAL_ID, "socialCount"], + ]); + + const dateTo = new Date(); + const dateFrom = new Date(dateTo.getTime() - 42 * 24 * 60 * 60 * 1000); + return lazy.TrackingDBService.getEventsByDateRange(dateFrom, dateTo).then( + eventsByDate => { + let totalEvents = {}; + for (let blockedType of idToTextMap.values()) { + totalEvents[blockedType] = 0; + } + + return eventsByDate.reduce((acc, day) => { + const type = day.getResultByName("type"); + const count = day.getResultByName("count"); + acc[idToTextMap.get(type)] = acc[idToTextMap.get(type)] + count; + return acc; + }, totalEvents); + } + ); + }, + get attachedFxAOAuthClients() { + return this.usesFirefoxSync + ? QueryCache.queries.ListAttachedOAuthClients.get() + : []; + }, + get platformName() { + return AppConstants.platform; + }, + get isChinaRepack() { + return ( + Services.prefs + .getDefaultBranch(null) + .getCharPref(DISTRIBUTION_ID_PREF, "default") === + DISTRIBUTION_ID_CHINA_REPACK + ); + }, + get userId() { + return lazy.ClientEnvironment.userId; + }, + get profileRestartCount() { + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + if (bts?.isBackgroundTaskMode) { + return 0; + } + // Counter starts at 1 when a profile is created, substract 1 so the value + // returned matches expectations + return ( + lazy.TelemetrySession.getMetadata("targeting").profileSubsessionCounter - + 1 + ); + }, + get homePageSettings() { + const url = lazy.HomePage.get(); + const { isWebExt, isCustomUrl, urls } = parseAboutPageURL(url); + + return { + isWebExt, + isCustomUrl, + urls, + isDefault: lazy.HomePage.isDefault, + isLocked: lazy.HomePage.locked, + }; + }, + get newtabSettings() { + const url = lazy.AboutNewTab.newTabURL; + const { isWebExt, isCustomUrl, urls } = parseAboutPageURL(url); + + return { + isWebExt, + isCustomUrl, + isDefault: lazy.AboutNewTab.activityStreamEnabled, + url: urls[0].url, + host: urls[0].host, + }; + }, + get activeNotifications() { + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + if (bts?.isBackgroundTaskMode) { + // This might need to hook into the alert service to enumerate relevant + // persistent native notifications. + return false; + } + + let window = lazy.BrowserWindowTracker.getTopWindow(); + + // Technically this doesn't mean we have active notifications, + // but because we use !activeNotifications to check for conflicts, this should return true + if (!window) { + return true; + } + + if ( + window.gURLBar?.view.isOpen || + window.gNotificationBox?.currentNotification || + window.gBrowser.getNotificationBox()?.currentNotification + ) { + return true; + } + + return false; + }, + + get isMajorUpgrade() { + return lazy.BrowserHandler.majorUpgrade; + }, + + get hasActiveEnterprisePolicies() { + return Services.policies.status === Services.policies.ACTIVE; + }, + + get userMonthlyActivity() { + return QueryCache.queries.UserMonthlyActivity.get(); + }, + + get doesAppNeedPin() { + return QueryCache.getters.doesAppNeedPin.get(); + }, + + get doesAppNeedPrivatePin() { + return QueryCache.getters.doesAppNeedPrivatePin.get(); + }, + + /** + * Is this invocation running in background task mode? + * + * @return {boolean} `true` if running in background task mode. + */ + get isBackgroundTaskMode() { + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + return !!bts?.isBackgroundTaskMode; + }, + + /** + * A non-empty task name if this invocation is running in background + * task mode, or `null` if this invocation is not running in + * background task mode. + * + * @return {string|null} background task name or `null`. + */ + get backgroundTaskName() { + let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService( + Ci.nsIBackgroundTasks + ); + return bts?.backgroundTaskName(); + }, + + get userPrefersReducedMotion() { + let window = Services.appShell.hiddenDOMWindow; + return window?.matchMedia("(prefers-reduced-motion: reduce)")?.matches; + }, + + /** + * Whether or not the user is in the Major Release 2022 holdback study. + */ + get inMr2022Holdback() { + return ( + lazy.NimbusFeatures.majorRelease2022.getVariable("onboarding") === false + ); + }, + + /** + * The distribution id, if any. + * @return {string} + */ + get distributionId() { + return Services.prefs + .getDefaultBranch(null) + .getCharPref("distribution.id", ""); + }, + + /** Where the Firefox View button is shown, if at all. + * @return {string} container of the button if it is shown in the toolbar/overflow menu + * @return {string} `null` if the button has been removed + */ + get fxViewButtonAreaType() { + let button = lazy.CustomizableUI.getWidget("firefox-view-button"); + return button.areaType; + }, + + isDefaultHandler: { + get html() { + return QueryCache.getters.isDefaultHTMLHandler.get(); + }, + get pdf() { + return QueryCache.getters.isDefaultPDFHandler.get(); + }, + }, + + get defaultPDFHandler() { + return QueryCache.getters.defaultPDFHandler.get(); + }, + + get creditCardsSaved() { + return getAutofillRecords({ collectionName: "creditCards" }); + }, + + get addressesSaved() { + return getAutofillRecords({ collectionName: "addresses" }); + }, + + /** + * Has the user ever used the Migration Wizard to migrate bookmarks? + * @return {boolean} `true` if bookmark migration has occurred. + */ + get hasMigratedBookmarks() { + return lazy.hasMigratedBookmarks; + }, + + /** + * Has the user ever used the Migration Wizard to migrate history? + * @return {boolean} `true` if history migration has occurred. + */ + get hasMigratedHistory() { + return lazy.hasMigratedHistory; + }, + + /** + * Has the user ever used the Migration Wizard to migrate passwords? + * @return {boolean} `true` if password migration has occurred. + */ + get hasMigratedPasswords() { + return lazy.hasMigratedPasswords; + }, + + /** + * Returns true if the user is configured to use the embedded migration + * wizard in about:welcome by having + * "browser.migrate.content-modal.about-welcome-behavior" be equal to + * "embedded". + * @return {boolean} `true` if the embedded migration wizard is enabled. + */ + get useEmbeddedMigrationWizard() { + return lazy.useEmbeddedMigrationWizard; + }, + + /** + * Whether the user installed Firefox via the RTAMO flow. + * @return {boolean} `true` when RTAMO has been used to download Firefox, + * `false` otherwise. + */ + get isRTAMO() { + const { attributionData } = this; + + return ( + attributionData?.source === "addons.mozilla.org" && + !!decodeAttributionValue(attributionData?.content)?.startsWith("rta:") + ); + }, + + /** + * Whether the user installed via the device migration flow. + * @return {boolean} `true` when the link to download the browser was part + * of guidance for device migration. `false` otherwise. + */ + get isDeviceMigration() { + const { attributionData } = this; + + return attributionData?.campaign === "migration"; + }, +}; + +const ASRouterTargeting = { + Environment: TargetingGetters, + + /** + * Snapshot the current targeting environment. + * + * Asynchronous getters are handled. Getters that throw or reject + * are ignored. + * + * @param {object} target - the environment to snapshot. + * @return {object} snapshot of target with `environment` object and `version` + * integer. + */ + async getEnvironmentSnapshot(target = ASRouterTargeting.Environment) { + async function resolve(object) { + if (typeof object === "object" && object !== null) { + if (Array.isArray(object)) { + return Promise.all(object.map(async item => resolve(await item))); + } + + if (object instanceof Date) { + return object; + } + + // One promise for each named property. Label promises with property name. + const promises = Object.keys(object).map(async key => { + // Each promise needs to check if we're shutting down when it is evaluated. + if (Services.startup.shuttingDown) { + throw new Error( + "shutting down, so not querying targeting environment" + ); + } + + const value = await resolve(await object[key]); + + return [key, value]; + }); + + const resolved = {}; + for (const result of await Promise.allSettled(promises)) { + // Ignore properties that are rejected. + if (result.status === "fulfilled") { + const [key, value] = result.value; + resolved[key] = value; + } + } + + return resolved; + } + + return object; + } + + const environment = await resolve(target); + + // Should we need to migrate in the future. + const snapshot = { environment, version: 1 }; + + return snapshot; + }, + + isTriggerMatch(trigger = {}, candidateMessageTrigger = {}) { + if (trigger.id !== candidateMessageTrigger.id) { + return false; + } else if ( + !candidateMessageTrigger.params && + !candidateMessageTrigger.patterns + ) { + return true; + } + + if (!trigger.param) { + return false; + } + + return ( + (candidateMessageTrigger.params && + trigger.param.host && + candidateMessageTrigger.params.includes(trigger.param.host)) || + (candidateMessageTrigger.params && + trigger.param.type && + candidateMessageTrigger.params.filter(t => t === trigger.param.type) + .length) || + (candidateMessageTrigger.params && + trigger.param.type && + candidateMessageTrigger.params.filter( + t => (t & trigger.param.type) === t + ).length) || + (candidateMessageTrigger.patterns && + trigger.param.url && + new MatchPatternSet(candidateMessageTrigger.patterns).matches( + trigger.param.url + )) + ); + }, + + /** + * getCachedEvaluation - Return a cached jexl evaluation if available + * + * @param {string} targeting JEXL expression to lookup + * @returns {obj|null} Object with value result or null if not available + */ + getCachedEvaluation(targeting) { + if (jexlEvaluationCache.has(targeting)) { + const { timestamp, value } = jexlEvaluationCache.get(targeting); + if (Date.now() - timestamp <= CACHE_EXPIRATION) { + return { value }; + } + jexlEvaluationCache.delete(targeting); + } + + return null; + }, + + /** + * checkMessageTargeting - Checks is a message's targeting parameters are satisfied + * + * @param {*} message An AS router message + * @param {obj} targetingContext a TargetingContext instance complete with eval environment + * @param {func} onError A function to handle errors (takes two params; error, message) + * @param {boolean} shouldCache Should the JEXL evaluations be cached and reused. + * @returns + */ + async checkMessageTargeting(message, targetingContext, onError, shouldCache) { + lazy.ASRouterPreferences.console.debug( + "in checkMessageTargeting, arguments = ", + Array.from(arguments) // eslint-disable-line prefer-rest-params + ); + + // If no targeting is specified, + if (!message.targeting) { + return true; + } + let result; + try { + if (shouldCache) { + result = this.getCachedEvaluation(message.targeting); + if (result) { + return result.value; + } + } + // Used to report the source of the targeting error in the case of + // undesired events + targetingContext.setTelemetrySource(message.id); + result = await targetingContext.evalWithDefault(message.targeting); + if (shouldCache) { + jexlEvaluationCache.set(message.targeting, { + timestamp: Date.now(), + value: result, + }); + } + } catch (error) { + if (onError) { + onError(error, message); + } + console.error(error); + result = false; + } + return result; + }, + + _isMessageMatch( + message, + trigger, + targetingContext, + onError, + shouldCache = false + ) { + return ( + message && + (trigger + ? this.isTriggerMatch(trigger, message.trigger) + : !message.trigger) && + // If a trigger expression was passed to this function, the message should match it. + // Otherwise, we should choose a message with no trigger property (i.e. a message that can show up at any time) + this.checkMessageTargeting( + message, + targetingContext, + onError, + shouldCache + ) + ); + }, + + /** + * findMatchingMessage - Given an array of messages, returns one message + * whos targeting expression evaluates to true + * + * @param {Array<Message>} messages An array of AS router messages + * @param {trigger} string A trigger expression if a message for that trigger is desired + * @param {obj|null} context A FilterExpression context. Defaults to TargetingGetters above. + * @param {func} onError A function to handle errors (takes two params; error, message) + * @param {func} ordered An optional param when true sort message by order specified in message + * @param {boolean} shouldCache Should the JEXL evaluations be cached and reused. + * @param {boolean} returnAll Should we return all matching messages, not just the first one found. + * @returns {obj|Array<Message>} If returnAll is false, a single message. If returnAll is true, an array of messages. + */ + async findMatchingMessage({ + messages, + trigger = {}, + context = {}, + onError, + ordered = false, + shouldCache = false, + returnAll = false, + }) { + const sortedMessages = getSortedMessages(messages, { ordered }); + lazy.ASRouterPreferences.console.debug( + "in findMatchingMessage, sortedMessages = ", + sortedMessages + ); + const matching = returnAll ? [] : null; + const targetingContext = new lazy.TargetingContext( + lazy.TargetingContext.combineContexts( + context, + this.Environment, + trigger.context || {} + ) + ); + + const isMatch = candidate => + this._isMessageMatch( + candidate, + trigger, + targetingContext, + onError, + shouldCache + ); + + for (const candidate of sortedMessages) { + if (await isMatch(candidate)) { + // If not returnAll, we should return the first message we find that matches. + if (!returnAll) { + return candidate; + } + + matching.push(candidate); + } + } + return matching; + }, +}; + +const EXPORTED_SYMBOLS = [ + "ASRouterTargeting", + "QueryCache", + "CachedTargetingGetter", + "getSortedMessages", +]; diff --git a/browser/components/newtab/lib/ASRouterTriggerListeners.jsm b/browser/components/newtab/lib/ASRouterTriggerListeners.jsm new file mode 100644 index 0000000000..6f3ddd32ca --- /dev/null +++ b/browser/components/newtab/lib/ASRouterTriggerListeners.jsm @@ -0,0 +1,1087 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AboutReaderParent: "resource:///actors/AboutReaderParent.sys.mjs", + BrowserUtils: "resource://gre/modules/BrowserUtils.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + EveryWindow: "resource:///modules/EveryWindow.jsm", +}); + +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("ASRouterTriggerListeners"); +}); + +const FEW_MINUTES = 15 * 60 * 1000; // 15 mins + +function isPrivateWindow(win) { + return ( + !(win instanceof Ci.nsIDOMWindow) || + win.closed || + lazy.PrivateBrowsingUtils.isWindowPrivate(win) + ); +} + +/** + * Check current location against the list of allowed hosts + * Additionally verify for redirects and check original request URL against + * the list. + * + * @returns {object} - {host, url} pair that matched the list of allowed hosts + */ +function checkURLMatch(aLocationURI, { hosts, matchPatternSet }, aRequest) { + // If checks pass we return a match + let match; + try { + match = { host: aLocationURI.host, url: aLocationURI.spec }; + } catch (e) { + // nsIURI.host can throw for non-nsStandardURL nsIURIs + return false; + } + + // Check current location against allowed hosts + if (hosts.has(match.host)) { + return match; + } + + if (matchPatternSet) { + if (matchPatternSet.matches(match.url)) { + return match; + } + } + + // Nothing else to check, return early + if (!aRequest) { + return false; + } + + // The original URL at the start of the request + const originalLocation = aRequest.QueryInterface(Ci.nsIChannel).originalURI; + // We have been redirected + if (originalLocation.spec !== aLocationURI.spec) { + return ( + hosts.has(originalLocation.host) && { + host: originalLocation.host, + url: originalLocation.spec, + } + ); + } + + return false; +} + +function createMatchPatternSet(patterns, flags) { + try { + return new MatchPatternSet(new Set(patterns), flags); + } catch (e) { + console.error(e); + } + return new MatchPatternSet([]); +} + +/** + * A Map from trigger IDs to singleton trigger listeners. Each listener must + * have idempotent `init` and `uninit` methods. + */ +const ASRouterTriggerListeners = new Map([ + [ + "openArticleURL", + { + id: "openArticleURL", + _initialized: false, + _triggerHandler: null, + _hosts: new Set(), + _matchPatternSet: null, + readerModeEvent: "Reader:UpdateReaderButton", + + init(triggerHandler, hosts, patterns) { + if (!this._initialized) { + this.receiveMessage = this.receiveMessage.bind(this); + lazy.AboutReaderParent.addMessageListener(this.readerModeEvent, this); + this._triggerHandler = triggerHandler; + this._initialized = true; + } + if (patterns) { + this._matchPatternSet = createMatchPatternSet([ + ...(this._matchPatternSet ? this._matchPatternSet.patterns : []), + ...patterns, + ]); + } + if (hosts) { + hosts.forEach(h => this._hosts.add(h)); + } + }, + + receiveMessage({ data, target }) { + if (data && data.isArticle) { + const match = checkURLMatch(target.currentURI, { + hosts: this._hosts, + matchPatternSet: this._matchPatternSet, + }); + if (match) { + this._triggerHandler(target, { id: this.id, param: match }); + } + } + }, + + uninit() { + if (this._initialized) { + lazy.AboutReaderParent.removeMessageListener( + this.readerModeEvent, + this + ); + this._initialized = false; + this._triggerHandler = null; + this._hosts = new Set(); + this._matchPatternSet = null; + } + }, + }, + ], + [ + "openBookmarkedURL", + { + id: "openBookmarkedURL", + _initialized: false, + _triggerHandler: null, + _hosts: new Set(), + bookmarkEvent: "bookmark-icon-updated", + + init(triggerHandler) { + if (!this._initialized) { + Services.obs.addObserver(this, this.bookmarkEvent); + this._triggerHandler = triggerHandler; + this._initialized = true; + } + }, + + observe(subject, topic, data) { + if (topic === this.bookmarkEvent && data === "starred") { + const browser = Services.wm.getMostRecentBrowserWindow(); + if (browser) { + this._triggerHandler(browser.gBrowser.selectedBrowser, { + id: this.id, + }); + } + } + }, + + uninit() { + if (this._initialized) { + Services.obs.removeObserver(this, this.bookmarkEvent); + this._initialized = false; + this._triggerHandler = null; + this._hosts = new Set(); + } + }, + }, + ], + [ + "frequentVisits", + { + id: "frequentVisits", + _initialized: false, + _triggerHandler: null, + _hosts: null, + _matchPatternSet: null, + _visits: null, + + init(triggerHandler, hosts = [], patterns) { + if (!this._initialized) { + this.onTabSwitch = this.onTabSwitch.bind(this); + lazy.EveryWindow.registerCallback( + this.id, + win => { + if (!isPrivateWindow(win)) { + win.addEventListener("TabSelect", this.onTabSwitch); + win.gBrowser.addTabsProgressListener(this); + } + }, + win => { + if (!isPrivateWindow(win)) { + win.removeEventListener("TabSelect", this.onTabSwitch); + win.gBrowser.removeTabsProgressListener(this); + } + } + ); + this._visits = new Map(); + this._initialized = true; + } + this._triggerHandler = triggerHandler; + if (patterns) { + this._matchPatternSet = createMatchPatternSet([ + ...(this._matchPatternSet ? this._matchPatternSet.patterns : []), + ...patterns, + ]); + } + if (this._hosts) { + hosts.forEach(h => this._hosts.add(h)); + } else { + this._hosts = new Set(hosts); // Clone the hosts to avoid unexpected behaviour + } + }, + + /* _updateVisits - Record visit timestamps for websites that match `this._hosts` and only + * if it's been more than FEW_MINUTES since the last visit. + * @param {string} host - Location host of current selected tab + * @returns {boolean} - If the new visit has been recorded + */ + _updateVisits(host) { + const visits = this._visits.get(host); + + if (visits && Date.now() - visits[0] > FEW_MINUTES) { + this._visits.set(host, [Date.now(), ...visits]); + return true; + } + if (!visits) { + this._visits.set(host, [Date.now()]); + return true; + } + + return false; + }, + + onTabSwitch(event) { + if (!event.target.ownerGlobal.gBrowser) { + return; + } + + const { gBrowser } = event.target.ownerGlobal; + const match = checkURLMatch(gBrowser.currentURI, { + hosts: this._hosts, + matchPatternSet: this._matchPatternSet, + }); + if (match) { + this.triggerHandler(gBrowser.selectedBrowser, match); + } + }, + + triggerHandler(aBrowser, match) { + const updated = this._updateVisits(match.host); + + // If the previous visit happend less than FEW_MINUTES ago + // no updates were made, no need to trigger the handler + if (!updated) { + return; + } + + this._triggerHandler(aBrowser, { + id: this.id, + param: match, + context: { + // Remapped to {host, timestamp} because JEXL operators can only + // filter over collections (arrays of objects) + recentVisits: this._visits + .get(match.host) + .map(timestamp => ({ host: match.host, timestamp })), + }, + }); + }, + + onLocationChange(aBrowser, aWebProgress, aRequest, aLocationURI, aFlags) { + // Some websites trigger redirect events after they finish loading even + // though the location remains the same. This results in onLocationChange + // events to be fired twice. + const isSameDocument = !!( + aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT + ); + if (aWebProgress.isTopLevel && !isSameDocument) { + const match = checkURLMatch( + aLocationURI, + { hosts: this._hosts, matchPatternSet: this._matchPatternSet }, + aRequest + ); + if (match) { + this.triggerHandler(aBrowser, match); + } + } + }, + + uninit() { + if (this._initialized) { + lazy.EveryWindow.unregisterCallback(this.id); + + this._initialized = false; + this._triggerHandler = null; + this._hosts = null; + this._matchPatternSet = null; + this._visits = null; + } + }, + }, + ], + + /** + * Attach listeners to every browser window to detect location changes, and + * notify the trigger handler whenever we navigate to a URL with a hostname + * we're looking for. + */ + [ + "openURL", + { + id: "openURL", + _initialized: false, + _triggerHandler: null, + _hosts: null, + _matchPatternSet: null, + _visits: null, + + /* + * If the listener is already initialised, `init` will replace the trigger + * handler and add any new hosts to `this._hosts`. + */ + init(triggerHandler, hosts = [], patterns) { + if (!this._initialized) { + this.onLocationChange = this.onLocationChange.bind(this); + lazy.EveryWindow.registerCallback( + this.id, + win => { + if (!isPrivateWindow(win)) { + win.addEventListener("TabSelect", this.onTabSwitch); + win.gBrowser.addTabsProgressListener(this); + } + }, + win => { + if (!isPrivateWindow(win)) { + win.removeEventListener("TabSelect", this.onTabSwitch); + win.gBrowser.removeTabsProgressListener(this); + } + } + ); + + this._visits = new Map(); + this._initialized = true; + } + this._triggerHandler = triggerHandler; + if (patterns) { + this._matchPatternSet = createMatchPatternSet([ + ...(this._matchPatternSet ? this._matchPatternSet.patterns : []), + ...patterns, + ]); + } + if (this._hosts) { + hosts.forEach(h => this._hosts.add(h)); + } else { + this._hosts = new Set(hosts); // Clone the hosts to avoid unexpected behaviour + } + }, + + uninit() { + if (this._initialized) { + lazy.EveryWindow.unregisterCallback(this.id); + + this._initialized = false; + this._triggerHandler = null; + this._hosts = null; + this._matchPatternSet = null; + this._visits = null; + } + }, + + onLocationChange(aBrowser, aWebProgress, aRequest, aLocationURI, aFlags) { + // Some websites trigger redirect events after they finish loading even + // though the location remains the same. This results in onLocationChange + // events to be fired twice. + const isSameDocument = !!( + aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT + ); + if (aWebProgress.isTopLevel && !isSameDocument) { + const match = checkURLMatch( + aLocationURI, + { hosts: this._hosts, matchPatternSet: this._matchPatternSet }, + aRequest + ); + if (match) { + let visitsCount = (this._visits.get(match.url) || 0) + 1; + this._visits.set(match.url, visitsCount); + this._triggerHandler(aBrowser, { + id: this.id, + param: match, + context: { visitsCount }, + }); + } + } + }, + }, + ], + + /** + * Add an observer notification to notify the trigger handler whenever the user + * saves or updates a login via the login capture doorhanger. + */ + [ + "newSavedLogin", + { + _initialized: false, + _triggerHandler: null, + + /** + * If the listener is already initialised, `init` will replace the trigger + * handler. + */ + init(triggerHandler) { + if (!this._initialized) { + Services.obs.addObserver(this, "LoginStats:NewSavedPassword"); + Services.obs.addObserver(this, "LoginStats:LoginUpdateSaved"); + this._initialized = true; + } + this._triggerHandler = triggerHandler; + }, + + uninit() { + if (this._initialized) { + Services.obs.removeObserver(this, "LoginStats:NewSavedPassword"); + Services.obs.removeObserver(this, "LoginStats:LoginUpdateSaved"); + + this._initialized = false; + this._triggerHandler = null; + } + }, + + observe(aSubject, aTopic, aData) { + if (aSubject.currentURI.asciiHost === "accounts.firefox.com") { + // Don't notify about saved logins on the FxA login origin since this + // trigger is used to promote login Sync and getting a recommendation + // to enable Sync during the sign up process is a bad UX. + return; + } + + switch (aTopic) { + case "LoginStats:NewSavedPassword": { + this._triggerHandler(aSubject, { + id: "newSavedLogin", + context: { type: "save" }, + }); + break; + } + case "LoginStats:LoginUpdateSaved": { + this._triggerHandler(aSubject, { + id: "newSavedLogin", + context: { type: "update" }, + }); + break; + } + default: { + throw new Error(`Unexpected observer notification: ${aTopic}`); + } + } + }, + }, + ], + [ + "formAutofill", + { + id: "formAutofill", + _initialized: false, + _triggerHandler: null, + _triggerDelay: 10000, // 10 second delay before triggering + _topic: "formautofill-storage-changed", + _events: ["add", "update", "notifyUsed"] /** @see AutofillRecords */, + _collections: ["addresses", "creditCards"] /** @see AutofillRecords */, + + init(triggerHandler) { + if (!this._initialized) { + Services.obs.addObserver(this, this._topic); + this._initialized = true; + } + this._triggerHandler = triggerHandler; + }, + + uninit() { + if (this._initialized) { + Services.obs.removeObserver(this, this._topic); + this._initialized = false; + this._triggerHandler = null; + } + }, + + observe(subject, topic, data) { + const browser = + Services.wm.getMostRecentBrowserWindow()?.gBrowser.selectedBrowser; + if ( + !browser || + topic !== this._topic || + !subject.wrappedJSObject || + // Ignore changes caused by manual edits in the credit card/address + // managers in about:preferences. + browser.contentWindow?.gSubDialog?.dialogs.length + ) { + return; + } + let { sourceSync, collectionName } = subject.wrappedJSObject; + // Ignore changes from sync and changes to untracked collections. + if (sourceSync || !this._collections.includes(collectionName)) { + return; + } + if (this._events.includes(data)) { + let event = data; + let type = collectionName; + if (event === "notifyUsed") { + event = "use"; + } + if (type === "creditCards") { + type = "card"; + } + if (type === "addresses") { + type = "address"; + } + lazy.setTimeout(() => { + if ( + this._initialized && + // Make sure the browser still exists and is still selected. + browser.isConnectedAndReady && + browser === + Services.wm.getMostRecentBrowserWindow()?.gBrowser + .selectedBrowser + ) { + this._triggerHandler(browser, { + id: this.id, + context: { event, type }, + }); + } + }, this._triggerDelay); + } + }, + }, + ], + + [ + "contentBlocking", + { + _initialized: false, + _triggerHandler: null, + _events: [], + _sessionPageLoad: 0, + onLocationChange: null, + + init(triggerHandler, params, patterns) { + params.forEach(p => this._events.push(p)); + + if (!this._initialized) { + Services.obs.addObserver(this, "SiteProtection:ContentBlockingEvent"); + Services.obs.addObserver( + this, + "SiteProtection:ContentBlockingMilestone" + ); + this.onLocationChange = this._onLocationChange.bind(this); + lazy.EveryWindow.registerCallback( + this.id, + win => { + if (!isPrivateWindow(win)) { + win.gBrowser.addTabsProgressListener(this); + } + }, + win => { + if (!isPrivateWindow(win)) { + win.gBrowser.removeTabsProgressListener(this); + } + } + ); + + this._initialized = true; + } + this._triggerHandler = triggerHandler; + }, + + uninit() { + if (this._initialized) { + Services.obs.removeObserver( + this, + "SiteProtection:ContentBlockingEvent" + ); + Services.obs.removeObserver( + this, + "SiteProtection:ContentBlockingMilestone" + ); + lazy.EveryWindow.unregisterCallback(this.id); + this.onLocationChange = null; + this._initialized = false; + } + this._triggerHandler = null; + this._events = []; + this._sessionPageLoad = 0; + }, + + observe(aSubject, aTopic, aData) { + switch (aTopic) { + case "SiteProtection:ContentBlockingEvent": + const { browser, host, event } = aSubject.wrappedJSObject; + if (this._events.filter(e => (e & event) === e).length) { + this._triggerHandler(browser, { + id: "contentBlocking", + param: { + host, + type: event, + }, + context: { + pageLoad: this._sessionPageLoad, + }, + }); + } + break; + case "SiteProtection:ContentBlockingMilestone": + if (this._events.includes(aSubject.wrappedJSObject.event)) { + this._triggerHandler( + Services.wm.getMostRecentBrowserWindow().gBrowser + .selectedBrowser, + { + id: "contentBlocking", + context: { + pageLoad: this._sessionPageLoad, + }, + param: { + type: aSubject.wrappedJSObject.event, + }, + } + ); + } + break; + } + }, + + _onLocationChange( + aBrowser, + aWebProgress, + aRequest, + aLocationURI, + aFlags + ) { + // Some websites trigger redirect events after they finish loading even + // though the location remains the same. This results in onLocationChange + // events to be fired twice. + const isSameDocument = !!( + aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT + ); + if ( + ["http", "https"].includes(aLocationURI.scheme) && + aWebProgress.isTopLevel && + !isSameDocument + ) { + this._sessionPageLoad += 1; + } + }, + }, + ], + + [ + "captivePortalLogin", + { + id: "captivePortalLogin", + _initialized: false, + _triggerHandler: null, + + _shouldShowCaptivePortalVPNPromo() { + return lazy.BrowserUtils.shouldShowVPNPromo(); + }, + + init(triggerHandler) { + if (!this._initialized) { + Services.obs.addObserver(this, "captive-portal-login-success"); + this._initialized = true; + } + this._triggerHandler = triggerHandler; + }, + + observe(aSubject, aTopic, aData) { + switch (aTopic) { + case "captive-portal-login-success": + const browser = Services.wm.getMostRecentBrowserWindow(); + // The check is here rather than in init because some + // folks leave their browsers running for a long time, + // eg from before leaving on a plane trip to after landing + // in the new destination, and the current region may have + // changed since init time. + if (browser && this._shouldShowCaptivePortalVPNPromo()) { + this._triggerHandler(browser.gBrowser.selectedBrowser, { + id: this.id, + }); + } + break; + } + }, + + uninit() { + if (this._initialized) { + this._triggerHandler = null; + this._initialized = false; + Services.obs.removeObserver(this, "captive-portal-login-success"); + } + }, + }, + ], + + [ + "preferenceObserver", + { + id: "preferenceObserver", + _initialized: false, + _triggerHandler: null, + _observedPrefs: [], + + init(triggerHandler, prefs) { + if (!this._initialized) { + this._triggerHandler = triggerHandler; + this._initialized = true; + } + prefs.forEach(pref => { + this._observedPrefs.push(pref); + Services.prefs.addObserver(pref, this); + }); + }, + + observe(aSubject, aTopic, aData) { + switch (aTopic) { + case "nsPref:changed": + const browser = Services.wm.getMostRecentBrowserWindow(); + if (browser && this._observedPrefs.includes(aData)) { + this._triggerHandler(browser.gBrowser.selectedBrowser, { + id: this.id, + param: { + type: aData, + }, + }); + } + break; + } + }, + + uninit() { + if (this._initialized) { + this._observedPrefs.forEach(pref => + Services.prefs.removeObserver(pref, this) + ); + this._initialized = false; + this._triggerHandler = null; + this._observedPrefs = []; + } + }, + }, + ], + [ + "nthTabClosed", + { + id: "nthTabClosed", + _initialized: false, + _triggerHandler: null, + // Number of tabs the user closed this session + _closedTabs: 0, + + init(triggerHandler) { + this._triggerHandler = triggerHandler; + if (!this._initialized) { + lazy.EveryWindow.registerCallback( + this.id, + win => { + win.addEventListener("TabClose", this); + }, + win => { + win.removeEventListener("TabClose", this); + } + ); + this._initialized = true; + } + }, + handleEvent(event) { + if (this._initialized) { + if (!event.target.ownerGlobal.gBrowser) { + return; + } + const { gBrowser } = event.target.ownerGlobal; + this._closedTabs++; + this._triggerHandler(gBrowser.selectedBrowser, { + id: this.id, + context: { tabsClosedCount: this._closedTabs }, + }); + } + }, + uninit() { + if (this._initialized) { + lazy.EveryWindow.unregisterCallback(this.id); + this._initialized = false; + this._triggerHandler = null; + this._closedTabs = 0; + } + }, + }, + ], + [ + "activityAfterIdle", + { + id: "activityAfterIdle", + _initialized: false, + _triggerHandler: null, + _idleService: null, + // Optimization - only report idle state after one minute of idle time. + // This represents a minimum idleForMilliseconds of 60000. + _idleThreshold: 60, + _idleSince: null, + _quietSince: null, + _awaitingVisibilityChange: false, + // Fire the trigger 2 seconds after activity resumes to ensure user is + // actively using the browser when it fires. + _triggerDelay: 2000, + _triggerTimeout: null, + // We may get an idle notification immediately after waking from sleep. + // The idle time in such a case will be the amount of time since the last + // user interaction, which was before the computer went to sleep. We want + // to ignore them in that case, so we ignore idle notifications that + // happen within 1 second of the last wake notification. + _wakeDelay: 1000, + _lastWakeTime: null, + _listenedEvents: ["visibilitychange", "TabClose", "TabAttrModified"], + // When the OS goes to sleep or the process is suspended, we want to drop + // the idle time, since the time between sleep and wake is expected to be + // very long (e.g. overnight). Otherwise, this would trigger on the first + // activity after waking/resuming, counting sleep as idle time. This + // basically means each session starts with a fresh idle time. + _observedTopics: [ + "sleep_notification", + "suspend_process_notification", + "wake_notification", + "resume_process_notification", + "mac_app_activate", + ], + + get _isVisible() { + return [...Services.wm.getEnumerator("navigator:browser")].some( + win => !win.closed && !win.document?.hidden + ); + }, + get _soundPlaying() { + return [...Services.wm.getEnumerator("navigator:browser")].some(win => + win.gBrowser?.tabs.some(tab => tab.soundPlaying) + ); + }, + init(triggerHandler) { + this._triggerHandler = triggerHandler; + // Instantiate this here instead of with a lazy service getter so we can + // stub it in tests (otherwise we'd have to wait up to 6 minutes for an + // idle notification in certain test environments). + if (!this._idleService) { + this._idleService = Cc[ + "@mozilla.org/widget/useridleservice;1" + ].getService(Ci.nsIUserIdleService); + } + if ( + !this._initialized && + !lazy.PrivateBrowsingUtils.permanentPrivateBrowsing + ) { + this._idleService.addIdleObserver(this, this._idleThreshold); + for (let topic of this._observedTopics) { + Services.obs.addObserver(this, topic); + } + lazy.EveryWindow.registerCallback( + this.id, + win => { + for (let ev of this._listenedEvents) { + win.addEventListener(ev, this); + } + }, + win => { + for (let ev of this._listenedEvents) { + win.removeEventListener(ev, this); + } + } + ); + if (!this._soundPlaying) { + this._quietSince = Date.now(); + } + this._initialized = true; + this.log("Initialized: ", { + idleTime: this._idleService.idleTime, + quietSince: this._quietSince, + }); + } + }, + observe(subject, topic, data) { + if (this._initialized) { + this.log("Heard observer notification: ", { + subject, + topic, + data, + idleTime: this._idleService.idleTime, + idleSince: this._idleSince, + quietSince: this._quietSince, + lastWakeTime: this._lastWakeTime, + }); + switch (topic) { + case "idle": + const now = Date.now(); + // If the idle notification is within 1 second of the last wake + // notification, ignore it. We do this to avoid counting time the + // computer spent asleep as "idle time" + const isImmediatelyAfterWake = + this._lastWakeTime && + now - this._lastWakeTime < this._wakeDelay; + if (!isImmediatelyAfterWake) { + this._idleSince = now - subject.idleTime; + } + break; + case "active": + // Trigger when user returns from being idle. + if (this._isVisible) { + this._onActive(); + this._idleSince = null; + this._lastWakeTime = null; + } else if (this._idleSince) { + // If the window is not visible, we want to wait until it is + // visible before triggering. + this._awaitingVisibilityChange = true; + } + break; + // OS/process notifications + case "wake_notification": + case "resume_process_notification": + case "mac_app_activate": + this._lastWakeTime = Date.now(); + // Fall through to reset idle time. + default: + this._idleSince = null; + } + } + }, + handleEvent(event) { + if (this._initialized) { + switch (event.type) { + case "visibilitychange": + if (this._awaitingVisibilityChange && this._isVisible) { + this._onActive(); + this._idleSince = null; + this._lastWakeTime = null; + this._awaitingVisibilityChange = false; + } + break; + case "TabAttrModified": + // Listen for DOMAudioPlayback* events. + if (!event.detail?.changed?.includes("soundplaying")) { + break; + } + // fall through + case "TabClose": + this.log("Tab sound changed: ", { + event, + idleTime: this._idleService.idleTime, + idleSince: this._idleSince, + quietSince: this._quietSince, + }); + // Maybe update time if a tab closes with sound playing. + if (this._soundPlaying) { + this._quietSince = null; + } else if (!this._quietSince) { + this._quietSince = Date.now(); + } + } + } + }, + _onActive() { + this.log("User is active: ", { + idleTime: this._idleService.idleTime, + idleSince: this._idleSince, + quietSince: this._quietSince, + lastWakeTime: this._lastWakeTime, + }); + if (this._idleSince && this._quietSince) { + const win = Services.wm.getMostRecentBrowserWindow(); + if (win && !isPrivateWindow(win) && !this._triggerTimeout) { + // Number of ms since the last user interaction/audio playback + const idleForMilliseconds = + Date.now() - Math.min(this._idleSince, this._quietSince); + this._triggerTimeout = lazy.setTimeout(() => { + this._triggerHandler(win.gBrowser.selectedBrowser, { + id: this.id, + context: { idleForMilliseconds }, + }); + this._triggerTimeout = null; + }, this._triggerDelay); + } + } + }, + uninit() { + if (this._initialized) { + this._idleService.removeIdleObserver(this, this._idleThreshold); + for (let topic of this._observedTopics) { + Services.obs.removeObserver(this, topic); + } + lazy.EveryWindow.unregisterCallback(this.id); + lazy.clearTimeout(this._triggerTimeout); + this._triggerTimeout = null; + this._initialized = false; + this._triggerHandler = null; + this._idleSince = null; + this._quietSince = null; + this._lastWakeTime = null; + this._awaitingVisibilityChange = false; + this.log("Uninitialized"); + } + }, + log(...args) { + lazy.log.debug("Idle trigger :>>", ...args); + }, + + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + }, + ], + [ + "cookieBannerDetected", + { + id: "cookieBannerDetected", + _initialized: false, + _triggerHandler: null, + + init(triggerHandler) { + this._triggerHandler = triggerHandler; + if (!this._initialized) { + lazy.EveryWindow.registerCallback( + this.id, + win => { + win.addEventListener("cookiebannerdetected", this); + }, + win => { + win.removeEventListener("cookiebannerdetected", this); + } + ); + this._initialized = true; + } + }, + handleEvent(event) { + if (this._initialized) { + const win = event.target || Services.wm.getMostRecentBrowserWindow(); + if (!win) { + return; + } + this._triggerHandler(win.gBrowser.selectedBrowser, { + id: this.id, + }); + } + }, + uninit() { + if (this._initialized) { + lazy.EveryWindow.unregisterCallback(this.id); + this._initialized = false; + this._triggerHandler = null; + } + }, + }, + ], +]); + +const EXPORTED_SYMBOLS = ["ASRouterTriggerListeners"]; diff --git a/browser/components/newtab/lib/AWScreenUtils.jsm b/browser/components/newtab/lib/AWScreenUtils.jsm new file mode 100644 index 0000000000..93da92f630 --- /dev/null +++ b/browser/components/newtab/lib/AWScreenUtils.jsm @@ -0,0 +1,79 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyModuleGetters(lazy, { + ASRouter: "resource://activity-stream/lib/ASRouter.jsm", + ASRouterTargeting: "resource://activity-stream/lib/ASRouterTargeting.jsm", +}); + +const AWScreenUtils = { + /** + * Filter the given screens in place with a predicate. + * + * @param {object[]} screens - The screens to filter. + * @param {Function} callback - The predicate for filtering the screens. + */ + async removeScreens(screens, callback) { + for (let i = 0; i < screens?.length; i++) { + if (await callback(screens[i], i)) { + screens.splice(i--, 1); + } + } + }, + /** + * Given a JEXL expression, returns the evaluation of the expression or returns + * true if the expression did not evaluate successfully + * @param {string} targeting - The JEXL expression that will be evaluated + * @returns {boolean} + */ + async evaluateScreenTargeting(targeting) { + const result = await lazy.ASRouter.evaluateExpression({ + expression: targeting, + context: lazy.ASRouterTargeting.Environment, + }); + if (result?.evaluationStatus?.success) { + return result.evaluationStatus.result; + } + + return true; + }, + /** + * Filter out screens whose targeting do not match. + * + * Given an array of screens, each screen will have it's `targeting` property + * evaluated, and removed if it's targeting evaluates to false + * + * @param {object[]} screens - An array of screens that will be looped + * through to be evaluated for removal + * @returns {object[]} - A new array containing the screens that were not removed + */ + async evaluateTargetingAndRemoveScreens(screens) { + const filteredScreens = [...screens]; + await this.removeScreens(filteredScreens, async screen => { + if (screen.targeting === undefined) { + // Don't remove the screen if we don't have a targeting property + return false; + } + + const result = await this.evaluateScreenTargeting(screen.targeting); + // Flipping the value because a true evaluation means we + // don't want to remove the screen, while false means we do + return !result; + }); + + return filteredScreens; + }, + + async addScreenImpression(screen) { + await lazy.ASRouter.addScreenImpression(screen); + }, +}; + +const EXPORTED_SYMBOLS = ["AWScreenUtils"]; diff --git a/browser/components/newtab/lib/AboutPreferences.jsm b/browser/components/newtab/lib/AboutPreferences.jsm new file mode 100644 index 0000000000..26eb4d5efe --- /dev/null +++ b/browser/components/newtab/lib/AboutPreferences.jsm @@ -0,0 +1,312 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const HTML_NS = "http://www.w3.org/1999/xhtml"; +const PREFERENCES_LOADED_EVENT = "home-pane-loaded"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", +}); + +// These "section" objects are formatted in a way to be similar to the ones from +// SectionsManager to construct the preferences view. +const PREFS_BEFORE_SECTIONS = () => [ + { + id: "search", + pref: { + feed: "showSearch", + titleString: "home-prefs-search-header", + }, + icon: "chrome://global/skin/icons/search-glass.svg", + }, + { + id: "topsites", + pref: { + feed: "feeds.topsites", + titleString: "home-prefs-shortcuts-header", + descString: "home-prefs-shortcuts-description", + get nestedPrefs() { + return Services.prefs.getBoolPref("browser.topsites.useRemoteSetting") + ? [ + { + name: "showSponsoredTopSites", + titleString: "home-prefs-shortcuts-by-option-sponsored", + eventSource: "SPONSORED_TOP_SITES", + }, + ] + : []; + }, + }, + icon: "chrome://browser/skin/topsites.svg", + maxRows: 4, + rowsPref: "topSitesRows", + eventSource: "TOP_SITES", + }, +]; + +const PREFS_AFTER_SECTIONS = () => [ + { + id: "snippets", + pref: { + feed: "feeds.snippets", + titleString: "home-prefs-snippets-header", + descString: "home-prefs-snippets-description-new", + }, + icon: "chrome://global/skin/icons/info.svg", + eventSource: "SNIPPETS", + }, +]; + +class AboutPreferences { + init() { + Services.obs.addObserver(this, PREFERENCES_LOADED_EVENT); + } + + uninit() { + Services.obs.removeObserver(this, PREFERENCES_LOADED_EVENT); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.uninit(); + break; + case at.SETTINGS_OPEN: + action._target.browser.ownerGlobal.openPreferences("paneHome"); + break; + // This is used to open the web extension settings page for an extension + case at.OPEN_WEBEXT_SETTINGS: + action._target.browser.ownerGlobal.BrowserOpenAddonsMgr( + `addons://detail/${encodeURIComponent(action.data)}` + ); + break; + } + } + + handleDiscoverySettings(sections) { + // Deep copy object to not modify original Sections state in store + let sectionsCopy = JSON.parse(JSON.stringify(sections)); + sectionsCopy.forEach(obj => { + if (obj.id === "topstories") { + obj.rowsPref = ""; + } + }); + return sectionsCopy; + } + + setupUserEvent(element, eventSource) { + element.addEventListener("command", e => { + const { checked } = e.target; + if (typeof checked === "boolean") { + this.store.dispatch( + ac.UserEvent({ + event: "PREF_CHANGED", + source: eventSource, + value: { status: checked, menu_source: "ABOUT_PREFERENCES" }, + }) + ); + } + }); + } + + observe(window) { + const discoveryStreamConfig = this.store.getState().DiscoveryStream.config; + let sections = this.store.getState().Sections; + + if (discoveryStreamConfig.enabled) { + sections = this.handleDiscoverySettings(sections); + } + + const featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + + this.renderPreferences(window, [ + ...PREFS_BEFORE_SECTIONS(featureConfig), + ...sections, + ...PREFS_AFTER_SECTIONS(featureConfig), + ]); + } + + /** + * Render preferences to an about:preferences content window with the provided + * preferences structure. + */ + renderPreferences({ document, Preferences, gHomePane }, prefStructure) { + // Helper to create a new element and append it + const createAppend = (tag, parent, options) => + parent.appendChild(document.createXULElement(tag, options)); + + // Helper to get fluentIDs sometimes encase in an object + const getString = message => + typeof message !== "object" ? message : message.id; + + // Helper to link a UI element to a preference for updating + const linkPref = (element, name, type) => { + const fullPref = `browser.newtabpage.activity-stream.${name}`; + element.setAttribute("preference", fullPref); + Preferences.add({ id: fullPref, type }); + + // Prevent changing the UI if the preference can't be changed + element.disabled = Preferences.get(fullPref).locked; + }; + + // Insert a new group immediately after the homepage one + const homeGroup = document.getElementById("homepageGroup"); + const contentsGroup = homeGroup.insertAdjacentElement( + "afterend", + homeGroup.cloneNode() + ); + contentsGroup.id = "homeContentsGroup"; + contentsGroup.setAttribute("data-subcategory", "contents"); + const homeHeader = createAppend("label", contentsGroup).appendChild( + document.createElementNS(HTML_NS, "h2") + ); + document.l10n.setAttributes(homeHeader, "home-prefs-content-header2"); + + const homeDescription = createAppend("description", contentsGroup); + document.l10n.setAttributes( + homeDescription, + "home-prefs-content-description2" + ); + + // Add preferences for each section + prefStructure.forEach(sectionData => { + const { + id, + pref: prefData, + icon = "webextension", + maxRows, + rowsPref, + shouldHidePref, + eventSource, + } = sectionData; + const { + feed: name, + titleString = {}, + descString, + nestedPrefs = [], + } = prefData || {}; + + // Don't show any sections that we don't want to expose in preferences UI + if (shouldHidePref) { + return; + } + + // Use full icon spec for certain protocols or fall back to packaged icon + const iconUrl = !icon.search(/^(chrome|moz-extension|resource):/) + ? icon + : `chrome://activity-stream/content/data/content/assets/glyph-${icon}-16.svg`; + + // Add the main preference for turning on/off a section + const sectionVbox = createAppend("vbox", contentsGroup); + sectionVbox.setAttribute("data-subcategory", id); + const checkbox = createAppend("checkbox", sectionVbox); + checkbox.classList.add("section-checkbox"); + checkbox.setAttribute("src", iconUrl); + // Setup a user event if we have an event source for this pref. + if (eventSource) { + this.setupUserEvent(checkbox, eventSource); + } + document.l10n.setAttributes( + checkbox, + getString(titleString), + titleString.values + ); + + linkPref(checkbox, name, "bool"); + + // Specially add a link for stories + if (id === "topstories") { + const sponsoredHbox = createAppend("hbox", sectionVbox); + sponsoredHbox.setAttribute("align", "center"); + sponsoredHbox.appendChild(checkbox); + checkbox.classList.add("tail-with-learn-more"); + + const link = createAppend("label", sponsoredHbox, { is: "text-link" }); + link.classList.add("learn-sponsored"); + link.setAttribute("href", sectionData.pref.learnMore.link.href); + document.l10n.setAttributes(link, sectionData.pref.learnMore.link.id); + } + + // Add more details for the section (e.g., description, more prefs) + const detailVbox = createAppend("vbox", sectionVbox); + detailVbox.classList.add("indent"); + if (descString) { + const label = createAppend("label", detailVbox); + label.classList.add("indent"); + document.l10n.setAttributes( + label, + getString(descString), + descString.values + ); + + // Add a rows dropdown if we have a pref to control and a maximum + if (rowsPref && maxRows) { + const detailHbox = createAppend("hbox", detailVbox); + detailHbox.setAttribute("align", "center"); + label.setAttribute("flex", 1); + detailHbox.appendChild(label); + + // Add box so the search tooltip is positioned correctly + const tooltipBox = createAppend("hbox", detailHbox); + + // Add appropriate number of localized entries to the dropdown + const menulist = createAppend("menulist", tooltipBox); + menulist.setAttribute("crop", "none"); + const menupopup = createAppend("menupopup", menulist); + for (let num = 1; num <= maxRows; num++) { + const item = createAppend("menuitem", menupopup); + document.l10n.setAttributes( + item, + "home-prefs-sections-rows-option", + { num } + ); + item.setAttribute("value", num); + } + linkPref(menulist, rowsPref, "int"); + } + } + + const subChecks = []; + const fullName = `browser.newtabpage.activity-stream.${sectionData.pref.feed}`; + const pref = Preferences.get(fullName); + + // Add a checkbox pref for any nested preferences + nestedPrefs.forEach(nested => { + const subcheck = createAppend("checkbox", detailVbox); + // Setup a user event if we have an event source for this pref. + if (nested.eventSource) { + this.setupUserEvent(subcheck, nested.eventSource); + } + subcheck.classList.add("indent"); + document.l10n.setAttributes(subcheck, nested.titleString); + linkPref(subcheck, nested.name, "bool"); + subChecks.push(subcheck); + subcheck.disabled = !pref._value; + subcheck.hidden = nested.hidden; + }); + + // Disable any nested checkboxes if the parent pref is not enabled. + pref.on("change", () => { + subChecks.forEach(subcheck => { + subcheck.disabled = !pref._value; + }); + }); + }); + + // Update the visibility of the Restore Defaults btn based on checked prefs + gHomePane.toggleRestoreDefaultsBtn(); + } +} + +const EXPORTED_SYMBOLS = ["AboutPreferences", "PREFERENCES_LOADED_EVENT"]; diff --git a/browser/components/newtab/lib/ActivityStream.jsm b/browser/components/newtab/lib/ActivityStream.jsm new file mode 100644 index 0000000000..bbfe6b3c72 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStream.jsm @@ -0,0 +1,758 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DEFAULT_SITES: "resource://activity-stream/lib/DefaultSites.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", +}); + +// NB: Eagerly load modules that will be loaded/constructed/initialized in the +// common case to avoid the overhead of wrapping and detecting lazy loading. +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +ChromeUtils.defineModuleGetter( + lazy, + "AboutPreferences", + "resource://activity-stream/lib/AboutPreferences.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "DefaultPrefs", + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "NewTabInit", + "resource://activity-stream/lib/NewTabInit.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "SectionsFeed", + "resource://activity-stream/lib/SectionsManager.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "RecommendationProvider", + "resource://activity-stream/lib/RecommendationProvider.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "PlacesFeed", + "resource://activity-stream/lib/PlacesFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "PrefsFeed", + "resource://activity-stream/lib/PrefsFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "Store", + "resource://activity-stream/lib/Store.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "SystemTickFeed", + "resource://activity-stream/lib/SystemTickFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "TelemetryFeed", + "resource://activity-stream/lib/TelemetryFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "FaviconFeed", + "resource://activity-stream/lib/FaviconFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "TopSitesFeed", + "resource://activity-stream/lib/TopSitesFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "TopStoriesFeed", + "resource://activity-stream/lib/TopStoriesFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "HighlightsFeed", + "resource://activity-stream/lib/HighlightsFeed.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "DiscoveryStreamFeed", + "resource://activity-stream/lib/DiscoveryStreamFeed.jsm" +); + +const REGION_BASIC_CONFIG = + "browser.newtabpage.activity-stream.discoverystream.region-basic-config"; + +// Determine if spocs should be shown for a geo/locale +function showSpocs({ geo }) { + const spocsGeoString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionSpocsConfig") || ""; + const spocsGeo = spocsGeoString.split(",").map(s => s.trim()); + return spocsGeo.includes(geo); +} + +// Configure default Activity Stream prefs with a plain `value` or a `getValue` +// that computes a value. A `value_local_dev` is used for development defaults. +const PREFS_CONFIG = new Map([ + [ + "default.sites", + { + title: + "Comma-separated list of default top sites to fill in behind visited sites", + getValue: ({ geo }) => + lazy.DEFAULT_SITES.get(lazy.DEFAULT_SITES.has(geo) ? geo : ""), + }, + ], + [ + "feeds.section.topstories.options", + { + title: "Configuration options for top stories feed", + // This is a dynamic pref as it depends on the feed being shown or not + getValue: args => + JSON.stringify({ + api_key_pref: "extensions.pocket.oAuthConsumerKey", + // Use the opposite value as what default value the feed would have used + hidden: !PREFS_CONFIG.get("feeds.system.topstories").getValue(args), + provider_icon: "chrome://global/skin/icons/pocket.svg", + provider_name: "Pocket", + read_more_endpoint: + "https://getpocket.com/explore/trending?src=fx_new_tab", + stories_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=${ + args.locale + }&feed_variant=${ + showSpocs(args) ? "default_spocs_on" : "default_spocs_off" + }`, + stories_referrer: "https://getpocket.com/recommendations", + topics_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/trending-topics?version=2&consumer_key=$apiKey&locale_lang=${args.locale}`, + show_spocs: showSpocs(args), + }), + }, + ], + [ + "feeds.topsites", + { + title: "Displays Top Sites on the New Tab Page", + value: true, + }, + ], + [ + "hideTopSitesTitle", + { + title: + "Hide the top sites section's title, including the section and collapse icons", + value: false, + }, + ], + [ + "showSponsored", + { + title: + "Show sponsored cards in spoc experiment (show_spocs in topstories.options has to be set to true as well)", + value: true, + }, + ], + [ + "showSponsoredTopSites", + { + title: "Show sponsored top sites", + value: true, + }, + ], + [ + "pocketCta", + { + title: "Pocket cta and button for logged out users.", + value: JSON.stringify({ + cta_button: "", + cta_text: "", + cta_url: "", + use_cta: false, + }), + }, + ], + [ + "showSearch", + { + title: "Show the Search bar", + value: true, + }, + ], + [ + "feeds.snippets", + { + title: "Show snippets on activity stream", + value: false, + }, + ], + [ + "topSitesRows", + { + title: "Number of rows of Top Sites to display", + value: 1, + }, + ], + [ + "telemetry", + { + title: "Enable system error and usage data collection", + value: true, + value_local_dev: false, + }, + ], + [ + "telemetry.ut.events", + { + title: "Enable Unified Telemetry event data collection", + value: AppConstants.EARLY_BETA_OR_EARLIER, + value_local_dev: false, + }, + ], + [ + "telemetry.structuredIngestion.endpoint", + { + title: "Structured Ingestion telemetry server endpoint", + value: "https://incoming.telemetry.mozilla.org/submit", + }, + ], + [ + "section.highlights.includeVisited", + { + title: + "Boolean flag that decides whether or not to show visited pages in highlights.", + value: true, + }, + ], + [ + "section.highlights.includeBookmarks", + { + title: + "Boolean flag that decides whether or not to show bookmarks in highlights.", + value: true, + }, + ], + [ + "section.highlights.includePocket", + { + title: + "Boolean flag that decides whether or not to show saved Pocket stories in highlights.", + value: true, + }, + ], + [ + "section.highlights.includeDownloads", + { + title: + "Boolean flag that decides whether or not to show saved recent Downloads in highlights.", + value: true, + }, + ], + [ + "section.highlights.rows", + { + title: "Number of rows of Highlights to display", + value: 1, + }, + ], + [ + "section.topstories.rows", + { + title: "Number of rows of Top Stories to display", + value: 1, + }, + ], + [ + "sectionOrder", + { + title: "The rendering order for the sections", + value: "topsites,topstories,highlights", + }, + ], + [ + "improvesearch.noDefaultSearchTile", + { + title: "Remove tiles that are the same as the default search", + value: true, + }, + ], + [ + "improvesearch.topSiteSearchShortcuts.searchEngines", + { + title: + "An ordered, comma-delimited list of search shortcuts that we should try and pin", + // This pref is dynamic as the shortcuts vary depending on the region + getValue: ({ geo }) => { + if (!geo) { + return ""; + } + const searchShortcuts = []; + if (geo === "CN") { + searchShortcuts.push("baidu"); + } else if (["BY", "KZ", "RU", "TR"].includes(geo)) { + searchShortcuts.push("yandex"); + } else { + searchShortcuts.push("google"); + } + if (["DE", "FR", "GB", "IT", "JP", "US"].includes(geo)) { + searchShortcuts.push("amazon"); + } + return searchShortcuts.join(","); + }, + }, + ], + [ + "improvesearch.topSiteSearchShortcuts.havePinned", + { + title: + "A comma-delimited list of search shortcuts that have previously been pinned", + value: "", + }, + ], + [ + "asrouter.devtoolsEnabled", + { + title: "Are the asrouter devtools enabled?", + value: false, + }, + ], + [ + "asrouter.providers.onboarding", + { + title: "Configuration for onboarding provider", + value: JSON.stringify({ + id: "onboarding", + type: "local", + localProvider: "OnboardingMessageProvider", + enabled: true, + // Block specific messages from this local provider + exclude: [], + }), + }, + ], + // See browser/app/profile/firefox.js for other ASR preferences. They must be defined there to enable roll-outs. + [ + "discoverystream.flight.blocks", + { + title: "Track flight blocks", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "discoverystream.config", + { + title: "Configuration for the new pocket new tab", + getValue: ({ geo, locale }) => { + return JSON.stringify({ + api_key_pref: "extensions.pocket.oAuthConsumerKey", + collapsible: true, + enabled: true, + show_spocs: showSpocs({ geo }), + hardcoded_layout: true, + // This is currently an exmple layout used for dev purposes. + layout_endpoint: + "https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic", + }); + }, + }, + ], + [ + "discoverystream.endpoints", + { + title: + "Endpoint prefixes (comma-separated) that are allowed to be requested", + value: + "https://getpocket.cdn.mozilla.net/,https://firefox-api-proxy.cdn.mozilla.net/,https://spocs.getpocket.com/", + }, + ], + [ + "discoverystream.isCollectionDismissible", + { + title: "Allows Pocket story collections to be dismissed", + value: false, + }, + ], + [ + "discoverystream.onboardingExperience.dismissed", + { + title: "Allows the user to dismiss the new Pocket onboarding experience", + skipBroadcast: true, + alsoToPreloaded: true, + value: false, + }, + ], + [ + "discoverystream.region-basic-layout", + { + title: "Decision to use basic layout based on region.", + getValue: ({ geo }) => { + const preffedRegionsString = + Services.prefs.getStringPref(REGION_BASIC_CONFIG) || ""; + // If no regions are set to basic, + // we don't need to bother checking against the region. + // We are also not concerned if geo is not set, + // because stories are going to be empty until we have geo. + if (!preffedRegionsString) { + return false; + } + const preffedRegions = preffedRegionsString + .split(",") + .map(s => s.trim()); + + return preffedRegions.includes(geo); + }, + }, + ], + [ + "discoverystream.spoc.impressions", + { + title: "Track spoc impressions", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "discoverystream.endpointSpocsClear", + { + title: + "Endpoint for when a user opts-out of sponsored content to delete the user's data from the ad server.", + value: "https://spocs.getpocket.com/user", + }, + ], + [ + "discoverystream.rec.impressions", + { + title: "Track rec impressions", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "showRecentSaves", + { + title: "Control whether a user wants recent saves visible on Newtab", + value: true, + }, + ], +]); + +// Array of each feed's FEEDS_CONFIG factory and values to add to PREFS_CONFIG +const FEEDS_DATA = [ + { + name: "aboutpreferences", + factory: () => new lazy.AboutPreferences(), + title: "about:preferences rendering", + value: true, + }, + { + name: "newtabinit", + factory: () => new lazy.NewTabInit(), + title: "Sends a copy of the state to each new tab that is opened", + value: true, + }, + { + name: "places", + factory: () => new lazy.PlacesFeed(), + title: "Listens for and relays various Places-related events", + value: true, + }, + { + name: "prefs", + factory: () => new lazy.PrefsFeed(PREFS_CONFIG), + title: "Preferences", + value: true, + }, + { + name: "sections", + factory: () => new lazy.SectionsFeed(), + title: "Manages sections", + value: true, + }, + { + name: "section.highlights", + factory: () => new lazy.HighlightsFeed(), + title: "Fetches content recommendations from places db", + value: false, + }, + { + name: "system.topstories", + factory: () => + new lazy.TopStoriesFeed(PREFS_CONFIG.get("discoverystream.config")), + title: + "System pref that fetches content recommendations from a configurable content provider", + // Dynamically determine if Pocket should be shown for a geo / locale + getValue: ({ geo, locale }) => { + // If we don't have geo, we don't want to flash the screen with stories while geo loads. + // Best to display nothing until geo is ready. + if (!geo) { + return false; + } + const preffedRegionsBlockString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionStoriesBlock") || + ""; + const preffedRegionsString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionStoriesConfig") || + ""; + const preffedLocaleListString = + lazy.NimbusFeatures.pocketNewtab.getVariable("localeListConfig") || ""; + const preffedBlockRegions = preffedRegionsBlockString + .split(",") + .map(s => s.trim()); + const preffedRegions = preffedRegionsString.split(",").map(s => s.trim()); + const preffedLocales = preffedLocaleListString + .split(",") + .map(s => s.trim()); + const locales = { + US: ["en-CA", "en-GB", "en-US"], + CA: ["en-CA", "en-GB", "en-US"], + GB: ["en-CA", "en-GB", "en-US"], + AU: ["en-CA", "en-GB", "en-US"], + NZ: ["en-CA", "en-GB", "en-US"], + IN: ["en-CA", "en-GB", "en-US"], + IE: ["en-CA", "en-GB", "en-US"], + ZA: ["en-CA", "en-GB", "en-US"], + CH: ["de"], + BE: ["de"], + DE: ["de"], + AT: ["de"], + IT: ["it"], + FR: ["fr"], + ES: ["es-ES"], + PL: ["pl"], + JP: ["ja", "ja-JP-mac"], + }[geo]; + + const regionBlocked = preffedBlockRegions.includes(geo); + const localeEnabled = locale && preffedLocales.includes(locale); + const regionEnabled = + preffedRegions.includes(geo) && !!locales && locales.includes(locale); + return !regionBlocked && (localeEnabled || regionEnabled); + }, + }, + { + name: "systemtick", + factory: () => new lazy.SystemTickFeed(), + title: "Produces system tick events to periodically check for data expiry", + value: true, + }, + { + name: "telemetry", + factory: () => new lazy.TelemetryFeed(), + title: "Relays telemetry-related actions to PingCentre", + value: true, + }, + { + name: "favicon", + factory: () => new lazy.FaviconFeed(), + title: "Fetches tippy top manifests from remote service", + value: true, + }, + { + name: "system.topsites", + factory: () => new lazy.TopSitesFeed(), + title: "Queries places and gets metadata for Top Sites section", + value: true, + }, + { + name: "recommendationprovider", + factory: () => new lazy.RecommendationProvider(), + title: "Handles setup and interaction for the personality provider", + value: true, + }, + { + name: "discoverystreamfeed", + factory: () => new lazy.DiscoveryStreamFeed(), + title: "Handles new pocket ui for the new tab page", + value: true, + }, +]; + +const FEEDS_CONFIG = new Map(); +for (const config of FEEDS_DATA) { + const pref = `feeds.${config.name}`; + FEEDS_CONFIG.set(pref, config.factory); + PREFS_CONFIG.set(pref, config); +} + +class ActivityStream { + /** + * constructor - Initializes an instance of ActivityStream + */ + constructor() { + this.initialized = false; + this.store = new lazy.Store(); + this.feeds = FEEDS_CONFIG; + this._defaultPrefs = new lazy.DefaultPrefs(PREFS_CONFIG); + } + + init() { + try { + this._updateDynamicPrefs(); + this._defaultPrefs.init(); + Services.obs.addObserver(this, "intl:app-locales-changed"); + + // Look for outdated user pref values that might have been accidentally + // persisted when restoring the original pref value at the end of an + // experiment across versions with a different default value. + const DS_CONFIG = + "browser.newtabpage.activity-stream.discoverystream.config"; + if ( + Services.prefs.prefHasUserValue(DS_CONFIG) && + [ + // Firefox 66 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.com/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + // Firefox 67 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + // Firefox 68 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","collapsible":true,"enabled":false,"show_spocs":true,"hardcoded_layout":true,"personalized":false,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + ].includes(Services.prefs.getStringPref(DS_CONFIG)) + ) { + Services.prefs.clearUserPref(DS_CONFIG); + } + + // Hook up the store and let all feeds and pages initialize + this.store.init( + this.feeds, + ac.BroadcastToContent({ + type: at.INIT, + data: { + locale: this.locale, + }, + meta: { + isStartup: true, + }, + }), + { type: at.UNINIT } + ); + + this.initialized = true; + } catch (e) { + // TelemetryFeed could be unavailable if the telemetry is disabled, or + // the telemetry feed is not yet initialized. + const telemetryFeed = this.store.feeds.get("feeds.telemetry"); + if (telemetryFeed) { + telemetryFeed.handleUndesiredEvent({ + data: { event: "ADDON_INIT_FAILED" }, + }); + } + throw e; + } + } + + /** + * Check if an old pref has a custom value to migrate. Clears the pref so that + * it's the default after migrating (to avoid future need to migrate). + * + * @param oldPrefName {string} Pref to check and migrate + * @param cbIfNotDefault {function} Callback that gets the current pref value + */ + _migratePref(oldPrefName, cbIfNotDefault) { + // Nothing to do if the user doesn't have a custom value + if (!Services.prefs.prefHasUserValue(oldPrefName)) { + return; + } + + // Figure out what kind of pref getter to use + let prefGetter; + switch (Services.prefs.getPrefType(oldPrefName)) { + case Services.prefs.PREF_BOOL: + prefGetter = "getBoolPref"; + break; + case Services.prefs.PREF_INT: + prefGetter = "getIntPref"; + break; + case Services.prefs.PREF_STRING: + prefGetter = "getStringPref"; + break; + } + + // Give the callback the current value then clear the pref + cbIfNotDefault(Services.prefs[prefGetter](oldPrefName)); + Services.prefs.clearUserPref(oldPrefName); + } + + uninit() { + if (this.geo === "") { + Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC); + } + + Services.obs.removeObserver(this, "intl:app-locales-changed"); + + this.store.uninit(); + this.initialized = false; + } + + _updateDynamicPrefs() { + // Save the geo pref if we have it + if (lazy.Region.home) { + this.geo = lazy.Region.home; + } else if (this.geo !== "") { + // Watch for geo changes and use a dummy value for now + Services.obs.addObserver(this, lazy.Region.REGION_TOPIC); + this.geo = ""; + } + + this.locale = Services.locale.appLocaleAsBCP47; + + // Update the pref config of those with dynamic values + for (const pref of PREFS_CONFIG.keys()) { + // Only need to process dynamic prefs + const prefConfig = PREFS_CONFIG.get(pref); + if (!prefConfig.getValue) { + continue; + } + + // Have the dynamic pref just reuse using existing default, e.g., those + // set via Autoconfig or policy + try { + const existingDefault = this._defaultPrefs.get(pref); + if (existingDefault !== undefined && prefConfig.value === undefined) { + prefConfig.getValue = () => existingDefault; + } + } catch (ex) { + // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing + // default branch to believe there's a type) but no actual default value + } + + // Compute the dynamic value (potentially generic based on dummy geo) + const newValue = prefConfig.getValue({ + geo: this.geo, + locale: this.locale, + }); + + // If there's an existing value and it has changed, that means we need to + // overwrite the default with the new value. + if (prefConfig.value !== undefined && prefConfig.value !== newValue) { + this._defaultPrefs.set(pref, newValue); + } + + prefConfig.value = newValue; + } + } + + observe(subject, topic, data) { + switch (topic) { + case "intl:app-locales-changed": + case lazy.Region.REGION_TOPIC: + this._updateDynamicPrefs(); + break; + } + } +} + +const EXPORTED_SYMBOLS = ["ActivityStream", "PREFS_CONFIG"]; diff --git a/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm b/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm new file mode 100644 index 0000000000..c4dafc4fa7 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm @@ -0,0 +1,332 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AboutHomeStartupCache: "resource:///modules/BrowserGlue.sys.mjs", + AboutNewTabParent: "resource:///actors/AboutNewTabParent.sys.mjs", +}); + +const { + actionCreators: ac, + actionTypes: at, + actionUtils: au, +} = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const ABOUT_NEW_TAB_URL = "about:newtab"; + +const DEFAULT_OPTIONS = { + dispatch(action) { + throw new Error( + `\nMessageChannel: Received action ${action.type}, but no dispatcher was defined.\n` + ); + }, + pageURL: ABOUT_NEW_TAB_URL, + outgoingMessageName: "ActivityStream:MainToContent", + incomingMessageName: "ActivityStream:ContentToMain", +}; + +class ActivityStreamMessageChannel { + /** + * ActivityStreamMessageChannel - This module connects a Redux store to the new tab page actor. + * You should use the BroadcastToContent, AlsoToOneContent, and AlsoToMain action creators + * in common/Actions.sys.mjs to help you create actions that will be automatically routed + * to the correct location. + * + * @param {object} options + * @param {function} options.dispatch The dispatch method from a Redux store + * @param {string} options.pageURL The URL to which the channel is attached, such as about:newtab. + * @param {string} options.outgoingMessageName The name of the message sent to child processes + * @param {string} options.incomingMessageName The name of the message received from child processes + * @return {ActivityStreamMessageChannel} + */ + constructor(options = {}) { + Object.assign(this, DEFAULT_OPTIONS, options); + + this.middleware = this.middleware.bind(this); + this.onMessage = this.onMessage.bind(this); + this.onNewTabLoad = this.onNewTabLoad.bind(this); + this.onNewTabUnload = this.onNewTabUnload.bind(this); + this.onNewTabInit = this.onNewTabInit.bind(this); + } + + /** + * Get an iterator over the loaded tab objects. + */ + get loadedTabs() { + // In the test, AboutNewTabParent is not defined. + return lazy.AboutNewTabParent?.loadedTabs || new Map(); + } + + /** + * middleware - Redux middleware that looks for AlsoToOneContent and BroadcastToContent type + * actions, and sends them out. + * + * @param {object} store A redux store + * @return {function} Redux middleware + */ + middleware(store) { + return next => action => { + const skipMain = action.meta && action.meta.skipMain; + if (au.isSendToOneContent(action)) { + this.send(action); + } else if (au.isBroadcastToContent(action)) { + this.broadcast(action); + } else if (au.isSendToPreloaded(action)) { + this.sendToPreloaded(action); + } + + if (!skipMain) { + next(action); + } + }; + } + + /** + * onActionFromContent - Handler for actions from a content processes + * + * @param {object} action A Redux action + * @param {string} targetId The portID of the port that sent the message + */ + onActionFromContent(action, targetId) { + this.dispatch(ac.AlsoToMain(action, this.validatePortID(targetId))); + } + + /** + * broadcast - Sends an action to all ports + * + * @param {object} action A Redux action + */ + broadcast(action) { + // We're trying to update all tabs, so signal the AboutHomeStartupCache + // that its likely time to refresh the cache. + lazy.AboutHomeStartupCache.onPreloadedNewTabMessage(); + + for (let { actor } of this.loadedTabs.values()) { + try { + actor.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The target page is closed/closing by the user or test, so just ignore. + } + } + } + + /** + * send - Sends an action to a specific port + * + * @param {obj} action A redux action; it should contain a portID in the meta.toTarget property + */ + send(action) { + const targetId = action.meta && action.meta.toTarget; + const target = this.getTargetById(targetId); + try { + target.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The target page is closed/closing by the user or test, so just ignore. + } + } + + /** + * A valid portID is a combination of process id and a port number. + * It is generated in AboutNewTabChild.sys.mjs. + */ + validatePortID(id) { + if (typeof id !== "string" || !id.includes(":")) { + console.error("Invalid portID"); + } + + return id; + } + + /** + * getTargetById - Retrieve the message target by portID, if it exists + * + * @param {string} id A portID + * @return {obj|null} The message target, if it exists. + */ + getTargetById(id) { + this.validatePortID(id); + + for (let { portID, actor } of this.loadedTabs.values()) { + if (portID === id) { + return actor; + } + } + return null; + } + + /** + * sendToPreloaded - Sends an action to each preloaded browser, if any + * + * @param {obj} action A redux action + */ + sendToPreloaded(action) { + // We're trying to update the preloaded about:newtab, so signal + // the AboutHomeStartupCache that its likely time to refresh + // the cache. + lazy.AboutHomeStartupCache.onPreloadedNewTabMessage(); + + const preloadedActors = this.getPreloadedActors(); + if (preloadedActors && action.data) { + for (let preloadedActor of preloadedActors) { + try { + preloadedActor.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The preloaded page is no longer available, so just ignore. + } + } + } + } + + /** + * getPreloadedActors - Retrieve the preloaded actors + * + * @return {Array|null} An array of actors belonging to the preloaded browsers, or null + * if there aren't any preloaded browsers + */ + getPreloadedActors() { + let preloadedActors = []; + for (let { actor, browser } of this.loadedTabs.values()) { + if (this.isPreloadedBrowser(browser)) { + preloadedActors.push(actor); + } + } + return preloadedActors.length ? preloadedActors : null; + } + + /** + * isPreloadedBrowser - Returns true if the passed browser has been preloaded + * for faster rendering of new tabs. + * + * @param {<browser>} A <browser> to check. + * @return {bool} True if the browser is preloaded. + * if there aren't any preloaded browsers + */ + isPreloadedBrowser(browser) { + return browser.getAttribute("preloadedState") === "preloaded"; + } + + simulateMessagesForExistingTabs() { + // Some pages might have already loaded, so we won't get the usual message + for (const loadedTab of this.loadedTabs.values()) { + let simulatedDetails = { + actor: loadedTab.actor, + browser: loadedTab.browser, + browsingContext: loadedTab.browsingContext, + portID: loadedTab.portID, + url: loadedTab.url, + simulated: true, + }; + + this.onActionFromContent( + { + type: at.NEW_TAB_INIT, + data: simulatedDetails, + }, + loadedTab.portID + ); + + if (loadedTab.loaded) { + this.tabLoaded(simulatedDetails); + } + } + } + + /** + * onNewTabInit - Handler for special RemotePage:Init message fired + * on initialization. + * + * @param {obj} msg The messsage from a page that was just initialized + * @param {obj} tabDetails details about a loaded tab + * + * tabDetails contains: + * actor, browser, browsingContext, portID, url + */ + onNewTabInit(msg, tabDetails) { + this.onActionFromContent( + { + type: at.NEW_TAB_INIT, + data: tabDetails, + }, + msg.data.portID + ); + } + + /** + * onNewTabLoad - Handler for special RemotePage:Load message fired on page load. + * + * @param {obj} msg The messsage from a page that was just loaded + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onNewTabLoad(msg, tabDetails) { + this.tabLoaded(tabDetails); + } + + tabLoaded(tabDetails) { + tabDetails.loaded = true; + + let { browser } = tabDetails; + if ( + this.isPreloadedBrowser(browser) && + browser.ownerGlobal.windowState !== browser.ownerGlobal.STATE_MINIMIZED && + !browser.ownerGlobal.isFullyOccluded + ) { + // As a perceived performance optimization, if this loaded Activity Stream + // happens to be a preloaded browser in a window that is not minimized or + // occluded, have it render its layers to the compositor now to increase + // the odds that by the time we switch to the tab, the layers are already + // ready to present to the user. + browser.renderLayers = true; + } + + this.onActionFromContent({ type: at.NEW_TAB_LOAD }, tabDetails.portID); + } + + /** + * onNewTabUnloadLoad - Handler for special RemotePage:Unload message fired + * on page unload. + * + * @param {obj} msg The messsage from a page that was just unloaded + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onNewTabUnload(msg, tabDetails) { + this.onActionFromContent({ type: at.NEW_TAB_UNLOAD }, tabDetails.portID); + } + + /** + * onMessage - Handles custom messages from content. It expects all messages to + * be formatted as Redux actions, and dispatches them to this.store + * + * @param {obj} msg A custom message from content + * @param {obj} msg.action A Redux action (e.g. {type: "HELLO_WORLD"}) + * @param {obj} msg.target A message target + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onMessage(msg, tabDetails) { + if (!msg.data || !msg.data.type) { + console.error( + new Error( + `Received an improperly formatted message from ${tabDetails.portID}` + ) + ); + return; + } + let action = {}; + Object.assign(action, msg.data); + // target is used to access a browser reference that came from the content + // and should only be used in feeds (not reducers) + action._target = { + browser: tabDetails.browser, + }; + + this.onActionFromContent(action, tabDetails.portID); + } +} + +const EXPORTED_SYMBOLS = ["ActivityStreamMessageChannel", "DEFAULT_OPTIONS"]; diff --git a/browser/components/newtab/lib/ActivityStreamPrefs.jsm b/browser/components/newtab/lib/ActivityStreamPrefs.jsm new file mode 100644 index 0000000000..8614671903 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamPrefs.jsm @@ -0,0 +1,95 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); + +const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream."; + +class Prefs extends Preferences { + /** + * Prefs - A wrapper around Preferences that always sets the branch to + * ACTIVITY_STREAM_PREF_BRANCH + */ + constructor(branch = ACTIVITY_STREAM_PREF_BRANCH) { + super({ branch }); + this._branchObservers = new Map(); + } + + ignoreBranch(listener) { + const observer = this._branchObservers.get(listener); + this._prefBranch.removeObserver("", observer); + this._branchObservers.delete(listener); + } + + observeBranch(listener) { + const observer = (subject, topic, pref) => { + listener.onPrefChanged(pref, this.get(pref)); + }; + this._prefBranch.addObserver("", observer); + this._branchObservers.set(listener, observer); + } +} + +class DefaultPrefs extends Preferences { + /** + * DefaultPrefs - A helper for setting and resetting default prefs for the add-on + * + * @param {Map} config A Map with {string} key of the pref name and {object} + * value with the following pref properties: + * {string} .title (optional) A description of the pref + * {bool|string|number} .value The default value for the pref + * @param {string} branch (optional) The pref branch (defaults to ACTIVITY_STREAM_PREF_BRANCH) + */ + constructor(config, branch = ACTIVITY_STREAM_PREF_BRANCH) { + super({ + branch, + defaultBranch: true, + }); + this._config = config; + } + + /** + * init - Set default prefs for all prefs in the config + */ + init() { + // Local developer builds (with the default mozconfig) aren't OFFICIAL + const IS_UNOFFICIAL_BUILD = !AppConstants.MOZILLA_OFFICIAL; + + for (const pref of this._config.keys()) { + try { + // Avoid replacing existing valid default pref values, e.g., those set + // via Autoconfig or policy + if (this.get(pref) !== undefined) { + continue; + } + } catch (ex) { + // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing + // default branch to believe there's a type) but no actual default value + } + + const prefConfig = this._config.get(pref); + let value; + if (IS_UNOFFICIAL_BUILD && "value_local_dev" in prefConfig) { + value = prefConfig.value_local_dev; + } else { + value = prefConfig.value; + } + + try { + this.set(pref, value); + } catch (ex) { + // Potentially the user somehow set an unexpected value type, so we fail + // to set a default of our expected type + } + } + } +} + +const EXPORTED_SYMBOLS = ["DefaultPrefs", "Prefs"]; diff --git a/browser/components/newtab/lib/ActivityStreamStorage.jsm b/browser/components/newtab/lib/ActivityStreamStorage.jsm new file mode 100644 index 0000000000..f34e289f8a --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamStorage.jsm @@ -0,0 +1,121 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + IndexedDB: "resource://gre/modules/IndexedDB.sys.mjs", +}); + +class ActivityStreamStorage { + /** + * @param storeNames Array of strings used to create all the required stores + */ + constructor({ storeNames, telemetry }) { + if (!storeNames) { + throw new Error("storeNames required"); + } + + this.dbName = "ActivityStream"; + this.dbVersion = 3; + this.storeNames = storeNames; + this.telemetry = telemetry; + } + + get db() { + return this._db || (this._db = this.createOrOpenDb()); + } + + /** + * Public method that binds the store required by the consumer and exposes + * the private db getters and setters. + * + * @param storeName String name of desired store + */ + getDbTable(storeName) { + if (this.storeNames.includes(storeName)) { + return { + get: this._get.bind(this, storeName), + getAll: this._getAll.bind(this, storeName), + set: this._set.bind(this, storeName), + }; + } + + throw new Error(`Store name ${storeName} does not exist.`); + } + + async _getStore(storeName) { + return (await this.db).objectStore(storeName, "readwrite"); + } + + _get(storeName, key) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).get(key) + ); + } + + _getAll(storeName) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).getAll() + ); + } + + _set(storeName, key, value) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).put(value, key) + ); + } + + _openDatabase() { + return lazy.IndexedDB.open(this.dbName, { version: this.dbVersion }, db => { + // If provided with array of objectStore names we need to create all the + // individual stores + this.storeNames.forEach(store => { + if (!db.objectStoreNames.contains(store)) { + this._requestWrapper(() => db.createObjectStore(store)); + } + }); + }); + } + + /** + * createOrOpenDb - Open a db (with this.dbName) if it exists. + * If it does not exist, create it. + * If an error occurs, deleted the db and attempt to + * re-create it. + * @returns Promise that resolves with a db instance + */ + async createOrOpenDb() { + try { + const db = await this._openDatabase(); + return db; + } catch (e) { + if (this.telemetry) { + this.telemetry.handleUndesiredEvent({ event: "INDEXEDDB_OPEN_FAILED" }); + } + await lazy.IndexedDB.deleteDatabase(this.dbName); + return this._openDatabase(); + } + } + + async _requestWrapper(request) { + let result = null; + try { + result = await request(); + } catch (e) { + if (this.telemetry) { + this.telemetry.handleUndesiredEvent({ event: "TRANSACTION_FAILED" }); + } + throw e; + } + + return result; + } +} + +function getDefaultOptions(options) { + return { collapsed: !!options.collapsed }; +} + +const EXPORTED_SYMBOLS = ["ActivityStreamStorage", "getDefaultOptions"]; diff --git a/browser/components/newtab/lib/CFRMessageProvider.sys.mjs b/browser/components/newtab/lib/CFRMessageProvider.sys.mjs new file mode 100644 index 0000000000..d183400f82 --- /dev/null +++ b/browser/components/newtab/lib/CFRMessageProvider.sys.mjs @@ -0,0 +1,821 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const FACEBOOK_CONTAINER_PARAMS = { + existing_addons: [ + "@contain-facebook", + "{bb1b80be-e6b3-40a1-9b6e-9d4073343f0b}", + "{a50d61ca-d27b-437a-8b52-5fd801a0a88b}", + ], + open_urls: ["www.facebook.com", "facebook.com"], + sumo_path: "extensionrecommendations", + min_frecency: 10000, +}; +const GOOGLE_TRANSLATE_PARAMS = { + existing_addons: [ + "jid1-93WyvpgvxzGATw@jetpack", + "{087ef4e1-4286-4be6-9aa3-8d6c420ee1db}", + "{4170faaa-ee87-4a0e-b57a-1aec49282887}", + "jid1-TMndP6cdKgxLcQ@jetpack", + "s3google@translator", + "{9c63d15c-b4d9-43bd-b223-37f0a1f22e2a}", + "translator@zoli.bod", + "{8cda9ce6-7893-4f47-ac70-a65215cec288}", + "simple-translate@sienori", + "@translatenow", + "{a79fafce-8da6-4685-923f-7ba1015b8748})", + "{8a802b5a-eeab-11e2-a41d-b0096288709b}", + "jid0-fbHwsGfb6kJyq2hj65KnbGte3yT@jetpack", + "storetranslate.plugin@gmail.com", + "jid1-r2tWDbSkq8AZK1@jetpack", + "{b384b75c-c978-4c4d-b3cf-62a82d8f8f12}", + "jid1-f7dnBeTj8ElpWQ@jetpack", + "{dac8a935-4775-4918-9205-5c0600087dc4}", + "gtranslation2@slam.com", + "{e20e0de5-1667-4df4-bd69-705720e37391}", + "{09e26ae9-e9c1-477c-80a6-99934212f2fe}", + "mgxtranslator@magemagix.com", + "gtranslatewins@mozilla.org", + ], + open_urls: ["translate.google.com"], + sumo_path: "extensionrecommendations", + min_frecency: 10000, +}; +const YOUTUBE_ENHANCE_PARAMS = { + existing_addons: [ + "enhancerforyoutube@maximerf.addons.mozilla.org", + "{dc8f61ab-5e98-4027-98ef-bb2ff6060d71}", + "{7b1bf0b6-a1b9-42b0-b75d-252036438bdc}", + "jid0-UVAeBCfd34Kk5usS8A1CBiobvM8@jetpack", + "iridium@particlecore.github.io", + "jid1-ss6kLNCbNz6u0g@jetpack", + "{1cf918d2-f4ea-4b4f-b34e-455283fef19f}", + ], + open_urls: ["www.youtube.com", "youtube.com"], + sumo_path: "extensionrecommendations", + min_frecency: 10000, +}; +const WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS = { + existing_addons: [ + "@wikipediacontextmenusearch", + "{ebf47fc8-01d8-4dba-aa04-2118402f4b20}", + "{5737a280-b359-4e26-95b0-adec5915a854}", + "olivier.debroqueville@gmail.com", + "{3923146e-98cb-472b-9c13-f6849d34d6b8}", + ], + open_urls: ["www.wikipedia.org", "wikipedia.org"], + sumo_path: "extensionrecommendations", + min_frecency: 10000, +}; +const REDDIT_ENHANCEMENT_PARAMS = { + existing_addons: ["jid1-xUfzOsOFlzSOXg@jetpack"], + open_urls: ["www.reddit.com", "reddit.com"], + sumo_path: "extensionrecommendations", + min_frecency: 10000, +}; + +const CFR_MESSAGES = [ + { + id: "FACEBOOK_CONTAINER_3", + template: "cfr_doorhanger", + groups: ["cfr-message-provider"], + content: { + layout: "addon_recommendation", + category: "cfrAddons", + bucket_id: "CFR_M1", + notification_text: { + string_id: "cfr-doorhanger-extension-notification2", + }, + heading_text: { string_id: "cfr-doorhanger-extension-heading" }, + info_icon: { + label: { string_id: "cfr-doorhanger-extension-sumo-link" }, + sumo_path: FACEBOOK_CONTAINER_PARAMS.sumo_path, + }, + addon: { + id: "954390", + title: "Facebook Container", + icon: "chrome://browser/skin/addons/addon-install-downloading.svg", + rating: "4.6", + users: "299019", + author: "Mozilla", + amo_url: "https://addons.mozilla.org/firefox/addon/facebook-container/", + }, + text: "Stop Facebook from tracking your activity across the web. Use Facebook the way you normally do without annoying ads following you around.", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-extension-ok-button" }, + action: { + type: "INSTALL_ADDON_FROM_URL", + data: { url: "https://example.com", telemetrySource: "amo" }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfraddons" }, + }, + }, + ], + }, + }, + frequency: { lifetime: 3 }, + targeting: ` + localeLanguageCode == "en" && + (xpinstallEnabled == true) && + (${JSON.stringify( + FACEBOOK_CONTAINER_PARAMS.existing_addons + )} intersect addonsInfo.addons|keys)|length == 0 && + (${JSON.stringify( + FACEBOOK_CONTAINER_PARAMS.open_urls + )} intersect topFrecentSites[.frecency >= ${ + FACEBOOK_CONTAINER_PARAMS.min_frecency + }]|mapToProperty('host'))|length > 0`, + trigger: { id: "openURL", params: FACEBOOK_CONTAINER_PARAMS.open_urls }, + }, + { + id: "GOOGLE_TRANSLATE_3", + groups: ["cfr-message-provider"], + template: "cfr_doorhanger", + content: { + layout: "addon_recommendation", + category: "cfrAddons", + bucket_id: "CFR_M1", + notification_text: { + string_id: "cfr-doorhanger-extension-notification2", + }, + heading_text: { string_id: "cfr-doorhanger-extension-heading" }, + info_icon: { + label: { string_id: "cfr-doorhanger-extension-sumo-link" }, + sumo_path: GOOGLE_TRANSLATE_PARAMS.sumo_path, + }, + addon: { + id: "445852", + title: "To Google Translate", + icon: "chrome://browser/skin/addons/addon-install-downloading.svg", + rating: "4.1", + users: "313474", + author: "Juan Escobar", + amo_url: + "https://addons.mozilla.org/firefox/addon/to-google-translate/", + }, + text: "Instantly translate any webpage text. Simply highlight the text, right-click to open the context menu, and choose a text or aural translation.", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-extension-ok-button" }, + action: { + type: "INSTALL_ADDON_FROM_URL", + data: { url: "https://example.com", telemetrySource: "amo" }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfraddons" }, + }, + }, + ], + }, + }, + frequency: { lifetime: 3 }, + targeting: ` + localeLanguageCode == "en" && + (xpinstallEnabled == true) && + (${JSON.stringify( + GOOGLE_TRANSLATE_PARAMS.existing_addons + )} intersect addonsInfo.addons|keys)|length == 0 && + (${JSON.stringify( + GOOGLE_TRANSLATE_PARAMS.open_urls + )} intersect topFrecentSites[.frecency >= ${ + GOOGLE_TRANSLATE_PARAMS.min_frecency + }]|mapToProperty('host'))|length > 0`, + trigger: { id: "openURL", params: GOOGLE_TRANSLATE_PARAMS.open_urls }, + }, + { + id: "YOUTUBE_ENHANCE_3", + groups: ["cfr-message-provider"], + template: "cfr_doorhanger", + content: { + layout: "addon_recommendation", + category: "cfrAddons", + bucket_id: "CFR_M1", + notification_text: { + string_id: "cfr-doorhanger-extension-notification2", + }, + heading_text: { string_id: "cfr-doorhanger-extension-heading" }, + info_icon: { + label: { string_id: "cfr-doorhanger-extension-sumo-link" }, + sumo_path: YOUTUBE_ENHANCE_PARAMS.sumo_path, + }, + addon: { + id: "700308", + title: "Enhancer for YouTube\u2122", + icon: "chrome://browser/skin/addons/addon-install-downloading.svg", + rating: "4.8", + users: "357328", + author: "Maxime RF", + amo_url: + "https://addons.mozilla.org/firefox/addon/enhancer-for-youtube/", + }, + text: "Take control of your YouTube experience. Automatically block annoying ads, set playback speed and volume, remove annotations, and more.", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-extension-ok-button" }, + action: { + type: "INSTALL_ADDON_FROM_URL", + data: { url: "https://example.com", telemetrySource: "amo" }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfraddons" }, + }, + }, + ], + }, + }, + frequency: { lifetime: 3 }, + targeting: ` + localeLanguageCode == "en" && + (xpinstallEnabled == true) && + (${JSON.stringify( + YOUTUBE_ENHANCE_PARAMS.existing_addons + )} intersect addonsInfo.addons|keys)|length == 0 && + (${JSON.stringify( + YOUTUBE_ENHANCE_PARAMS.open_urls + )} intersect topFrecentSites[.frecency >= ${ + YOUTUBE_ENHANCE_PARAMS.min_frecency + }]|mapToProperty('host'))|length > 0`, + trigger: { id: "openURL", params: YOUTUBE_ENHANCE_PARAMS.open_urls }, + }, + { + id: "WIKIPEDIA_CONTEXT_MENU_SEARCH_3", + groups: ["cfr-message-provider"], + template: "cfr_doorhanger", + exclude: true, + content: { + layout: "addon_recommendation", + category: "cfrAddons", + bucket_id: "CFR_M1", + notification_text: { + string_id: "cfr-doorhanger-extension-notification2", + }, + heading_text: { string_id: "cfr-doorhanger-extension-heading" }, + info_icon: { + label: { string_id: "cfr-doorhanger-extension-sumo-link" }, + sumo_path: WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.sumo_path, + }, + addon: { + id: "659026", + title: "Wikipedia Context Menu Search", + icon: "chrome://browser/skin/addons/addon-install-downloading.svg", + rating: "4.9", + users: "3095", + author: "Nick Diedrich", + amo_url: + "https://addons.mozilla.org/firefox/addon/wikipedia-context-menu-search/", + }, + text: "Get to a Wikipedia page fast, from anywhere on the web. Just highlight any webpage text and right-click to open the context menu to start a Wikipedia search.", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-extension-ok-button" }, + action: { + type: "INSTALL_ADDON_FROM_URL", + data: { url: "https://example.com", telemetrySource: "amo" }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfraddons" }, + }, + }, + ], + }, + }, + frequency: { lifetime: 3 }, + targeting: ` + localeLanguageCode == "en" && + (xpinstallEnabled == true) && + (${JSON.stringify( + WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.existing_addons + )} intersect addonsInfo.addons|keys)|length == 0 && + (${JSON.stringify( + WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.open_urls + )} intersect topFrecentSites[.frecency >= ${ + WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.min_frecency + }]|mapToProperty('host'))|length > 0`, + trigger: { + id: "openURL", + params: WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.open_urls, + }, + }, + { + id: "REDDIT_ENHANCEMENT_3", + groups: ["cfr-message-provider"], + template: "cfr_doorhanger", + exclude: true, + content: { + layout: "addon_recommendation", + category: "cfrAddons", + bucket_id: "CFR_M1", + notification_text: { + string_id: "cfr-doorhanger-extension-notification2", + }, + heading_text: { string_id: "cfr-doorhanger-extension-heading" }, + info_icon: { + label: { string_id: "cfr-doorhanger-extension-sumo-link" }, + sumo_path: REDDIT_ENHANCEMENT_PARAMS.sumo_path, + }, + addon: { + id: "387429", + title: "Reddit Enhancement Suite", + icon: "chrome://browser/skin/addons/addon-install-downloading.svg", + rating: "4.6", + users: "258129", + author: "honestbleeps", + amo_url: + "https://addons.mozilla.org/firefox/addon/reddit-enhancement-suite/", + }, + text: "New features include Inline Image Viewer, Never Ending Reddit (never click 'next page' again), Keyboard Navigation, Account Switcher, and User Tagger.", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-extension-ok-button" }, + action: { + type: "INSTALL_ADDON_FROM_URL", + data: { url: "https://example.com", telemetrySource: "amo" }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfraddons" }, + }, + }, + ], + }, + }, + frequency: { lifetime: 3 }, + targeting: ` + localeLanguageCode == "en" && + (xpinstallEnabled == true) && + (${JSON.stringify( + REDDIT_ENHANCEMENT_PARAMS.existing_addons + )} intersect addonsInfo.addons|keys)|length == 0 && + (${JSON.stringify( + REDDIT_ENHANCEMENT_PARAMS.open_urls + )} intersect topFrecentSites[.frecency >= ${ + REDDIT_ENHANCEMENT_PARAMS.min_frecency + }]|mapToProperty('host'))|length > 0`, + trigger: { id: "openURL", params: REDDIT_ENHANCEMENT_PARAMS.open_urls }, + }, + { + id: "DOH_ROLLOUT_CONFIRMATION", + groups: ["cfr-message-provider"], + targeting: ` + "doh-rollout.enabled"|preferenceValue && + !"doh-rollout.disable-heuristics"|preferenceValue && + !"doh-rollout.skipHeuristicsCheck"|preferenceValue && + !"doh-rollout.doorhanger-decision"|preferenceValue + `, + template: "cfr_doorhanger", + content: { + skip_address_bar_notifier: true, + persistent_doorhanger: true, + anchor_id: "PanelUI-menu-button", + layout: "icon_and_message", + text: { string_id: "cfr-doorhanger-doh-body" }, + icon: "chrome://global/skin/icons/security.svg", + buttons: { + secondary: [ + { + label: { string_id: "cfr-doorhanger-doh-secondary-button" }, + action: { + type: "DISABLE_DOH", + }, + }, + ], + primary: { + label: { string_id: "cfr-doorhanger-doh-primary-button-2" }, + action: { + type: "ACCEPT_DOH", + }, + }, + }, + bucket_id: "DOH_ROLLOUT_CONFIRMATION", + heading_text: { string_id: "cfr-doorhanger-doh-header" }, + info_icon: { + label: { + string_id: "cfr-doorhanger-extension-sumo-link", + }, + sumo_path: "extensionrecommendations", + }, + notification_text: "Message from Firefox", + category: "cfrFeatures", + }, + trigger: { + id: "openURL", + patterns: ["*://*/*"], + }, + }, + { + id: "SAVE_LOGIN", + groups: ["cfr-message-provider"], + frequency: { + lifetime: 3, + }, + targeting: + "(!type || type == 'save') && isFxAEnabled == true && usesFirefoxSync == false", + template: "cfr_doorhanger", + content: { + layout: "icon_and_message", + text: "Securely store and sync your passwords to all your devices.", + icon: "chrome://browser/content/aboutlogins/icons/intro-illustration.svg", + icon_class: "cfr-doorhanger-large-icon", + buttons: { + secondary: [ + { + label: { + string_id: "cfr-doorhanger-extension-cancel-button", + }, + action: { + type: "CANCEL", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { + category: "general-cfrfeatures", + }, + }, + }, + ], + primary: { + label: { + value: "Turn on Sync", + attributes: { accesskey: "T" }, + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { + category: "sync", + entrypoint: "cfr-save-login", + }, + }, + }, + }, + bucket_id: "CFR_SAVE_LOGIN", + heading_text: "Never Lose a Password Again", + info_icon: { + label: { + string_id: "cfr-doorhanger-extension-sumo-link", + }, + sumo_path: "extensionrecommendations", + }, + notification_text: { + string_id: "cfr-doorhanger-feature-notification", + }, + category: "cfrFeatures", + }, + trigger: { + id: "newSavedLogin", + }, + }, + { + id: "UPDATE_LOGIN", + groups: ["cfr-message-provider"], + frequency: { + lifetime: 3, + }, + targeting: + "type == 'update' && isFxAEnabled == true && usesFirefoxSync == false", + template: "cfr_doorhanger", + content: { + layout: "icon_and_message", + text: "Securely store and sync your passwords to all your devices.", + icon: "chrome://browser/content/aboutlogins/icons/intro-illustration.svg", + icon_class: "cfr-doorhanger-large-icon", + buttons: { + secondary: [ + { + label: { + string_id: "cfr-doorhanger-extension-cancel-button", + }, + action: { + type: "CANCEL", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { + category: "general-cfrfeatures", + }, + }, + }, + ], + primary: { + label: { + value: "Turn on Sync", + attributes: { accesskey: "T" }, + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { + category: "sync", + entrypoint: "cfr-update-login", + }, + }, + }, + }, + bucket_id: "CFR_UPDATE_LOGIN", + heading_text: "Never Lose a Password Again", + info_icon: { + label: { + string_id: "cfr-doorhanger-extension-sumo-link", + }, + sumo_path: "extensionrecommendations", + }, + notification_text: { + string_id: "cfr-doorhanger-feature-notification", + }, + category: "cfrFeatures", + }, + trigger: { + id: "newSavedLogin", + }, + }, + { + id: "MILESTONE_MESSAGE", + groups: ["cfr-message-provider"], + template: "milestone_message", + content: { + layout: "short_message", + category: "cfrFeatures", + anchor_id: "tracking-protection-icon-container", + skip_address_bar_notifier: true, + bucket_id: "CFR_MILESTONE_MESSAGE", + heading_text: { string_id: "cfr-doorhanger-milestone-heading2" }, + notification_text: "", + text: "", + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-milestone-ok-button" }, + action: { type: "OPEN_PROTECTION_REPORT" }, + event: "PROTECTION", + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-milestone-close-button" }, + action: { type: "CANCEL" }, + event: "DISMISS", + }, + ], + }, + }, + targeting: "pageLoad >= 1", + frequency: { + lifetime: 7, // Length of privacy.contentBlocking.cfr-milestone.milestones pref + }, + trigger: { + id: "contentBlocking", + params: ["ContentBlockingMilestone"], + }, + }, + { + id: "HEARTBEAT_TACTIC_2", + groups: ["cfr-message-provider"], + template: "cfr_urlbar_chiclet", + content: { + layout: "chiclet_open_url", + category: "cfrHeartbeat", + bucket_id: "HEARTBEAT_TACTIC_2", + notification_text: "Improve Firefox", + active_color: "#595e91", + action: { + url: "http://example.com/%VERSION%/", + where: "tabshifted", + }, + }, + targeting: "false", + frequency: { + lifetime: 3, + }, + trigger: { + id: "openURL", + patterns: ["*://*/*"], + }, + }, + { + id: "HOMEPAGE_REMEDIATION_82", + groups: ["cfr-message-provider"], + frequency: { + lifetime: 3, + }, + targeting: + "!homePageSettings.isDefault && homePageSettings.isCustomUrl && homePageSettings.urls[.host == 'google.com']|length > 0 && visitsCount >= 3 && userPrefs.cfrFeatures", + template: "cfr_doorhanger", + content: { + layout: "icon_and_message", + text: "Update your homepage to search Google while also being able to search your Firefox history and bookmarks.", + icon: "chrome://global/skin/icons/search-glass.svg", + buttons: { + secondary: [ + { + label: { + string_id: "cfr-doorhanger-extension-cancel-button", + }, + action: { + type: "CANCEL", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { + category: "general-cfrfeatures", + }, + }, + }, + ], + primary: { + label: { + value: "Activate now", + attributes: { + accesskey: "A", + }, + }, + action: { + type: "CONFIGURE_HOMEPAGE", + data: { + homePage: "default", + newtab: "default", + layout: { + search: true, + topsites: false, + highlights: false, + topstories: false, + snippets: false, + }, + }, + }, + }, + }, + bucket_id: "HOMEPAGE_REMEDIATION_82", + heading_text: "A better search experience", + info_icon: { + label: { + string_id: "cfr-doorhanger-extension-sumo-link", + }, + sumo_path: "extensionrecommendations", + }, + notification_text: { + string_id: "cfr-doorhanger-feature-notification", + }, + category: "cfrFeatures", + }, + trigger: { + id: "openURL", + params: ["google.com", "www.google.com"], + }, + }, + { + id: "INFOBAR_ACTION_86", + groups: ["cfr-message-provider"], + targeting: "false", + template: "infobar", + content: { + type: "global", + text: { string_id: "default-browser-notification-message" }, + buttons: [ + { + label: { string_id: "default-browser-notification-button" }, + primary: true, + accessKey: "O", + action: { + type: "SET_DEFAULT_BROWSER", + }, + }, + ], + }, + trigger: { id: "defaultBrowserCheck" }, + }, + { + id: "PREF_OBSERVER_MESSAGE_94", + groups: ["cfr-message-provider"], + targeting: "true", + template: "infobar", + content: { + type: "global", + text: "This is a message triggered when a pref value changes", + buttons: [ + { + label: "OK", + primary: true, + accessKey: "O", + action: { + type: "CANCEL", + }, + }, + ], + }, + trigger: { id: "preferenceObserver", params: ["foo.bar"] }, + }, +]; + +export const CFRMessageProvider = { + getMessages() { + return Promise.resolve(CFR_MESSAGES.filter(msg => !msg.exclude)); + }, +}; diff --git a/browser/components/newtab/lib/CFRPageActions.jsm b/browser/components/newtab/lib/CFRPageActions.jsm new file mode 100644 index 0000000000..2051e5f5f7 --- /dev/null +++ b/browser/components/newtab/lib/CFRPageActions.jsm @@ -0,0 +1,1049 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + CustomizableUI: "resource:///modules/CustomizableUI.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + RemoteL10n: "resource://activity-stream/lib/RemoteL10n.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "TrackingDBService", + "@mozilla.org/tracking-db-service;1", + "nsITrackingDBService" +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "milestones", + "browser.contentblocking.cfr-milestone.milestones", + "[]", + null, + JSON.parse +); + +const POPUP_NOTIFICATION_ID = "contextual-feature-recommendation"; +const SUMO_BASE_URL = Services.urlFormatter.formatURLPref( + "app.support.baseURL" +); +const ADDONS_API_URL = + "https://services.addons.mozilla.org/api/v4/addons/addon"; + +const DELAY_BEFORE_EXPAND_MS = 1000; +const CATEGORY_ICONS = { + cfrAddons: "webextensions-icon", + cfrFeatures: "recommendations-icon", + cfrHeartbeat: "highlights-icon", +}; + +/** + * A WeakMap from browsers to {host, recommendation} pairs. Recommendations are + * defined in the ExtensionDoorhanger.schema.json. + * + * A recommendation is specific to a browser and host and is active until the + * given browser is closed or the user navigates (within that browser) away from + * the host. + */ +let RecommendationMap = new WeakMap(); + +/** + * A WeakMap from windows to their CFR PageAction. + */ +let PageActionMap = new WeakMap(); + +/** + * We need one PageAction for each window + */ +class PageAction { + constructor(win, dispatchCFRAction) { + this.window = win; + + this.urlbar = win.gURLBar; // The global URLBar object + this.urlbarinput = win.gURLBar.textbox; // The URLBar DOM node + + this.container = win.document.getElementById( + "contextual-feature-recommendation" + ); + this.button = win.document.getElementById("cfr-button"); + this.label = win.document.getElementById("cfr-label"); + + // This should NOT be use directly to dispatch message-defined actions attached to buttons. + // Please use dispatchUserAction instead. + this._dispatchCFRAction = dispatchCFRAction; + + this._popupStateChange = this._popupStateChange.bind(this); + this._collapse = this._collapse.bind(this); + this._cfrUrlbarButtonClick = this._cfrUrlbarButtonClick.bind(this); + this._executeNotifierAction = this._executeNotifierAction.bind(this); + this.dispatchUserAction = this.dispatchUserAction.bind(this); + + // Saved timeout IDs for scheduled state changes, so they can be cancelled + this.stateTransitionTimeoutIDs = []; + + XPCOMUtils.defineLazyGetter(this, "isDarkTheme", () => { + try { + return this.window.document.documentElement.hasAttribute( + "lwt-toolbar-field-brighttext" + ); + } catch (e) { + return false; + } + }); + } + + addImpression(recommendation) { + this._dispatchImpression(recommendation); + // Only send an impression ping upon the first expansion. + // Note that when the user clicks on the "show" button on the asrouter admin + // page (both `bucket_id` and `id` will be set as null), we don't want to send + // the impression ping in that case. + if (!!recommendation.id && !!recommendation.content.bucket_id) { + this._sendTelemetry({ + message_id: recommendation.id, + bucket_id: recommendation.content.bucket_id, + event: "IMPRESSION", + }); + } + } + + reloadL10n() { + lazy.RemoteL10n.reloadL10n(); + } + + async showAddressBarNotifier(recommendation, shouldExpand = false) { + this.container.hidden = false; + + let notificationText = await this.getStrings( + recommendation.content.notification_text + ); + this.label.value = notificationText; + if (notificationText.attributes) { + this.button.setAttribute( + "tooltiptext", + notificationText.attributes.tooltiptext + ); + // For a11y, we want the more descriptive text. + this.container.setAttribute( + "aria-label", + notificationText.attributes.tooltiptext + ); + } + this.container.setAttribute( + "data-cfr-icon", + CATEGORY_ICONS[recommendation.content.category] + ); + if (recommendation.content.active_color) { + this.container.style.setProperty( + "--cfr-active-color", + recommendation.content.active_color + ); + } + + // Wait for layout to flush to avoid a synchronous reflow then calculate the + // label width. We can safely get the width even though the recommendation is + // collapsed; the label itself remains full width (with its overflow hidden) + let [{ width }] = await this.window.promiseDocumentFlushed(() => + this.label.getClientRects() + ); + this.urlbarinput.style.setProperty("--cfr-label-width", `${width}px`); + + this.container.addEventListener("click", this._cfrUrlbarButtonClick); + // Collapse the recommendation on url bar focus in order to free up more + // space to display and edit the url + this.urlbar.addEventListener("focus", this._collapse); + + if (shouldExpand) { + this._clearScheduledStateChanges(); + + // After one second, expand + this._expand(DELAY_BEFORE_EXPAND_MS); + + this.addImpression(recommendation); + } + + if (notificationText.attributes) { + this.window.A11yUtils.announce({ + raw: notificationText.attributes["a11y-announcement"], + source: this.container, + }); + } + } + + hideAddressBarNotifier() { + this.container.hidden = true; + this._clearScheduledStateChanges(); + this.urlbarinput.removeAttribute("cfr-recommendation-state"); + this.container.removeEventListener("click", this._cfrUrlbarButtonClick); + this.urlbar.removeEventListener("focus", this._collapse); + if (this.currentNotification) { + this.window.PopupNotifications.remove(this.currentNotification); + this.currentNotification = null; + } + } + + _expand(delay) { + if (delay > 0) { + this.stateTransitionTimeoutIDs.push( + this.window.setTimeout(() => { + this.urlbarinput.setAttribute("cfr-recommendation-state", "expanded"); + }, delay) + ); + } else { + // Non-delayed state change overrides any scheduled state changes + this._clearScheduledStateChanges(); + this.urlbarinput.setAttribute("cfr-recommendation-state", "expanded"); + } + } + + _collapse(delay) { + if (delay > 0) { + this.stateTransitionTimeoutIDs.push( + this.window.setTimeout(() => { + if ( + this.urlbarinput.getAttribute("cfr-recommendation-state") === + "expanded" + ) { + this.urlbarinput.setAttribute( + "cfr-recommendation-state", + "collapsed" + ); + } + }, delay) + ); + } else { + // Non-delayed state change overrides any scheduled state changes + this._clearScheduledStateChanges(); + if ( + this.urlbarinput.getAttribute("cfr-recommendation-state") === "expanded" + ) { + this.urlbarinput.setAttribute("cfr-recommendation-state", "collapsed"); + } + } + } + + _clearScheduledStateChanges() { + while (this.stateTransitionTimeoutIDs.length) { + // clearTimeout is safe even with invalid/expired IDs + this.window.clearTimeout(this.stateTransitionTimeoutIDs.pop()); + } + } + + // This is called when the popup closes as a result of interaction _outside_ + // the popup, e.g. by hitting <esc> + _popupStateChange(state) { + if (state === "shown") { + if (this._autoFocus) { + this.window.document.commandDispatcher.advanceFocusIntoSubtree( + this.currentNotification.owner.panel + ); + this._autoFocus = false; + } + } else if (state === "removed") { + if (this.currentNotification) { + this.window.PopupNotifications.remove(this.currentNotification); + this.currentNotification = null; + } + } else if (state === "dismissed") { + const message = RecommendationMap.get(this.currentNotification?.browser); + this._sendTelemetry({ + message_id: message?.id, + bucket_id: message?.content.bucket_id, + event: "DISMISS", + }); + this._collapse(); + } + } + + shouldShowDoorhanger(recommendation) { + if (recommendation.content.layout === "chiclet_open_url") { + return false; + } + + return true; + } + + dispatchUserAction(action) { + this._dispatchCFRAction( + { type: "USER_ACTION", data: action }, + this.window.gBrowser.selectedBrowser + ); + } + + _dispatchImpression(message) { + this._dispatchCFRAction({ type: "IMPRESSION", data: message }); + } + + _sendTelemetry(ping) { + const data = { action: "cfr_user_event", source: "CFR", ...ping }; + if (lazy.PrivateBrowsingUtils.isWindowPrivate(this.window)) { + data.is_private = true; + } + this._dispatchCFRAction({ + type: "DOORHANGER_TELEMETRY", + data, + }); + } + + _blockMessage(messageID) { + this._dispatchCFRAction({ + type: "BLOCK_MESSAGE_BY_ID", + data: { id: messageID }, + }); + } + + maybeLoadCustomElement(win) { + if (!win.customElements.get("remote-text")) { + Services.scriptloader.loadSubScript( + "resource://activity-stream/data/custom-elements/paragraph.js", + win + ); + } + } + + /** + * getStrings - Handles getting the localized strings vs message overrides. + * If string_id is not defined it assumes you passed in an override + * message and it just returns it. + * If subAttribute is provided, the string for it is returned. + * @return A string. One of 1) passed in string 2) a String object with + * attributes property if there are attributes 3) the sub attribute. + */ + async getStrings(string, subAttribute = "") { + if (!string.string_id) { + if (subAttribute) { + if (string.attributes) { + return string.attributes[subAttribute]; + } + + console.error(`String ${string.value} does not contain any attributes`); + return subAttribute; + } + + if (typeof string.value === "string") { + const stringWithAttributes = new String(string.value); // eslint-disable-line no-new-wrappers + stringWithAttributes.attributes = string.attributes; + return stringWithAttributes; + } + + return string; + } + + const [localeStrings] = await lazy.RemoteL10n.l10n.formatMessages([ + { + id: string.string_id, + args: string.args, + }, + ]); + + const mainString = new String(localeStrings.value); // eslint-disable-line no-new-wrappers + if (localeStrings.attributes) { + const attributes = localeStrings.attributes.reduce((acc, attribute) => { + acc[attribute.name] = attribute.value; + return acc; + }, {}); + mainString.attributes = attributes; + } + + return subAttribute ? mainString.attributes[subAttribute] : mainString; + } + + async _setAddonRating(document, content) { + const footerFilledStars = this.window.document.getElementById( + "cfr-notification-footer-filled-stars" + ); + const footerEmptyStars = this.window.document.getElementById( + "cfr-notification-footer-empty-stars" + ); + const footerUsers = this.window.document.getElementById( + "cfr-notification-footer-users" + ); + + const { rating } = content.addon; + if (rating) { + const MAX_RATING = 5; + const STARS_WIDTH = 16 * MAX_RATING; + const calcWidth = stars => `${(stars / MAX_RATING) * STARS_WIDTH}px`; + const filledWidth = + rating <= MAX_RATING ? calcWidth(rating) : calcWidth(MAX_RATING); + const emptyWidth = + rating <= MAX_RATING ? calcWidth(MAX_RATING - rating) : calcWidth(0); + + footerFilledStars.style.width = filledWidth; + footerEmptyStars.style.width = emptyWidth; + + const ratingString = await this.getStrings( + { + string_id: "cfr-doorhanger-extension-rating", + args: { total: rating }, + }, + "tooltiptext" + ); + footerFilledStars.setAttribute("tooltiptext", ratingString); + footerEmptyStars.setAttribute("tooltiptext", ratingString); + } else { + footerFilledStars.style.width = ""; + footerEmptyStars.style.width = ""; + footerFilledStars.removeAttribute("tooltiptext"); + footerEmptyStars.removeAttribute("tooltiptext"); + } + + const { users } = content.addon; + if (users) { + footerUsers.setAttribute("value", users); + footerUsers.hidden = false; + } else { + // Prevent whitespace around empty label from affecting other spacing + footerUsers.hidden = true; + footerUsers.removeAttribute("value"); + } + } + + _createElementAndAppend({ type, id }, parent) { + let element = this.window.document.createXULElement(type); + if (id) { + element.setAttribute("id", id); + } + parent.appendChild(element); + return element; + } + + async _renderMilestonePopup(message, browser) { + this.maybeLoadCustomElement(this.window); + + let { content, id } = message; + let { primary, secondary } = content.buttons; + let earliestDate = await lazy.TrackingDBService.getEarliestRecordedDate(); + let timestamp = new Date().getTime(earliestDate); + let panelTitle = ""; + let headerLabel = this.window.document.getElementById( + "cfr-notification-header-label" + ); + let reachedMilestone = 0; + let totalSaved = await lazy.TrackingDBService.sumAllEvents(); + for (let milestone of lazy.milestones) { + if (totalSaved >= milestone) { + reachedMilestone = milestone; + } + } + if (headerLabel.firstChild) { + headerLabel.firstChild.remove(); + } + headerLabel.appendChild( + lazy.RemoteL10n.createElement(this.window.document, "span", { + content: message.content.heading_text, + attributes: { + blockedCount: reachedMilestone, + date: timestamp, + }, + }) + ); + + // Use the message layout as a CSS selector to hide different parts of the + // notification template markup + this.window.document + .getElementById("contextual-feature-recommendation-notification") + .setAttribute("data-notification-category", content.layout); + this.window.document + .getElementById("contextual-feature-recommendation-notification") + .setAttribute("data-notification-bucket", content.bucket_id); + + let primaryBtnString = await this.getStrings(primary.label); + let primaryActionCallback = () => { + this.dispatchUserAction(primary.action); + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "CLICK_BUTTON", + }); + + RecommendationMap.delete(browser); + // Invalidate the pref after the user interacts with the button. + // We don't need to show the illustration in the privacy panel. + Services.prefs.clearUserPref( + "browser.contentblocking.cfr-milestone.milestone-shown-time" + ); + }; + + let secondaryBtnString = await this.getStrings(secondary[0].label); + let secondaryActionsCallback = () => { + this.dispatchUserAction(secondary[0].action); + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "DISMISS", + }); + RecommendationMap.delete(browser); + }; + + let mainAction = { + label: primaryBtnString, + accessKey: primaryBtnString.attributes.accesskey, + callback: primaryActionCallback, + }; + + let secondaryActions = [ + { + label: secondaryBtnString, + accessKey: secondaryBtnString.attributes.accesskey, + callback: secondaryActionsCallback, + }, + ]; + + // Actually show the notification + this.currentNotification = this.window.PopupNotifications.show( + browser, + POPUP_NOTIFICATION_ID, + panelTitle, + "cfr", + mainAction, + secondaryActions, + { + hideClose: true, + persistWhileVisible: true, + recordTelemetryInPrivateBrowsing: content.show_in_private_browsing, + } + ); + Services.prefs.setIntPref( + "browser.contentblocking.cfr-milestone.milestone-achieved", + reachedMilestone + ); + Services.prefs.setStringPref( + "browser.contentblocking.cfr-milestone.milestone-shown-time", + Date.now().toString() + ); + } + + // eslint-disable-next-line max-statements + async _renderPopup(message, browser) { + this.maybeLoadCustomElement(this.window); + + const { id, content } = message; + + const headerLabel = this.window.document.getElementById( + "cfr-notification-header-label" + ); + const headerLink = this.window.document.getElementById( + "cfr-notification-header-link" + ); + const headerImage = this.window.document.getElementById( + "cfr-notification-header-image" + ); + const footerText = this.window.document.getElementById( + "cfr-notification-footer-text" + ); + const footerLink = this.window.document.getElementById( + "cfr-notification-footer-learn-more-link" + ); + const { primary, secondary } = content.buttons; + let primaryActionCallback; + let persistent = !!content.persistent_doorhanger; + let options = { + persistent, + persistWhileVisible: persistent, + recordTelemetryInPrivateBrowsing: content.show_in_private_browsing, + }; + let panelTitle; + + headerLabel.value = await this.getStrings(content.heading_text); + if (content.info_icon) { + headerLink.setAttribute( + "href", + SUMO_BASE_URL + content.info_icon.sumo_path + ); + headerImage.setAttribute( + "tooltiptext", + await this.getStrings(content.info_icon.label, "tooltiptext") + ); + } + headerLink.onclick = () => + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "RATIONALE", + }); + // Use the message layout as a CSS selector to hide different parts of the + // notification template markup + this.window.document + .getElementById("contextual-feature-recommendation-notification") + .setAttribute("data-notification-category", content.layout); + this.window.document + .getElementById("contextual-feature-recommendation-notification") + .setAttribute("data-notification-bucket", content.bucket_id); + + switch (content.layout) { + case "icon_and_message": + const author = this.window.document.getElementById( + "cfr-notification-author" + ); + if (author.firstChild) { + author.firstChild.remove(); + } + author.appendChild( + lazy.RemoteL10n.createElement(this.window.document, "span", { + content: content.text, + }) + ); + primaryActionCallback = () => { + this._blockMessage(id); + this.dispatchUserAction(primary.action); + this.hideAddressBarNotifier(); + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "ENABLE", + }); + RecommendationMap.delete(browser); + }; + + let getIcon = () => { + if (content.icon_dark_theme && this.isDarkTheme) { + return content.icon_dark_theme; + } + return content.icon; + }; + + let learnMoreURL = content.learn_more + ? SUMO_BASE_URL + content.learn_more + : null; + + panelTitle = await this.getStrings(content.heading_text); + options = { + popupIconURL: getIcon(), + popupIconClass: content.icon_class, + learnMoreURL, + ...options, + }; + break; + default: + const authorText = await this.getStrings({ + string_id: "cfr-doorhanger-extension-author", + args: { name: content.addon.author }, + }); + panelTitle = await this.getStrings(content.addon.title); + await this._setAddonRating(this.window.document, content); + if (footerText.firstChild) { + footerText.firstChild.remove(); + } + if (footerText.lastChild) { + footerText.lastChild.remove(); + } + + // Main body content of the dropdown + footerText.appendChild( + lazy.RemoteL10n.createElement(this.window.document, "span", { + content: content.text, + }) + ); + + footerLink.value = await this.getStrings({ + string_id: "cfr-doorhanger-extension-learn-more-link", + }); + footerLink.setAttribute("href", content.addon.amo_url); + footerLink.onclick = () => + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "LEARN_MORE", + }); + + footerText.appendChild(footerLink); + options = { + popupIconURL: content.addon.icon, + popupIconClass: content.icon_class, + name: authorText, + ...options, + }; + + primaryActionCallback = async () => { + primary.action.data.url = + // eslint-disable-next-line no-use-before-define + await CFRPageActions._fetchLatestAddonVersion(content.addon.id); + this._blockMessage(id); + this.dispatchUserAction(primary.action); + this.hideAddressBarNotifier(); + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "INSTALL", + }); + RecommendationMap.delete(browser); + }; + } + + const primaryBtnStrings = await this.getStrings(primary.label); + const mainAction = { + label: primaryBtnStrings, + accessKey: primaryBtnStrings.attributes.accesskey, + callback: primaryActionCallback, + }; + + let _renderSecondaryButtonAction = async (event, button) => { + let label = await this.getStrings(button.label); + let { attributes } = label; + + return { + label, + accessKey: attributes.accesskey, + callback: () => { + if (button.action) { + this.dispatchUserAction(button.action); + } else { + this._blockMessage(id); + this.hideAddressBarNotifier(); + RecommendationMap.delete(browser); + } + + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event, + }); + // We want to collapse if needed when we dismiss + this._collapse(); + }, + }; + }; + + // For each secondary action, define default telemetry event + const defaultSecondaryEvent = ["DISMISS", "BLOCK", "MANAGE"]; + const secondaryActions = await Promise.all( + secondary.map((button, i) => { + return _renderSecondaryButtonAction( + button.event || defaultSecondaryEvent[i], + button + ); + }) + ); + + // If the recommendation button is focused, it was probably activated via + // the keyboard. Therefore, focus the first element in the notification when + // it appears. + // We don't use the autofocus option provided by PopupNotifications.show + // because it doesn't focus the first element; i.e. the user still has to + // press tab once. That's not good enough, especially for screen reader + // users. Instead, we handle this ourselves in _popupStateChange. + this._autoFocus = this.window.document.activeElement === this.container; + + // Actually show the notification + this.currentNotification = this.window.PopupNotifications.show( + browser, + POPUP_NOTIFICATION_ID, + panelTitle, + "cfr", + mainAction, + secondaryActions, + { + ...options, + hideClose: true, + eventCallback: this._popupStateChange, + } + ); + } + + _executeNotifierAction(browser, message) { + switch (message.content.layout) { + case "chiclet_open_url": + this._dispatchCFRAction( + { + type: "USER_ACTION", + data: { + type: "OPEN_URL", + data: { + args: message.content.action.url, + where: message.content.action.where, + }, + }, + }, + this.window + ); + break; + } + + this._blockMessage(message.id); + this.hideAddressBarNotifier(); + RecommendationMap.delete(browser); + } + + /** + * Respond to a user click on the recommendation by showing a doorhanger/ + * popup notification or running the action defined in the message + */ + async _cfrUrlbarButtonClick(event) { + const browser = this.window.gBrowser.selectedBrowser; + if (!RecommendationMap.has(browser)) { + // There's no recommendation for this browser, so the user shouldn't have + // been able to click + this.hideAddressBarNotifier(); + return; + } + const message = RecommendationMap.get(browser); + const { id, content } = message; + + this._sendTelemetry({ + message_id: id, + bucket_id: content.bucket_id, + event: "CLICK_DOORHANGER", + }); + + if (this.shouldShowDoorhanger(message)) { + // The recommendation should remain either collapsed or expanded while the + // doorhanger is showing + this._clearScheduledStateChanges(browser, message); + await this.showPopup(); + } else { + await this._executeNotifierAction(browser, message); + } + } + + async showPopup() { + const browser = this.window.gBrowser.selectedBrowser; + const message = RecommendationMap.get(browser); + const { content } = message; + let anchor; + + // A hacky way of setting the popup anchor outside the usual url bar icon box + // See https://searchfox.org/mozilla-central/rev/847b64cc28b74b44c379f9bff4f415b97da1c6d7/toolkit/modules/PopupNotifications.jsm#42 + //If the anchor has been moved to the overflow menu ('menu-panel') and an alt_anchor_id has been provided, we want to use the alt_anchor_id + + if ( + content.alt_anchor_id && + lazy.CustomizableUI.getWidget(content.anchor_id).areaType.includes( + "panel" + ) + ) { + anchor = this.window.document.getElementById(content.alt_anchor_id); + } else { + anchor = + this.window.document.getElementById(content.anchor_id) || + this.container; + } + browser.cfrpopupnotificationanchor = anchor; + + await this._renderPopup(message, browser); + } + + async showMilestonePopup() { + const browser = this.window.gBrowser.selectedBrowser; + const message = RecommendationMap.get(browser); + const { content } = message; + + // A hacky way of setting the popup anchor outside the usual url bar icon box + // See https://searchfox.org/mozilla-central/rev/c5c002f81f08a73e04868e0c2bf0eb113f200b03/toolkit/modules/PopupNotifications.sys.mjs#40 + browser.cfrpopupnotificationanchor = + this.window.document.getElementById(content.anchor_id) || this.container; + + await this._renderMilestonePopup(message, browser); + return true; + } +} + +function isHostMatch(browser, host) { + return ( + browser.documentURI.scheme.startsWith("http") && + browser.documentURI.host === host + ); +} + +const CFRPageActions = { + // For testing purposes + RecommendationMap, + PageActionMap, + + /** + * To be called from browser.js on a location change, passing in the browser + * that's been updated + */ + updatePageActions(browser) { + const win = browser.ownerGlobal; + const pageAction = PageActionMap.get(win); + if (!pageAction || browser !== win.gBrowser.selectedBrowser) { + return; + } + if (RecommendationMap.has(browser)) { + const recommendation = RecommendationMap.get(browser); + if ( + !recommendation.content.skip_address_bar_notifier && + (isHostMatch(browser, recommendation.host) || + // If there is no host associated we assume we're back on a tab + // that had a CFR message so we should show it again + !recommendation.host) + ) { + // The browser has a recommendation specified with this host, so show + // the page action + pageAction.showAddressBarNotifier(recommendation); + } else if (!recommendation.content.persistent_doorhanger) { + if (recommendation.retain) { + // Keep the recommendation first time the user navigates away just in + // case they will go back to the previous page + pageAction.hideAddressBarNotifier(); + recommendation.retain = false; + } else { + // The user has navigated away from the specified host in the given + // browser, so the recommendation is no longer valid and should be removed + RecommendationMap.delete(browser); + pageAction.hideAddressBarNotifier(); + } + } + } else { + // There's no recommendation specified for this browser, so hide the page action + pageAction.hideAddressBarNotifier(); + } + }, + + /** + * Fetch the URL to the latest add-on xpi so the recommendation can download it. + * @param id The add-on ID + * @return A string for the URL that was fetched + */ + async _fetchLatestAddonVersion(id) { + let url = null; + try { + const response = await fetch(`${ADDONS_API_URL}/${id}/`, { + credentials: "omit", + }); + if (response.status !== 204 && response.ok) { + const json = await response.json(); + url = json.current_version.files[0].url; + } + } catch (e) { + console.error( + "Failed to get the latest add-on version for this recommendation" + ); + } + return url; + }, + + /** + * Force a recommendation to be shown. Should only happen via the Admin page. + * @param browser The browser for the recommendation + * @param recommendation The recommendation to show + * @param dispatchCFRAction A function to dispatch resulting actions to + * @return Did adding the recommendation succeed? + */ + async forceRecommendation(browser, recommendation, dispatchCFRAction) { + if (!browser) { + return false; + } + // If we are forcing via the Admin page, the browser comes in a different format + const win = browser.ownerGlobal; + const { id, content } = recommendation; + RecommendationMap.set(browser, { + id, + content, + retain: true, + }); + if (!PageActionMap.has(win)) { + PageActionMap.set(win, new PageAction(win, dispatchCFRAction)); + } + + if (content.skip_address_bar_notifier) { + if (recommendation.template === "milestone_message") { + await PageActionMap.get(win).showMilestonePopup(); + PageActionMap.get(win).addImpression(recommendation); + } else { + await PageActionMap.get(win).showPopup(); + PageActionMap.get(win).addImpression(recommendation); + } + } else { + await PageActionMap.get(win).showAddressBarNotifier(recommendation, true); + } + return true; + }, + + /** + * Add a recommendation specific to the given browser and host. + * @param browser The browser for the recommendation + * @param host The host for the recommendation + * @param recommendation The recommendation to show + * @param dispatchCFRAction A function to dispatch resulting actions to + * @return Did adding the recommendation succeed? + */ + async addRecommendation(browser, host, recommendation, dispatchCFRAction) { + if (!browser) { + return false; + } + const win = browser.ownerGlobal; + if ( + browser !== win.gBrowser.selectedBrowser || + // We can have recommendations without URL restrictions + (host && !isHostMatch(browser, host)) + ) { + return false; + } + if (RecommendationMap.has(browser)) { + // Don't replace an existing message + return false; + } + const { id, content } = recommendation; + if ( + !content.show_in_private_browsing && + lazy.PrivateBrowsingUtils.isWindowPrivate(win) + ) { + return false; + } + RecommendationMap.set(browser, { + id, + host, + content, + retain: true, + }); + if (!PageActionMap.has(win)) { + PageActionMap.set(win, new PageAction(win, dispatchCFRAction)); + } + + if (content.skip_address_bar_notifier) { + if (recommendation.template === "milestone_message") { + await PageActionMap.get(win).showMilestonePopup(); + PageActionMap.get(win).addImpression(recommendation); + } else { + // Tracking protection messages + await PageActionMap.get(win).showPopup(); + PageActionMap.get(win).addImpression(recommendation); + } + } else { + // Doorhanger messages + await PageActionMap.get(win).showAddressBarNotifier(recommendation, true); + } + return true; + }, + + /** + * Clear all recommendations and hide all PageActions + */ + clearRecommendations() { + // WeakMaps aren't iterable so we have to test all existing windows + for (const win of Services.wm.getEnumerator("navigator:browser")) { + if (win.closed || !PageActionMap.has(win)) { + continue; + } + PageActionMap.get(win).hideAddressBarNotifier(); + } + // WeakMaps don't have a `clear` method + PageActionMap = new WeakMap(); + RecommendationMap = new WeakMap(); + this.PageActionMap = PageActionMap; + this.RecommendationMap = RecommendationMap; + }, + + /** + * Reload the l10n Fluent files for all PageActions + */ + reloadL10n() { + for (const win of Services.wm.getEnumerator("navigator:browser")) { + if (win.closed || !PageActionMap.has(win)) { + continue; + } + PageActionMap.get(win).reloadL10n(); + } + }, +}; + +const EXPORTED_SYMBOLS = ["CFRPageActions", "PageAction"]; diff --git a/browser/components/newtab/lib/DefaultSites.sys.mjs b/browser/components/newtab/lib/DefaultSites.sys.mjs new file mode 100644 index 0000000000..ea49cccc03 --- /dev/null +++ b/browser/components/newtab/lib/DefaultSites.sys.mjs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const DEFAULT_SITES_MAP = new Map([ + // This first item is the global list fallback for any unexpected geos + [ + "", + "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.reddit.com/,https://www.amazon.com/,https://twitter.com/", + ], + [ + "US", + "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://twitter.com/", + ], + [ + "CA", + "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://www.amazon.ca/,https://twitter.com/", + ], + [ + "DE", + "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.de/,https://www.ebay.de/,https://www.wikipedia.org/,https://www.reddit.com/", + ], + [ + "PL", + "https://www.youtube.com/,https://www.facebook.com/,https://allegro.pl/,https://www.wikipedia.org/,https://www.olx.pl/,https://www.wykop.pl/", + ], + [ + "RU", + "https://vk.com/,https://www.youtube.com/,https://ok.ru/,https://www.avito.ru/,https://www.aliexpress.com/,https://www.wikipedia.org/", + ], + [ + "GB", + "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.amazon.co.uk/,https://www.bbc.co.uk/,https://www.ebay.co.uk/", + ], + [ + "FR", + "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.amazon.fr/,https://www.leboncoin.fr/,https://twitter.com/", + ], + [ + "CN", + "https://www.baidu.com/,https://www.zhihu.com/,https://www.ifeng.com/,https://weibo.com/,https://www.ctrip.com/,https://www.iqiyi.com/", + ], +]); + +// Immutable for export. +export const DEFAULT_SITES = Object.freeze(DEFAULT_SITES_MAP); diff --git a/browser/components/newtab/lib/DiscoveryStreamFeed.jsm b/browser/components/newtab/lib/DiscoveryStreamFeed.jsm new file mode 100644 index 0000000000..0297ca0b4c --- /dev/null +++ b/browser/components/newtab/lib/DiscoveryStreamFeed.jsm @@ -0,0 +1,2427 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", + PersistentCache: "resource://activity-stream/lib/PersistentCache.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", +}); +const { setTimeout, clearTimeout } = ChromeUtils.importESModule( + "resource://gre/modules/Timer.sys.mjs" +); +const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const CACHE_KEY = "discovery_stream"; +const LAYOUT_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const STARTUP_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week +const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour +const MIN_PERSONALIZATION_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours +const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server +const FETCH_TIMEOUT = 45 * 1000; +const SPOCS_URL = "https://spocs.getpocket.com/spocs"; +const FEED_URL = + "https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=$locale®ion=$region&count=30"; +const PREF_CONFIG = "discoverystream.config"; +const PREF_ENDPOINTS = "discoverystream.endpoints"; +const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId"; +const PREF_ENABLED = "discoverystream.enabled"; +const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout"; +const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint"; +const PREF_SPOCS_ENDPOINT_QUERY = "discoverystream.spocs-endpoint-query"; +const PREF_REGION_BASIC_LAYOUT = "discoverystream.region-basic-layout"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories"; +const PREF_USER_TOPSITES = "feeds.topsites"; +const PREF_SYSTEM_TOPSITES = "feeds.system.topsites"; +const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear"; +const PREF_SHOW_SPONSORED = "showSponsored"; +const PREF_SHOW_SPONSORED_TOPSITES = "showSponsoredTopSites"; +const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions"; +const PREF_FLIGHT_BLOCKS = "discoverystream.flight.blocks"; +const PREF_REC_IMPRESSIONS = "discoverystream.rec.impressions"; +const PREF_COLLECTIONS_ENABLED = + "discoverystream.sponsored-collections.enabled"; +const PREF_POCKET_BUTTON = "extensions.pocket.enabled"; +const PREF_COLLECTION_DISMISSIBLE = "discoverystream.isCollectionDismissible"; +const PREF_PERSONALIZATION = "discoverystream.personalization.enabled"; +const PREF_PERSONALIZATION_OVERRIDE = + "discoverystream.personalization.override"; + +let getHardcodedLayout; + +class DiscoveryStreamFeed { + constructor() { + // Internal state for checking if we've intialized all our data + this.loaded = false; + + // Persistent cache for remote endpoint data. + this.cache = new lazy.PersistentCache(CACHE_KEY, true); + this.locale = Services.locale.appLocaleAsBCP47; + this._impressionId = this.getOrCreateImpressionId(); + // Internal in-memory cache for parsing json prefs. + this._prefCache = {}; + } + + getOrCreateImpressionId() { + let impressionId = Services.prefs.getCharPref(PREF_IMPRESSION_ID, ""); + if (!impressionId) { + impressionId = String(Services.uuid.generateUUID()); + Services.prefs.setCharPref(PREF_IMPRESSION_ID, impressionId); + } + return impressionId; + } + + finalLayoutEndpoint(url, apiKey) { + if (url.includes("$apiKey") && !apiKey) { + throw new Error( + `Layout Endpoint - An API key was specified but none configured: ${url}` + ); + } + return url.replace("$apiKey", apiKey); + } + + get config() { + if (this._prefCache.config) { + return this._prefCache.config; + } + try { + this._prefCache.config = JSON.parse( + this.store.getState().Prefs.values[PREF_CONFIG] + ); + const layoutUrl = this._prefCache.config.layout_endpoint; + + const apiKeyPref = this._prefCache.config.api_key_pref; + if (layoutUrl && apiKeyPref) { + const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); + this._prefCache.config.layout_endpoint = this.finalLayoutEndpoint( + layoutUrl, + apiKey + ); + } + } catch (e) { + // istanbul ignore next + this._prefCache.config = {}; + // istanbul ignore next + console.error( + `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config. ${e}` + ); + } + this._prefCache.config.enabled = + this._prefCache.config.enabled && + this.store.getState().Prefs.values[PREF_ENABLED]; + + return this._prefCache.config; + } + + resetConfigDefauts() { + this.store.dispatch({ + type: at.CLEAR_PREF, + data: { + name: PREF_CONFIG, + }, + }); + } + + get region() { + return lazy.Region.home; + } + + get isBff() { + if (this._isBff === undefined) { + const pocketConfig = + this.store.getState().Prefs.values?.pocketConfig || {}; + + const preffedLocaleListString = pocketConfig.localeListConfig || ""; + const preffedLocales = preffedLocaleListString + .split(",") + .map(s => s.trim()); + const localeEnabled = this.locale && preffedLocales.includes(this.locale); + + const preffedRegionBffConfigString = pocketConfig.regionBffConfig || ""; + const preffedRegionBffConfig = preffedRegionBffConfigString + .split(",") + .map(s => s.trim()); + const regionBff = preffedRegionBffConfig.includes(this.region); + this._isBff = !localeEnabled && regionBff; + } + + return this._isBff; + } + + get showSpocs() { + // High level overall sponsored check, if one of these is true, + // we know we need some sort of spoc control setup. + return this.showSponsoredStories || this.showSponsoredTopsites; + } + + get showSponsoredStories() { + // Combine user-set sponsored opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SHOW_SPONSORED] && + this.config.show_spocs + ); + } + + get showSponsoredTopsites() { + const placements = this.getPlacements(); + // Combine user-set sponsored opt-out with placement data + return !!( + this.store.getState().Prefs.values[PREF_SHOW_SPONSORED_TOPSITES] && + placements.find(placement => placement.name === "sponsored-topsites") + ); + } + + get showStories() { + // Combine user-set stories opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] && + this.store.getState().Prefs.values[PREF_USER_TOPSTORIES] + ); + } + + get showTopsites() { + // Combine user-set topsites opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SYSTEM_TOPSITES] && + this.store.getState().Prefs.values[PREF_USER_TOPSITES] + ); + } + + get personalized() { + // If stories are not displayed, no point in trying to personalize them. + if (!this.showStories) { + return false; + } + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + const personalization = + this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + + // There is a server sent flag to keep personalization on. + // If the server stops sending this, we turn personalization off, + // until the server starts returning the signal. + const overrideState = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_OVERRIDE]; + + return ( + personalization && + !overrideState && + !!this.recommendationProvider && + (spocsPersonalized || recsPersonalized) + ); + } + + get recommendationProvider() { + if (this._recommendationProvider) { + return this._recommendationProvider; + } + this._recommendationProvider = this.store.feeds.get( + "feeds.recommendationprovider" + ); + return this._recommendationProvider; + } + + setupConfig(isStartup = false) { + // Send the initial state of the pref on our reducer + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_SETUP, + data: this.config, + meta: { + isStartup, + }, + }) + ); + } + + setupPrefs(isStartup = false) { + const pocketNewtabExperiment = lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "pocketNewtab", + }); + + const pocketNewtabRollout = lazy.ExperimentAPI.getRolloutMetaData({ + featureId: "pocketNewtab", + }); + + // We want to know if the user is in an experiment or rollout, + // but we prioritize experiments over rollouts. + const experimentMetaData = pocketNewtabExperiment || pocketNewtabRollout; + + let utmSource = "pocket-newtab"; + let utmCampaign = experimentMetaData?.slug; + let utmContent = experimentMetaData?.branch?.slug; + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_EXPERIMENT_DATA, + data: { + utmSource, + utmCampaign, + utmContent, + }, + meta: { + isStartup, + }, + }) + ); + + const pocketButtonEnabled = Services.prefs.getBoolPref(PREF_POCKET_BUTTON); + + const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {}; + const { region } = this.store.getState().Prefs.values; + + this.setupSpocsCacheUpdateTime(); + const saveToPocketCardRegions = nimbusConfig.saveToPocketCardRegions + ?.split(",") + .map(s => s.trim()); + const saveToPocketCard = + pocketButtonEnabled && + (nimbusConfig.saveToPocketCard || + saveToPocketCardRegions?.includes(region)); + + const hideDescriptionsRegions = nimbusConfig.hideDescriptionsRegions + ?.split(",") + .map(s => s.trim()); + const hideDescriptions = + nimbusConfig.hideDescriptions || + hideDescriptionsRegions?.includes(region); + + // We don't BroadcastToContent for this, as the changes may + // shift around elements on an open newtab the user is currently reading. + // So instead we AlsoToPreloaded so the next tab is updated. + // This is because setupPrefs is called by the system and not a user interaction. + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_PREFS_SETUP, + data: { + recentSavesEnabled: nimbusConfig.recentSavesEnabled, + pocketButtonEnabled, + saveToPocketCard, + hideDescriptions, + compactImages: nimbusConfig.compactImages, + imageGradient: nimbusConfig.imageGradient, + newSponsoredLabel: nimbusConfig.newSponsoredLabel, + titleLines: nimbusConfig.titleLines, + descLines: nimbusConfig.descLines, + readTime: nimbusConfig.readTime, + }, + meta: { + isStartup, + }, + }) + ); + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE, + data: { + value: + this.store.getState().Prefs.values[PREF_COLLECTION_DISMISSIBLE], + }, + meta: { + isStartup, + }, + }) + ); + } + + async setupPocketState(target) { + let dispatch = action => + this.store.dispatch(ac.OnlyToOneContent(action, target)); + const isUserLoggedIn = lazy.pktApi.isUserLoggedIn(); + dispatch({ + type: at.DISCOVERY_STREAM_POCKET_STATE_SET, + data: { + isUserLoggedIn, + }, + }); + + // If we're not logged in, don't bother fetching recent saves, we're done. + if (isUserLoggedIn) { + let recentSaves = await lazy.pktApi.getRecentSavesCache(); + if (recentSaves) { + // We have cache, so we can use those. + dispatch({ + type: at.DISCOVERY_STREAM_RECENT_SAVES, + data: { + recentSaves, + }, + }); + } else { + // We don't have cache, so fetch fresh stories. + lazy.pktApi.getRecentSaves({ + success(data) { + dispatch({ + type: at.DISCOVERY_STREAM_RECENT_SAVES, + data: { + recentSaves: data, + }, + }); + }, + error(error) {}, + }); + } + } + } + + uninitPrefs() { + // Reset in-memory cache + this._prefCache = {}; + } + + async fetchFromEndpoint(rawEndpoint, options = {}) { + if (!rawEndpoint) { + console.error("Tried to fetch endpoint but none was configured."); + return null; + } + + const apiKeyPref = this._prefCache.config.api_key_pref; + const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); + + // The server somtimes returns this value already replaced, but we try this for two reasons: + // 1. Layout endpoints are not from the server. + // 2. Hardcoded layouts don't have this already done for us. + const endpoint = rawEndpoint + .replace("$apiKey", apiKey) + .replace("$locale", this.locale) + .replace("$region", this.region); + + try { + // Make sure the requested endpoint is allowed + const allowed = this.store + .getState() + .Prefs.values[PREF_ENDPOINTS].split(","); + if (!allowed.some(prefix => endpoint.startsWith(prefix))) { + throw new Error(`Not one of allowed prefixes (${allowed})`); + } + + const controller = new AbortController(); + const { signal } = controller; + + const fetchPromise = fetch(endpoint, { + ...options, + credentials: "omit", + signal, + }); + // istanbul ignore next + const timeoutId = setTimeout(() => { + controller.abort(); + }, FETCH_TIMEOUT); + + const response = await fetchPromise; + if (!response.ok) { + throw new Error(`Unexpected status (${response.status})`); + } + clearTimeout(timeoutId); + + return response.json(); + } catch (error) { + console.error(`Failed to fetch ${endpoint}: ${error.message}`); + } + return null; + } + + get spocsCacheUpdateTime() { + if (this._spocsCacheUpdateTime) { + return this._spocsCacheUpdateTime; + } + this.setupSpocsCacheUpdateTime(); + return this._spocsCacheUpdateTime; + } + + setupSpocsCacheUpdateTime() { + const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {}; + const { spocsCacheTimeout } = nimbusConfig; + const MAX_TIMEOUT = 30; + const MIN_TIMEOUT = 5; + // We do a bit of min max checking the the configured value is between + // 5 and 30 minutes, to protect against unreasonable values. + if ( + spocsCacheTimeout && + spocsCacheTimeout <= MAX_TIMEOUT && + spocsCacheTimeout >= MIN_TIMEOUT + ) { + // This value is in minutes, but we want ms. + this._spocsCacheUpdateTime = spocsCacheTimeout * 60 * 1000; + } else { + // The const is already in ms. + this._spocsCacheUpdateTime = SPOCS_FEEDS_UPDATE_TIME; + } + } + + /** + * Returns true if data in the cache for a particular key has expired or is missing. + * @param {object} cachedData data returned from cache.get() + * @param {string} key a cache key + * @param {string?} url for "feed" only, the URL of the feed. + * @param {boolean} is this check done at initial browser load + */ + isExpired({ cachedData, key, url, isStartup }) { + const { layout, spocs, feeds } = cachedData; + const updateTimePerComponent = { + layout: LAYOUT_UPDATE_TIME, + spocs: this.spocsCacheUpdateTime, + feed: COMPONENT_FEEDS_UPDATE_TIME, + }; + const EXPIRATION_TIME = isStartup + ? STARTUP_CACHE_EXPIRE_TIME + : updateTimePerComponent[key]; + switch (key) { + case "layout": + // This never needs to expire, as it's not expected to change. + if (this.config.hardcoded_layout) { + return false; + } + return !layout || !(Date.now() - layout.lastUpdated < EXPIRATION_TIME); + case "spocs": + return !spocs || !(Date.now() - spocs.lastUpdated < EXPIRATION_TIME); + case "feed": + return ( + !feeds || + !feeds[url] || + !(Date.now() - feeds[url].lastUpdated < EXPIRATION_TIME) + ); + default: + // istanbul ignore next + throw new Error(`${key} is not a valid key`); + } + } + + async _checkExpirationPerComponent() { + const cachedData = (await this.cache.get()) || {}; + const { feeds } = cachedData; + return { + layout: this.isExpired({ cachedData, key: "layout" }), + spocs: this.showSpocs && this.isExpired({ cachedData, key: "spocs" }), + feeds: + this.showStories && + (!feeds || + Object.keys(feeds).some(url => + this.isExpired({ cachedData, key: "feed", url }) + )), + }; + } + + /** + * Returns true if any data for the cached endpoints has expired or is missing. + */ + async checkIfAnyCacheExpired() { + const expirationPerComponent = await this._checkExpirationPerComponent(); + return ( + expirationPerComponent.layout || + expirationPerComponent.spocs || + expirationPerComponent.feeds + ); + } + + async fetchLayout(isStartup) { + const cachedData = (await this.cache.get()) || {}; + let { layout } = cachedData; + if (this.isExpired({ cachedData, key: "layout", isStartup })) { + const layoutResponse = await this.fetchFromEndpoint( + this.config.layout_endpoint + ); + if (layoutResponse && layoutResponse.layout) { + layout = { + lastUpdated: Date.now(), + spocs: layoutResponse.spocs, + layout: layoutResponse.layout, + status: "success", + }; + + await this.cache.set("layout", layout); + } else { + console.error("No response for response.layout prop"); + } + } + return layout; + } + + updatePlacements(sendUpdate, layout, isStartup = false) { + const placements = []; + const placementsMap = {}; + for (const row of layout.filter(r => r.components && r.components.length)) { + for (const component of row.components.filter( + c => c.placement && c.spocs + )) { + // If we find a valid placement, we set it to this value. + let placement; + + // We need to check to see if this placement is on or not. + // If this placement has a prefs array, check against that. + if (component.spocs.prefs) { + // Check every pref in the array to see if this placement is turned on. + if ( + component.spocs.prefs.length && + component.spocs.prefs.every( + p => this.store.getState().Prefs.values[p] + ) + ) { + // This placement is on. + placement = component.placement; + } + } else if (this.showSponsoredStories) { + // If we do not have a prefs array, use old check. + // This is because Pocket spocs uses an old non pref method. + placement = component.placement; + } + + // Validate this placement and check for dupes. + if (placement?.name && !placementsMap[placement.name]) { + placementsMap[placement.name] = placement; + placements.push(placement); + } + } + } + + // Update placements data. + // Even if we have no placements, we still want to update it to clear it. + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_PLACEMENTS, + data: { placements }, + meta: { + isStartup, + }, + }); + } + + /** + * Adds a query string to a URL. + * A query can be any string literal accepted by https://developer.mozilla.org/docs/Web/API/URLSearchParams + * Examples: "?foo=1&bar=2", "&foo=1&bar=2", "foo=1&bar=2", "?bar=2" or "bar=2" + */ + addEndpointQuery(url, query) { + if (!query) { + return url; + } + + const urlObject = new URL(url); + const params = new URLSearchParams(query); + + for (let [key, val] of params.entries()) { + urlObject.searchParams.append(key, val); + } + + return urlObject.toString(); + } + + parseGridPositions(csvPositions) { + let gridPositions; + + // Only accept parseable non-negative integers + try { + gridPositions = csvPositions.map(index => { + let parsedInt = parseInt(index, 10); + + if (!isNaN(parsedInt) && parsedInt >= 0) { + return parsedInt; + } + + throw new Error("Bad input"); + }); + } catch (e) { + // Catch spoc positions that are not numbers or negative, and do nothing. + // We have hard coded backup positions. + gridPositions = undefined; + } + + return gridPositions; + } + + async loadLayout(sendUpdate, isStartup) { + let layoutResp = {}; + let url = ""; + + if (!this.config.hardcoded_layout) { + layoutResp = await this.fetchLayout(isStartup); + } + + if (!layoutResp || !layoutResp.layout) { + const isBasicLayout = + this.config.hardcoded_basic_layout || + this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT] || + this.store.getState().Prefs.values[PREF_REGION_BASIC_LAYOUT]; + + const sponsoredCollectionsEnabled = + this.store.getState().Prefs.values[PREF_COLLECTIONS_ENABLED]; + + const pocketConfig = + this.store.getState().Prefs.values?.pocketConfig || {}; + const onboardingExperience = + this.isBff && pocketConfig.onboardingExperience; + + let items = isBasicLayout ? 3 : 21; + if (pocketConfig.fourCardLayout || pocketConfig.hybridLayout) { + items = isBasicLayout ? 4 : 24; + } + + const prepConfArr = arr => { + return arr + ?.split(",") + .filter(item => item) + .map(item => parseInt(item, 10)); + }; + + const spocAdTypes = prepConfArr(pocketConfig.spocAdTypes); + const spocZoneIds = prepConfArr(pocketConfig.spocZoneIds); + const spocTopsitesAdTypes = prepConfArr(pocketConfig.spocTopsitesAdTypes); + const spocTopsitesZoneIds = prepConfArr(pocketConfig.spocTopsitesZoneIds); + const { spocSiteId } = pocketConfig; + let spocPlacementData; + let spocTopsitesPlacementData; + let spocsUrl; + + if (spocAdTypes?.length && spocZoneIds?.length) { + spocPlacementData = { + ad_types: spocAdTypes, + zone_ids: spocZoneIds, + }; + } + + if (spocTopsitesAdTypes?.length && spocTopsitesZoneIds?.length) { + spocTopsitesPlacementData = { + ad_types: spocTopsitesAdTypes, + zone_ids: spocTopsitesZoneIds, + }; + } + + if (spocSiteId) { + const newUrl = new URL(SPOCS_URL); + newUrl.searchParams.set("site", spocSiteId); + spocsUrl = newUrl.href; + } + + let feedUrl = FEED_URL; + + if (this.isBff) { + feedUrl = `https://${lazy.NimbusFeatures.saveToPocket.getVariable( + "bffApi" + )}/desktop/v1/recommendations?locale=$locale®ion=$region&count=30`; + } + + // Set a hardcoded layout if one is needed. + // Changing values in this layout in memory object is unnecessary. + layoutResp = getHardcodedLayout({ + spocsUrl, + feedUrl, + items, + sponsoredCollectionsEnabled, + spocPlacementData, + spocTopsitesPlacementData, + spocPositions: this.parseGridPositions( + pocketConfig.spocPositions?.split(`,`) + ), + spocTopsitesPositions: this.parseGridPositions( + pocketConfig.spocTopsitesPositions?.split(`,`) + ), + widgetPositions: this.parseGridPositions( + pocketConfig.widgetPositions?.split(`,`) + ), + widgetData: [ + ...(this.locale.startsWith("en-") ? [{ type: "TopicsWidget" }] : []), + ], + hybridLayout: pocketConfig.hybridLayout, + hideCardBackground: pocketConfig.hideCardBackground, + fourCardLayout: pocketConfig.fourCardLayout, + newFooterSection: pocketConfig.newFooterSection, + compactGrid: pocketConfig.compactGrid, + // For now essentialReadsHeader and editorsPicksHeader are English only. + essentialReadsHeader: + this.locale.startsWith("en-") && pocketConfig.essentialReadsHeader, + editorsPicksHeader: + this.locale.startsWith("en-") && pocketConfig.editorsPicksHeader, + onboardingExperience, + }); + } + + sendUpdate({ + type: at.DISCOVERY_STREAM_LAYOUT_UPDATE, + data: layoutResp, + meta: { + isStartup, + }, + }); + + if (layoutResp.spocs) { + url = + this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] || + this.config.spocs_endpoint || + layoutResp.spocs.url; + + const spocsEndpointQuery = + this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT_QUERY]; + + // For QA, testing, or debugging purposes, there may be a query string to add. + url = this.addEndpointQuery(url, spocsEndpointQuery); + + if ( + url && + url !== this.store.getState().DiscoveryStream.spocs.spocs_endpoint + ) { + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT, + data: { + url, + }, + meta: { + isStartup, + }, + }); + this.updatePlacements(sendUpdate, layoutResp.layout, isStartup); + } + } + } + + /** + * buildFeedPromise - Adds the promise result to newFeeds and + * pushes a promise to newsFeedsPromises. + * @param {Object} Has both newFeedsPromises (Array) and newFeeds (Object) + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Function} We return a function so we can contain + * the scope for isStartup and the promises object. + * Combines feed results and promises for each component with a feed. + */ + buildFeedPromise( + { newFeedsPromises, newFeeds }, + isStartup = false, + sendUpdate + ) { + return component => { + const { url } = component.feed; + + if (!newFeeds[url]) { + // We initially stub this out so we don't fetch dupes, + // we then fill in with the proper object inside the promise. + newFeeds[url] = {}; + const feedPromise = this.getComponentFeed(url, isStartup); + + feedPromise + .then(feed => { + // If we stored the result of filter in feed cache as it happened, + // I think we could reduce doing this for cache fetches. + // Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606277 + newFeeds[url] = this.filterRecommendations(feed); + sendUpdate({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed: newFeeds[url], + url, + }, + meta: { + isStartup, + }, + }); + }) + .catch( + /* istanbul ignore next */ error => { + console.error( + `Error trying to load component feed ${url}: ${error}` + ); + } + ); + newFeedsPromises.push(feedPromise); + } + }; + } + + filterRecommendations(feed) { + if ( + feed && + feed.data && + feed.data.recommendations && + feed.data.recommendations.length + ) { + const { data: recommendations } = this.filterBlocked( + feed.data.recommendations + ); + return { + ...feed, + data: { + ...feed.data, + recommendations, + }, + }; + } + return feed; + } + + /** + * reduceFeedComponents - Filters out components with no feeds, and combines + * all feeds on this component with the feeds from other components. + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Function} We return a function so we can contain the scope for isStartup. + * Reduces feeds into promises and feed data. + */ + reduceFeedComponents(isStartup, sendUpdate) { + return (accumulator, row) => { + row.components + .filter(component => component && component.feed) + .forEach(this.buildFeedPromise(accumulator, isStartup, sendUpdate)); + return accumulator; + }; + } + + /** + * buildFeedPromises - Filters out rows with no components, + * and gets us a promise for each unique feed. + * @param {Object} layout This is the Discovery Stream layout object. + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Object} An object with newFeedsPromises (Array) and newFeeds (Object), + * we can Promise.all newFeedsPromises to get completed data in newFeeds. + */ + buildFeedPromises(layout, isStartup, sendUpdate) { + const initialData = { + newFeedsPromises: [], + newFeeds: {}, + }; + return layout + .filter(row => row && row.components) + .reduce(this.reduceFeedComponents(isStartup, sendUpdate), initialData); + } + + async loadComponentFeeds(sendUpdate, isStartup = false) { + const { DiscoveryStream } = this.store.getState(); + + if (!DiscoveryStream || !DiscoveryStream.layout) { + return; + } + + // Reset the flag that indicates whether or not at least one API request + // was issued to fetch the component feed in `getComponentFeed()`. + this.componentFeedFetched = false; + const { newFeedsPromises, newFeeds } = this.buildFeedPromises( + DiscoveryStream.layout, + isStartup, + sendUpdate + ); + + // Each promise has a catch already built in, so no need to catch here. + await Promise.all(newFeedsPromises); + + if (this.componentFeedFetched) { + this.cleanUpTopRecImpressionPref(newFeeds); + } + await this.cache.set("feeds", newFeeds); + sendUpdate({ + type: at.DISCOVERY_STREAM_FEEDS_UPDATE, + meta: { + isStartup, + }, + }); + } + + getPlacements() { + const { placements } = this.store.getState().DiscoveryStream.spocs; + return placements; + } + + // I wonder, can this be better as a reducer? + // See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717 + placementsForEach(callback) { + this.getPlacements().forEach(callback); + } + + // Bug 1567271 introduced meta data on a list of spocs. + // This involved moving the spocs array into an items prop. + // However, old data could still be returned, and cached data might also be old. + // For ths reason, we want to ensure if we don't find an items array, + // we use the previous array placement, and then stub out title and context to empty strings. + // We need to do this *after* both fresh fetches and cached data to reduce repetition. + normalizeSpocsItems(spocs) { + const items = spocs.items || spocs; + const title = spocs.title || ""; + const context = spocs.context || ""; + const sponsor = spocs.sponsor || ""; + // We do not stub sponsored_by_override with an empty string. It is an override, and an empty string + // explicitly means to override the client to display an empty string. + // An empty string is not an no op in this case. Undefined is the proper no op here. + const { sponsored_by_override } = spocs; + // Undefined is fine here. It's optional and only used by collections. + // If we leave it out, you get a collection that cannot be dismissed. + const { flight_id } = spocs; + return { + items, + title, + context, + sponsor, + sponsored_by_override, + ...(flight_id ? { flight_id } : {}), + }; + } + + // This turns personalization on/off if the server sends the override command. + // The server sends a true signal to keep personalization on. So a malfunctioning + // server would more likely mistakenly turn off personalization, and not turn it on. + // This is safer, because the override is for cases where personalization is causing issues. + // So having it mistakenly go off is safe, but it mistakenly going on could be bad. + personalizationOverride(overrideCommand) { + // Are we currently in an override state. + // This is useful to know if we want to do a cleanup. + const overrideState = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_OVERRIDE]; + + // Is this profile currently set to be personalized. + const personalization = + this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + + // If we have an override command, profile is currently personalized, + // and is not currently being overridden, we can set the override pref. + if (overrideCommand && personalization && !overrideState) { + this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION_OVERRIDE, true)); + } + + // This is if we need to revert an override and do cleanup. + // We do this if we are in an override state, + // but not currently receiving the override signal. + if (!overrideCommand && overrideState) { + this.store.dispatch({ + type: at.CLEAR_PREF, + data: { name: PREF_PERSONALIZATION_OVERRIDE }, + }); + } + } + + updateSponsoredCollectionsPref(collectionEnabled = false) { + const currentState = + this.store.getState().Prefs.values[PREF_COLLECTIONS_ENABLED]; + + // If the current state does not match the new state, update the pref. + if (currentState !== collectionEnabled) { + this.store.dispatch( + ac.SetPref(PREF_COLLECTIONS_ENABLED, collectionEnabled) + ); + } + } + + async loadSpocs(sendUpdate, isStartup) { + const cachedData = (await this.cache.get()) || {}; + let spocsState; + + const placements = this.getPlacements(); + + if (this.showSpocs && placements?.length) { + spocsState = cachedData.spocs; + if (this.isExpired({ cachedData, key: "spocs", isStartup })) { + const endpoint = + this.store.getState().DiscoveryStream.spocs.spocs_endpoint; + + const headers = new Headers(); + headers.append("content-type", "application/json"); + + const apiKeyPref = this._prefCache.config.api_key_pref; + const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); + + const spocsResponse = await this.fetchFromEndpoint(endpoint, { + method: "POST", + headers, + body: JSON.stringify({ + pocket_id: this._impressionId, + version: 2, + consumer_key: apiKey, + ...(placements.length ? { placements } : {}), + }), + }); + + if (spocsResponse) { + spocsState = { + lastUpdated: Date.now(), + spocs: { + ...spocsResponse, + }, + }; + + if (spocsResponse.settings && spocsResponse.settings.feature_flags) { + this.personalizationOverride( + // The server's old signal was for a version override. + // When we removed version 1, version 2 was now the defacto only version. + // Without a version 1, the override is now a command to turn off personalization. + !spocsResponse.settings.feature_flags.spoc_v2 + ); + this.updateSponsoredCollectionsPref( + spocsResponse.settings.feature_flags.collections + ); + } + + const spocsResultPromises = this.getPlacements().map( + async placement => { + const freshSpocs = spocsState.spocs[placement.name]; + + if (!freshSpocs) { + return; + } + + // spocs can be returns as an array, or an object with an items array. + // We want to normalize this so all our spocs have an items array. + // There can also be some meta data for title and context. + // This is mostly because of backwards compat. + const { + items: normalizedSpocsItems, + title, + context, + sponsor, + sponsored_by_override, + } = this.normalizeSpocsItems(freshSpocs); + + if (!normalizedSpocsItems || !normalizedSpocsItems.length) { + // In the case of old data, we still want to ensure we normalize the data structure, + // even if it's empty. We expect the empty data to be an object with items array, + // and not just an empty array. + spocsState.spocs = { + ...spocsState.spocs, + [placement.name]: { + title, + context, + items: [], + }, + }; + return; + } + + // Migrate flight_id + const { data: migratedSpocs } = + this.migrateFlightId(normalizedSpocsItems); + + const { data: capResult } = this.frequencyCapSpocs(migratedSpocs); + + const { data: blockedResults } = this.filterBlocked(capResult); + + const { data: scoredResults } = await this.scoreItems( + blockedResults, + "spocs" + ); + + spocsState.spocs = { + ...spocsState.spocs, + [placement.name]: { + title, + context, + sponsor, + sponsored_by_override, + items: scoredResults, + }, + }; + } + ); + await Promise.all(spocsResultPromises); + + this.cleanUpFlightImpressionPref(spocsState.spocs); + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.spocs, + }); + } else { + console.error("No response for spocs_endpoint prop"); + } + } + } + + // Use good data if we have it, otherwise nothing. + // We can have no data if spocs set to off. + // We can have no data if request fails and there is no good cache. + // We want to send an update spocs or not, so client can render something. + spocsState = + spocsState && spocsState.spocs + ? spocsState + : { + lastUpdated: Date.now(), + spocs: {}, + }; + + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.spocs, + }, + meta: { + isStartup, + }, + }); + } + + async clearSpocs() { + const endpoint = + this.store.getState().Prefs.values[PREF_SPOCS_CLEAR_ENDPOINT]; + if (!endpoint) { + return; + } + const headers = new Headers(); + headers.append("content-type", "application/json"); + + await this.fetchFromEndpoint(endpoint, { + method: "DELETE", + headers, + body: JSON.stringify({ + pocket_id: this._impressionId, + }), + }); + } + + /* + * This just re hydrates the provider from cache. + * We can call this on startup because it's generally fast. + * It reports to devtools the last time the data in the cache was updated. + */ + async loadPersonalizationScoresCache(isStartup = false) { + const cachedData = (await this.cache.get()) || {}; + const { personalization } = cachedData; + + if (this.personalized && personalization && personalization.scores) { + this.recommendationProvider.setProvider(personalization.scores); + + this.personalizationLastUpdated = personalization._timestamp; + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, + data: { + lastUpdated: this.personalizationLastUpdated, + }, + meta: { + isStartup, + }, + }) + ); + } + } + + /* + * This creates a new recommendationProvider using fresh data, + * It's run on a last updated timer. This is the opposite of loadPersonalizationScoresCache. + * This is also much slower so we only trigger this in the background on idle-daily. + * It causes new profiles to pick up personalization slowly because the first time + * a new profile is run you don't have any old cache to use, so it needs to wait for the first + * idle-daily. Older profiles can rely on cache during the idle-daily gap. Idle-daily is + * usually run once every 24 hours. + */ + async updatePersonalizationScores() { + if ( + !this.personalized || + Date.now() - this.personalizationLastUpdated < + MIN_PERSONALIZATION_UPDATE_TIME + ) { + return; + } + + this.recommendationProvider.setProvider(); + + await this.recommendationProvider.init(); + + const personalization = { scores: this.recommendationProvider.getScores() }; + this.personalizationLastUpdated = Date.now(); + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, + data: { + lastUpdated: this.personalizationLastUpdated, + }, + }) + ); + personalization._timestamp = this.personalizationLastUpdated; + this.cache.set("personalization", personalization); + } + + observe(subject, topic, data) { + switch (topic) { + case "idle-daily": + this.updatePersonalizationScores(); + break; + case "nsPref:changed": + // If the Pocket button was turned on or off, we need to update the cards + // because cards show menu options for the Pocket button that need to be removed. + if (data === PREF_POCKET_BUTTON) { + this.configReset(); + } + break; + } + } + + /* + * This function is used to sort any type of story, both spocs and recs. + * This uses hierarchical sorting, first sorting by priority, then by score within a priority. + * This function could be sorting an array of spocs or an array of recs. + * A rec would have priority undefined, and a spoc would probably have a priority set. + * Priority is sorted ascending, so low numbers are the highest priority. + * Score is sorted descending, so high numbers are the highest score. + * Undefined priority values are considered the lowest priority. + * A negative priority is considered the same as undefined, lowest priority. + * A negative priority is unlikely and not currently supported or expected. + * A negative score is a possible use case. + */ + sortItem(a, b) { + // If the priorities are the same, sort based on score. + // If both item priorities are undefined, + // we can safely sort via score. + if (a.priority === b.priority) { + return b.score - a.score; + } else if (!a.priority || a.priority <= 0) { + // If priority is undefined or an unexpected value, + // consider it lowest priority. + return 1; + } else if (!b.priority || b.priority <= 0) { + // Also consider this case lowest priority. + return -1; + } + // Our primary sort for items with priority. + return a.priority - b.priority; + } + + async scoreItems(items, type) { + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + const personalizedByType = + type === "feed" ? recsPersonalized : spocsPersonalized; + + const data = ( + await Promise.all( + items.map(item => this.scoreItem(item, personalizedByType)) + ) + ) + // Sort by highest scores. + .sort(this.sortItem); + + return { data }; + } + + async scoreItem(item, personalizedByType) { + item.score = item.item_score; + if (item.score !== 0 && !item.score) { + item.score = 1; + } + if (this.personalized && personalizedByType) { + await this.recommendationProvider.calculateItemRelevanceScore(item); + } + return item; + } + + filterBlocked(data) { + if (data && data.length) { + let flights = this.readDataPref(PREF_FLIGHT_BLOCKS); + const filteredItems = data.filter(item => { + const blocked = + lazy.NewTabUtils.blockedLinks.isBlocked({ url: item.url }) || + flights[item.flight_id]; + return !blocked; + }); + return { data: filteredItems }; + } + return { data }; + } + + // For backwards compatibility, older spoc endpoint don't have flight_id, + // but instead had campaign_id we can use + // + // @param {Object} data An object that might have a SPOCS array. + // @returns {Object} An object with a property `data` as the result. + migrateFlightId(spocs) { + if (spocs && spocs.length) { + return { + data: spocs.map(s => { + return { + ...s, + ...(s.flight_id || s.campaign_id + ? { + flight_id: s.flight_id || s.campaign_id, + } + : {}), + ...(s.caps + ? { + caps: { + ...s.caps, + flight: s.caps.flight || s.caps.campaign, + }, + } + : {}), + }; + }), + }; + } + return { data: spocs }; + } + + // Filter spocs based on frequency caps + // + // @param {Object} data An object that might have a SPOCS array. + // @returns {Object} An object with a property `data` as the result, and a property + // `filterItems` as the frequency capped items. + frequencyCapSpocs(spocs) { + if (spocs && spocs.length) { + const impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); + const caps = []; + const result = spocs.filter(s => { + const isBelow = this.isBelowFrequencyCap(impressions, s); + if (!isBelow) { + caps.push(s); + } + return isBelow; + }); + // send caps to redux if any. + if (caps.length) { + this.store.dispatch({ + type: at.DISCOVERY_STREAM_SPOCS_CAPS, + data: caps, + }); + } + return { data: result, filtered: caps }; + } + return { data: spocs, filtered: [] }; + } + + // Frequency caps are based on flight, which may include multiple spocs. + // We currently support two types of frequency caps: + // - lifetime: Indicates how many times spocs from a flight can be shown in total + // - period: Indicates how many times spocs from a flight can be shown within a period + // + // So, for example, the feed configuration below defines that for flight 1 no more + // than 5 spocs can be shown in total, and no more than 2 per hour. + // "flight_id": 1, + // "caps": { + // "lifetime": 5, + // "flight": { + // "count": 2, + // "period": 3600 + // } + // } + isBelowFrequencyCap(impressions, spoc) { + const flightImpressions = impressions[spoc.flight_id]; + if (!flightImpressions) { + return true; + } + + const lifetime = spoc.caps && spoc.caps.lifetime; + + const lifeTimeCap = Math.min( + lifetime || MAX_LIFETIME_CAP, + MAX_LIFETIME_CAP + ); + const lifeTimeCapExceeded = flightImpressions.length >= lifeTimeCap; + if (lifeTimeCapExceeded) { + return false; + } + + const flightCap = spoc.caps && spoc.caps.flight; + if (flightCap) { + const flightCapExceeded = + flightImpressions.filter(i => Date.now() - i < flightCap.period * 1000) + .length >= flightCap.count; + return !flightCapExceeded; + } + return true; + } + + async retryFeed(feed) { + const { url } = feed; + const result = await this.getComponentFeed(url); + const newFeed = this.filterRecommendations(result); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed: newFeed, + url, + }, + }) + ); + } + + async getComponentFeed(feedUrl, isStartup) { + const cachedData = (await this.cache.get()) || {}; + const { feeds } = cachedData; + + let feed = feeds ? feeds[feedUrl] : null; + if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) { + let options = {}; + if (this.isBff) { + const headers = new Headers(); + const oAuthConsumerKey = lazy.NimbusFeatures.saveToPocket.getVariable( + "oAuthConsumerKeyBff" + ); + headers.append("consumer_key", oAuthConsumerKey); + options = { + method: "GET", + headers, + }; + } + + const feedResponse = await this.fetchFromEndpoint(feedUrl, options); + if (feedResponse) { + const { settings = {} } = feedResponse; + let { recommendations } = feedResponse; + if (this.isBff) { + recommendations = feedResponse.data.map(item => ({ + id: item.tileId, + url: item.url, + title: item.title, + excerpt: item.excerpt, + publisher: item.publisher, + raw_image_src: item.imageUrl, + })); + } + const { data: scoredItems } = await this.scoreItems( + recommendations, + "feed" + ); + const { recsExpireTime } = settings; + const rotatedItems = this.rotate(scoredItems, recsExpireTime); + this.componentFeedFetched = true; + feed = { + lastUpdated: Date.now(), + data: { + settings, + recommendations: rotatedItems, + status: "success", + }, + }; + } else { + console.error("No response for feed"); + } + } + + // If we have no feed at this point, both fetch and cache failed for some reason. + return ( + feed || { + data: { + status: "failed", + }, + } + ); + } + + /** + * Called at startup to update cached data in the background. + */ + async _maybeUpdateCachedData() { + const expirationPerComponent = await this._checkExpirationPerComponent(); + // Pass in `store.dispatch` to send the updates only to main + if (expirationPerComponent.layout) { + await this.loadLayout(this.store.dispatch); + } + if (expirationPerComponent.spocs) { + await this.loadSpocs(this.store.dispatch); + } + if (expirationPerComponent.feeds) { + await this.loadComponentFeeds(this.store.dispatch); + } + } + + /** + * @typedef {Object} RefreshAll + * @property {boolean} updateOpenTabs - Sends updates to open tabs immediately if true, + * updates in background if false + * @property {boolean} isStartup - When the function is called at browser startup + * + * Refreshes layout, component feeds, and spocs in order if caches have expired. + * @param {RefreshAll} options + */ + async refreshAll(options = {}) { + const personalizationCacheLoadPromise = this.loadPersonalizationScoresCache( + options.isStartup + ); + + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + + let expirationPerComponent = {}; + if (this.personalized) { + // We store this before we refresh content. + // This way, we can know what and if something got updated, + // so we can know to score the results. + expirationPerComponent = await this._checkExpirationPerComponent(); + } + await this.refreshContent(options); + + if (this.personalized) { + // personalizationCacheLoadPromise is probably done, because of the refreshContent await above, + // but to be sure, we should check that it's done, without making the parent function wait. + personalizationCacheLoadPromise.then(() => { + // If we don't have expired stories or feeds, we don't need to score after init. + // If we do have expired stories, we want to score after init. + // In both cases, we don't want these to block the parent function. + // This is why we store the promise, and call then to do our scoring work. + const initPromise = this.recommendationProvider.init(); + initPromise.then(() => { + // Both scoreFeeds and scoreSpocs are promises, + // but they don't need to wait for each other. + // We can just fire them and forget at this point. + const { feeds, spocs } = this.store.getState().DiscoveryStream; + if ( + recsPersonalized && + feeds.loaded && + expirationPerComponent.feeds + ) { + this.scoreFeeds(feeds); + } + if ( + spocsPersonalized && + spocs.loaded && + expirationPerComponent.spocs + ) { + this.scoreSpocs(spocs); + } + }); + }); + } + } + + async scoreFeeds(feedsState) { + if (feedsState.data) { + const feeds = {}; + const feedsPromises = Object.keys(feedsState.data).map(url => { + let feed = feedsState.data[url]; + const feedPromise = this.scoreItems(feed.data.recommendations, "feed"); + feedPromise.then(({ data: scoredItems }) => { + const { recsExpireTime } = feed.data.settings; + const recommendations = this.rotate(scoredItems, recsExpireTime); + feed = { + ...feed, + data: { + ...feed.data, + recommendations, + }, + }; + + feeds[url] = feed; + + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed, + url, + }, + }) + ); + }); + return feedPromise; + }); + await Promise.all(feedsPromises); + await this.cache.set("feeds", feeds); + } + } + + async scoreSpocs(spocsState) { + const spocsResultPromises = this.getPlacements().map(async placement => { + const nextSpocs = spocsState.data[placement.name] || {}; + const { items } = nextSpocs; + + if (!items || !items.length) { + return; + } + + const { data: scoreResult } = await this.scoreItems(items, "spocs"); + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...nextSpocs, + items: scoreResult, + }, + }; + }); + await Promise.all(spocsResultPromises); + + // Update cache here so we don't need to re calculate scores on loads from cache. + // Related Bug 1606276 + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }, + }) + ); + } + + async refreshContent(options = {}) { + const { updateOpenTabs, isStartup } = options; + + const dispatch = updateOpenTabs + ? action => this.store.dispatch(ac.BroadcastToContent(action)) + : this.store.dispatch; + + await this.loadLayout(dispatch, isStartup); + if (this.showStories || this.showTopsites) { + const promises = []; + // We could potentially have either or both sponsored topsites or stories. + // We only make one fetch, and control which to request when we fetch. + // So for now we only care if we need to make this request at all. + const spocsPromise = this.loadSpocs(dispatch, isStartup).catch(error => + console.error(`Error trying to load spocs feeds: ${error}`) + ); + promises.push(spocsPromise); + if (this.showStories) { + const storiesPromise = this.loadComponentFeeds( + dispatch, + isStartup + ).catch(error => + console.error(`Error trying to load component feeds: ${error}`) + ); + promises.push(storiesPromise); + } + await Promise.all(promises); + if (isStartup) { + await this._maybeUpdateCachedData(); + } + } + } + + // We have to rotate stories on the client so that + // active stories are at the front of the list, followed by stories that have expired + // impressions i.e. have been displayed for longer than recsExpireTime. + rotate(recommendations, recsExpireTime) { + const maxImpressionAge = Math.max( + recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, + DEFAULT_RECS_EXPIRE_TIME + ); + const impressions = this.readDataPref(PREF_REC_IMPRESSIONS); + const expired = []; + const active = []; + for (const item of recommendations) { + if ( + impressions[item.id] && + Date.now() - impressions[item.id] >= maxImpressionAge + ) { + expired.push(item); + } else { + active.push(item); + } + } + return active.concat(expired); + } + + enableStories() { + if (this.config.enabled && this.loaded) { + // If stories are being re enabled, ensure we have stories. + this.refreshAll({ updateOpenTabs: true }); + } + } + + async enable() { + await this.refreshAll({ updateOpenTabs: true, isStartup: true }); + Services.obs.addObserver(this, "idle-daily"); + this.loaded = true; + } + + async reset() { + this.resetDataPrefs(); + await this.resetCache(); + if (this.loaded) { + Services.obs.removeObserver(this, "idle-daily"); + } + this.resetState(); + } + + async resetCache() { + await this.resetAllCache(); + } + + async resetContentCache() { + await this.cache.set("layout", {}); + await this.cache.set("feeds", {}); + await this.cache.set("spocs", {}); + } + + async resetAllCache() { + await this.resetContentCache(); + await this.cache.set("personalization", {}); + // Reset in-memory caches. + this._isBff = undefined; + this._spocsCacheUpdateTime = undefined; + } + + resetDataPrefs() { + this.writeDataPref(PREF_SPOC_IMPRESSIONS, {}); + this.writeDataPref(PREF_REC_IMPRESSIONS, {}); + this.writeDataPref(PREF_FLIGHT_BLOCKS, {}); + } + + resetState() { + // Reset reducer + this.store.dispatch( + ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET }) + ); + this.setupPrefs(false /* isStartup */); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE, + data: { + value: + this.store.getState().Prefs.values[PREF_COLLECTION_DISMISSIBLE], + }, + }) + ); + this.personalizationLastUpdated = null; + this.loaded = false; + } + + async onPrefChange() { + // We always want to clear the cache/state if the pref has changed + await this.reset(); + if (this.config.enabled) { + // Load data from all endpoints + await this.enable(); + } + } + + // This is a request to change the config from somewhere. + // Can be from a specific pref related to Discovery Stream, + // or can be a generic request from an external feed that + // something changed. + configReset() { + this._prefCache.config = null; + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_CHANGE, + data: this.config, + }) + ); + } + + recordFlightImpression(flightId) { + let impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); + + const timeStamps = impressions[flightId] || []; + timeStamps.push(Date.now()); + impressions = { ...impressions, [flightId]: timeStamps }; + + this.writeDataPref(PREF_SPOC_IMPRESSIONS, impressions); + } + + recordTopRecImpressions(recId) { + let impressions = this.readDataPref(PREF_REC_IMPRESSIONS); + if (!impressions[recId]) { + impressions = { ...impressions, [recId]: Date.now() }; + this.writeDataPref(PREF_REC_IMPRESSIONS, impressions); + } + } + + recordBlockFlightId(flightId) { + const flights = this.readDataPref(PREF_FLIGHT_BLOCKS); + if (!flights[flightId]) { + flights[flightId] = 1; + this.writeDataPref(PREF_FLIGHT_BLOCKS, flights); + } + } + + cleanUpFlightImpressionPref(data) { + let flightIds = []; + this.placementsForEach(placement => { + const newSpocs = data[placement.name]; + if (!newSpocs) { + return; + } + + const items = newSpocs.items || []; + flightIds = [...flightIds, ...items.map(s => `${s.flight_id}`)]; + }); + if (flightIds && flightIds.length) { + this.cleanUpImpressionPref( + id => !flightIds.includes(id), + PREF_SPOC_IMPRESSIONS + ); + } + } + + // Clean up rec impression pref by removing all stories that are no + // longer part of the response. + cleanUpTopRecImpressionPref(newFeeds) { + // Need to build a single list of stories. + const activeStories = Object.keys(newFeeds) + .filter(currentValue => newFeeds[currentValue].data) + .reduce((accumulator, currentValue) => { + const { recommendations } = newFeeds[currentValue].data; + return accumulator.concat(recommendations.map(i => `${i.id}`)); + }, []); + this.cleanUpImpressionPref( + id => !activeStories.includes(id), + PREF_REC_IMPRESSIONS + ); + } + + writeDataPref(pref, impressions) { + this.store.dispatch(ac.SetPref(pref, JSON.stringify(impressions))); + } + + readDataPref(pref) { + const prefVal = this.store.getState().Prefs.values[pref]; + return prefVal ? JSON.parse(prefVal) : {}; + } + + cleanUpImpressionPref(isExpired, pref) { + const impressions = this.readDataPref(pref); + let changed = false; + + Object.keys(impressions).forEach(id => { + if (isExpired(id)) { + changed = true; + delete impressions[id]; + } + }); + + if (changed) { + this.writeDataPref(pref, impressions); + } + } + + onCollectionsChanged() { + // Update layout, and reload any off screen tabs. + // This does not change any existing open tabs. + // It also doesn't update any spoc or rec data, just the layout. + const dispatch = action => this.store.dispatch(ac.AlsoToPreloaded(action)); + this.loadLayout(dispatch, false); + } + + async onPrefChangedAction(action) { + switch (action.data.name) { + case PREF_CONFIG: + case PREF_ENABLED: + case PREF_HARDCODED_BASIC_LAYOUT: + case PREF_SPOCS_ENDPOINT: + case PREF_SPOCS_ENDPOINT_QUERY: + case PREF_PERSONALIZATION: + // This is a config reset directly related to Discovery Stream pref. + this.configReset(); + break; + case PREF_COLLECTIONS_ENABLED: + this.onCollectionsChanged(); + break; + case PREF_USER_TOPSITES: + case PREF_SYSTEM_TOPSITES: + if ( + !( + this.showTopsites || + (this.showStories && this.showSponsoredStories) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + break; + case PREF_USER_TOPSTORIES: + case PREF_SYSTEM_TOPSTORIES: + if ( + !( + this.showStories || + (this.showTopsites && this.showSponsoredTopsites) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + if (action.data.value) { + this.enableStories(); + } + break; + // Check if spocs was disabled. Remove them if they were. + case PREF_SHOW_SPONSORED: + case PREF_SHOW_SPONSORED_TOPSITES: + const dispatch = update => + this.store.dispatch(ac.BroadcastToContent(update)); + // We refresh placements data because one of the spocs were turned off. + this.updatePlacements( + dispatch, + this.store.getState().DiscoveryStream.layout + ); + // Currently the order of this is important. + // We need to check this after updatePlacements is called, + // because some of the spoc logic depends on the result of placement updates. + if ( + !( + (this.showSponsoredStories || + (this.showTopSites && this.showSponsoredTopSites)) && + (this.showSponsoredTopsites || + (this.showStories && this.showSponsoredStories)) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + // Placements have changed so consider spocs expired, and reload them. + await this.cache.set("spocs", {}); + await this.loadSpocs(dispatch); + break; + } + } + + async onAction(action) { + switch (action.type) { + case at.INIT: + // During the initialization of Firefox: + // 1. Set-up listeners and initialize the redux state for config; + this.setupConfig(true /* isStartup */); + this.setupPrefs(true /* isStartup */); + // 2. If config.enabled is true, start loading data. + if (this.config.enabled) { + await this.enable(); + } + Services.prefs.addObserver(PREF_POCKET_BUTTON, this); + break; + case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK: + case at.SYSTEM_TICK: + // Only refresh if we loaded once in .enable() + if ( + this.config.enabled && + this.loaded && + (await this.checkIfAnyCacheExpired()) + ) { + await this.refreshAll({ updateOpenTabs: false }); + } + break; + case at.DISCOVERY_STREAM_DEV_IDLE_DAILY: + Services.obs.notifyObservers(null, "idle-daily"); + break; + case at.DISCOVERY_STREAM_DEV_SYNC_RS: + lazy.RemoteSettings.pollChanges(); + break; + case at.DISCOVERY_STREAM_DEV_EXPIRE_CACHE: + // Personalization scores update at a slower interval than content, so in order to debug, + // we want to be able to expire just content to trigger the earlier expire times. + await this.resetContentCache(); + break; + case at.DISCOVERY_STREAM_CONFIG_SET_VALUE: + // Use the original string pref to then set a value instead of + // this.config which has some modifications + this.store.dispatch( + ac.SetPref( + PREF_CONFIG, + JSON.stringify({ + ...JSON.parse(this.store.getState().Prefs.values[PREF_CONFIG]), + [action.data.name]: action.data.value, + }) + ) + ); + break; + case at.DISCOVERY_STREAM_POCKET_STATE_INIT: + this.setupPocketState(action.meta.fromTarget); + break; + case at.DISCOVERY_STREAM_CONFIG_RESET: + // This is a generic config reset likely related to an external feed pref. + this.configReset(); + break; + case at.DISCOVERY_STREAM_CONFIG_RESET_DEFAULTS: + this.resetConfigDefauts(); + break; + case at.DISCOVERY_STREAM_RETRY_FEED: + this.retryFeed(action.data.feed); + break; + case at.DISCOVERY_STREAM_CONFIG_CHANGE: + // When the config pref changes, load or unload data as needed. + await this.onPrefChange(); + break; + case at.DISCOVERY_STREAM_IMPRESSION_STATS: + if ( + action.data.tiles && + action.data.tiles[0] && + action.data.tiles[0].id + ) { + this.recordTopRecImpressions(action.data.tiles[0].id); + } + break; + case at.DISCOVERY_STREAM_SPOC_IMPRESSION: + if (this.showSpocs) { + this.recordFlightImpression(action.data.flightId); + + // Apply frequency capping to SPOCs in the redux store, only update the + // store if the SPOCs are changed. + const spocsState = this.store.getState().DiscoveryStream.spocs; + + let frequencyCapped = []; + this.placementsForEach(placement => { + const spocs = spocsState.data[placement.name]; + if (!spocs || !spocs.items) { + return; + } + + const { data: capResult, filtered } = this.frequencyCapSpocs( + spocs.items + ); + frequencyCapped = [...frequencyCapped, ...filtered]; + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...spocs, + items: capResult, + }, + }; + }); + + if (frequencyCapped.length) { + // Update cache here so we don't need to re calculate frequency caps on loads from cache. + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }, + }) + ); + } + } + break; + // This is fired from the browser, it has no concept of spocs, flight or pocket. + // We match the blocked url with our available spoc urls to see if there is a match. + // I suspect we *could* instead do this in BLOCK_URL but I'm not sure. + case at.PLACES_LINK_BLOCKED: + if (this.showSpocs) { + let blockedItems = []; + const spocsState = this.store.getState().DiscoveryStream.spocs; + + this.placementsForEach(placement => { + const spocs = spocsState.data[placement.name]; + if (spocs && spocs.items && spocs.items.length) { + const blockedResults = []; + const blocks = spocs.items.filter(s => { + const blocked = s.url === action.data.url; + if (!blocked) { + blockedResults.push(s); + } + return blocked; + }); + + blockedItems = [...blockedItems, ...blocks]; + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...spocs, + items: blockedResults, + }, + }; + } + }); + + if (blockedItems.length) { + // Update cache here so we don't need to re calculate blocks on loads from cache. + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + + // If we're blocking a spoc, we want open tabs to have + // a slightly different treatment from future tabs. + // AlsoToPreloaded updates the source data and preloaded tabs with a new spoc. + // BroadcastToContent updates open tabs with a non spoc instead of a new spoc. + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_LINK_BLOCKED, + data: action.data, + }) + ); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_SPOC_BLOCKED, + data: action.data, + }) + ); + break; + } + } + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_LINK_BLOCKED, + data: action.data, + }) + ); + break; + case at.UNINIT: + // When this feed is shutting down: + this.uninitPrefs(); + this._recommendationProvider = null; + Services.prefs.removeObserver(PREF_POCKET_BUTTON, this); + break; + case at.BLOCK_URL: { + // If we block a story that also has a flight_id + // we want to record that as blocked too. + // This is because a single flight might have slightly different urls. + action.data.forEach(site => { + const { flight_id } = site; + if (flight_id) { + this.recordBlockFlightId(flight_id); + } + }); + break; + } + case at.PREF_CHANGED: + await this.onPrefChangedAction(action); + if (action.data.name === "pocketConfig") { + await this.onPrefChange(); + this.setupPrefs(false /* isStartup */); + } + break; + } + } +} + +/* This function generates a hardcoded layout each call. + This is because modifying the original object would + persist across pref changes and system_tick updates. + + NOTE: There is some branching logic in the template. + `spocsUrl` Changing the url for spocs is used for adding a siteId query param. + `feedUrl` Where to fetch stories from. + `items` How many items to include in the primary card grid. + `spocPositions` Changes the position of spoc cards. + `spocTopsitesPositions` Changes the position of spoc topsites. + `spocPlacementData` Used to set the spoc content. + `spocTopsitesPlacementData` Used to set spoc content for topsites. + `sponsoredCollectionsEnabled` Tuns on and off the sponsored collection section. + `hybridLayout` Changes cards to smaller more compact cards only for specific breakpoints. + `hideCardBackground` Removes Pocket card background and borders. + `fourCardLayout` Enable four Pocket cards per row. + `newFooterSection` Changes the layout of the topics section. + `compactGrid` Reduce the number of pixels between the Pocket cards. + `essentialReadsHeader` Updates the Pocket section header and title to say "Today’s Essential Reads", moves the "Recommended by Pocket" header to the right side. + `editorsPicksHeader` Updates the Pocket section header and title to say "Editor’s Picks", if used with essentialReadsHeader, creates a second section 2 rows down for editorsPicks. + `onboardingExperience` Show new users some UI explaining Pocket above the Pocket section. +*/ +getHardcodedLayout = ({ + spocsUrl = SPOCS_URL, + feedUrl = FEED_URL, + items = 21, + spocPositions = [1, 5, 7, 11, 18, 20], + spocTopsitesPositions = [1], + spocPlacementData = { ad_types: [3617], zone_ids: [217758, 217995] }, + spocTopsitesPlacementData, + widgetPositions = [], + widgetData = [], + sponsoredCollectionsEnabled = false, + hybridLayout = false, + hideCardBackground = false, + fourCardLayout = false, + newFooterSection = false, + compactGrid = false, + essentialReadsHeader = false, + editorsPicksHeader = false, + onboardingExperience = false, +}) => ({ + lastUpdate: Date.now(), + spocs: { + url: spocsUrl, + }, + layout: [ + { + width: 12, + components: [ + { + type: "TopSites", + header: { + title: { + id: "newtab-section-header-topsites", + }, + }, + ...(spocTopsitesPlacementData + ? { + placement: { + name: "sponsored-topsites", + ad_types: spocTopsitesPlacementData.ad_types, + zone_ids: spocTopsitesPlacementData.zone_ids, + }, + spocs: { + probability: 1, + prefs: [PREF_SHOW_SPONSORED_TOPSITES], + positions: spocTopsitesPositions.map(position => { + return { index: position }; + }), + }, + } + : {}), + properties: {}, + }, + ...(sponsoredCollectionsEnabled + ? [ + { + type: "CollectionCardGrid", + properties: { + items: 3, + }, + header: { + title: "", + }, + placement: { + name: "sponsored-collection", + ad_types: [3617], + zone_ids: [217759, 218031], + }, + spocs: { + probability: 1, + positions: [ + { + index: 0, + }, + { + index: 1, + }, + { + index: 2, + }, + ], + }, + }, + ] + : []), + { + type: "Message", + essentialReadsHeader, + editorsPicksHeader, + header: { + title: { + id: "newtab-section-header-pocket", + values: { provider: "Pocket" }, + }, + subtitle: "", + link_text: { + id: "newtab-pocket-learn-more", + }, + link_url: "https://getpocket.com/firefox/new_tab_learn_more", + icon: "chrome://global/skin/icons/pocket.svg", + }, + properties: {}, + styles: { + ".ds-message": "margin-bottom: -20px", + }, + }, + { + type: "CardGrid", + properties: { + items, + hybridLayout, + hideCardBackground, + fourCardLayout, + compactGrid, + essentialReadsHeader, + editorsPicksHeader, + onboardingExperience, + }, + widgets: { + positions: widgetPositions.map(position => { + return { index: position }; + }), + data: widgetData, + }, + cta_variant: "link", + header: { + title: "", + }, + placement: { + name: "spocs", + ad_types: spocPlacementData.ad_types, + zone_ids: spocPlacementData.zone_ids, + }, + feed: { + embed_reference: null, + url: feedUrl, + }, + spocs: { + probability: 1, + positions: spocPositions.map(position => { + return { index: position }; + }), + }, + }, + { + type: "Navigation", + newFooterSection, + properties: { + alignment: "left-align", + links: [ + { + name: "Self improvement", + url: "https://getpocket.com/explore/self-improvement?utm_source=pocket-newtab", + }, + { + name: "Food", + url: "https://getpocket.com/explore/food?utm_source=pocket-newtab", + }, + { + name: "Entertainment", + url: "https://getpocket.com/explore/entertainment?utm_source=pocket-newtab", + }, + { + name: "Health & fitness", + url: "https://getpocket.com/explore/health?utm_source=pocket-newtab", + }, + { + name: "Science", + url: "https://getpocket.com/explore/science?utm_source=pocket-newtab", + }, + { + name: "More recommendations ›", + url: "https://getpocket.com/explore?utm_source=pocket-newtab", + }, + ], + extraLinks: [ + { + name: "Career", + url: "https://getpocket.com/explore/career?utm_source=pocket-newtab", + }, + { + name: "Technology", + url: "https://getpocket.com/explore/technology?utm_source=pocket-newtab", + }, + ], + privacyNoticeURL: { + url: "https://www.mozilla.org/privacy/firefox/#suggest-relevant-content", + title: { + id: "newtab-section-menu-privacy-notice", + }, + }, + }, + header: { + title: { + id: "newtab-pocket-read-more", + }, + }, + styles: { + ".ds-navigation": "margin-top: -10px;", + }, + }, + ...(newFooterSection + ? [ + { + type: "PrivacyLink", + properties: { + url: "https://www.mozilla.org/privacy/firefox/", + title: { + id: "newtab-section-menu-privacy-notice", + }, + }, + }, + ] + : []), + ], + }, + ], +}); + +const EXPORTED_SYMBOLS = ["DiscoveryStreamFeed"]; diff --git a/browser/components/newtab/lib/DownloadsManager.jsm b/browser/components/newtab/lib/DownloadsManager.jsm new file mode 100644 index 0000000000..6c7b756e0d --- /dev/null +++ b/browser/components/newtab/lib/DownloadsManager.jsm @@ -0,0 +1,191 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DownloadsCommon: "resource:///modules/DownloadsCommon.sys.mjs", + DownloadsViewUI: "resource:///modules/DownloadsViewUI.sys.mjs", + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", +}); + +const DOWNLOAD_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for downloads changed events + +class DownloadsManager { + constructor(store) { + this._downloadData = null; + this._store = null; + this._downloadItems = new Map(); + this._downloadTimer = null; + } + + setTimeout(callback, delay) { + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT); + return timer; + } + + formatDownload(download) { + let referrer = download.source.referrerInfo?.originalReferrer?.spec || null; + return { + hostname: new URL(download.source.url).hostname, + url: download.source.url, + path: download.target.path, + title: lazy.DownloadsViewUI.getDisplayName(download), + description: + lazy.DownloadsViewUI.getSizeWithUnits(download) || + lazy.DownloadsCommon.strings.sizeUnknown, + referrer, + date_added: download.endTime, + }; + } + + init(store) { + this._store = store; + this._downloadData = lazy.DownloadsCommon.getData( + null /* null for non-private downloads */, + true, + false, + true + ); + this._downloadData.addView(this); + } + + onDownloadAdded(download) { + if (!this._downloadItems.has(download.source.url)) { + this._downloadItems.set(download.source.url, download); + + // On startup, all existing downloads fire this notification, so debounce them + if (this._downloadTimer) { + this._downloadTimer.delay = DOWNLOAD_CHANGED_DELAY_TIME; + } else { + this._downloadTimer = this.setTimeout(() => { + this._downloadTimer = null; + this._store.dispatch({ type: at.DOWNLOAD_CHANGED }); + }, DOWNLOAD_CHANGED_DELAY_TIME); + } + } + } + + onDownloadRemoved(download) { + if (this._downloadItems.has(download.source.url)) { + this._downloadItems.delete(download.source.url); + this._store.dispatch({ type: at.DOWNLOAD_CHANGED }); + } + } + + async getDownloads( + threshold, + { + numItems = this._downloadItems.size, + onlySucceeded = false, + onlyExists = false, + } + ) { + if (!threshold) { + return []; + } + let results = []; + + // Only get downloads within the time threshold specified and sort by recency + const downloadThreshold = Date.now() - threshold; + let downloads = [...this._downloadItems.values()] + .filter(download => download.endTime > downloadThreshold) + .sort((download1, download2) => download1.endTime < download2.endTime); + + for (const download of downloads) { + // Ignore blocked links, but allow long (data:) uris to avoid high CPU + if ( + download.source.url.length < 10000 && + lazy.NewTabUtils.blockedLinks.isBlocked(download.source) + ) { + continue; + } + + // Only include downloads where the file still exists + if (onlyExists) { + // Refresh download to ensure the 'exists' attribute is up to date + await download.refresh(); + if (!download.target.exists) { + continue; + } + } + // Only include downloads that were completed successfully + if (onlySucceeded) { + if (!download.succeeded) { + continue; + } + } + const formattedDownloadForHighlights = this.formatDownload(download); + results.push(formattedDownloadForHighlights); + if (results.length === numItems) { + break; + } + } + return results; + } + + uninit() { + if (this._downloadData) { + this._downloadData.removeView(this); + this._downloadData = null; + } + if (this._downloadTimer) { + this._downloadTimer.cancel(); + this._downloadTimer = null; + } + } + + onAction(action) { + let doDownloadAction = callback => { + let download = this._downloadItems.get(action.data.url); + if (download) { + callback(download); + } + }; + + switch (action.type) { + case at.COPY_DOWNLOAD_LINK: + doDownloadAction(download => { + lazy.DownloadsCommon.copyDownloadLink(download); + }); + break; + case at.REMOVE_DOWNLOAD_FILE: + doDownloadAction(download => { + lazy.DownloadsCommon.deleteDownload(download).catch(console.error); + }); + break; + case at.SHOW_DOWNLOAD_FILE: + doDownloadAction(download => { + lazy.DownloadsCommon.showDownloadedFile( + new lazy.FileUtils.File(download.target.path) + ); + }); + break; + case at.OPEN_DOWNLOAD_FILE: + const win = action._target.browser.ownerGlobal; + const openWhere = + action.data.event && win.whereToOpenLink(action.data.event); + doDownloadAction(download => { + lazy.DownloadsCommon.openDownload(download, { + // Replace "current" or unknown value with "tab" as the default behavior + // for opening downloads when handled internally + openWhere: ["window", "tab", "tabshifted"].includes(openWhere) + ? openWhere + : "tab", + }); + }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} +const EXPORTED_SYMBOLS = ["DownloadsManager"]; diff --git a/browser/components/newtab/lib/FaviconFeed.jsm b/browser/components/newtab/lib/FaviconFeed.jsm new file mode 100644 index 0000000000..0311582919 --- /dev/null +++ b/browser/components/newtab/lib/FaviconFeed.jsm @@ -0,0 +1,197 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { getDomain } = ChromeUtils.importESModule( + "resource://activity-stream/lib/TippyTopProvider.sys.mjs" +); +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +const MIN_FAVICON_SIZE = 96; + +/** + * Get favicon info (uri and size) for a uri from Places. + * + * @param uri {nsIURI} Page to check for favicon data + * @returns A promise of an object (possibly null) containing the data + */ +function getFaviconInfo(uri) { + return new Promise(resolve => + lazy.PlacesUtils.favicons.getFaviconDataForPage( + uri, + // Package up the icon data in an object if we have it; otherwise null + (iconUri, faviconLength, favicon, mimeType, faviconSize) => + resolve(iconUri ? { iconUri, faviconSize } : null), + lazy.NewTabUtils.activityStreamProvider.THUMB_FAVICON_SIZE + ) + ); +} + +/** + * Fetches visit paths for a given URL from its most recent visit in Places. + * + * Note that this includes the URL itself as well as all the following + * permenent&temporary redirected URLs if any. + * + * @param {String} a URL string + * + * @returns {Array} Returns an array containing objects as + * {int} visit_id: ID of the visit in moz_historyvisits. + * {String} url: URL of the redirected URL. + */ +async function fetchVisitPaths(url) { + const query = ` + WITH RECURSIVE path(visit_id) + AS ( + SELECT v.id + FROM moz_places h + JOIN moz_historyvisits v + ON v.place_id = h.id + WHERE h.url_hash = hash(:url) AND h.url = :url + AND v.visit_date = h.last_visit_date + + UNION + + SELECT id + FROM moz_historyvisits + JOIN path + ON visit_id = from_visit + WHERE visit_type IN + (${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_PERMANENT}, + ${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_TEMPORARY}) + ) + SELECT visit_id, ( + SELECT ( + SELECT url + FROM moz_places + WHERE id = place_id) + FROM moz_historyvisits + WHERE id = visit_id) AS url + FROM path + `; + + const visits = + await lazy.NewTabUtils.activityStreamProvider.executePlacesQuery(query, { + columns: ["visit_id", "url"], + params: { url }, + }); + return visits; +} + +/** + * Fetch favicon for a url by following its redirects in Places. + * + * This can improve the rich icon coverage for Top Sites since Places only + * associates the favicon to the final url if the original one gets redirected. + * Note this is not an urgent request, hence it is dispatched to the main + * thread idle handler to avoid any possible performance impact. + */ +async function fetchIconFromRedirects(url) { + const visitPaths = await fetchVisitPaths(url); + if (visitPaths.length > 1) { + const lastVisit = visitPaths.pop(); + const redirectedUri = Services.io.newURI(lastVisit.url); + const iconInfo = await getFaviconInfo(redirectedUri); + if (iconInfo && iconInfo.faviconSize >= MIN_FAVICON_SIZE) { + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(url), + iconInfo.iconUri, + false, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } + } +} + +class FaviconFeed { + constructor() { + this._queryForRedirects = new Set(); + } + + /** + * fetchIcon attempts to fetch a rich icon for the given url from two sources. + * First, it looks up the tippy top feed, if it's still missing, then it queries + * the places for rich icon with its most recent visit in order to deal with + * the redirected visit. See Bug 1421428 for more details. + */ + async fetchIcon(url) { + // Avoid initializing and fetching icons if prefs are turned off + if (!this.shouldFetchIcons) { + return; + } + + const site = await this.getSite(getDomain(url)); + if (!site) { + if (!this._queryForRedirects.has(url)) { + this._queryForRedirects.add(url); + Services.tm.idleDispatchToMainThread(() => fetchIconFromRedirects(url)); + } + return; + } + + let iconUri = Services.io.newURI(site.image_url); + // The #tippytop is to be able to identify them for telemetry. + iconUri = iconUri.mutate().setRef("tippytop").finalize(); + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(url), + iconUri, + false, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } + + /** + * Get the site tippy top data from Remote Settings. + */ + async getSite(domain) { + const sites = await this.tippyTop.get({ + filters: { domain }, + syncIfEmpty: false, + }); + return sites.length ? sites[0] : null; + } + + /** + * Get the tippy top collection from Remote Settings. + */ + get tippyTop() { + if (!this._tippyTop) { + this._tippyTop = RemoteSettings("tippytop"); + } + return this._tippyTop; + } + + /** + * Determine if we should be fetching and saving icons. + */ + get shouldFetchIcons() { + return Services.prefs.getBoolPref("browser.chrome.site_icons"); + } + + onAction(action) { + switch (action.type) { + case at.RICH_ICON_MISSING: + this.fetchIcon(action.data.url); + break; + } + } +} + +const EXPORTED_SYMBOLS = ["FaviconFeed", "fetchIconFromRedirects"]; diff --git a/browser/components/newtab/lib/FeatureCalloutMessages.sys.mjs b/browser/components/newtab/lib/FeatureCalloutMessages.sys.mjs new file mode 100644 index 0000000000..3dbba0c212 --- /dev/null +++ b/browser/components/newtab/lib/FeatureCalloutMessages.sys.mjs @@ -0,0 +1,648 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Eventually, make this a messaging system +// provider instead of adding these message +// into OnboardingMessageProvider.jsm +const FIREFOX_VIEW_PREF = "browser.firefox-view.feature-tour"; +const PDFJS_PREF = "browser.pdfjs.feature-tour"; +// Empty screens are included as placeholders to ensure step +// indicator shows the correct number of total steps in the tour +const EMPTY_SCREEN = { content: {} }; +const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000; + +// Generate a JEXL targeting string based on the current screen +// id found in a given Feature Callout tour progress preference +// and the `complete` property being true +const matchCurrentScreenTargeting = (prefName, screenId) => { + const prefVal = `'${prefName}' | preferenceValue`; + //regExpMatch() is a JEXL filter expression. Here we check if 'screen' and 'complete' exist in the pref's value (which is stringified JSON), and return their values. Returns null otherwise + const screenRegEx = '(?<=screen":)"(.*)(?=",)'; + const completeRegEx = '(?<=complete":)(.*)(?=})'; + + const screenMatch = `${prefVal} | regExpMatch('${screenRegEx}')`; + const completeMatch = `${prefVal} | regExpMatch('${completeRegEx}')`; + //We are checking the return of regExpMatch() here. If it's truthy, we grab the matched string and compare it to the desired value + const screenVal = `(${screenMatch}) ? (${screenMatch}[1] == '${screenId}') : false`; + const completeVal = `(${completeMatch}) ? (${completeMatch}[1] != "true") : false`; + + return `(${screenVal}) && (${completeVal})`; +}; + +/** + * add24HourImpressionJEXLTargeting - + * Creates a "hasn't been viewed in > 24 hours" + * JEXL string and adds it to each message specified + * + * @param {array} messageIds - IDs of messages that the targeting string will be added to + * @param {string} prefix - The prefix of messageIDs that will used to create the JEXL string + * @param {array} messages - The array of messages that will be edited + * @returns {array} - The array of messages with the appropriate targeting strings edited + */ +function add24HourImpressionJEXLTargeting( + messageIds, + prefix, + uneditedMessages +) { + let noImpressionsIn24HoursString = uneditedMessages + .filter(message => message.id.startsWith(prefix)) + .map( + message => + // If the last impression is null or if epoch time + // of the impression is < current time - 24hours worth of MS + `(messageImpressions.${message.id}[messageImpressions.${ + message.id + } | length - 1] == null || messageImpressions.${ + message.id + }[messageImpressions.${message.id} | length - 1] < ${ + Date.now() - ONE_DAY_IN_MS + })` + ) + .join(" && "); + + // We're appending the string here instead of using + // template strings to avoid a recursion error from + // using the 'messages' variable within itself + return uneditedMessages.map(message => { + if (messageIds.includes(message.id)) { + message.targeting += `&& ${noImpressionsIn24HoursString}`; + } + + return message; + }); +} + +// Exporting the about:firefoxview messages as a method here +// acts as a safety guard against mutations of the original objects +const MESSAGES = () => { + let messages = [ + { + id: "FIREFOX_VIEW_SPOTLIGHT", + template: "spotlight", + content: { + id: "FIREFOX_VIEW_PROMO", + template: "multistage", + modal: "tab", + screens: [ + { + id: "DEFAULT_MODAL_UI", + content: { + title: { + fontSize: "32px", + fontWeight: 400, + string_id: "firefoxview-spotlight-promo-title", + }, + subtitle: { + fontSize: "15px", + fontWeight: 400, + marginBlock: "10px", + marginInline: "40px", + string_id: "firefoxview-spotlight-promo-subtitle", + }, + logo: { height: "48px" }, + primary_button: { + label: { + string_id: "firefoxview-spotlight-promo-primarybutton", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "FEATURE_CALLOUT_1", + complete: false, + }), + }, + }, + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "firefoxview-spotlight-promo-secondarybutton", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + navigate: true, + }, + }, + }, + }, + ], + }, + priority: 3, + trigger: { + id: "featureCalloutCheck", + }, + frequency: { + // Add the highest possible cap to ensure impressions are recorded while allowing the Spotlight to sync across windows/tabs with Firefox View open + lifetime: 100, + }, + targeting: `!inMr2022Holdback && source == "about:firefoxview" && + !'browser.newtabpage.activity-stream.asrouter.providers.cfr'|preferenceIsUserSet && + 'browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features'|preferenceValue && + ${matchCurrentScreenTargeting( + FIREFOX_VIEW_PREF, + "FIREFOX_VIEW_SPOTLIGHT" + )}`, + }, + { + id: "FIREFOX_VIEW_FEATURE_TOUR_1_NO_CWS", + template: "feature_callout", + content: { + id: "FIREFOX_VIEW_FEATURE_TOUR", + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + { + id: "FEATURE_CALLOUT_1", + parent_selector: "#tab-pickup-container", + content: { + position: "callout", + arrow_position: "top", + title: { + string_id: "callout-firefox-view-tab-pickup-title", + }, + subtitle: { + string_id: "callout-firefox-view-tab-pickup-subtitle", + }, + logo: { + imageURL: "chrome://browser/content/callout-tab-pickup.svg", + darkModeImageURL: + "chrome://browser/content/callout-tab-pickup-dark.svg", + height: "128px", + }, + primary_button: { + label: { + string_id: "callout-primary-advance-button-label", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "FEATURE_CALLOUT_2", + complete: false, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + EMPTY_SCREEN, + ], + }, + priority: 3, + targeting: `!inMr2022Holdback && source == "about:firefoxview" && ${matchCurrentScreenTargeting( + FIREFOX_VIEW_PREF, + "FEATURE_CALLOUT_1" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "FIREFOX_VIEW_FEATURE_TOUR_2_NO_CWS", + template: "feature_callout", + content: { + id: "FIREFOX_VIEW_FEATURE_TOUR", + startScreen: 1, + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + EMPTY_SCREEN, + { + id: "FEATURE_CALLOUT_2", + parent_selector: "#recently-closed-tabs-container", + content: { + position: "callout", + arrow_position: "bottom", + title: { + string_id: "callout-firefox-view-recently-closed-title", + }, + subtitle: { + string_id: "callout-firefox-view-recently-closed-subtitle", + }, + primary_button: { + label: { + string_id: "callout-primary-complete-button-label", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: FIREFOX_VIEW_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + ], + }, + priority: 3, + targeting: `!inMr2022Holdback && source == "about:firefoxview" && ${matchCurrentScreenTargeting( + FIREFOX_VIEW_PREF, + "FEATURE_CALLOUT_2" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER", + template: "feature_callout", + content: { + id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER", + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + { + id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER", + parent_selector: "#tab-pickup-container", + content: { + position: "callout", + arrow_position: "top", + title: { + string_id: + "continuous-onboarding-firefox-view-tab-pickup-title", + }, + subtitle: { + string_id: + "continuous-onboarding-firefox-view-tab-pickup-subtitle", + }, + logo: { + imageURL: "chrome://browser/content/callout-tab-pickup.svg", + darkModeImageURL: + "chrome://browser/content/callout-tab-pickup-dark.svg", + height: "128px", + }, + primary_button: { + label: { + string_id: "mr1-onboarding-get-started-primary-button-label", + }, + action: { + type: "CLICK_ELEMENT", + navigate: true, + data: { + selector: + "#tab-pickup-container button.primary:not(#error-state-button)", + }, + }, + }, + dismiss_button: { + action: { + navigate: true, + }, + }, + }, + }, + ], + }, + priority: 2, + targeting: `!inMr2022Holdback && source == "about:firefoxview" && "browser.firefox-view.view-count" | preferenceValue > 2 + && (("identity.fxaccounts.enabled" | preferenceValue == false) || !(("services.sync.engine.tabs" | preferenceValue == true) && ("services.sync.username" | preferenceValue))) && (!messageImpressions.FIREFOX_VIEW_SPOTLIGHT[messageImpressions.FIREFOX_VIEW_SPOTLIGHT | length - 1] || messageImpressions.FIREFOX_VIEW_SPOTLIGHT[messageImpressions.FIREFOX_VIEW_SPOTLIGHT | length - 1] < currentDate|date - ${ONE_DAY_IN_MS})`, + frequency: { + lifetime: 1, + }, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "PDFJS_FEATURE_TOUR_1_A", + template: "feature_callout", + content: { + id: "PDFJS_FEATURE_TOUR", + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + { + id: "FEATURE_CALLOUT_1_A", + parent_selector: "hbox#browser", + content: { + position: "callout", + callout_position_override: { + top: "45px", + right: "55px", + }, + arrow_position: "top-end", + title: { + string_id: "callout-pdfjs-edit-title", + }, + subtitle: { + string_id: "callout-pdfjs-edit-body-a", + }, + primary_button: { + label: { + string_id: "callout-pdfjs-edit-button", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "FEATURE_CALLOUT_2_A", + complete: false, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + EMPTY_SCREEN, + ], + }, + priority: 1, + targeting: `source == "chrome" && ${matchCurrentScreenTargeting( + PDFJS_PREF, + "FEATURE_CALLOUT_1_A" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "PDFJS_FEATURE_TOUR_2_A", + template: "feature_callout", + content: { + id: "PDFJS_FEATURE_TOUR", + startScreen: 1, + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + EMPTY_SCREEN, + { + id: "FEATURE_CALLOUT_2_A", + parent_selector: "hbox#browser", + content: { + position: "callout", + callout_position_override: { + top: "45px", + right: "25px", + }, + arrow_position: "top-end", + title: { + string_id: "callout-pdfjs-draw-title", + }, + subtitle: { + string_id: "callout-pdfjs-draw-body-a", + }, + primary_button: { + label: { + string_id: "callout-pdfjs-draw-button", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + ], + }, + priority: 1, + targeting: `source == "chrome" && ${matchCurrentScreenTargeting( + PDFJS_PREF, + "FEATURE_CALLOUT_2_A" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "PDFJS_FEATURE_TOUR_1_B", + template: "feature_callout", + content: { + id: "PDFJS_FEATURE_TOUR", + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + { + id: "FEATURE_CALLOUT_1_B", + parent_selector: "hbox#browser", + content: { + position: "callout", + callout_position_override: { + top: "45px", + right: "55px", + }, + arrow_position: "top-end", + title: { + string_id: "callout-pdfjs-edit-title", + }, + subtitle: { + string_id: "callout-pdfjs-edit-body-b", + }, + primary_button: { + label: { + string_id: "callout-pdfjs-edit-button", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "FEATURE_CALLOUT_2_B", + complete: false, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + EMPTY_SCREEN, + ], + }, + priority: 1, + targeting: `source == "chrome" && ${matchCurrentScreenTargeting( + PDFJS_PREF, + "FEATURE_CALLOUT_1_B" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + { + id: "PDFJS_FEATURE_TOUR_2_B", + template: "feature_callout", + content: { + id: "PDFJS_FEATURE_TOUR", + startScreen: 1, + template: "multistage", + backdrop: "transparent", + transitions: false, + disableHistoryUpdates: true, + screens: [ + EMPTY_SCREEN, + { + id: "FEATURE_CALLOUT_2_B", + parent_selector: "hbox#browser", + content: { + position: "callout", + callout_position_override: { + top: "45px", + right: "25px", + }, + arrow_position: "top-end", + title: { + string_id: "callout-pdfjs-draw-title", + }, + subtitle: { + string_id: "callout-pdfjs-draw-body-b", + }, + primary_button: { + label: { + string_id: "callout-pdfjs-draw-button", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + dismiss_button: { + action: { + type: "SET_PREF", + data: { + pref: { + name: PDFJS_PREF, + value: JSON.stringify({ + screen: "", + complete: true, + }), + }, + }, + }, + }, + }, + }, + ], + }, + priority: 1, + targeting: `source == "chrome" && ${matchCurrentScreenTargeting( + PDFJS_PREF, + "FEATURE_CALLOUT_2_B" + )}`, + trigger: { id: "featureCalloutCheck" }, + }, + ]; + messages = add24HourImpressionJEXLTargeting( + ["FIREFOX_VIEW_TAB_PICKUP_REMINDER"], + "FIREFOX_VIEW", + messages + ); + return messages; +}; + +export const FeatureCalloutMessages = { + getMessages() { + return MESSAGES(); + }, +}; diff --git a/browser/components/newtab/lib/FilterAdult.jsm b/browser/components/newtab/lib/FilterAdult.jsm new file mode 100644 index 0000000000..2ee9c4c467 --- /dev/null +++ b/browser/components/newtab/lib/FilterAdult.jsm @@ -0,0 +1,3036 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "gFilterAdultEnabled", + "browser.newtabpage.activity-stream.filterAdult", + true +); + +// Keep a Set of adult base domains for lookup (initialized at end of file) +let gAdultSet; + +// Keep a hasher for repeated hashings +let gCryptoHash = null; + +/** + * Run some text through md5 and return the base64 result. + */ +function md5Hash(text) { + // Lazily create a reusable hasher + if (gCryptoHash === null) { + gCryptoHash = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + } + + gCryptoHash.init(gCryptoHash.MD5); + + // Convert the text to a byte array for hashing + gCryptoHash.update( + text.split("").map(c => c.charCodeAt(0)), + text.length + ); + + // Request the has result as ASCII base64 + return gCryptoHash.finish(true); +} + +const FilterAdult = { + /** + * Filter out any link objects that have a url with an adult base domain. + * + * @param {string[]} links + * An array of links to test. + * @returns {string[]} + * A filtered array without adult links. + */ + filter(links) { + if (!lazy.gFilterAdultEnabled) { + return links; + } + + return links.filter(({ url }) => { + try { + const uri = Services.io.newURI(url); + return !gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri))); + } catch (ex) { + return true; + } + }); + }, + + /** + * Determine if the supplied url is an adult url or not. + * + * @param {string} url + * The url to test. + * @returns {boolean} + * True if it is an adult url. + */ + isAdultUrl(url) { + if (!lazy.gFilterAdultEnabled) { + return false; + } + try { + const uri = Services.io.newURI(url); + return gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri))); + } catch (ex) { + return false; + } + }, + + /** + * For tests, adds a domain to the adult list. + */ + addDomainToList(url) { + gAdultSet.add( + md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url))) + ); + }, + + /** + * For tests, removes a domain to the adult list. + */ + removeDomainFromList(url) { + gAdultSet.delete( + md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url))) + ); + }, +}; + +const EXPORTED_SYMBOLS = ["FilterAdult"]; + +// These are md5 hashes of base domains to be filtered out. Originally from: +// https://hg.mozilla.org/mozilla-central/log/default/browser/base/content/newtab/newTab.inadjacent.json +gAdultSet = new Set([ + "+/UCpAhZhz368iGioEO8aQ==", + "+1e7jvUo8f2/2l0TFrQqfA==", + "+1gcqAqaRZwCj5BGiZp3CA==", + "+25t/2lo0FUEtWYK8LdQZQ==", + "+8PiQt6O7pJI/nIvQpDaAg==", + "+CLf5witKkuOvPCulTlkqw==", + "+CvLiih/gf2ugXAF+LgWqw==", + "+DWs0vvFGt6d3mzdcsdsyA==", + "+H0Rglt/HnhZwdty2hsDHg==", + "+L1FDsr5VQtuYc2Is5QGjw==", + "+LJYVZl1iPrdMU3L5+nxZw==", + "+Mp+JIyO0XC5urvMyi3wvQ==", + "+NMUaQ7XPsAi0rk7tTT9wQ==", + "+NmjwjsPhGJh9bM10SFkLw==", + "+OERSmo7OQUUjudkccSMOA==", + "+OLntmlsMBBYPREPnS6iVw==", + "+OXdvbTxHtSoLg7bZMho4w==", + "+P5q4YD1Rr5SX26Xr+tzlw==", + "+PUVXkoTqHxJHO18z4KMfw==", + "+Pl0bSMBAdXpRIA+zE02JA==", + "+QosBAnSM2h4lsKuBlqEZw==", + "+S+WXgVDSU1oGmCzGwuT3g==", + "+SclwwY8R2RPrnX54Z+A6w==", + "+VfRcTBQ80KSeJRdg0cDfw==", + "+WpF8+poKmHPUBB4UYh/ig==", + "+YVxSyViJfrme/ENe1zA7A==", + "+YrqTEJlJCv0A2RHQ8tr1A==", + "+ZozWaPWw8ws1cE5DJACeg==", + "+aF4ilbjQbLpAuFXQEYMWQ==", + "+dBv88reDrjEz6a2xX3Hzw==", + "+dIEf5FBrHpkjmwUmGS6eg==", + "+edqJYGvcy1AH2mEjJtSIg==", + "+fcjH2kZKNj8quOytUk4nQ==", + "+gO0bg8LY+py2dLM1sM7Ag==", + "+gbitI/gpxebN/rK7qj8Fw==", + "+gpHnUj2GWocP74t5XWz4w==", + "+jVN/3ASc2O44sX6ab8/cg==", + "+mJLK+6qq8xFv7O/mbILTw==", + "+n0K7OB2ItzhySZ4rhUrMg==", + "+p8pofUlwn8vV6Rp6+sz9g==", + "+tuUmnRDRWVLA+1k0dcUvg==", + "+zBkeHF4P8vLzk1iO1Zn3Q==", + "//eHwmDOQRSrv+k9C/k3ZQ==", + "/2Chaw2M9DzsadFFkCu6WQ==", + "/2c4oNniwhL3z5IOngfggg==", + "/2jGyMekNu7U136K+2N3Jg==", + "/Bwpt5fllzDHq2Ul6v86fA==", + "/DJgKE9ouibewuZ2QEnk6w==", + "/DiUApY7cVp5W9o24rkgRA==", + "/FchS2nPezycB8Bcqc2dbg==", + "/FdZzSprPnNDPwbhV1C0Cg==", + "/FsJYFNe+7UvsSkiotNJEQ==", + "/G26n5Xoviqldr5sg/Jl3w==", + "/HU2+fBqfWTEuqINc0UZSA==", + "/IarsLzJB8bf0AupJJ+/Eg==", + "/KYZdUWrkfxSsIrp46xxow==", + "/MEOgAhwb7F0nBnV4tIRZA==", + "/MeHciFhvFzQsCIw39xIZA==", + "/Ph/6l/lFNVqxAje1+PgFA==", + "/SP6pOdYFzcAl2OL05z4uQ==", + "/TSsi/AwKHtP6kQaeReI3w==", + "/VnKh/NDv7y/bfO6CWsLaQ==", + "/XC/FmMIOdhMTPqmy4DfUA==", + "/XjB6c5fxFGcKVAQ4o+OMw==", + "/YuQw7oAF08KDptxJEBS9g==", + "/a+bLXOq02sa/s8h7PhUTg==", + "/a9O7kWeXa0le45ab3+nVw==", + "/c34NtdUZAHWIwGl3JM8Tw==", + "/cJ0Nn5YbXeUpOHMfWXNHQ==", + "/cdR1i5TuQvO+u3Ov3b0KQ==", + "/gi3UZmunVOIXhZSktZ8zQ==", + "/hFhjFGJx2wRfz6hyrIpvA==", + "/jDVt9dRIn+o4IQ1DPwbsg==", + "/jH6imhTPZ/tHI4gYz2+HA==", + "/kGxvyEokQsVz0xlKzCn2A==", + "/mFp3GFkGNLhx2CiDvJv4A==", + "/mrqas0eDX+sFUNJvCQY8g==", + "/n1RLTTVpygre1dl36PDwQ==", + "/ngbFuKIAVpdSwsA3VxvNw==", + "/p/aCTIhi1bU0/liuO/a2Q==", + "/u5W2Gab4GgCMIc4KTp2mg==", + "/wIZAye9h1TUiZmDW0ZmYA==", + "/wiA2ltAuWyBhIvQAYBTQw==", + "/y/jHHEpUu5TR+R2o96kXA==", + "/zFLRvi75UL8qvg+a6zqGg==", + "00TVKawojyqrJkC7YqT41Q==", + "022B0oiRMx8Xb4Af98mTvQ==", + "02im2RooJQ/9UfUrh5LO+A==", + "0G93AxGPVwmr66ZOleM90A==", + "0HN6MIGtkdzNPsrGs611xA==", + "0K4NBxqEa3RYpnrkrD/XjQ==", + "0L0FVcH5Dlj3oL8+e9Na7g==", + "0NrvBuyjcJ2q6yaHpz/FOA==", + "0ODJyWKJSfObo+FNdRQkkA==", + "0QB0OUW5x2JLHfrtmpZQ+w==", + "0QCQORCYfLuSbq94Sbt0bQ==", + "0QbH4oI8IjZ9BRcqRyvvDQ==", + "0QxPAqRF8inBuFEEzNmLjA==", + "0SkC/4PtnX1bMYgD6r6CLA==", + "0TxcYwG72dT7Tg+eG8pP1w==", + "0UeRwDID2RBIikInqFI7uw==", + "0VsaJHR0Ms8zegsCpAKoyg==", + "0Y6iiZjCwPDwD/CwJzfioQ==", + "0ZEC3hy411LkOhKblvTcqg==", + "0ZRGz+oj2infCAkuKKuHiQ==", + "0a4SafpDIe8V4FlFWYkMHw==", + "0b/xj6fd0x+aB8EB0LC4SA==", + "0bj069wXgEJbw7dpiPr8Tg==", + "0dIeIM5Zvm5nSVWLy94LWg==", + "0e8hM3E5tnABRyy29A8yFw==", + "0egBaMnAf0CQEXf1pCIKnA==", + "0fN+eHlbRS6mVZBbH/B9FQ==", + "0fnruVOCxEczscBuv4yL9A==", + "0fpe9E6m3eLp/5j5rLrz2Q==", + "0klouNfZRHFFpdHi4ZR2hA==", + "0nOg18ZJ/NicqVUz5Jr0Hg==", + "0ofMbUCA3/v5L8lHnX4S5w==", + "0p1jMr06OyBoXQuSLYN4aQ==", + "0p8YbEMxeb73HbAfvPLQRw==", + "0q+erphtrB+6HBnnYg7O6w==", + "0rTYcuVYdilO7zEfKrxY3A==", + "0rfG4gRugAwVP0i3AGVxxg==", + "0u+0WHr7WI6IlVBBgiRi6w==", + "0yJ7TQYzcp3DXVSvwavr+w==", + "1+A9FCGP3bZhk6gU3LQtNg==", + "1+XWdu4qCqLLVjqkKz3nmA==", + "1+qmrbC8c7MJ6pxmDMcKuA==", + "1/Hxu8M9N/oNwk8bCj4FNQ==", + "1/SGIab+NnizimUmNDC4wA==", + "1/ZheMsbojazxt31j/l3iA==", + "10OltdxPXOvfatJuwPVKbQ==", + "11FE2kknwYi2Qu0JUKMn3A==", + "11U5XEwfMI7avx014LfC8g==", + "16d+fhFlgayu3ttKVV/pbg==", + "16iT/jCcPDrJEfi2bE5F+Q==", + "18RKixTv12q3xoBLz6eKiA==", + "18ndtDM9UaNfBR1cr3SHdA==", + "19yQHaBemtlgo2QkU5M6jQ==", + "1AeReq55UQotRQVKJ66pmg==", + "1ApqwW7pE+XUB2Cs2M6y7g==", + "1B5gxGQSGzVKoNd5Ol4N7g==", + "1BjsijOzgHt/0i36ZGffoQ==", + "1C50kisi9nvyVJNfq2hOEQ==", + "1E3pMgAHOnHx3ALdNoHr8Q==", + "1EI9aa955ejNo1dJepcZJw==", + "1FSrgkUXgZot2CsmbAtkPw==", + "1Gpj4TPXhdPEI4zfQFsOCg==", + "1HDgfU7xU7LWO/BXsODZAQ==", + "1I+UVx3krrD4NhzO7dgfHQ==", + "1JI9bT92UzxI8txjhst9LQ==", + "1JRgSHnfAQFQtSkFTttkqQ==", + "1LPC0BzhJbepHTSAiZ3QTw==", + "1MIn73MLroxXirrb+vyg2Q==", + "1Oykse0jQVbuR3MvW5ot4A==", + "1Pmnur6TbZ9cmemvu0+dSA==", + "1PvTn90xwZJPoVfyT5/uIQ==", + "1QGhj9NONF2rC44UdO+Izw==", + "1RQZ2pWSxT+RKyhBigtSFg==", + "1Vtrv6QUAfiYQjlLTpNovg==", + "1WIi4I62GqkjDXOYqHWJfQ==", + "1Wc8jQlDSB4Dp32wkL2odw==", + "1X14kHeKwGmLeYqpe60XEA==", + "1YO9G8qAhLIu2rShvekedw==", + "1Ym0lyBJ9aFjhJb/GdUPvQ==", + "1b2uf+CdVjufqiVpUShvHw==", + "1buQEv2YlH/ljTgH0uJEtw==", + "1cj1Fpd3+UiBAOahEhsluA==", + "1d7RPHdZ9qzAbG3Vi9BdFA==", + "1dhq3ozNCx0o4dV1syLVDA==", + "1dsKN1nG6upj7kKTKuJWsQ==", + "1eCHcz4swFH+uRhiilOinQ==", + "1eRUCdIJe3YGD5jOMbkkOg==", + "1fztTtQWNMIMSAc5Hr6jMQ==", + "1gA65t5FiBTEgMELTQFUPQ==", + "1jBaRO8Bg5l6TH7qJ8EPiw==", + "1k8tL2xmGFVYMgKUcmDcEw==", + "1lCcQWGDePPYco4vYrA5vw==", + "1m1yD4L9A7Q1Ot+wCsrxJQ==", + "1mw6LfTiirFyfjejf8QNGA==", + "1nXByug2eKq0kR3H3VjnWQ==", + "1tpM0qgdo7JDFwvT0TD78g==", + "1vqRt79ukuvdJNyIlIag8Q==", + "1wBuHqS1ciup31WTfm3NPg==", + "1xWx5V3G9murZP7srljFmA==", + "1zDfWw5LdG20ClNP1HYxgw==", + "203EqmJI9Q4tWxTJaBdSzA==", + "23C4eh3yBb5n/RNZeTyJkA==", + "23d9B9Gz5kUOi1I//EYsSQ==", + "24H9q+E8pgCEdFS7JO5kzQ==", + "25w3ZRUzCvJwAVHYCIO5uw==", + "26+yXbqI+fmIZsYl4UhUzw==", + "26Wmdp6SkKN74W0/XPcnmA==", + "29EybnMEO95Ng4l/qK4NWQ==", + "2Ct+pLXrK6Ku1f4qehjurQ==", + "2D6yhuABiaFFoXz0Lh0C+w==", + "2DNbXVgesUa7PgYQ4zX5Lw==", + "2E41e0MgM3WhFx2oasIQeA==", + "2HHqeGRMfzf3RXwVybx+ZQ==", + "2Hc5oyl0AYRy2VzcDKy+VA==", + "2QQtKtBAm2AjJ5c0WQ6BQA==", + "2QS/6OBA1T01NlIbfkTYJg==", + "2RFaMPlSbVuoEqKXgkIa5A==", + "2SI4F7Vvde2yjzMLAwxOog==", + "2SwIiUwT4vRZPrg7+vZqDA==", + "2W6lz1Z7PhkvObEAg2XKJw==", + "2Wvk/kouEEOY0evUkQLhOQ==", + "2XrR2hjDEvx8MQpHk9dnjw==", + "2aDK0tGNgMLyxT+BQPDE8Q==", + "2aIx9UdMxxZWvrfeJ+DcTw==", + "2abfl3N46tznOpr+94VONQ==", + "2bsIpvnGcFhTCSrK9EW1FQ==", + "2hEzujfG3mR5uQJXbvOPTQ==", + "2j83jrPwPfYlpJJ2clEBYQ==", + "2ksediOVrh4asSBxKcudTg==", + "2melaInV0wnhBpiI3da6/A==", + "2nSTEYzLK77h5Rgyti+ULQ==", + "2os5s7j7Tl46ZmoZJH8FjA==", + "2rOkEVl90EPqfHOF5q2FYw==", + "2rhjiY0O0Lo36wTHjmlNyw==", + "2vm7g3rk1ACJOTCXkLB3zA==", + "2wesXiib76wM9sqRZ7JYwQ==", + "2ywo4t5PPSVUCWDwUlOVwQ==", + "3++dZXzZ6AFEz7hK+i5hww==", + "3+9nURtBK3FKn0J9DQDa3g==", + "3+zsjCi7TnJhti//YXK35w==", + "3/1puZTGSrD9qNKPGaUZww==", + "300hoYyMR/mk1mfWJxS8/w==", + "301utVPZ93AnPLYbsiJggw==", + "312g8iTB9oJgk/OqcgR7Cw==", + "342VOUOxoLHUqtHANt83Hw==", + "36XDmX6j542q+Oei1/x0gw==", + "37Nkh06O979nt7xzspOFyQ==", + "3AKEYQqpkfW7CZMFQZoxOw==", + "3AVYtcIv7A5mVbVnQMaCeA==", + "3BjLFon1Il0SsjxHE2A1LQ==", + "3CJbrUdW68E3Drhe4ahUnQ==", + "3EhLkC9NqD3A6ApV6idmgg==", + "3Ejtsqw3Iep/UQd0tXnSlg==", + "3FH4D31nKV13sC9RpRZFIg==", + "3Gg9N7vjAfQEYOtQKuF/Eg==", + "3HPOzIZxoaQAmWRy9OkoSg==", + "3JhnM6G4L06NHt31lR0zXA==", + "3L3KEBHhgDwH615w4OvgZA==", + "3Leu2Sc+YOntJFlrvhaXeg==", + "3P2aJxV8Trll2GH9ptElYA==", + "3RTtSaMp1TZegJo5gFtwwA==", + "3TbRZtFtsh9ez8hqZuTDeA==", + "3TjntNWtpG7VqBt3729L6Q==", + "3UBYBMejKInSbCHRoJJ7dg==", + "3UNJ37f+gnNyYk9yLFeoYA==", + "3WVBP9fyAiBPZAq3DpMwOQ==", + "3Wfj05vCLFAB9vII5AU9tw==", + "3WwITQML938W9+MUM56a3A==", + "3XyoREdvhmSbyvAbgw2y/A==", + "3Y4w0nETru3SiSVUMcWXqw==", + "3Y6/HqS1trYc9Dh778sefg==", + "3YXp1PmMldUjBz3hC6ItbA==", + "3djRJvkZk9O2bZeUTe+7xQ==", + "3go7bJ9WqH/PPUTjNP3q/Q==", + "3hVslsq98QCDIiO40JNOuA==", + "3iC21ByW/YVL+pSyppanWw==", + "3itfXtlLPRmPCSYaSvc39Q==", + "3j0kFUZ6g+yeeEljx+WXGg==", + "3jmCreW5ytSuGfmeLv7NfQ==", + "3jqsY8/xTWELmu/az3Daug==", + "3kREs/qaMX0AwFXN0LO5ow==", + "3ltw31yJuAl4VT6MieEXXw==", + "3nthUmLZ30HxQrzr2d7xFA==", + "3oMTbWf7Bv83KRlfjNWQZA==", + "3pi3aNVq1QNJmu1j0iyL0g==", + "3rbml1D0gfXnwOs5jRZ3gA==", + "3sNJJIx1NnjYcgJhjOLJOg==", + "3v09RHCPTLUztqapThYaHg==", + "3xw8+0/WU51Yz4TWIMK8mw==", + "3y5Xk65ShGvWFbQxcZaQAQ==", + "3yDD+xT8iRfUVdxcc7RxKw==", + "3yavzOJ1mM44pOSFLLszgA==", + "4+htiqjEz9oq0YcI/ErBVg==", + "40HzgVKYnqIb6NJhpSIF0A==", + "40gCrW4YWi+2lkqMSPKBPg==", + "41WEjhYUlG6jp2UPGj11eQ==", + "444F9T6Y7J67Y9sULG81qg==", + "46FCwqh+eMkf+czjhjworw==", + "46piyANQVvvLqcoMq5G8tQ==", + "49jZr/mEW6fvnyzskyN40w==", + "49z/15Nx9Og7dN9ebVqIzg==", + "4A+RHIw+aDzw0rSRYfbc7g==", + "4BkqgraeXY7yaI1FE07Evw==", + "4CfEP8TeMKX33ktwgifGgA==", + "4DIPP/yWRgRuFqVeqIyxMQ==", + "4FBBtWPvqJ3dv4w25tRHiQ==", + "4ID0PHTzIMZz2rQqDGBVfA==", + "4KJZPCE9NKTfzFxl76GWjg==", + "4LtQrahKXVtsbXrEzYU1zQ==", + "4LvQSicqsgxQFWauqlcEjw==", + "4NHQwbb3zWq2klqbT/pG6g==", + "4NP8EFFJyPcuQKnBSxzKgQ==", + "4PBaoeEwUj79njftnYYqLg==", + "4Qinl7cWmVeLJgah8bcNkw==", + "4SdHWowXgCpCDL28jEFpAw==", + "4TQkMnRsXBobbtnBmfPKnA==", + "4VR5LiXLew6Nyn91zH9L4w==", + "4WO6eT0Rh6sokb29zSJQnQ==", + "4WRdAjiUmOQg2MahsunjAg==", + "4WcFEswYU/HHQPw77DYnyA==", + "4XNUmgwxsqDYsNmPkgNQYQ==", + "4Xh/B3C16rrjbES+FM1W8g==", + "4ZFYKa7ZgvHyZLS6WpM8gA==", + "4aPU6053cfMLHgLwAZJRNg==", + "4ekt4m38G9m599xJCmhlug==", + "4erEA42TqGA9K4iFKkxMMA==", + "4ifNsmjYf1iOn2YpMfzihg==", + "4iiCq+HhC+hPMldNQMt0NA==", + "4itEKfbRCJvqlgKnyEdIOQ==", + "4jeOFKuKpCmMXUVJSh9y0g==", + "4kXlJNuT79XXf1HuuFOlHw==", + "4kj0S8XlmhHXoUP7dQItUw==", + "4mQVNv7FHj+/O6XFqWFt/Q==", + "4mig4AMLUw+T/ect9p4CfA==", + "4qMSNAxichi3ori/pR+o0w==", + "4rrSL6N0wyucuxeRELfAmw==", + "4u3eyKc+y3uRnkASrgBVUw==", + "4wnUAbPT3AHRJrPwTTEjyw==", + "4xojeUxTFmMLGm6jiMYh/Q==", + "4yEkKp2FYZ09mAhw2IcrrA==", + "4yVqq66iHYQjiTSxGgX2oA==", + "4yrFNgqWq17zVCyffULocA==", + "50jASqzGm4VyHJbFv8qVRA==", + "50xwiYvGQytEDyVgeeOnMg==", + "51yLpfEdvqXmtB6+q27/AQ==", + "520wTzrysiRi2Td92Zq0HQ==", + "53UccFNzMi9mKmdeD82vAw==", + "54XELlPm8gBvx8D5bN3aUg==", + "59ipbMH7cKBsF9bNf4PLeQ==", + "5CMadLqS2KWwwMCpzlDmLw==", + "5DDb7fFJQEb3XTc3YyOTjg==", + "5HovoyHtul8lXh+z8ywq9A==", + "5I/heFSQG/UpWGx0uhAqGQ==", + "5KOgetfZR+O2wHQSKt41BQ==", + "5LJqHFRyIwQKA4HbtqAYQQ==", + "5LuFDNKzMd2BzpWEIYO2Ww==", + "5M3dFrAOemzQ0MAbA8bI5w==", + "5N2oi2pB69NxeNt08yPLhw==", + "5NEP7Xt7ynj6xCzWzt21hQ==", + "5Nk2Z94DhlIdfG5HNgvBbQ==", + "5PfGtbH9fmVuNnq83xIIgQ==", + "5Q/Y2V0iSVTK8HE8JerEig==", + "5S5/asYfWjOwnzYpbK6JDw==", + "5SbwLDNT6sBOy6nONtUcTg==", + "5T39s5CtSrK5awMPUcEWJg==", + "5VO1inwXMvLDBQSOahT6rg==", + "5VY++KiWgo7jXSdFJsPN3A==", + "5Wcq+6hgnWsQZ/bojERpUw==", + "5Yrj6uevT8wHRyqqgnSfeg==", + "5dUry23poD+0wxZ3hH6WmA==", + "5eHStFN7wEmIE+uuRwIlPQ==", + "5eXpiczlRdmqMYSaodOUiQ==", + "5gGoDPTc/sOIDLngmlEq4A==", + "5jHgQF4SfO/zy9xy9t+9dw==", + "5jyuDp82Fux+B0+zlx8EXw==", + "5kvyy902llnYGQdn2Py04w==", + "5l6kDfjtZjkTZPJvNNOVFw==", + "5lfLJAk1L3QzGMML3fOuSw==", + "5m1ijXEW+4RTNGZsDA/rxQ==", + "5oD/aGqoakxaezq43x0Tvw==", + "5pje7qyz8BRsa8U4a4rmoA==", + "5pqqzC/YmRIMA9tMFPi7rg==", + "5r1ZsGkrzNQEpgt/gENibw==", + "5u2PdDcIY3RQgtchSGDCGg==", + "5ugVOraop5P5z5XLlYPJyQ==", + "5w/c9WkI/FA+4lOtdPxoww==", + "5w4FbRhWACP7k2WnNitiHg==", + "6+jhreeBLfw64tJ+Nhyipw==", + "600bwlyhcy754W1E6tuyYg==", + "600mjiWke4u0CDaSQKLOOg==", + "60suecbWRfexSh7C67RENA==", + "61V74uIjaSfZM8au1dxr1A==", + "62RHCbpGU8Hb+Ubn+SCTBg==", + "63OTPaKM0xCfJOy9EDto+Q==", + "64AA4jLHXc1Dp15aMaGVcA==", + "64QzHOYX0A9++FqRzZRHlQ==", + "64YsV2qeDxk2Q6WK/h7OqA==", + "65KhGKUBFQubRRIEdh9SwQ==", + "6706ncrH1OANFnaK6DUMqQ==", + "68jPYo3znYoU4uWI7FH3/g==", + "68nqDtXOuxF7DSw6muEZvg==", + "6ACvJNfryPSjGOK39ov8Qg==", + "6CjtF1S2Y6RCbhl7hMsD+g==", + "6G2bD3Y7qbGmfPqH9TqLFA==", + "6GXHGF62/+jZ7PfIBlMxZw==", + "6HGeEPyTAu9oiKhNVLjQnA==", + "6HnWgYNKohqhoa1tnjjU3A==", + "6M6QapJ5xtMXfiD3bMaiLA==", + "6NP81geiL14BeQW6TpLnUA==", + "6PzjncEw2wHZg7SP7SQk9w==", + "6QAtjOK9enNLRhcVa2iaTg==", + "6QUGE2S8oFYx4T4nW56cCw==", + "6W79FmpUN1ByNtv5IEXY4w==", + "6WhHPWlqEUqXC52rHGRHjA==", + "6XYqR2WvDzx4fWO7BIOTjA==", + "6Z9myGCF5ylWljgIYAmhqw==", + "6ZKmm7IW7IdWuVytLr68CQ==", + "6ZMs9vCzK9lsbS6eyzZlIA==", + "6b7ue29cBDsvmj1VSa5njw==", + "6c0iuya20Ys8BsvoI4iQaQ==", + "6cTETZ9iebhWl+4W5CB+YQ==", + "6dshA8knH5qqD+KmR/kdSQ==", + "6e8boFcyc8iF0/tHVje4eQ==", + "6erpZS36qZRXeZ9RN9L+kw==", + "6fWom3YoKvW6NIg6y9o9CQ==", + "6k2cuk0McTThSMW/QRHfjA==", + "6lVSzYUQ/r0ep4W2eCzFpg==", + "6leyDVmC5jglAa98NQ3+Hg==", + "6nwR+e9Qw0qp8qIwH9S/Mg==", + "6o5g9JfKLKQ2vBPqKs6kjg==", + "6rIWazDEWU5WPZHLkqznuQ==", + "6rqK8sjLPJUIp7ohkEwfZg==", + "6sBemZt4qY/TBwqk3YcLOQ==", + "6sNP0rzCCm3w976I2q2s/w==", + "6tfM6dx3R5TiVKaqYQjnCg==", + "6txm8z4/LGCH0cpaet/Hsg==", + "6uMF5i0b/xsk55DlPumT7A==", + "6uT7LZiWjLnnqnnSEW4e/Q==", + "6v3eTZtPYBfKFSjfOo2UaA==", + "6wkfN8hyKmKU6tG3YetCmw==", + "6z8CRivao3IMyV4p4gMh7g==", + "71w3aSvuh2mBLtdqJCN3wA==", + "734u4Y1R3u7UNUnD+wWUoA==", + "74FW/QYTzr/P1k6QwVHMcw==", + "778O1hdVKHLG2q9dycUS0Q==", + "78b8sDBp28zUlYPV5UTnYw==", + "79uTykH43voFC3XhHHUzKg==", + "7E6V6/zSjbtqraG7Umj+Jw==", + "7Ephy+mklG2Y3MFdqmXqlA==", + "7Eqzyb+Kep+dIahYJWNNxQ==", + "7GgNLBppgAKcgJCDSsRqOQ==", + "7J3FoFGuTIW36q0PZkgBiw==", + "7K8l6KoP0BH82/WMLntfrg==", + "7R5rFaXCxM3moIUtoCfM2g==", + "7Tauesu7bgs5lJmQROVFiQ==", + "7VHlLw20dWck+I8tCEZilA==", + "7W9aF7dxnL+E8lbS/F7brg==", + "7XRiYvytcwscemlxd9iXIQ==", + "7Y87wVJok20UfuwkGbXxLg==", + "7b0oo4+qphu6HRvJq6qkHQ==", + "7bM/pn4G7g7Zl6Xf1r62Lg==", + "7br49X11xc2GxQLSpZWjKQ==", + "7btpMFgeGkUsiTtsmNxGQA==", + "7cnUHeaPO8txZGGWHL9tKg==", + "7dz+W494zwU5sg63v5flCg==", + "7k5rBuh8FbTTI4TP87wBPQ==", + "7l0RMKbONGS/goW/M+gnMQ==", + "7mxU5fJl/c6dXss9H3vGcQ==", + "7nr3zyWL+HHtJhRrCPhYZA==", + "7p4NpnoNSQR7ISg+w+4yFg==", + "7pkUY2UzSbGnwLvyRrbxfA==", + "7sCJ4RxbxRqVnF4MBoKfuQ==", + "7w3b73nN/fIBvuLuGZDCYQ==", + "7w4PDRJxptG8HMe/ijL6cQ==", + "7wgT9WIiMVcrj48PVAMIgw==", + "7xDIG/80SnhgxAYPL9YJtg==", + "7xTKFcog69nTmMfr5qFUTA==", + "80C9TB9/XT1gGFfQDJxRoA==", + "80PCwYh4llIKAplcDvMj4g==", + "80UE+Ivby3nwplO/HA7cPw==", + "81ZH3SO0NrOO+xoR/Ngw1g==", + "81iQLU+YwxNwq4of6e9z7A==", + "81nkjWtpBhqhvOp6K8dcWg==", + "81pAhreEPxcKse+++h1qBg==", + "82hTTe1Nr4N2g7zwgGjxkw==", + "83ERX2XJV3ST4XwvN7YWCg==", + "83WGpQGWyt6mCV+emaomog==", + "83wtvSoSP9FVBsdWaiWfpA==", + "861mBNvjIkVgkBiocCUj/Q==", + "88PNi9+yn3Bp4/upgxtWGA==", + "88tB/HgUIUnqWXEX++b5Aw==", + "897ptlztTjr7yk+pk8MT0Q==", + "8AfCSZC0uasVON9Y/0P2Pw==", + "8B12CamjOGzJDnQ+RkUf4w==", + "8BLkvEkfnOizJq0OTCYGzw==", + "8CjmgWQSAAGcXX9kz3kssw==", + "8Cm19vJW8ivhFPy0oQXVNA==", + "8DtgIyYiNFqDc5qVrpFUng==", + "8GyPup4QAiolFJ9v80/Nkw==", + "8JVHFRwAd/SCLU0CRJYofg==", + "8LNNoHe6rEQyJ0ebl151Mw==", + "8M0kSvjn5KN8bjsMdUqKZQ==", + "8N3mhHt29FZDHn1P2WH1wQ==", + "8OFxXwnPmrogpNoueZlC4Q==", + "8QK7emHS6rAcAF5QQemW/A==", + "8RtLlzkGEiisy1v9Xo0sbw==", + "8VqeoQELbCs232+Mu+HblA==", + "8WU1vLKV1GhrL7oS9PpABg==", + "8ZBiwr842ZMKphlqmNngHw==", + "8ZFPMJJYVJHsfRpU4DigSg==", + "8ZqmPJDnQSOFXvNMRQYG2Q==", + "8c+lvG5sZNimvx9NKNH3ug==", + "8cXqZub6rjgJXmh1CYJBOg==", + "8dBIsHMEAk7aoArLZKDZtg==", + "8dUcSkd2qnX5lD9B+fUe+Q==", + "8dbyfox/isKLsnVjQNsEXg==", + "8fJLQeIHaTnJ8wGqUiKU6g==", + "8g08gjG/QtvAYer32xgNAg==", + "8hsfXqi4uiuL+bV1VrHqCw==", + "8iYdEleTXGM+Wc85/7vU9w==", + "8j9GVPiFdfIRm/+ho7hpoA==", + "8nOTDhFyZ8YUA4b6M5p84w==", + "8snljTGo/uICl9q0Hxy7/A==", + "8uP4HUnSodw88yoiWXOIcw==", + "8vLA9MOdmLTo3Qg+/2GzLA==", + "8vr+ERVrM99dp+IGnCWDGQ==", + "8ylI1AS3QJpAi3I/NLMYdg==", + "9+hjTVMQUsvVKs7Tmp52tg==", + "90dtIMq0ozJXezT2r79vMQ==", + "91+Yms6Oy/rP0rVjha5z9w==", + "91LQuW6bMSxl10J/UDX23A==", + "91SdBFJEZ65M+ixGaprY/A==", + "91VcAVv7YDzkC1XtluPigw==", + "91vfsZ7Lx9x5gqWTOdM4sg==", + "96ORaz1JRHY1Gk8H74+C2g==", + "99+SBN45LwKCPfrjUKRPmw==", + "9Bet5waJF5/ZvsYaHUVEjQ==", + "9DRHdyX8ECKHUoEsGuqR4Q==", + "9DtM1vls4rFTdrSnQ7uWXw==", + "9FdpxlIFu11qIPdO7WC5nw==", + "9Gkw+hvsR/tFY1cO89topg==", + "9J53kk+InE3CKa7cPyCXMw==", + "9JKIJrlQjhNSC46H3Cstcw==", + "9L6yLO93sRN70+3qq3ObfA==", + "9MDG0WeBPpjGJLEmUJgBWg==", + "9QFYrCXsGsInUb4SClS3cQ==", + "9RGIQ2qyevNbSSEF36xk/A==", + "9RXymE9kCkDvBzWGyMgIWA==", + "9SUOfKtfKmkGICJnvbIDMg==", + "9SgfpAY0UhNC6sYGus9GgQ==", + "9T7gB0ZkdWB0VpbKIXiujQ==", + "9TalxEyFgy6hFCM73hgb7Q==", + "9UhKmKtr4vMzXTEn74BEhg==", + "9W57pTzc572EvSURqwrRhw==", + "9Y1ZmfiHJd9vCiZ6KfO1xQ==", + "9aKH1u5+4lgYhhLztQ4KWA==", + "9ajIS45NTicqRANzRhDWFA==", + "9bAWYElyRN1oJ6eJwPtCtQ==", + "9cvHJmim9e0pOaoUEtiM6A==", + "9dbn0Kzwr9adCEfBJh78uQ==", + "9iB7+VwXRbi6HLkWyh9/kg==", + "9inw7xzbqAnZDKOl/MfCqA==", + "9jxA/t3TQx8dQ+FBsn/YCg==", + "9k17UqdR1HzlF7OBAjpREA==", + "9k1u/5TgPmXrsx3/NsYUhg==", + "9lLhHcrPWI4EsA4fHIIXuw==", + "9nMltdrrBmM5ESBY2FRjGA==", + "9oQ/SVNJ4Ye9lq8AaguGAQ==", + "9oUawSwUGOmb0sDn3XS6og==", + "9onh6QKp70glZk9cX3s34A==", + "9pdeedz1UZUlv8jPfPeZ1g==", + "9pk75mBzhmcdT+koHvgDlw==", + "9qWLbRLXWIBJUXYjYhY2pg==", + "9rL8nC/VbSqrvnUtH9WsxQ==", + "9reBKZ1Rp6xcdH1pFQacjw==", + "9s3ar9q32Y5A3tla5GW/2Q==", + "9sYLg75/hudZaBA3FrzKHw==", + "9tiibT8V9VwnPOErWGNT3w==", + "9vEgJVJLEfed6wJ7hBUGgQ==", + "9viAzLFGYYudBYFu7kFamg==", + "9vmJUS7WIVOlhMqwipAknQ==", + "9wUIeSgNN36SFxy8v2unVg==", + "9xIgKpZGqq0/OU6wM5ZSHw==", + "9xmtuClkFlpz/X5E9JBWBA==", + "A+DLpIlYyCb9DaarpLN76g==", + "A2ODff+ImIkreJtDPUVrlg==", + "A3dX2ShyL9+WOi6MNJBoYQ==", + "A6TLWhipfymkjPYq8kaoDQ==", + "AChOz8avRYsvxlbWcorQ3w==", + "AEpTVUQhIEJGlXJB6rS26A==", + "AFdelaqvxRj6T3YdLgCFyg==", + "AGd0rcLnQ0n+meYyJur1Pw==", + "AGoVLd0QPcXnTedT5T95JQ==", + "ALJWKUImVE40MbEooqsrng==", + "ALlGgVDO8So71ccX0D6u2g==", + "AMfL0rH+g8c0VqOUSgNzQw==", + "ARCWkHAnVgBOIkCDQ19ZuA==", + "ARKIvf4+zRF8eCvUITWPng==", + "ATmMzriwGLl+M3ppkfcZNA==", + "AUGmvZkpkKBry5bHZn4DJA==", + "AV/YJfdoDUdRcrXVwinhQg==", + "AVjwqrTBQH1VREuBlOyUOg==", + "AX1HxQKXD12Yv5HWi39aPQ==", + "AYxGETZs477n2sa1Ulu/RQ==", + "AZs3v4KJYxdi8T1gjVjI2Q==", + "AcKwfS8FRVqb72uSkDNY/Q==", + "AcbG0e6xN8pZfYAv7QJe1Q==", + "Af9j1naGtnZf0u1LyYmK1w==", + "AfVPdxD3FyfwwNrQnVNQ7A==", + "AgDJsaW0LkpGE65Kxk5+IA==", + "Ahpi9+nl13kPTdzL+jgqMw==", + "AiMtfedwGcddA+XYNc+21g==", + "AjHz9GkRTFPjrqBokCDzFw==", + "Ak3rlzEOds6ykivfg39xmw==", + "AkAes5oErTaJiGD2I4A1Pw==", + "AklOdt9/2//3ylUhWebHRw==", + "Al8+d/dlOA5BXsUc5GL8Tg==", + "Ao1Zc0h5AdSHtYt1caWZnQ==", + "AoN/pnK4KEUaGw4V9SFjpg==", + "ApiuEPWr8UjuRyJjsYZQBw==", + "AqHVaj3JcR44hnMzUPvVYg==", + "Ar1Eb/f/LtuIjXnnVPYQlA==", + "Ar9N1VYgE7riwmcrM3bA2Q==", + "AsAHrIkMgc3RRWnklY9lJw==", + "AvdeYb9XNOUFWiiz+XGfng==", + "AwPTZpC28NJQhf5fNiJuLA==", + "AxEjImKz4tMFieSo7m60Sg==", + "AyWlT+EGzIXc395zTlEU5Q==", + "B+TsxQZf0IiQrU8X9S4dsQ==", + "B0TaUQ6dKhPfSc5V/MjLEQ==", + "B1VVUbl8pU0Phyl1RYrmBg==", + "B6reUwMkQFaCHb9BYZExpw==", + "BA18GEAOOyVXO2yZt2U35w==", + "BAJ+/jbk2HyobezZyB9LiQ==", + "BB/R8oQOcoE4j63Hrh8ifg==", + "BB9PTlwKAWkExt3kKC/Wog==", + "BDNM1u/9mefjuW1YM2DuBg==", + "BDbfe/xa9Mz1lVD82ZYRGA==", + "BH+rkZWQjTp7au6vtll/CQ==", + "BL3buzSCV78rCXNEhUhuKQ==", + "BLJk9wA88z6e0IQNrWJIVw==", + "BLbTFLSb4mkxMaq4/B2khg==", + "BMOi5JmFUg5sCkbTTffXHw==", + "BMZB1FwvAuEqyrd0rZrEzw==", + "BPT4PQxeQcsZsUQl33VGmg==", + "BTiGLT6XdZIpFBc91IJY6g==", + "BV1moliPL15M14xkL+H1zw==", + "BW0A06zoQw7S+YMGaegT7g==", + "BXGlq54wIH6R3OdYfSSDRw==", + "BYpHADmEnzBsegdYTv8B5Q==", + "BYz52gYI/Z6AbYbjWefcEA==", + "BZTzHJGhzhs3mCXHDqMjnQ==", + "BaRwTrc5ulyKbW4+QqD0dw==", + "BhKO1s1O693Fjy1LItR/Jw==", + "BjfOelfc1IBgmUxMJFjlbQ==", + "BlCgDd7EYDIqnoAiKOXX6Q==", + "BophnnMszW5o+ywgb+3Qbw==", + "Bq82MoMcDjIo/exqd/6UoA==", + "BuDVDLl0OGdomEcr+73XhQ==", + "BuENxPg7JNrWXcCxBltOPg==", + "Bv4mNIC72KppYw/nHQxfpQ==", + "Bvk8NX4l6WktLcRDRKsK/A==", + "BwRA+tMtwEvth28IwpZx+w==", + "BxFP+4o6PSlGN78eSVT1pA==", + "BxsDnI8jXr4lBwDbyHaYXw==", + "Byhi4ymFqqH8uIeoMRvPug==", + "BzkNYH03gF/mQY71RwO3VA==", + "C+Ssp+v1r+00+qiTy2d7kA==", + "C4QEzQKGxyRi2rjwioHttA==", + "C65PZm8rZxJ6tTEb6d08Eg==", + "C7UaoIEXsVRxjeA0u99Qmw==", + "CBAGa5l95f3hVzNi6MPWeQ==", + "CCK+6Dr72G3WlNCzV7nmqw==", + "CDsanJz7e3r/eQe+ZYFeVQ==", + "CF1sAlhjDQY/KWOBnSSveA==", + "CHLHizLruvCrVi9chj9sXA==", + "CHsFJfsvZkPWDXkA6ZMsDQ==", + "CJoZn5wdTXbhrWO5LkiW0g==", + "CLPzjXKGGpJ0VrkSJp7wPQ==", + "CPDs+We/1wvsGdaiqxzeCQ==", + "CQ0PPwgdG3N6Ohfwx1C8xA==", + "CQpJFrpOvcQhsTXIlJli+Q==", + "CRiL6zpjfznhGXhCIbz8pQ==", + "CRmAj3JcasAb4iZ9ZbNIbw==", + "CT3ldhWpS1SEEmPtjejR/Q==", + "CT9g8mKsIN/VeHLSTFJcNQ==", + "CUCjG2UaEBmiYWQc6+AS1Q==", + "CUEueo8QXRxkfVdfNIk/gg==", + "CWBGcRFYwZ0va6115vV/oQ==", + "CX/N/lHckmAtHKysYtGdZA==", + "CXMKIdGvm60bgfsNc+Imvg==", + "CYJB3qy5GalPLAv1KGFEZA==", + "CZNoTy26VUQirvYxSPc/5A==", + "CZbd+UoTz0Qu1kkCS3k8Xg==", + "CazLJMJjQMeHhYLwXW7YNg==", + "Ci7sS7Yi1+IwAM3VMAB4ew==", + "CiiUeJ0LeWfm7+gmEmYXtg==", + "CkDIoAFLlIRXra78bxT/ZA==", + "CkZUmKBAGu0FLpgPDrybpw==", + "Cl1u5nGyXaoGyDmNdt38Bw==", + "CmBf5qchS1V3C2mS6Rl4bw==", + "CmVD6nh8b/04/6JV9SovlA==", + "CmkmWcMK4eqPBcRbdnQvhw==", + "CnIwpRVC2URVfoiymnsdYQ==", + "CoLvjQDQGldGDqRxfQo+WQ==", + "CrJDgdfzOea2M2hVedTrIg==", + "CsPkyTZADMnKcgSuNu1qxg==", + "CtDj/h2Q/lRey20G8dzSgA==", + "CuGIxWhRLN7AalafBZLCKQ==", + "Cv079ZF55RnbsDT27MOQIA==", + "Cz1G77hsDtAjpe0WzEgQog==", + "CzP13PM/mNpJcJg8JD3s6w==", + "CzSumIcYrZlxOUwUnLR2Zw==", + "CzWhuxwYbNB/Ffj/uSCtbw==", + "D09afzGpwCEH0EgZUSmIZA==", + "D0Qt9sRlMaPnOv1xaq+XUg==", + "D0W5F7gKMljoG5rlue1jrg==", + "D175i+2bZ7aWa4quSSkQpA==", + "D2JcY4zWwqaCKebLM8lPiQ==", + "D31ZticrjGWAO45l5hFh7A==", + "D5ibbo8UJMfFZ48RffuhgQ==", + "D5jaV+HtXkSpSxJPmaBDXg==", + "D66Suu3tWBD+eurBpPXfjA==", + "D7piVoB2NJlBxK5owyo4+g==", + "D7wN7b5u5PKkMaLJBP9Ksw==", + "DA+3fjr7mgpwf6BZcExj0w==", + "DB706G73NpBSRS8TKQOVZw==", + "DBKrdpCE0awppxST4o/zzg==", + "DCjgaGV5hgSVtFY5tcwkuA==", + "DCvI9byhw0wOFwF1uP6xIQ==", + "DDitrRSvovaiXe2nfAtp4g==", + "DEaZD/8aWV6+zkiLSVN/gA==", + "DG2Qe2DqPs5MkZPOqX363Q==", + "DJ+a37tCaGF5OgUhG+T0NA==", + "DJmrmNRKARzsTCKSMLmcNA==", + "DJoy1NSZZw87oxWGlNHhfg==", + "DJscTYNFPyPmTb57g/1w+Q==", + "DKApp/alXiaPSRNm3MfSuA==", + "DLzHkTjjuH6LpWHo2ITD0Q==", + "DMHmyn2U2n+UXxkqdvKpnA==", + "DO1/jfP/xBI9N0RJNqB2Rw==", + "DQJRsUwO1fOuGlkgJavcwQ==", + "DQQB/l55iPN9XcySieNX3A==", + "DQeib845UqBMEl96sqsaSg==", + "DQlZWBgdTCoYB1tJrNS5YQ==", + "DRiFNojs7wM8sfkWcmLnhQ==", + "DWKsPfKDAtfuwgmc2dKUNg==", + "DY0IolKTYlW+jbKLPAlYjQ==", + "DYWCPUq/hpjr6puBE7KBHg==", + "DbWQI3H2tcJsVJThszfHGA==", + "DdaT4JLC7U0EkF50LzIj9w==", + "DdiNGiOSoIZxrMrGNvqkXw==", + "DinJuuBX9OKsK5fUtcaTcQ==", + "DjHszpS8Dgocv3oQkW/VZQ==", + "DjeSrUoWW2QAZOAybeLGJg==", + "Dk0L/lQizPEb3Qud6VHb1Q==", + "DmxgZsQg+Qy1GP0fPkW3VA==", + "Dmyb+a7/QFsU4d2cVQsxDw==", + "DnF6TYSJxlc+cwdfevLYng==", + "Do3aqbRKtmlQI2fXtSZfxQ==", + "DoiItHSms0B9gYmunVbRkQ==", + "DqzWt1gfyu/e7RQl5zWnuQ==", + "Dt6hvhPJu94CJpiyJ5uUkg==", + "Dt8Q5ORzTmpPR2Wdk0k+Aw==", + "DuEKxykezAvyaFO2/5ZmKQ==", + "Dulw855DfgIwiK7hr3X8vg==", + "Duz/8Ebbd0w6oHwOs0Wnwg==", + "DwOTyyCoUfaSShHZx9u6xg==", + "DwP0MQf71VsqvAbAMtC3QQ==", + "DwrNdmU5VFFf3TwCCcptPA==", + "Dz90OhYEjpaJ/pxwg1Qxhg==", + "E+02smwQGBIxv42LIF2Y4Q==", + "E1CvxFbuu9AYW604mnpGTw==", + "E2LR1aZ3DcdCBuVT7BhReA==", + "E2v8Kk60qVpQ232YzjS2ow==", + "E3jMjAgXwvwR8PA53g4+PQ==", + "E4NtzxQruLcetC23zKVIng==", + "E4ojRDwGsIiyuxBuXHsKBA==", + "E8yMPK7W0SIGTK6gIqhxiQ==", + "E9IlDyULLdeaVUzN6eky8g==", + "E9ajQQMe02gyUiW3YLjO/A==", + "E9yeifEZtpqlD0N3pomnGw==", + "EATnlYm0p3h04cLAL95JgA==", + "EC0+iUdSZvmIEzipXgj7Gg==", + "EGLOaMe6Nvzs/cmb7pNpbg==", + "EJgedRYsZPc4cT9rlwaZhg==", + "EKU3OVlT4b/8j3MTBqpMNg==", + "ENFfP93LA257G6pXQkmIdg==", + "EUXQZwLgnDG+C8qxVoBNdw==", + "EXveRXjzsjh8zbbQY2pM9g==", + "EZVQGsXTZvht1qedRLF8bQ==", + "EbGG4X18upaiVQmPfwKytg==", + "EdvIAKdRAXj7e42mMlFOGQ==", + "Ee4A3lTMLQ7iDQ7b8QP8Qg==", + "EfXDc6h69aBPE6qsB+6+Ig==", + "Egs14xVbRWjfBBX7X5Z60g==", + "Ej7W3+67kCIng3yulXGpRQ==", + "ElTNyMR4Rg8ApKrPw88WPg==", + "Epm0d/DvXkOFeM4hoPCBrg==", + "EqMlrz1to7HG4GIFTPaehQ==", + "EqYq2aVOrdX5r7hBqUJP7g==", + "Err1mbWJud80JNsDEmXcYg==", + "EuGWtIbyKToOe6DN3NkVpQ==", + "Ev/xjTi7akYBI7IeZJ4Igw==", + "EvSB+rCggob2RBeXyDQRvQ==", + "Ex3x5HeDPhgO2S9jjCFy4g==", + "EyIsYQxgFa4huyo/Lomv7g==", + "EzjbinBHx3Wr08eXpH3HXA==", + "F50iXjRo1aSTr37GQQXuJA==", + "F58ktE4O0f7C9HdsXYm+lw==", + "F5FcNti7lUa9DyF2iEpBug==", + "F5bs0GGWBx9eBwcJJpXbqg==", + "F8l+Qd9TZgzV+r8G584lKA==", + "F8tEIT5EhcvLNRU5f0zlXQ==", + "FA+nK6mpFWdD0kLFcEdhxA==", + "FAXzjjIr8l1nsQFPpgxM/g==", + "FCLQocqxxhJeleARZ6kSPg==", + "FH5Z60RXXUiDk+dSZBxD3g==", + "FHvI0IVNvih8tC7JgzvCOw==", + "FI2WhaSMb3guFLe3e9il8Q==", + "FIOCTEbzb2+KMCnEdJ7jZw==", + "FL/j3GJBuXdAo54JYiWklQ==", + "FLvED9nB9FEl9LqPn7OOrA==", + "FN7oLGBQGHXXn5dLnr/ElA==", + "FNvQqYoe0s/SogpAB7Hr1Q==", + "FUQySDFodnRhr+NUsWt0KA==", + "FV/D5uSco+Iz8L+5t7E8SA==", + "FWphIPZMumqnXr1glnbK4w==", + "FXzaxi3nAXBc8WZfFElQeA==", + "FbxScyuRacAQkdQ034ShTA==", + "FcFcn4qmPse5mJCX5yNlsA==", + "FcKjlHKfQAGoovtpf+DxWQ==", + "Fd0c8f2eykUp9GYhqOcKoA==", + "Fd2fYFs8vtjws2kx1gf6Rw==", + "FeRovookFQIsXmHXUJhGOw==", + "FhthAO5IkMyW4dFwpFS7RA==", + "Fiy3hkcGZQjNKSQP9vRqyA==", + "FltEN+7NKvzt+XAktHpfHA==", + "FnVNxl5AFH1AieYru2ZG+A==", + "FoJZ61VrU8i084pAuoWhDQ==", + "FpWDTLTDmkUhH/Sgo+g1Gg==", + "FpgdsQ2OG+bVEy3AeuLXFQ==", + "FqWLkhWl0iiD/u2cp+XK9A==", + "FrTgaF5YZCNkyfR1kVzTLQ==", + "Ft2wXUokFdUf6d2Y/lwriw==", + "FtxpWdhEmC6MT61qQv4DGA==", + "FuWspiqu5g8Eeli5Az+BkA==", + "FxnbKnuDct4OWcnFMT/a5w==", + "Fz8EI+ZpYlbcttSHs5PfpA==", + "FzqIpOcTsckSNHExrl+9jg==", + "Fzuq+Wg7clo6DTujNrxsSA==", + "G+sGF13VXPH4Ih6XgFEXxg==", + "G/PA+kt0N+jXDVKjR/054A==", + "G0LChrb0OE5YFqsfTpIL1Q==", + "G0MlFNCbRjXk4ekcPO/chQ==", + "G2UponGde3/Z+9b2m9abpQ==", + "G37U8XTFyshfCs7qzFxATg==", + "G3PmmPGHaWHpPW30xQgm3Q==", + "G4qzBI1sFP2faN+tlRL/Bw==", + "G736AX070whraDxChqUrqw==", + "G7J/za99BFbAZH+Q+/B8WA==", + "G8LFBop8u6IIng+gQuVg3w==", + "GA8k6GQ20DGduVoC+gieRA==", + "GCYI9Dn1h3gOuueKc7pdKA==", + "GDMqfhPQN0PxfJPnK1Bb9A==", + "GF0lY77rx1NQzAsZpFtXIQ==", + "GF2yvI9UWf1WY7V7HXmKPA==", + "GFRJoPcXlkKSvJRuBOAYHQ==", + "GG8a3BlwGrYIwZH9j3cnPA==", + "GHEdXgGWOeOa6RuPMF0xXg==", + "GIHKW6plyLra0BmMOurFgA==", + "GKzs8mlnQQc58CyOBTlfIg==", + "GLDNTSwygNBmuFwCIm7HtA==", + "GLmWLXURlUOJ+PMjpWEXVA==", + "GLnS9wDCje7TOMvBX9jJVA==", + "GNak/LFeoHWlTdLW1iU4eg==", + "GNrMvNXQkW7PydlyJa+f1w==", + "GQJxu1SoMBH14KPV/G/KrQ==", + "GSWncBq4nwomZCBoxCULww==", + "GT6WUDXiheKAM7tPg3he9A==", + "GTNttXfMniNhrbhn92Aykg==", + "GUiinC3vgBjbQC2ybMrMNQ==", + "GW1Uaq622QamiiF24QUA0g==", + "GWwJ32SZqD5wldrXUdNTLA==", + "GdTanUprpE3X/YjJDPpkhQ==", + "Gdf4VEDLBrKJNQ8qzDsIyw==", + "GglPoW5fvr4JSM3Zv99oiA==", + "GhpJfRSWZigLg/azTssyVA==", + "Ghuj9hAyfehmYgebBktfgA==", + "GmC+0rNDMIR+YbUudoNUXw==", + "GnJKlRzmgKN9vWyGfMq3aA==", + "GncGQgmWpI/fZyb/6zaFCg==", + "GrSbnecYAC3j5gtoKntL0A==", + "Gt4/MMrLBErhbFjGbiNqQQ==", + "GzbeM7snhe+M+J7X+gAsQw==", + "H+NHjk/GJDh/GaNzMQSzjg==", + "H+yPRiooEh5J7lAJB4RZ7Q==", + "H0UMAUfHFQH92A2AXRCBKA==", + "H1NJEI+fvOQbI51kaNQQjQ==", + "H1y2iXVaQYwP0SakN6sa+Q==", + "H1zH9I8RwfEy5DGz3z+dHw==", + "H6HPFAcdHFbQUNrYnB74dA==", + "H6j2nPbBaxHecXruxiWYkA==", + "HBRzLacCVYfwUVGzrefZYg==", + "HCbHUfsTDl6+bxPjT57lrA==", + "HCu4ZMrcLMZbPXbTlWuvvQ==", + "HDxGhvdQwGh0aLRYEGFqnw==", + "HEcOaEd9zCoOVbEmroSvJg==", + "HEghmKg3GN60K7otpeNhaA==", + "HFCQEiZf7/SNc+oNSkkwlA==", + "HFHMGgfOeO0UPrray1G+Zw==", + "HGxe+5/kkh6R9GXzEOOFHA==", + "HHxn4iIQ7m0tF1rSd+BZBg==", + "HI4ZIE5s8ez8Rb+Mv39FxA==", + "HITIVoFoWNg04NExe13dNA==", + "HJYgUxFZ66fRT8Ka73RaUg==", + "HK0yf7F97bkf1VYCrEFoWA==", + "HK9xG03FjgCy8vSR+hx8+Q==", + "HLesnV3DL+FhWF3h6RXe8g==", + "HLxROy6fx/mLXFTDSX4eLA==", + "HMQarkPWOUDIg5+5ja2dBQ==", + "HMWOlMmzocOIiJ7yG1YaDQ==", + "HOi+vsGAae4vhr+lJ5ATnQ==", + "HPvYV94ufwiNHEImu4OYvQ==", + "HRF3WL/ue3/QlYyu7NUTrA==", + "HRWYX2XOdsOqYzCcqkwIyw==", + "HYylUirJRqLm+dkp39fSOQ==", + "HaHTsLzx7V3G1SFknXpGxA==", + "HaIRV9SNPRTPDOSX9sK/bg==", + "HaSc7MZphCMysTy2JbTJkw==", + "Hb+pdSavvJ9lUXkSVZW8Og==", + "HbT6W1Ssd3W7ApKzrmsbcg==", + "HbXv8InyZqFT7i3VrllBgg==", + "HdB7Se47cWjPgpJN0pZuiA==", + "HdXg64DBy5WcL5fRRiUVOg==", + "HeQbUuBM9sqfXFXRBDISSw==", + "HfvsiCQN/3mT0FabCU5ygQ==", + "HgIFX42oUdRPu7sKAXhNWg==", + "HhBHt5lQauNl7EZXpsDHJA==", + "HiAgt86AyznvbI2pnLalVQ==", + "HjlPM2FQWdILUXHalIhQ5w==", + "HjyxyL0db2hGDq2ZjwOOhg==", + "HkbdaMuDTPBDnt3wAn5RpQ==", + "Hm6MG6BXbAGURVJKWRM6ZA==", + "HnVfyqgJ+1xSsN4deTXcIA==", + "HoaBBw2aPCyhh0f5GxF+/Q==", + "Hs3vUOOs2TWQdQZHs+FaQQ==", + "Hst3yfyTB7yBUinvVzYROQ==", + "HtDXgMuF8PJ1haWk88S0Ew==", + "HuDuxs2KiGqmeyY1s1PjpQ==", + "HwLSUie8bzH+pOJT3XQFyg==", + "HxEU37uBMeiR5y8q/pM42g==", + "Hy1nqC40l5ItxumkIC2LAA==", + "I+wVQA+jpPTJ6xEsAlYucg==", + "I07W2eDQwe6DVsm1zHKM8A==", + "I5qDndyelK4Njv4YrX7S6w==", + "I9KNZC1tijiG1T72C4cVqQ==", + "IA1jmtfpYkz/E2wD0+27WA==", + "IADk81pIu8NIL/+9Fi94pA==", + "IAMInfSYb76GxDlAr1dsTg==", + "ICPdBCdONUqPwD5BXU5lrw==", + "IEz72W2/W8xBx5aCobUFOQ==", + "IHhyR6+5sZXTH+/NrghIPg==", + "IHyIeMad23fSDisblwyfpA==", + "IKgNa2oPaFVGYnOsL+GC5Q==", + "INNBBin5ePwTyhPIyndHHg==", + "IPLD9nT5EEYG9ioaSIYuuA==", + "ITYL3tDwddEdWSD6J6ULaA==", + "ITZ3P47ALS0JguFms6/cDA==", + "IUZ5aGpkJ9rLgSg6oAmMlw==", + "IUwVHH6+8/0c+nOrjclOWA==", + "IWZnTJ3Hb9qw9HAK/M9gTw==", + "IYIP2UBRyWetVfYLRsi1SQ==", + "IYIbEaErHoFBn8sTT9ICIQ==", + "IbN736G1Px5bsYqE5gW1JQ==", + "IdadoCPmSgHDHzn1zyf8Jw==", + "IdmcpJXyVDajzeiGZixhSA==", + "IhHyHbHGyQS+VawxteLP0w==", + "IhpXs1TK7itQ3uTzZPRP5Q==", + "IindlAnepkazs5DssBCPhA==", + "IjmLaf3stWDAwvjzNbJpQA==", + "Ily2MKoFI1zr5LxBy93EmQ==", + "Iqszlv4R49UevjGxIPMhIA==", + "IrDuBrVu1HWm0BthAHyOLQ==", + "Is3uxoSNqoIo5I15z6Z2UQ==", + "IshzWega6zr3979khNVFQQ==", + "It+K/RCYMOfNrDZxo7lbcA==", + "IwLbkL33z+LdTjaFYh93kg==", + "IwfeA6d0cT4nDTCCRhK+pA==", + "J/PNYu4y6ZMWFFXsAhaoow==", + "J/eAtAPswMELIj8K2ai+Xg==", + "J0NauydfKsACUUEpMhQg8A==", + "J1nYqJ7tIQK1+a/3sMXI/Q==", + "J2NFyb8cXEpZyxWDthYQiA==", + "J4MC9He6oqjOWsYQh9nl3Q==", + "J8v2f6hWFu8oLuwhOeoQjA==", + "JATLdpQm//SQnkyCfI5x7Q==", + "JBkbaBiorCtFq9M9lSUdMg==", + "JC8Q+8yOJ52NvtVeyHo68w==", + "JFFeXsFsMA59iNtZey7LAA==", + "JFHutgSe1/SlcYKIbNNYwQ==", + "JFi6N1PlrpKaYECOnI7GFg==", + "JGEy6VP3sz3LHiyT2UwNHQ==", + "JGeqHRQpf4No74aCs+YTfA==", + "JGx8sTyvr4bLREIhSqpFkw==", + "JHBjKpCgSgrNNACZW1W+1w==", + "JIC8R48jGVqro6wmG2KXIw==", + "JJJkp1TpuDx5wrua2Wml7g==", + "JJbzQ/trOeqQomsKXKwUpQ==", + "JKg64m6mU7C/CkTwVn4ASg==", + "JKmZqz9cUnj6eTsWnFaB0A==", + "JKphO0UYjFqcbPr6EeBuqg==", + "JLq/DrW2f26NaRwfpDXIEA==", + "JPxEncA4IkfBDvpjHsQzig==", + "JQf9UmutPh3tAnu7FDk3nA==", + "JSr/lqDej81xqUvd/O2s7w==", + "JSyhTcHLTfzHsPrxJyiVrA==", + "JSyq2MIuObPnEgEUDyALjQ==", + "JVSLiwurnCelNBiG2nflpQ==", + "JXCYeWjFqcdSf6QwB54G+A==", + "JYJvOZ4CHktLrYJyAbdOnA==", + "JZRjdJLgZ+S0ieWVDj8IJg==", + "Ja3ECL7ClwDrWMTdcSQ6Ug==", + "JaYQXntiyznQzrTlEeZMIw==", + "Jbxl8Nw1vlHO9rtu0q/Fpg==", + "Jcxjli2tcIAjCe+5LyvqdQ==", + "Je1UESovkBa9T6wS0hevLw==", + "JgXSPXDqaS1G9NqmJXZG0A==", + "JgxNrUlL8wutG04ogKFPvw==", + "JipruVZx4ban3Zo5nNM37g==", + "Jit0X0srSNFnn8Ymi1EY+g==", + "Jj4IrSVpqQnhFrzNvylSzA==", + "Jm862vBTCYbv/V4T1t46+Q==", + "JnE6BK0vpWIhNkaeaYNUzw==", + "JoATsk/aJH0UcDchFMksWA==", + "JquDByOmaQEpFb47ZJ4+JA==", + "JrKGKAKdjfAaYeQH8Y2ZRQ==", + "Js7g8Dr6XsnGURA4UNF0Ug==", + "Jt4Eg6MJn8O4Ph/K2LeSUA==", + "Ju4YwtPw+MKzpbC0wJsZow==", + "JvXTdChcE3AqMbFYTT3/wg==", + "JyIDGL1m/w+pQDOyyeYupA==", + "JyUJEnU6hJu8x2NCnGrYFw==", + "JzW+yhrjXW1ivKu3mUXPXg==", + "K1CGbMfhlhIuS0YHLG30PQ==", + "K1RL+tLjICBvMupe7QppIQ==", + "K1RgR6HR5uDEQgZ32TAFgA==", + "K2gk9zWGd0lJFRMQ1AjQ/Q==", + "K3NBEG8jJTJbSrYSOC3FKw==", + "K4VS+DDkTdBblG93l2eNkA==", + "K4yZNVoqHjXNhrZzz2gTew==", + "K5lhaAIZkGeP5rH2ebSJFw==", + "K8PVQhEJCEH1ghwOdztjRw==", + "K9A87aMlJC8XB9LuFM913g==", + "KCJJfgLe00+tjSfP6EBcUg==", + "KGI/cXVz6v6CfL8H6akcUQ==", + "KI7tQFYW38zYHOzkKp9/lQ==", + "KO2XVYyNZadcQv8aCNn5JA==", + "KOm8PTa+ICgDrgK9QxCJZw==", + "KOmdvm+wJuZ/nT/o1+xOuw==", + "KPh6TwYpspne4KZA6NyMbw==", + "KQw25X4LnQ9is+qdqfxo0w==", + "KR401XBdgCrtVDSaXqPEiA==", + "KSorNz/PLR/YYkxaj1fuqw==", + "KSumhnbKxMXQDkZIpDSWmQ==", + "KTjwL+qswa+Bid8xLdjMTg==", + "KXuFON8tMBizNkCC48ICLA==", + "KXvdjZ3rRKn60djPTCENGA==", + "KYuUNrkTvjUWQovw9dNakA==", + "Kh/J1NpDBGoyDU+Mrnnxkg==", + "KhUT2buOXavGCpcDOcbOYg==", + "KhrIIHfqXl9zGE9aGrkRVg==", + "Kj1QI+s9261S3lTtPKd9eg==", + "KjfL7YyVqmCJGBGDFdJ0gw==", + "KjnL3x+56r3M2pDj1pPihA==", + "KkXlgPJPen6HLxbNn5llBw==", + "KkwQL0DeUM3nPFfHb2ej+A==", + "KlY5TGg0pR/57TVX+ik1KQ==", + "KmcGEE0pacQ/HDUgjlt7Pg==", + "KodYHHN62zESrXUye7M01g==", + "Koiog/hpN7ew5kgJbty34A==", + "Kt6BTG1zdeBZ3nlVk+BZKQ==", + "KuNY8qAJBce+yUIluW8AYw==", + "KujFdhhgB9q4oJfjYMSsLg==", + "KyLQxi5UP+qOiyZl0PoHNQ==", + "KzWdWPP2gH0DoMYV4ndJRg==", + "Kzs+/IZJO8v4uIv9mlyJ2Q==", + "L+N/6geuokiLPPSDXM9Qkg==", + "L2D7G0btrwxl9V4dP3XM5Q==", + "L2IeUnATZHqOPcrnW2APbA==", + "L2RofFWDO0fVgSz4D2mtdw==", + "L3Jt5dHQpWQk74IAuDOL8g==", + "L4+C6I7ausPl6JbIbmozAg==", + "LATQEY7f47i77M6p11wjWA==", + "LCj4hI520tA685Sscq6uLw==", + "LCvz/h9hbouXCmdWDPGWqg==", + "LDuBcL5r3PUuzKKZ9x6Kfw==", + "LEVYAE54618FrlXkDN01Kw==", + "LFcpCtnSnsCPD2gT/RA+Zg==", + "LGwcvetzQ3QqKjNh5vA8vw==", + "LHQETSI5zsejvDaPpsO29g==", + "LJeLdqmriyAQp+QjZGFkdQ==", + "LJtRcR70ug6UHiuqbT6NGw==", + "LKyOFgUKKGUU/PxpFYMILw==", + "LMCZqd3UoF/kHHwzTdj7Tw==", + "LMEtzh0+J27+4zORfcjITw==", + "LPYFDbTEp5nGtG6uO8epSw==", + "LQttmX92SI94+hDNVd8Gtw==", + "LSN9GmT6LUHlCAMFqpuPIA==", + "LUWxfy4lfgB5wUrqCOUisw==", + "LWWfRqgtph1XrpxF4N64TA==", + "LWd0+N3M94n81qd346LfJQ==", + "LZAKplVoNjeQgfaHqkyEJA==", + "La0gzdbDyXUq6YAXeKPuJA==", + "LawT9ZygiVtBk0XJ+KkQgQ==", + "LbPp1oL0t3K2BAlIN+l8DA==", + "LblwOqNiciHmt2NXjd89tg==", + "LcF0OqPWrcpHby8RwXz1Yg==", + "LcoJBEPTlSsQwfuoKQUxEw==", + "LhqRc9oewY4XaaXTcnXIHQ==", + "Lo1xTCEWSxVuIGEbBEkVxA==", + "LoUv/f2lcWpjftzpdivMww==", + "LpoayYsTO8WLFLCSh2kf2w==", + "Lqel4GdU0ZkfoJVXI5WC/Q==", + "LqgzKxbI6WTMz0AMIDJR5w==", + "LsmsPokAwWNCuC74MaqFCQ==", + "Lt/pVD4TFRoiikmgAxEWEw==", + "Lu02ic/E94s42A14m7NGCA==", + "LyPXOoOPMieqINtX8C9Zag==", + "LyYPOZKm8bBegMr5NTSBfg==", + "M/cQja3uIk1im9++brbBOA==", + "M0ESOGwJ4WZ4Ons1ljP0bQ==", + "M20iX2sUfw5SXaZLZYlTaA==", + "M2JMnViESVHTZaru6LDM6w==", + "M2suCoFHJ5fh9oKEpUG3xA==", + "M55eersiJuN9v61r8DoAjQ==", + "M98hjSxCwvZ27aBaJTGozQ==", + "M9oqlPb63e0kZE0zWOm+JQ==", + "MArbGuIAGnw4+fw6mZIxaw==", + "MBjMU/17AXBK0tqyARZP5w==", + "MFeXfNZy6Q9wBfZmPQy3xg==", + "MI+HSMRh8KTW+Afiaxd/Fw==", + "MJ1FuK8PXcmnBAG9meU84A==", + "MK7AqlJIGqK2+K5mCvMXRQ==", + "ML7ipnY/g8mA1PUIju1j8Q==", + "MLHt6Ak288G0RGhCVaOeqA==", + "MLlVniZ08FHAS5xe+ZKRaA==", + "MMaegl2Md9s/wOx5o9564w==", + "MN94B0r5CNAF9sl3Kccdbw==", + "MOrAbuJTyGKPC6MgYJlx5Q==", + "MQYM3BT77i35LG9HcqxY2Q==", + "MQvAr+OOfnYnr/Il/2Ubkg==", + "MUkRa/PjeWMhbCTq43g6Aw==", + "MVoxyIA+emaulH8Oks8Weg==", + "MWcV03ULc0vSt/pFPYPvFA==", + "MbI04HlTGCoc/6WDejwtaQ==", + "MdvhC1cuXqni/0mtQlSOCw==", + "MeKXnEfxeuQu9t3r/qWvcw==", + "MfkyURTBfkNZwB+wZKjP4g==", + "Mj87ajJ/yR41XwAbFzJbcA==", + "Ml3mi1lGS1IspHp3dYYClg==", + "MlKWxeEh8404vXenBLq4bw==", + "MlOOZOwcRGIkifaktEq0aQ==", + "MnStiFQAr3QlaRZ02SYGaQ==", + "Mofqu40zMRrlcGRLS42eBw==", + "MpAwWMt7bcs4eL7hCSLudQ==", + "MqqDg9Iyt4k3vYVW5F+LDw==", + "Mr5mCtC53+wwmwujOU/fWw==", + "MrbEUlTagbesBNg0OemHpw==", + "MrxR3cJaDHp0t3jQNThEyg==", + "MsCloSmTFoBpm7XWYb+ueQ==", + "Muf2Eafcf9G3U2ZvQ9OgtQ==", + "MvMbvZNKbXFe2XdN+HtnpQ==", + "N+K1ibXAOyMWdfYctNDSZQ==", + "N/HgDydvaXuJvTCBhG/KtA==", + "N2KovXW14hN/6+iWa1Yv3g==", + "N2X7KWekNN+fMmwyXgKD5w==", + "N3YDSkBUqSmrmNvZZx4a1Q==", + "N4/mQFyhDpPzmihjFJJn6w==", + "N65PqIWiQeS082D6qpfrAg==", + "N7fHwb397tuQHtBz1P80ZQ==", + "N8dXCawxSBX40fgRRSDqlQ==", + "N9nD7BGEM7LDwWIMDB+rEQ==", + "NBmB/cQfS+ipERd7j9+oVg==", + "ND2hYtAIQGMxBF7o7+u7nQ==", + "ND9l4JWcncRaSLATsq0LVw==", + "NDZWIhhixq7NT8baJUR4VQ==", + "NGApiVkDSwzO45GT57GDQw==", + "NKGY0ANVZ0gnUtzVx1pKSw==", + "NKRzJndo2uXNiNppVnqy1g==", + "NMbAjbnuK7EkVeY3CQI5VA==", + "NN/ymVQNa17JOTGr6ki3eQ==", + "NOmu8oZc6CcKLu+Wfz2YOQ==", + "NQVQfN3nIg9ipHiFh4BvfQ==", + "NRyFx6jqO/oo9ojvbYzsAg==", + "NSrzwNlB0bde3ph8k6ZQcQ==", + "NZtcY8fIpSKPso/KA6ZfzA==", + "Nc5kiwXCAyjpzt43G5RF1A==", + "NdULoUDGhIolzw1PyYKV0A==", + "NdVyHoTbBhX6Umz/9vbi0g==", + "Ndx5LDiVyyTz/Fh3oBTgvA==", + "Nf9fbRHm844KZ2sqUjNgkA==", + "NfxVYc3RNWZwzh2RmfXpiA==", + "Ng5v/B9Z10TTfsDFQ/XrXQ==", + "NhZbSq0CjDNOAIvBHBM9zA==", + "NiQ/m4DZXUbpca9aZdzWAw==", + "NiawWuMBDo0Q3P2xK/vnLQ==", + "NjeDgQ1nzH1XGRnLNqCmSg==", + "NmQrsmb8PVP05qnSulPe5Q==", + "NmWmDxwK5FpKlZbo0Rt8RA==", + "NoX8lkY+kd2GPuGjp+s0tQ==", + "NquRbPn8fFQhBrUCQeRRoQ==", + "Nr4zGo5VUrjXbI8Lr4YVWQ==", + "Nsd+DfRX6L54xs+iWeMjCQ==", + "NtwqUO3SKZE/9MXLbTJo/g==", + "NuBYjwlxadAH+vLWYRZ3bg==", + "NvkR0inSzAdetpI4SOXGhw==", + "NvurnIHin4O+wNP7MnrZ1w==", + "NxSdT2+MUkQN49pyNO2bJw==", + "NyF+4VRog7etp90B9FuEjA==", + "O/EizzJSuFY8MpusBRn7Tg==", + "O1ckWUwuhD44MswpaD6/rw==", + "O209ftgvu0vSr0UZywRFXA==", + "O538ibsrI4gkE5tfwjxjmg==", + "O5N2yd+QQggPBinQ+zIhtQ==", + "O7JiE0bbp583G6ZWRGBcfw==", + "O839JUrR+JS30/nOp428QA==", + "OChiB4BzcRE8Qxilu6TgJg==", + "OEJ40VmMDYzc2ESEMontRA==", + "OERGn45uzfDfglzFFn6JAg==", + "OFLn4wun6lq484I7f6yEwg==", + "OGpsXRHlaN8BvZftxh1e7A==", + "OHJBT2SEv5b5NxBpiAf7oQ==", + "OIwtfdq37eQ0qoXuB2j7Hw==", + "OMO4pqzfcbQ11YO4nkTXfg==", + "OONAvFS/kmH7+vPhAGTNSg==", + "OOS6wQCJsXH8CsWEidB35A==", + "OVHqwV8oQMC5KSMzd5VemA==", + "OaNpzwshdHUZMphQXa6i8w==", + "Oc3BqTF3ZBW3xE0QsnFn/A==", + "OlpA9HsF8MBh7b45WZSSlg==", + "OlwHO6Sg2zIwsCOCRu0HiQ==", + "Omi2ZB9kdR1HrVP2nueQkA==", + "Omr+zPWVucPCSfkgOzLmSQ==", + "OnmvXbyT2BYsSDJYZhLScA==", + "OpC/sL320wl5anx6AVEL+A==", + "OpL+vHwPasW30s2E1TYgpA==", + "OrqJKjRndcZ8OjE3cSQv7g==", + "Otz/PgYOEZ1CQDW54FWJIQ==", + "OwArFF1hpdBupCkanpwT+Q==", + "OwIGvTh8FPFqa4ijNkguAw==", + "Owg8qCpjZa+PmbhZew6/sw==", + "OzFRv+PzPqTNmOnvZGoo5g==", + "OzH7jTcyeM7RPVFtBdakpQ==", + "OzMR5D2LriC5yrVd5hchnA==", + "P0Pc8owrqt6spdf7FgBFSw==", + "P14k+fyz0TG9yIPdojp52w==", + "P3y5MoXrkRTSLhCdLlnc4A==", + "P430CeF2MDkuq11YdjvV8A==", + "P5WPQc5NOaK7WQiRtFabkw==", + "P5fucOJhtcRIoElFJS4ffg==", + "P5wS+xB8srW4a5KDp/JVkA==", + "P7eMlOz9YUcJO+pJy0Kpkw==", + "P8lUiLFoL100c9YSQWYqDA==", + "PAlx9+U+yQCAc5Fi0BOG0w==", + "PBULPuFXb6V3Di713n3Gug==", + "PCOGl7GIqbizAKj/sZmlwQ==", + "PD+yHtJxZJ2XEvjIPIJHsQ==", + "PF0lpolQQXlpc3qTLMBk8w==", + "PHwJ5ZAqqftZ4ypr8H1qiQ==", + "PKtXc4x4DEjM45dnmPWzyg==", + "PMCWKgog/G+GFZcIruSONw==", + "PMvG4NqJP76kMRAup6TSZA==", + "PPa7BDMpRdxJdBxkuWCxKA==", + "PTAm/jGkie7OlgVOvPKpaA==", + "PTW+fhZq/ErxHqpM0DZwHQ==", + "PXC6ZpdMH0ATis/jGW12iA==", + "PaROi5U16Tk35p0EKX5JpA==", + "ParhxI6RtLETBSwB0vwChQ==", + "PbDVq2Iw1eeM8c2o/XYdTA==", + "PbnxuVerGwHyshkumqAARg==", + "Pc+u0MAzp4lndTz4m6oQ5w==", + "PcdBtV8pfKU0YbDpsjPgwg==", + "PcoVtZrS1x1Q+6nfm4f80w==", + "PdBgXFq5mBqNxgCiqaRnkw==", + "PeJS+mXnAA6jQ0WxybRQ8w==", + "PfkWkSbAxIt1Iso0znW0+Q==", + "PggVPQL5YKqSU/1asihcrg==", + "PibGJQNw7VHPTgqeCzGUGA==", + "Po0lhBfiMaXhl+vYh1D8gA==", + "PolhKCedOsplEcaX4hQ0YQ==", + "Pp1ZMxJ8yajdbfKM4HAQxA==", + "PqLCd/pwc+q5GkL6MB0jTg==", + "Pt3i49uweYVgWze3OjkjJA==", + "Pu9pEf+Tek3J+3jmQNqrKw==", + "Pv9FWQEDLKnG/9K9EIz4Gw==", + "PwvPBc+4L73xK22S9kTrdA==", + "PxReytUUn/BbxYTFMu1r2Q==", + "PybPZhJErbRTuAafrrkb3g==", + "Q0TJZxpn3jk67L7N+YDaNA==", + "Q1pdQadt12anX1QRmU2Y/A==", + "Q3TpCE+wnmH/1h/EPWsBtQ==", + "Q4bfQslDSqU64MOQbBQEUw==", + "Q6vGRQiNwoyz7bDETGvi5g==", + "Q7Df6zGwvb4rC+EtIKfaSw==", + "Q7teXmTHAC5qBy+t7ugf0w==", + "Q8RVI/kRbKuXa8HAQD7zUA==", + "QAz7FA+jpz9GgLvwdoNTEQ==", + "QCpzCTReHxGm5lcLsgwPCA==", + "QGYFMpkv37CS2wmyp42ppg==", + "QH36wzyIhh6I56Vnx79hRA==", + "QH3lAwOYBAJ0Fd5pULAZqw==", + "QIKjir/ppRyS63BwUcHWmw==", + "QJEbr3+42P9yiAfrekKdRQ==", + "QTz21WkhpPjfK8YoBrpo+w==", + "QV0OG5bpjrjku4AzDvp9yw==", + "QVwuN66yPajcjiRnVk/V8g==", + "QWURrsEgxbJ8MWcaRmOWqw==", + "Qc+XYy2qyWJ5VVwd2PExbw==", + "Qf7JFJJuuacSzl6djUT2EQ==", + "Qg1ubGl+orphvT990e5ZPA==", + "QiozlNcQCbqXtwItWExqJQ==", + "QmSBVvdk0tqH9RAicXq2zA==", + "QmcURiMzmVeUNaYPSOtTTg==", + "QoUC9nyK1BAzoUVnBLV2zw==", + "QoqHzpHDHTwQD5UF30NruQ==", + "QozQL0DTtr+PXNKifv6l6g==", + "Qrh7OEHjp80IW+YzQwzlJg==", + "QsquNcCZL9wv7oZFqm64vQ==", + "QtD35QhE8sAccPrDnhtQmQ==", + "Qv6wWP4PpycDGxe7EZNSCw==", + "QvYZxsLdu+3nV/WhY1DsYg==", + "Qx6rVv9Xj8CBjqikWI9KFA==", + "QyyiJ5I/OZC50o89fa5EmQ==", + "R+beucURp/H5jLs4kW6wmg==", + "R/y6+JJP8rzz1KITJ4qWBw==", + "R1TCCfgltnXBvt5AiUnCtQ==", + "R2OOV18CV/YpWL1xzr/VQg==", + "R2Use39If2C0FVBP7KDerA==", + "R36O31Pj8jn0AWSuqI7X2Q==", + "R3ijnutzvK6IKV3AKHQZSA==", + "R5oOM58zdbVxFSDQnNWqeA==", + "R6Me6sSGP5xpNI8R0xGOWw==", + "R6cO8GzYfOGTIi773jtkXw==", + "R81DX/5a7DYKkS4CU+TL+w==", + "R8FxgXWKBpEVbnl41+tWEw==", + "R8ULpSNu9FcCwXZM0QedSg==", + "R906Kxp2VFVR3VD+o6Vxcw==", + "R97chlspND/sE9/HMScXjQ==", + "RAAw14BA1ws5Wu/rU7oegw==", + "RAECgYZmcF4WxcFcZ4A0Ww==", + "RBMv0IxXEO3o7MnV47Bzow==", + "RClzwwKh51rbB4ekl99EZA==", + "RDgGGxTtcPvRg/5KRRlz4w==", + "REnDNe9mGfqVGZt+GdsmjQ==", + "RHKCMAqrPjvUYt13BVcmvw==", + "RHToSGASrwEmvzjX6VPvNQ==", + "RIVYGO2smx9rmRoDVYMPXw==", + "RIZYDgXqsIdTf9o2Tp/S7g==", + "RJJqFMeiCZHdsqs72J17MQ==", + "RKVDdE1AkILTFndYWi9wFg==", + "RM5CpIiB94Sqxi462G7caA==", + "RNK9G1hfuz3ETY/RmA9+aA==", + "RNdyt6ZRGvwYG5Ws3QTuEA==", + "ROSt+NlEoiPFtpRqKtDUrQ==", + "RQOlmzHwQKFpafKPJj0D8w==", + "RQywrOLZEKw9+kG6qTzr3g==", + "RUmhye56tQu9xXs4SRJpOQ==", + "RVD3Ij6sRwwxTUDAxwELtA==", + "RWI0HfpP7643OSEZR8kxzw==", + "RYkDwwng6eeffPHxt8iD9A==", + "RZTpYKxOAH9JgF1QFGN+hw==", + "RfSwpO/ywQx4lfgeYlBr2w==", + "RgtwfY5pTolKrUGT+6Pp6g==", + "RhcqXY4OsZlVVF7ZlkTeRw==", + "RiahBXX2JbPzt8baPiP/8g==", + "RkQK9S1ezo+dFYHQP57qrw==", + "RlNPyhgYOIn28R4vKCVtYA==", + "RnOXOygwJFqrD+DlM3R5Ew==", + "RnxOYPSQdHS6fw4KkDJtrA==", + "RppDe/WGt1Ed6Vqg1+cCkQ==", + "RqYpA5AY7mKPaSxoQfI1CA==", + "RrE3B3X/SJi3CqCUlTYwaw==", + "Rrq0ak9YexLqqbSD4SSXlw==", + "Rs8deApkoosIJSfX7NXtAA==", + "RuLeQHP1wHsxhdmYMcgtrQ==", + "RvXWAFwM+mUAPW1MjPBaHA==", + "Rvchz/xjcY9uKiDAkRBMmA==", + "Rww3qkF3kWSd+AaMT0kfdw==", + "RxmdoO8ak8y/HzMSIm+yBQ==", + "Ry3zgZ6KHrpNyb7+Tt2Pkw==", + "RzeH+G3gvuK1z+nJGYqARQ==", + "S+b37XhKRm8cDwRb1gSsKQ==", + "S2MAIYeDQeJ1pl9vhtYtUg==", + "S3VQa6DH+BdlSrxT/g6B5g==", + "S47hklz3Ow+n5aY6+qsCoA==", + "S4RvORcJ3m6WhnAgV4YfYA==", + "S4rFuiKLFKZ+cL7ldiTwpg==", + "S7Vjy/gOWp0HozPP1RUOZw==", + "S8jlvuYuankCnvIvMVMzmg==", + "S9L29U2P5K8wNW+sWbiH7w==", + "SCO9nQncEcyVXGCtx30Jdg==", + "SChDh/Np1HyTPWfICfE1uA==", + "SDi5+FoP9bMyKYp+vVv1XA==", + "SEGu+cSbeeeZg4xWwsSErQ==", + "SEIZhyguLoyH7So0p1KY0A==", + "SESKbGF35rjO64gktmLTWA==", + "SElc2+YVi3afE1eG1MI7dQ==", + "SFn78uklZfMtKoz2N0xDaQ==", + "SIuKH/Qediq0TyvqUF93HQ==", + "SM7E98MyViSSS9G0Pwzwyw==", + "SNPYH4r/J9vpciGN2ybP5Q==", + "SOdpdrk2ayeyv0xWdNuy9g==", + "SPGpjEJrpflv1hF0qsFlPw==", + "SPHU6ES1WVm0Mu2LB+YjrA==", + "SSKhl2L3Mvy93DcZulADtA==", + "SUAwMWLMml8uGqagz5oqhQ==", + "SVFbcjXbV7HRg+7jUrzpwg==", + "SVLHWPCCH7GPVCF7QApPbw==", + "SVuEYfQ9FGyVMo1672n0Yg==", + "SbMjjI8/P8B9a9H2G0wHEQ==", + "Scto+9TWxj1eZgvNKo+a9A==", + "SfwnYZCKP1iUJyU1yq4eKg==", + "SiSlasZ+6U2IZYogqr2UPg==", + "Slu3z535ijcs5kzDnR7kfA==", + "SmRWEzqddY9ucGAP5jXjAg==", + "Sr9c0ReRpkDYGAiqSy683g==", + "Srl4HivgHMxMOUHyM3jvNw==", + "StDtLMlCI75g4XC59mESEQ==", + "StoXC7TBzyRViPzytAlzyQ==", + "StpQm/cQF8cT0LFzKUhC5w==", + "SusSOsWNoAerAIMBVWHtfA==", + "Swjn3YkWgj0uxbZ1Idtk+A==", + "SzCGM8ypE58FLaR1+1ccxQ==", + "Szko0IPE7RX2+mfsWczrMg==", + "T/6gSz2HwWJDFIVrmcm8Ug==", + "T1pMWdoNDpIsHF8nKuOn2A==", + "T6LA+daQqRI38iDKZTdg1A==", + "T7waQc3PvTFr0yWGKmFQdQ==", + "T9WoUJNwp8h4Yydixbx6nA==", + "TA9WjiLAFgJubLN4StPwLw==", + "TAD0Lk95CD86vbwrcRogaQ==", + "TBQpcKq2huNC5OmI2wzRQw==", + "TDrq23VUdzEU/8L5i8jRJQ==", + "TGB+FIzzKnouLh5bAiVOQg==", + "THfzE2G2NVKKfO+A2TjeFw==", + "THs1r8ZEPChSGrrhrNTlsA==", + "TI90EuS/bHq/CAlX32UFXg==", + "TIKadc6FAaRWSQUg5OATgg==", + "TIWSM78m0RprwgPGK/e0JA==", + "TLJbasOoVO435E5NE5JDcA==", + "TNyvLixb03aP2f8cDozzfA==", + "TSGL3iQYUgVg/O9SBKP9EA==", + "TSPFvkgw6uLsJh66Ou0H9w==", + "TVlHoi8J7sOZ2Ti7Dm92cQ==", + "TXab/hqNGWaSK+fXAoB2bg==", + "TYlnrwgyeZoRgOpBYneRAg==", + "TZ3ATPOFjNqFGSKY3vP2Hw==", + "TZT86wXfzFffjt0f95UF5w==", + "TafM7nTE5d+tBpRCsb8TjQ==", + "TahqPgS7kEg+y6Df0HBASw==", + "TcFinyBrUoAEcLzWdFymow==", + "TcGhAJHRr7eMwGeFgpFBhg==", + "TcyyXrSsQsnz0gJ36w4Dxw==", + "TeBGJCqSqbzvljIh9viAqA==", + "TfHvdbl2M4deg65QKBTPng==", + "TfNHjSTV8w6Pg6+FaGlxvA==", + "TgWe70YalDPyyUz6n88ujg==", + "Tk5MAqd1gyHpkYi8ErlbWg==", + "TlJizlASbPtShZhkPww4UA==", + "Tm4zk2Lmg8w4ITMI31NfTA==", + "Tmx0suRHzlUK4FdBivwOwA==", + "Tp52d1NndiC9w3crFqFm9g==", + "TrLmfgwaNATh24eSrOT+pw==", + "TrWS+reCJ0vbrDNT5HDR9w==", + "Tu6w6DtX2RJJ3Ym3o3QAWw==", + "TuaG3wRdM9BWKAxh2UmAsg==", + "Tud+AMyuFkWYYZ73yoJGpQ==", + "Tug3eh+28ttyf+U7jfpg5w==", + "U+bB5NjFIuQr/Y5UpXHwxA==", + "U+oTpcjhc0E+6UjP11OE/Q==", + "U0KmEI6e5zJkaI4YJyA5Ew==", + "U49SfOBeqQV9wzsNkboi8Q==", + "U6VQghxOXsydh3Naa5Nz4A==", + "U9kE50Wq5/EHO03c5hE4Ug==", + "UAqf4owQ+EmrE45hBcUMEw==", + "UEMwF4kwgIGxGT4jrBhMPQ==", + "UHpge5Bldt9oPGo2oxnYvQ==", + "UIXytIHyVODxlrg+eQoARA==", + "UK+R+hAoVeZ4xvsoZjdWpw==", + "UNRlg6+CYVOt68NwgufGNA==", + "UNdKik7Vy23LjjPzEdzNsg==", + "UNt7CNMtltJWq8giDciGyA==", + "UP7NXAE0uxHRXUAWPhto0w==", + "UP9mmAKzeQqGhod7NCqzhg==", + "UPYR575ASaBSZIR3aX1IgQ==", + "UPzS4LR3p/h0u69+7YemrQ==", + "UQTQk5rrs6lEb1a+nkLwfg==", + "USCvrMEm/Wqeu9oX6FrgcQ==", + "USq1iF90eUv41QBebs3bhw==", + "UTmTgvl+vGiCDQpLXyVgOg==", + "UVEZPoH9cysC+17MKHFraw==", + "UXUNYEOffgW3AdBs7zTMFA==", + "UZoibx+y1YJy/uRSa9Oa2w==", + "Ua6aO6HwM+rY4sPR19CNFA==", + "UbABE6ECnjB+9YvblE9CYw==", + "UbSFw5jtyLk5MealqJw++A==", + "Ugt8HVC/aUzyWpiHd0gCOQ==", + "UgvtdE2eBZBUCAJG/6c0og==", + "Uh1mvZNGehK1AaI4a1auKQ==", + "Uje3Ild84sN41JEg3PEHDg==", + "UjmDFO7uzjl4RZDPeMeNyg==", + "Um1ftRBycvb+363a90Osog==", + "Umd+5fTcxa3mzRFDL9Z8Ww==", + "Uo+FIhw1mfjF6/M8cE1c/Q==", + "Uo1ebgsOxc3eDRds1ah3ag==", + "UreSZCIdDgloih8KLeX7gg==", + "UtLYUlQJ02oKcjNR3l+ktg==", + "Uudn69Kcv2CGz2FbfJSSEA==", + "UvC1WADanMrhT+gPp/yVqA==", + "Uw6Iw+TP9ZdZGm2b/DAmkg==", + "UwqBVd4Wfias4ElOjk2BzQ==", + "Uy4QI8D2y1bq/HDNItCtAw==", + "UymZUnEEQWVnLDdRemv+Tw==", + "UzPPFSXgeV7KW4CN5GIQXA==", + "V+QzdKh5gxTPp2yPC9ZNEg==", + "V/xG5QFyx1pihimKmAo8ZA==", + "V1fvtnJ0L3sluj9nI5KzRw==", + "V2P75JFB4Se9h7TCUMfeNA==", + "V5HEaY3v9agOhsbYOAZgJA==", + "V5HKdaTHjA8IzvHNd9C51g==", + "V6CRKrKezPwsRdbm0DJ2Yg==", + "V6zyoX6MERIybGhhULnZiw==", + "V7eji28JSg3vTi30BCS7gw==", + "V8m51xgUgywRoV6BGKUrgg==", + "V8q+xz4ljszLZMrOMOngug==", + "V9G1we3DOIQGKXjjPqIppQ==", + "V9vkAanK+Pkc4FGAokJsTA==", + "VAg/aU5nl72O+cdNuPRO4g==", + "VCL3xfPVCL5RjihQM59fgg==", + "VE4sLM5bKlLdk85sslxiLQ==", + "VGRCSrgGTkBNb8sve0fYnQ==", + "VH70dN82yPCRctmAHMfCig==", + "VI8pgqBZeGWNaxkuqQVe7g==", + "VIC7inSiqzM6v9VqtXDyCw==", + "VIkS30v268x+M1GCcq/A8A==", + "VJt2kPVBLEBpGpgvuv1oUw==", + "VK95g27ws2C6J2h/7rC2qA==", + "VOB+9Bcfu8aHKGdNO0iMRw==", + "VOvrzqiZ1EHw+ZzzTWtpsw==", + "VPa7DG6v7KnzMvtJPb88LQ==", + "VPqyIomYm7HbK5biVDvlpw==", + "VQIpquUqmeyt/q6OgxzduQ==", + "VRnx+kd6VdxChwsfbo1oeQ==", + "VUDsc9RMS1fSM43c+Jo9dQ==", + "VWNDBOtjiiI4uVNntOlu/A==", + "VWb8U4jF/Ic0+wpoXi/y/g==", + "VWy9lB5t4fNCp4O/4n8S4w==", + "VX+cVXV8p9i5EBTMoiQOQQ==", + "VXu4ARjq7DS2IR/gT24Pfw==", + "VZX1FnyC8NS2k3W+RGQm4g==", + "VaJc9vtYlqJbRPGb5Tf0ow==", + "VbCoGr8apEcN7xfdaVwVXw==", + "VbHoWmtiiPdABvkbt+3XKQ==", + "Vg2E5qEDfC+QxZTZDCu9yQ==", + "VhYGC8KYe5Up+UJ2OTLKUw==", + "Vik8tGNxO0xfdV0pFmmFDw==", + "ViweSJuNWbx5Lc49ETEs/A==", + "VjclDY8HN4fSpB263jsEiQ==", + "VllbOAjeW3Dpbj5lp2OSmA==", + "VoPth5hDHhkQcrQTxHXbuw==", + "VpmBstwR7qPVqPgKYQTA3g==", + "VsXEBIaMkVftkxt1kIh7TA==", + "Vu0E+IJXBnc25x4n41kQig==", + "VzQ1NwNv9btxUzxwVqvHQg==", + "VznvTPAAwAev+yhl9oZT0w==", + "W+M4BcYNmjj7xAximDGWsA==", + "W/0s1x3Qm+wN8DhROk6FrQ==", + "W/5ThNLu43uT1O+fg0Fzwg==", + "W04GeDh+Tk/I1S85KlozRA==", + "W2x0SBzSIsTRgyWUCOZ/lg==", + "W4CfeVp9mXgk04flryL7iA==", + "W4utAK3ws0zjiba/3i91YA==", + "W5now3RWSzzMDAxsHSl++Q==", + "W8bATujVUT80v2XGJTKXDg==", + "W8y32OLHihfeV0XFw7LmOg==", + "WADmxH7R6B4LR+W6HqQQ6A==", + "WBu0gJmmjVdVbjDmQOkU6w==", + "WGKFTWJac8uehn3N59yHJw==", + "WHutPin+uUEqtrA7L8878A==", + "WKehT4nGF2T7aKuzABDMlA==", + "WLsh3UF4WXdHwgnbKEwRlQ==", + "WLwpjgr9KzevuogoHZaVUw==", + "WN7lFJfw4lSnTCcbmt5nsg==", + "WNfDNaWUOqABQ6c6kR+eyw==", + "WQMffxULFKJ+bun6NrCURA==", + "WQznrwqvMhUlM3CzmbhAOQ==", + "WRjYdKdtnd1G9e/vFXCt0g==", + "WRoJMO0BCJyn5V6qnpUi4Q==", + "WTr3q/gDkmB4Zyj7Ly20+w==", + "WVhfn2yJZ43qCTu0TVWJwA==", + "WWN44lbUnEdHmxSfMCZc6w==", + "WY7mCUGvpXrC8gkBB46euw==", + "WbAdlac/PhYUq7J2+n5f+w==", + "Wd0dOs7eIMqW5wnILTQBtg==", + "WdCWezJU4JK43EOZ9YHVdg==", + "Wf2olJCYZRGTTZxZoBePuQ==", + "WjDqf1LyFyhdd8qkwWk+MA==", + "WkSJpxBa45XJRWWZFee7hw==", + "Wn+Vj4eiWx0WPUHr3nFbyA==", + "WnHK5ZQDR6Da5cGODXeo0A==", + "WrJMOuXSLKKzgmIDALkyNw==", + "WtT0QAERZSiIt2SFDiAizg==", + "WwraoO97OTalvavjUsqhxQ==", + "Wx9jh/teM0LJHrvTScssyQ==", + "WyCFB4+6lVtlzu3ExHAGbQ==", + "WzjvUJ4jZAEK7sBqw+m07A==", + "X/Gha4Ajjm/GStp/tv+Jvw==", + "X1PaCfEDScclLtOTiF5JUw==", + "X2Tawm2Cra6H7WtXi1Z4Qw==", + "X2YfnPXgF2VHVX95ZcBaxQ==", + "X4hrgqMIcApsjA9qOWBoCw==", + "X4kdXUuhcUqMSduqhfLpxA==", + "X4o0OkTz0ec70mzgwRfltA==", + "X6Ln4si8G5aKar52ZH/FEQ==", + "X6ulLp4noBgefQTsbuIbYQ==", + "X9QAaNjgiOeAWSphrGtyVw==", + "XA2hUgq3GVPpxtRYiqnclg==", + "XAq/C+XyR6m3uzzLlMWO5Q==", + "XEwOJG24eaEtAuBWtMxhwg==", + "XF/yncdoT4ruPeXCxEhl9Q==", + "XGAXhUFjORwKmAq9gGEcRg==", + "XHHEg/8KZioW/4/wgSEkbQ==", + "XHjrTLXkm/bBY/BewmJcCQ==", + "XJihma9zSRrXLC+T+VcFDA==", + "XLq/nWX8lQqjxsK9jlCqUg==", + "XOG1PYgqoG8gVLIbVLTQgg==", + "XSb71ae0v+yDxNF5HJXGbQ==", + "XTCcsVfEvqxnjc0K5PLcyw==", + "XV13yK0QypJXmgI+dj4KYw==", + "XV5MYe0Q7YMtoBD6/iMdSw==", + "XVVy3e6dTnO3HpgD6BtwQw==", + "XXFr0WUuGsH5nXPas7hR3Q==", + "Xconi1dtldH90Wou9swggw==", + "XddlSluOH6VkR7spFIFmdQ==", + "XdkxmYYooeDKzy7PXVigBQ==", + "XePy/hhnQwHXFeXUQQ55Vg==", + "XfBOCJwi2dezYzLe316ivw==", + "XfY+QUriCAA1+3QAsswdgg==", + "XgPHx2+ULpm14IOZU2lrDg==", + "XjjrIpsmATV/lyln4tPb+g==", + "Xo8ZjXOIoXlBjFCGdlPuZw==", + "XpGXh76RDgXC4qnTCsnNHA==", + "XqFSbgvgZn0CpaZoZiRauQ==", + "XqTK/2QuGWj50tGmiDxysA==", + "XqUO7ULEYhDOuT/I2J8BOA==", + "XqW7UBTobbV4lt1yfh0LZw==", + "XrFDomoH2qFjQ2jJ2yp9lA==", + "XsF7R12agx/KkRWl0TyXRA==", + "Xv0mNYedaBc57RrcbHr9OA==", + "XwKWd03sAz8MmvJEuN08xA==", + "Y1Nm3omeWX2MXaCjDDYnWQ==", + "Y1flEyZZAYxauMo4cmtJ1w==", + "Y26jxXvl79RcffH8O8b9Ew==", + "Y5KKN7t/v9JSxG/m1GMPSA==", + "Y5XR8Igvau/h+c1pRgKayg==", + "Y5iDQySR2c3MK7RPMCgSrw==", + "Y78dviyBS3Jq9zoRD5sZtQ==", + "Y7OofF9eUvp7qlpgdrzvkg==", + "Y7XpxIwsGK3Lm/7jX/rRmg==", + "Y7iDCWYrO1coopM3RZWIPg==", + "YA+zdEC+yEgFWRIgS1Eiqw==", + "YA0kMTJ82PYuLA4pkn4rfw==", + "YHM6NNHjmodv+G0mRLK7kw==", + "YK+q7uJObkQZvOwQ9hplMg==", + "YLz+HA6qIneP+4naavq44Q==", + "YNqIHCmBp/EbCgaPKJ7phw==", + "YPgMthbpcBN2CMkugV60hQ==", + "YVlRQHQglkbj3J2nHiP/Hw==", + "YXHQ3JI9+oca8pc/jMH6mA==", + "YZ39RIXpeLAhyMgmW2vfkQ==", + "YZt6HwCvdI5DRQqndA/hBQ==", + "YaUKOTyByjUvp1XaoLiW5Q==", + "YfbfE3WyYOW7083Y8sGfwQ==", + "YgVpC5d5V6K/BpOD663yQA==", + "YhLEPsi/TNyeUJw69SPYzQ==", + "Yig+Wh18VIqdsmwtwfoUQw==", + "Yjm5tSq1ejZn3aWqqysNvA==", + "YmaksRzoU+OwlpiEaBDYaQ==", + "YmjZJyNfHN5FaTL/HAm8ww==", + "YodhkayN5wsgPZEYN7/KNA==", + "YrEP9z2WPQ8l7TY1qWncDA==", + "YtZ8CYfnIpMd2FFA5fJ+1Q==", + "Yw4ztKv6yqxK9U1L0noFXg==", + "Yy2pPhITTmkEwoudXizHqQ==", + "YzTV0esAxBFVls3e0qRsnA==", + "Z+bsbVP91KrJvxrujBLrrQ==", + "Z0sjccxzKylgEiPCFBqPSA==", + "Z2MkqmpQXdlctCTCUDPyzw==", + "Z2rwGmVEMCY6nCfHO3qOzw==", + "Z5B+uOmPZbpbFWHpI9WhPw==", + "Z8T1b9RsUWf59D06MUrXCQ==", + "Z9bDWIgcq6XwMoU2ECDR5Q==", + "ZAQHWU6RMg4IadOxuaukyw==", + "ZCdad3AwhVArttapWFwT/Q==", + "ZH5Es/4lJ+D5KEkF1BVSGg==", + "ZIZx4MehWTVXPN9cVQBmyA==", + "ZItMIn1vhGqAlpDHclg0Ig==", + "ZJY+hujfd58mTKTdsmHoQQ==", + "ZJc7GV0Yb6MrXkpDVIuc8g==", + "ZKXxq9yr7NGBOHidht34uQ==", + "ZKeTDCboOgCptrjSfgu0xw==", + "ZKvox7BaQg4/p5jIX69Umw==", + "ZNrjP1fLdQpGykFXoLBNPw==", + "ZQ0ZnTsZKWxbRj7Tilh24Q==", + "ZQSDYgpsimK+lYGdXBWE/w==", + "ZRWyfXyXqAaOEjkzWl949Q==", + "ZRnR6i+5WKMRfs3BDRBCJg==", + "ZSmN8mmI9lDEHkJqBBg0Nw==", + "ZV8mEgJweIYk0/l0BFKetA==", + "ZVnErH1Si4u51QoT0OT7pA==", + "ZWXfE3uGU91WpPMGyknmqw==", + "ZXeMG5eqQpZO/SGKC4WQkA==", + "ZYW30FfgwHmW6nAbUGmwzA==", + "ZZImGypBWwYOAW43xDRWCQ==", + "ZaPsR9X77SNt7dLjMJUh8A==", + "ZbLVNTQSVZQWTNgC4ZGfQg==", + "ZcuIvc8fDI+2uF0I0uLiVA==", + "ZfRlID+pC1Rr4IY14jolMw==", + "ZgdpqFrVGiaHkh9o3rDszg==", + "ZgjifTVKmxOieco81gnccQ==", + "ZiJ/kJ9GneF3TIEm08lfvQ==", + "ZlBNHAiYsfaEEiPQ1z+rCA==", + "ZlOAnCLV1PkR0kb3E+Nfuw==", + "ZmVpw1TUVuT13Zw/MNI5hQ==", + "ZmblZauRqO5tGysY3/0kDw==", + "ZoNSxARrRiKZF5Wvpg7bew==", + "Zqd6+81TwYuiIgLrToFOTQ==", + "ZqjnqxZE/BjOUY0CMdVl0g==", + "ZqkmoGB0p5uT5J6XBGh7Tw==", + "ZrCezGLz38xKmzAom6yCTQ==", + "ZrCnZB/U/vcqEtI1cSvnww==", + "ZtWvgitOSRDWq7LAKYYd4Q==", + "ZtmnX24AwYAXHb2ZDC6MeQ==", + "ZuayB6IpbeITokKGVi9R5w==", + "ZvvxwDd0I6MsYd7aobjLUA==", + "ZyDh3vCQWzS5DI1zSasXWA==", + "ZybIEGf1Rn/26vlHmuMxhw==", + "ZydKlOpn2ySBW0G3uAqwuw==", + "ZygAjaN62XhW5smlLkks+Q==", + "Zyo0fzewcqXiKe2mAwKx5g==", + "ZyoaR1cMiKAsElmYZqKjLA==", + "Zz/5VMbw1TqwazReplvsEg==", + "ZzT5b0dYQXkQHTXySpWEaA==", + "ZzduJxTnXLD9EPKMn1LI4Q==", + "a/Y6IAVFv0ykRs9WD+ming==", + "a1aL8zQ+ie3YPogE3hyFFg==", + "a4EYNljinYTx9vb1VvUA6A==", + "a4rPqbDWiMivVzaRxvAj7g==", + "a5gZ5uuRrXEAjgaoh7PXAg==", + "a6IszND1m+6w+W+CvseC7g==", + "a6vem8n6WmRZAalDrHNP0g==", + "a7Pv1SOWYnkhIUC22dhdDA==", + "aD4QvtMlr8Lk/zZgZ6zIMg==", + "aEnHUfn7UE/Euh6jsMuZ7g==", + "aFJuE/s+Kbge4ppn+wulkA==", + "aIPde9CtyZrhbHLK740bfw==", + "aJFbBhYtMbTyMFBFIz/dTA==", + "aK9nybtiIBUvxgs1iQFgsw==", + "aLY2pCT0WfFO5EJyinLpPg==", + "aLh1XEUrfR9W82gzusKcOg==", + "aMa1yVA71/w6Uf1Szc9rMA==", + "aMmrAzoRWLOMPHhBuxczKg==", + "aN5x46Gw1VihRalwCt1CGg==", + "aOeJZUIZM9YWjIEokFPnzQ==", + "aRpdnrOyu5mWB1P5YMbvOA==", + "aRrcmH+Ud3mF1vEXcpEm4w==", + "aTWiWjyeSDVY/q8y9xc2zg==", + "aWZRql2IUPVe9hS3dxgVfQ==", + "aXqiibI6BpW3qilV6izHaQ==", + "aXrbsro7KLV8s4I4NMi4Eg==", + "aXs9qTEXLTkN956ch3pnOA==", + "aY6B28XdPnuYnbOy9uSP8A==", + "adJAjAFyR2ne1puEgRiH+g==", + "adT+OjEB2kqpeYi4kQ6FPg==", + "afMd/Hr3rYz/l7a3CfdDjg==", + "ahAbmGJZvUOXrcK6OydNGQ==", + "alJtvTAD7dH/zss/Ek1DMQ==", + "alqHQBz8V446EdzuVfeY5Q==", + "anyANMnNkUqr3JuPJz5Qzw==", + "apWEPWUvMC24Y+2vTSLXoA==", + "aqcOby9QyEbizPsgO3g0yw==", + "ash1r2J6B0PUxJe8P0otVQ==", + "asouSfUjJa8yfMG7BBe+fA==", + "auvG6kWMnhCMi7c7e9eHrw==", + "avFTp3rS6z5zxQUZQuaBHQ==", + "avZp5K7zJvRvJvpLSldNAw==", + "aw4CzX8pYbPVMuNrGCEcWg==", + "axEl7xXt/bwlvxKhI7hx4g==", + "ayBGGPEy++biljvGcwIjXA==", + "aySnrShOW4/xRSzl/dtSKQ==", + "ays5/F7JANIgPHN0vp2dqQ==", + "b06KGv5zDYsTxyTbQ9/eyA==", + "b0vZfEyuTja2JYMa20Rtbg==", + "b16O4LF7sVqB7aLU2f3F1A==", + "b3BQG9/9qDNC/bNSTBY/sQ==", + "b3q8kjHJPj9DWrz3yNgwjQ==", + "b4BoZmzVErvuynxirLxn0w==", + "b4aFwwcWMXsSdgS1AdFOXA==", + "b53qqLnrTBthRXmmnuXWvw==", + "b6rrRA0W247O+FfvDHbVCQ==", + "b85nxzs8xiHxaqezuDVWvg==", + "b8BZV1NfBdLi70ir4vYvZg==", + "bA2kaTpeXflTElTnQRp6GQ==", + "bBEndaOStXBpAK79FrgHaw==", + "bG+P+p34t/IJ1ubRiWg6IA==", + "bGGUhiG9SqJMHQWitXTcYQ==", + "bIk7Fa6SW7X18hfDjTKowg==", + "bJ1cZW7KsXmoLw0BcoppJg==", + "bJgsuw29cO2WozqsGZxl7w==", + "bK045TkBlz+/3+6n6Qwvrg==", + "bL2FuwsPT7a7oserJQnPcw==", + "bLEntCrCHFy9pg3T3gbBzg==", + "bLd38ZNkVeuhf0joEAxnBQ==", + "bLsStF0DDebpO+xulqGNtg==", + "bMWFvjM8eVezU1ZXKmdgqw==", + "bMb1ia0rElr2ZpZVhva0Jw==", + "bNDKcFu8T5Y6OoLSV+o/Sw==", + "bNq/hj0Cjt4lkLQeVxDVdQ==", + "bO55S58bqDiRWXSAIUGJKw==", + "bPRX2zl+K1S0iWAWUn1DZw==", + "bQ7J5mebp38rfP/fuqQOsg==", + "bQKkL+/KUCsAXlwwIH0N3w==", + "bTNRjJm+FfSQVfd56nNNqQ==", + "bUF0JIfS4uKd3JZj2xotLQ==", + "bUxQBaqKyvlSHcuRL9whjg==", + "bV9r7j2kNJpDCEM5E2339Q==", + "bWwtTFlhO3xEh/pdw0uWaQ==", + "bb/U8UynPHwczew/hxLQxw==", + "bbBsi6tXMVWyq3SDVTIXUg==", + "beSrliUu0BOadCWmx+yZyA==", + "bfUD03N2PRDT+MZ+WFVtow==", + "bhVbgJ4Do4v56D9mBuR/EA==", + "birqO8GOwGEI97zYaHyAuw==", + "bjLZ7ot/X/vWSVx4EYwMCg==", + "bkRdUHAksJZGzE1gugizYQ==", + "blygTgAHZJ3NzyAT33Bfww==", + "bs2QG8yYWxPzhtyMqO6u3A==", + "bsHIShcLS134C+dTxFQHyA==", + "bvbMJZMHScwjJALxEyGIyg==", + "bvyB6OEwhwCIfJ6KRhjnRw==", + "bz294kSG4egZnH2dJ8HwEg==", + "bzVeU2qM9zHuzf7cVIsSZw==", + "bzXXzQGZs8ustv0K4leklA==", + "c1wbFbN7AdUERO/xVPJlgw==", + "c3WVxyC5ZFtzGeQlH5Gw+w==", + "c5Tc7rTFXNJqYyc0ppW+Iw==", + "c5q/8n7Oeffv3B1snHM/lA==", + "c5ymZKqx/td1MiS2ERiz9A==", + "c6Yhwy/q3j7skXq52l36Ww==", + "cBBOQn7ZjxDku0CUrxq2ng==", + "cFFE2R4GztNoftYkqalqUQ==", + "cHSj5dpQ04h/WyefjABfmQ==", + "cHkOsVd80Rgwepeweq4S1g==", + "cLR0Ry4/N5swqga1R6QDMw==", + "cMo6l1EQESx1rIo+R4Vogg==", + "cNsC9bH30eM1EZS6IdEdtQ==", + "cSHSg9xJz/3F6kc+hKXkwg==", + "cT3PwwS6ALZA/na9NjtdzA==", + "cTvDd8okNUx0RCMer6O8sw==", + "cUyqCa7Oue934riyC17F8g==", + "cVhdRFuZaW/09CYPmtNv5g==", + "cWUg7AfqhiiEmBIu+ryImA==", + "cWdlhVZD7NWHUGte24tMjg==", + "cXpfd6Io6Glj2/QzrDMCvA==", + "ca+kx+kf7JuZ3pfYKDwFlg==", + "caepyBOAFu0MxbcXrGf6TA==", + "catI+QUNk3uJ+mUBY3bY8Q==", + "cbBXgB1WQ/i8Xul0bYY2fg==", + "ccK42Lm8Tsv73YMVZRwL6A==", + "cchuqe+CWCJpoakjHLvUfA==", + "ccmy4GVuX967KaQyycmO0w==", + "ccy3Ke2k4+evIw0agHlh3w==", + "cdWUm6uLNzR/knuj2x75eA==", + "cffrYrBX3UQhfX1TbAF+GQ==", + "cfh5VZFmIqJH/bKboDvtlA==", + "cgSEbLqqvDsNUyeA3ryJ6Q==", + "chwv4+xbEAa93PHg8q9zgQ==", + "ck86G8HsbXflyrK7MBntLg==", + "ckugAisBNX18eQz+EnEjjw==", + "cl4t9FXabQg7tbh1g7a0OA==", + "coGEgMVs2b314qrXMjNumQ==", + "cszpMdGbsbe6BygqMlnC9Q==", + "ctJYJegZhG42i+vnPFWAWw==", + "cu4ZluwohhfIYLkWp72pqA==", + "cuQslgfqD2VOMhAdnApHrA==", + "cvMJ714elj/HUh89a9lzOQ==", + "cvOg7N4DmTM+ok1NBLyBiQ==", + "cvZT1pvNbIL8TWg+SoTZdA==", + "cvrGmub2LoJ+FaM5HTPt9A==", + "cw1gBLtxH/m4H7dSM7yvFg==", + "cwBNvZc0u4bGABo88YUsVQ==", + "cxpZ4bloGv734LBf4NpVhA==", + "cxqHS4UbPolcYUwMMzgoOA==", + "czBWiYsQtNFrksWwoQxlOw==", + "d+ctfXU0j07rpRRzb5/HDA==", + "d/Wd3Ma1xYyoMByPQnA9Cw==", + "d0NBFiwGlQNclKObRtGVMQ==", + "d0VAZLbLcDUgLgIfT1GmVQ==", + "d0qvm3bl38rRCpYdWqolCQ==", + "d13Rj3NJdcat0K/kxlHLFw==", + "dAq8/1JSQf1f4QPLUitp0g==", + "dCDaYYrgASXPMGFRV0RCGg==", + "dChBe9QR29ObPFu/9PusLg==", + "dFSavcNwGd8OaLUdWq3sng==", + "dFetwmFw+D6bPMAZodUMZQ==", + "dG98w8MynOoX7aWmkvt+jg==", + "dGjcKAOGBd4gIjJq7fL+qQ==", + "dGrf9SWJ13+eWS6BtmKCNw==", + "dJHKDkfMFJeoULg7U4wwDQ==", + "dK2DU3t1ns+DWDwfBvH3SQ==", + "dL6n/JsK+Iq6UTbQuo/GOw==", + "dM9up4vKQV5LeX82j//1jQ==", + "dMRx4Mf6LrN64tiJuyWmDw==", + "dNTU+/2DdZyGGTdc+3KMhQ==", + "dNq2InSVDGnYXjkxPNPRxA==", + "dOS+mVCy3rFX9FvpkTxGXA==", + "dRFCIbVu0Y8XbjG5i+UFCQ==", + "dTMoNd6DDr1Tu8tuZWLudw==", + "dUx1REyXKiDFAABooqrKEA==", + "dVh/XMTUIx1nYN4q1iH1bA==", + "dXDPnL1ggEoBqR13aaW9HA==", + "dZg5w8rFETMp9SgW7m0gfg==", + "dZgMquvZmfLqP4EcFaWCiA==", + "daBhAvmE9shDgmciDAC5eg==", + "dhTevyxTYAuKbdLWhG47Kw==", + "dihDsG7+6aocG6M9BWrCzQ==", + "dmAfbd9F0OJHRAhNMEkRsA==", + "dml2gqLPsKpbIZ93zTXwCQ==", + "dnvatwSEcl73ROwcZ4bbIQ==", + "dpSTNOCPFHN5yGoMpl1EUA==", + "dqVw2q2nhCvTcW82MT7z0g==", + "drfODfDI6GyMW7hzkmzQvA==", + "dsueq9eygFXILDC7ZpamuA==", + "dtnE401dC0zRWU0S/QOTAg==", + "duRFqmvqF93uf/vWn8aOmg==", + "dxWv00FN/2Cgmgq9U3NVDQ==", + "e/nWuo5YalCAFKsoJmFyFA==", + "e2xLFVavnZIUUtxJx+qa1g==", + "e369ZIQjxMZJtopA//G55Q==", + "e4B3HmWjW+6hQzcOLru6Xg==", + "e5KCqQ/1GAyVMRNgQpYf6g==", + "e5l9ZiNWXglpw6nVCtO8JQ==", + "e5txnNRcGs2a9+mBFcF1Qg==", + "e9GqAEnk8XI5ix6kJuieNQ==", + "eAOEgF5N80A/oDVnlZYRAw==", + "eBapvE+hdyFTsZ0y5yrahg==", + "eC/RcoCVQBlXdE9WtcgXIw==", + "eCy/T+a8kXggn1L8SQwgvA==", + "eDWsx4isnr2xPveBOGc7Hw==", + "eDcyiPaB954q5cPXcuxAQw==", + "eFimq+LuHi42byKnBeqnZQ==", + "eFkXKRd2dwu/KWI5ZFpEzw==", + "eJDUejE/Ez/7kV+S74PDYg==", + "eJFIQh/TR7JriMzYiTw4Sg==", + "eJLrGwPRa6NgWiOrw1pA7w==", + "eJlcN+gJnqAnctbWSIO9uA==", + "eKQCVzLuzoCLcB4im8147A==", + "eLYKLr4labZeLiRrDJ9mnA==", + "ePlsM/iOMme2jEUYwi15ng==", + "eQ45Mvf5in9xKrP6/qjYbg==", + "eRwaYiog2DdlGQyaltCMJg==", + "eS/vTdSlMUnpmnl1PbHjyw==", + "eTMPXa60OTGjSPmvR4IgGw==", + "eV+RwWPiGEB+76bqvw+hbA==", + "eWgLAqJOU+fdn8raHb9HCw==", + "eXFOya6x5inTdGwJx/xtUQ==", + "eYAQWuWZX2346VMCD6s7/A==", + "eYE9No9sN5kUZ5ePEyS3+Q==", + "eddhS+FkXxiUnbPoCd5JJw==", + "edlXkskLx287vOBZ9+gVYg==", + "ehfPlu6YctzzpQmFiQDxGA==", + "ehwc2vvwNUAI7MxU4MWQZw==", + "ejfikwrSPMqEHjZAk3DMkA==", + "emVLJVzha7ui5OFHPJzeRQ==", + "enj9VEzLbmeOyYugTmdGfQ==", + "epY+dsm5EMoXnZCnO4WSHw==", + "es/L9iW8wsyLeC5S4Q8t+g==", + "eshD40tvOA6bXb0Fs/cH3A==", + "etRjRvfL/IwceY/IJ1tgzQ==", + "euxzbIq4vfGYoY3s1QmLcw==", + "evaWFoxZNQcRszIRnxqB+A==", + "ewPT4dM12nDWEDoRfiZZnA==", + "ewe/P3pJLYu/kMb5tpvVog==", + "ezsm4aFd6+DO9FUxz0A8Pg==", + "f/BjtP5fmFw2dRHgocbFlg==", + "f07bdNVAe9x+cAMdF1bByQ==", + "f09F7+1LRolRL5nZTcfKGA==", + "f0H/AFSx2KLZi9kVx5BAZg==", + "f1+fHgR5rDPsCZOzqrHM7Q==", + "f1Gs++Iilgq9GHukcnBG3w==", + "f1h+Vp+xmdZsZIziHrB2+g==", + "f5Xo7F1uaiM760Qbt978iw==", + "f6Ye5F0Lkn34uLVDCzogFQ==", + "f6iLrMpxKhFxIlfRsFAuew==", + "f9ywiGXsz+PuEsLTV3zIbQ==", + "fAKFfwlCOyhtdBK6yNnsNg==", + "fDOUzPTU2ndpbH0vgkgrJQ==", + "fFvXa1dbMoOOoWZdHxPGjw==", + "fHL+fHtDxhALZFb9W/uHuw==", + "fHNpW230mNib08aB7IM3XQ==", + "fKalNdhsyxTt1w08bv9fJA==", + "fM5uYpkvJFArnYiQ3MrQnA==", + "fO0+6TsjL+45p9mSsMRiIg==", + "fOARCnIg/foF/6tm7m9+3w==", + "fQS0jnQMnHBn7+JZWkiE/g==", + "fS471/rN4K2m10mUwGFuLg==", + "fSANOaHD0Koaqg7AoieY9A==", + "fU32wmMeD44UsFSqFY0wBA==", + "fU5ZZ1bIVsV+eXxOpGWo/Q==", + "fUAy3f9bAglLvZWvkO2Lug==", + "fVCRaPsTCKEVLkoF4y3zEw==", + "fW3QZyq5UixIA1mP6eWgqQ==", + "fX4G68hFL7DmEmjbWlCBJQ==", + "fY9VATklOvceDfHZDDk57A==", + "fZrj3wGQSt8RXv0ykJROcQ==", + "fbTm027Ms0/tEzbGnKZMDA==", + "fdqt93OrpG13KAJ5cASvkg==", + "fgXfRuqFfAu8qxbTi4bmhA==", + "fgdUFvQPb5h+Rqz8pzLsmw==", + "fhcbn9xE/6zobqQ2niSBgA==", + "fiv0DJivQeqUkrzDNlluRw==", + "fmC+85h5WBuk8fDEUWPjtQ==", + "fo3JL+2kPgDWfP+CCrFlFw==", + "foPAmiABJ3IXBoed2EgQXA==", + "foXSDEUwMhfHWJSmSejsQg==", + "fpXijBOM3Ai1RkmHven5Ww==", + "fsW2DaKYTCC7gswCT+ByQQ==", + "fsoXIbq0T0nmSpW8b+bj+g==", + "fsrX00onlGvfsuiCc35pGg==", + "ftsf2qztw3NC78ep/CZXWQ==", + "fv/PW8oexJYWf5De30fdLQ==", + "fvm0IQfnbfZFETg9v3z/Fg==", + "fxg/vQq9WPpmQsqQ4RFYaA==", + "fy54Milpa7KZH/zgrDmMXQ==", + "fzkmVWKhJsxyCwiqB/ULnQ==", + "g/z9yk94XaeBRFj4hqPzdw==", + "g0GbRp2hFVIdc7ct7Ky7ag==", + "g0aTR8aJ0uVy3YvGYu5xrw==", + "g0kHTNRI7x/lAsr92EEppw==", + "g0lWrzEYMntVIahC7i0O2g==", + "g1ELwsk6hQ+RAY1BH640Pg==", + "g2nh2xENCFOpHZfdEXnoQA==", + "g5EzTJ0KA4sO3+Opss3LMg==", + "g6udffWh7qUnSIo1Ldn3eA==", + "g6zSo8BvLuKqdmBFM1ejLA==", + "g8TcogVxHpw7uhgNFt5VCQ==", + "gAoV4BZYdW1Wm712YXOhWQ==", + "gB8wkuIzvuDAIhDtNT1gyA==", + "gBgJF0PiGEfcUnXF0RO7/w==", + "gC7gUwGumN7GNlWwfIOjJQ==", + "gDLjxT7vm07arF4SRX5/Vg==", + "gDxqUdxxeXDYhJk9zcrNyA==", + "gEHGeR2F82OgBeAlnYhRSw==", + "gFEnTI8os2BfRGqx9p5x8w==", + "gGLz3Ss+amU7y6JF09jq7A==", + "gICaI06E9scnisonpvqCsA==", + "gK7dhke5ChQzlYc/bcIkcg==", + "gR0sgItXIH8hE4FVs9Q07w==", + "gR3B8usSEb0NLos51BmJQg==", + "gTB2zM3RPm27mUQRXc/YRg==", + "gTnsH3IzALFscTZ1JkA9pw==", + "gU3gu8Y5CYVPqHrZmLYHbQ==", + "gUNP5w7ANJm257qjFxSJrA==", + "gW0oKhtQQ7BxozxUWw5XvQ==", + "gXlb7bbRqHXusTE5deolGA==", + "gYGQBLo5TdMyXks0LsZhsQ==", + "gYgCu/qUpXWryubJauuPNw==", + "gYnznEt9r97haD/j2Cko7g==", + "gYvdNJCDDQmNhtJ6NKSuTA==", + "gZNJ1Qq6OcnwXqc+jXzMLQ==", + "gZWTFt5CuLqMz6OhWL+hqQ==", + "gaEtlJtD6ZjF5Ftx0IFt0A==", + "gf1Ypna/Tt+TZ08Y+GcvGg==", + "gfhkPuMvjoC3CGcnOvki3Q==", + "gfnbviaVhKvv1UvlRGznww==", + "ggIfX1J4dX3xQoHnHUI7VA==", + "gglLMohmJDPRGMY1XKndjQ==", + "ghp8sWGKWw20S/z1tbTxFg==", + "ginkFyNVMwkZLE49AbfqfA==", + "gkrg0NR0iCaL7edq0vtewA==", + "glnqaRfwm6NxivtB2nySzw==", + "gnAIpoCyl3mQytLFgBEgGA==", + "gnez1VrH+UHT8C/SB9qGdA==", + "gnkadeCgjdmLdlu/AjBZJg==", + "goSgZ8N5UbT5NMnW3PjIlQ==", + "gqehq46BhFX2YLknuMv02w==", + "gsC/mWD8KFblxB0JxNuqJw==", + "gvvyX5ATi4q9NhnwxRxC8w==", + "gwyVIrTk5o0YMKQq4lpJ+Q==", + "gxwbqZDHLbQVqXjaq42BCg==", + "h+KRDKIvyVUBmRjv1LcCyg==", + "h0MH5NGFfChgmRJ3E/R3HQ==", + "h13Xuonj+0dD1xH86IhSyQ==", + "h1NNwMy0RjQmLloSw1hvdg==", + "h2B0ty0GobQhDnFqmKOpKQ==", + "h2cnQQF2/R3Mq2hWdDdrTg==", + "h3vYYI9yhpSZV2MQMJtwFQ==", + "h5HsEsObPuPFqREfynVblw==", + "h7Fc+eT/GuC8iWI+YTD0UQ==", + "hCzsi1yDv9ja5/o7t94j9Q==", + "hDGa2yLwNvgBd/v6mxmQaQ==", + "hDILjSpTLqJpiSSSGu445A==", + "hIABph+vhtSF5kkZQtOCTA==", + "hIJA+1QGuKEj+3ijniyBSQ==", + "hIjgi20+km+Ks23NJ4VQ6Q==", + "hJ8leLNuJ6DK5V8scnDaZQ==", + "hJSP7CostefBkJrwVEjKHA==", + "hK8KhTFcR06onlIJjTji/Q==", + "hKOsXOBoFTl/K4xE+RNHDA==", + "hN9bmMHfmnVBVr+7Ibd2Ng==", + "hNHqznsrIVRSQdII6crkww==", + "hP7dSa8lLn9KTE/Z0s4GVQ==", + "hPnPQOhz4QKhZi02KD6C+A==", + "hRxbdeniAVFgKUgB9Q3Y+g==", + "hSNZWNKUtDtMo6otkXA/DA==", + "hSkY45CeB6Ilvh0Io4W6cg==", + "hUWqqG1QwYgGC5uXJpCvJw==", + "hW9DJA1YCxHmVUAF7rhSmQ==", + "hWoxz5HhE50oYBNRoPp1JQ==", + "hY82j+sUQQRpCi6CCGea5A==", + "hZlX6qOfwxW5SPfqtRqaMw==", + "hdzol5dk//Q6tCm4+OndIA==", + "hf9HFxWRNX2ucH8FLS7ytA==", + "hfcH5Az2M7rp+EjtVpPwsg==", + "hiYg+aVzdBUDCG0CXz9kCw==", + "hkOBNoHbno2iNR7t3/d4vg==", + "hlMumZ7RJFpILuKs09ABtw==", + "hlu7os0KtAkpBTBV6D2jyQ==", + "hlvtFGW8r0PkbUAYXEM+Hw==", + "hnCUnoxofUiqQvrxl73M8w==", + "hq35Fjgvrcx6I9e6egWS4w==", + "hqeSvwu8eqA072iidlJBAw==", + "htDbVu1xGhCRd8qoMlBoMg==", + "htNVAogFakQkTX6GHoCVXg==", + "hv5GrLEIjPb4bGOi8RSO0w==", + "hvsZ5JmVevK1zclFYmxHaw==", + "hy303iin+Wm7JA6MeelwiQ==", + "i2sSvrTh/RdLJX0uKhbrew==", + "i42XumprV/aDT5R0HcmfIQ==", + "i6ZYpFwsyWyMJNgqUMSV1A==", + "i6r+mZfyhZyqlYv56o0H+w==", + "i8XXN7jcrmhnrOVDV8a2Hw==", + "i9IRqAqKjBTppsxtPB7rdw==", + "iANKiuMqWzrHSk9nbPe3bQ==", + "iCF+GWw9/YGQXsOOPAnPHQ==", + "iCnm5fPmSmxsIzuRK6osrA==", + "iFtadcw8v6betKka9yaJfg==", + "iGI9uqMoBBAjPszpxjZBWQ==", + "iGuY4VxcotHvMFXuXum7KA==", + "iGykaF+h4p46HhrWqL8Ffg==", + "iIWxFdolLcnXqIjPMg+5kQ==", + "iIm8c9uDotr87Aij+4vnMw==", + "iJ2nT8w8LuK11IXYqBK+YA==", + "iK0dWKHjVVexuXvMWJV9pg==", + "iPwX3SbbG9ez9HoHsrHbKw==", + "iQ304I1hmLZktA1d1cuOJA==", + "iS9wumBV5ktCTefFzKYfkA==", + "iSeH0JFSGK73F470Rhtesw==", + "iUsUCB0mfRsE9KPEQctIzw==", + "iVDd2Zk7vwmEh97LkOONpQ==", + "iWNlSnwrtCmVF89B+DZqOQ==", + "ibsb1ncaLZXAYgGkMO7tjQ==", + "ieEAgvK9LsWh2t6DsQOpWA==", + "ifZM0gBm9g9L09YlL+vXBg==", + "ifuJCv9ZA84Vz1FYAPsyEA==", + "ilBBNK/IV69xKTShvI94fQ==", + "imZ+mwiT22sW2M9alcUFfg==", + "inrUwXyKikpOW0y2Kl1wGw==", + "ionqS0piAOY2LeSReAz4zg==", + "ipPPjxpXHS1tcykXmrHPMQ==", + "irnD9K8bsT+up/JUrxPw6A==", + "iruDC5MeywV4yA8o1tw/KQ==", + "isep9d+Q7DEUf0W7CJJYzw==", + "itPtn+JaO4i7wz2wOPOmDQ==", + "iu5csar0IQQBOTgw5OvJwQ==", + "iujlt9fXcUXEYc+T2s5UjA==", + "iwKBOGDTFzV4aXgDGfyUkw==", + "izeyFvXOumNgVyLrbKW45g==", + "j+8/VARfbQSYhHzj0KPurQ==", + "j+lDhAnWAyso+1N8cm85hQ==", + "j4FBMnNfdBwx0VsDeTvhFg==", + "j8nMH8mK/0Aae7ZkqyPgdg==", + "j8to4gtSIRYpCogv2TESuQ==", + "jCgdKXsBCgf7giUKnr6paQ==", + "jEdanvXKyZdZJG6mj/3FWw==", + "jEqP0dyHKHiUjZ9dNNGTlQ==", + "jGHMJqbj6X1NdTDyWmXYAQ==", + "jHOoSl3ldFYr9YErEBnD3w==", + "jKJn4czwUl/6wtZklcMsSg==", + "jLI3XpVfjJ6IzrwOc4g9Pw==", + "jLkmUZ6fV56GfhC0nkh4GA==", + "jMZKSMP2THqwpWqJNJRWdw==", + "jNJQ6otieHBYIXA9LjXprg==", + "jNcMS2zX1iSZN9uYnb2EIg==", + "jOPdd330tB6+7C29a9wn0Q==", + "jQVlDU+HjZ2OHSDBidxX5A==", + "jQjyjWCEo9nWFjP4O8lehw==", + "jS0JuioLGAVaHdo/96JFoQ==", + "jTg9Y6EfpON4CRFOq0QovA==", + "jTmPbq+wh30+yJ/dRXk1cA==", + "jV/D2B11NLXZRH77sG9lBw==", + "jWsC7kdp2YmIZpfXGUimiA==", + "jZMDIu95ITTjaUX0pk4V5g==", + "jd6IpPJwOJW1otHKtKZ5Gw==", + "jdRzkUJrWxrqoyNH9paHfQ==", + "jdVMQqApseHH3fd91NFhxg==", + "jfegbZSZWkDoPulFomVntA==", + "jgNijyoj2JrQNSlUv4gk4A==", + "ji+1YHlRvzevs3q5Uw1gfA==", + "ji306HRiq965zb8EZD2uig==", + "jiV+b/1EFMnHG6J0hHpzBg==", + "jjNMPXbmpFNsCpWY0cv3eg==", + "jkUpkLoIXuu7aSH8ZghIAQ==", + "joDXdLpXvRjOqkRiYaD/Sw==", + "jon1y9yMEGfiIBjsDeeJdA==", + "jp5Em/0Ml4Txr1ptTUQjpg==", + "jpNUgFnanr9Sxvj2xbBXZw==", + "jpjpNjL1IKzJdGqWujhxCw==", + "jqPQ0aOuvOJte/ghI1RVng==", + "jrRH0aTUYCOpPLZwzwPRfQ==", + "jrfRznO0nAz6tZM1mHOKIA==", + "jt9Ocr9D8EwGRgrXVz//aQ==", + "jx7rpxbm1NaUMcE2ktg5sA==", + "jz7QlwxCIzysP39Cgro8jg==", + "k+IBS52XdOe5/hLp28ufnA==", + "k/Aou2Jmyh8Bu3k8/+ndsQ==", + "k/OVIllJvW6BefaLEPq7DA==", + "k/pBSWE2BvUsvJhA9Zl5uw==", + "k0XIjxp2vFG7sTrKcfAihA==", + "k1DPiH6NkOFXP/r3N12GyA==", + "k2KP9oPMnHmFlZO6u6tgyw==", + "k6OmSlaSZ5CB0i7SD9LczQ==", + "k8eZxqwxiN/ievXdLSEL/w==", + "kBAB2PSjXwqoQOXNrv80AA==", + "kFrRjz7Cf2KvLtz9X6oD+w==", + "kGeXrHEN6o7h5qJYcThCPw==", + "kHcBZXoxnFJ+GMwBZ/xhfQ==", + "kIGxCUxSlNgsKZ45Al1lWw==", + "kJdY3XEdJS/hyHdR+IN0GA==", + "kMUdiwM7WR8KGOucLK4Brw==", + "kNGIV3+jQmJlZDTXy1pnyA==", + "kRnBEH6ILR5GNSmjHYOclw==", + "kSUectNPXpXNg+tIveTFRw==", + "kTCHqcb3Cos51o8cL+MXcg==", + "kUhyc3G8Zvx8+q5q5nVEhw==", + "kUudvRfA33uJDzHIShQd3Q==", + "kWPUUi7x9kKKa6nJ+FDR5Q==", + "kZ/mZZg9YSDmk2rCGChYAg==", + "kZ0D191c/uv4YMG15yVLDw==", + "kZkmDatUOdIqs7GzH3nI1A==", + "ka7pMp8eSiv92WgAsz2vdA==", + "kcJ1acgBv6FtUhV8KuWoow==", + "kgKWQJJQKLUuD2VYKIKvxA==", + "kggaIvN2tlbZdZRI8S5Apw==", + "kgyUtd8MFe0tuuxDEUZA9w==", + "kh51WUI5TRnKhur6ZEpRTQ==", + "kj5WqpRCjWAfjM7ULMcuPQ==", + "kjWYVC7Eok2w2YT4rrI+IA==", + "kkbX+a00dfiTgbMI+aJpMg==", + "kly/2kE4/7ffbO34WTgoGg==", + "knYKU74onR6NkGVjQLezZg==", + "kq26VyDyJTH/eM6QvS2cMw==", + "kr8tw1+3NxoPExnAtTmfxg==", + "ksOFI9C7IrDNk4OP6SpPgw==", + "kuWGANwzNRpG4XmY7KjjNg==", + "kvAaIJb+aRAfKK104dxFAA==", + "kwlAQhR2jPMmfLTAwcmoxw==", + "kydoXVaNcx1peR5g6i588g==", + "kzGNkWh3fz27cZer4BspUQ==", + "kzTl7WH/JXsX1fqgnuTOgw==", + "kzXsrxWRnWhkA82LsLRYog==", + "kzYddqiMsY3EYrpxve2/CQ==", + "l+x2QhxG8wb5AQbcRxXlmA==", + "l0E0U/CJsyCVSTsXW4Fp+w==", + "l2NppPcweAtmA1V2CNdk2Q==", + "l2ZB9TvT68rn8AAN4MdxWw==", + "l2mAbuFF3QBIUILDODiUHQ==", + "l4ddTxbTCW5UmZW+KRmx6A==", + "l5f3I6osM9oxLRAwnUnc5A==", + "l6QHU5JsJExNoOnqxBPVbw==", + "l6Ssc04/CnsqUua9ELu2iQ==", + "l8/KMItWaW3n4g1Yot/rcQ==", + "lC5EumoIcctvxYqwELqIqw==", + "lFUq6PGk9dBRtUuiEW7Cug==", + "lHN2dn2cUKJ8ocVL3vEhUQ==", + "lJFPmPWcDzDp5B2S8Ad8AA==", + "lK2xe+OuPutp4os0ZAZx5w==", + "lM/EhwTsbivA7MDecaVTPw==", + "lMaO8Yf+6YNowGyhDkPhQA==", + "lMjip5hbCjkD9JQjuhewDg==", + "lNF8PvUIN02NattcGi5u4g==", + "lON3WM0uMJ30F8poBMvAjQ==", + "lOPJhHqCtMRFZfWMX/vFZQ==", + "lTE6u9G/RzvmbuAzq2J2/Q==", + "lV70RNlE++04G1KFB3BMXA==", + "lY+tivtsfvU0LJzBQ6itYQ==", + "lacCCRiWdquNm4YRO7FoKA==", + "leDlMcM+B1mDE8k5SWtUeg==", + "lf1fwA0YoWUZaEybE+LyMQ==", + "lfOLLyZNbsWQgHRhicr4ag==", + "lffapwUUgaQOIqLz2QPbAg==", + "lhAOM81Ej6YZYBu45pQYgg==", + "lizovLQxu6L9sbafNQuShQ==", + "lkl6XkrTMUpXi46dPxTPxg==", + "lkzFdvtBx5bV6xZO0cxK7g==", + "ll2M0QQzBsj5OFi02fv3Yg==", + "llOvGOUDVfX68jKnAlvVRA==", + "llujnWE17U8MIHmx4SbrSA==", + "lqhgbgEqROAdfzEnJ17eXA==", + "lsBTMnse2BgPS6wvPbe7JA==", + "luO1R8dUM9gy1E2lojRQoA==", + "luR/kvHLwA6tSdLeTM4TzA==", + "lwYQm2ynA3ik2gE1m11IEg==", + "lyfqic/AbEJbCiw+wA01FA==", + "lz+SeifYXxamOLs1FsFmSQ==", + "lzUQ1o7JAbdJYpmEqi6KnQ==", + "m+eh+ZqS74w2q0vejBkjaw==", + "m/Lp4U75AQyk9c8cX14HJg==", + "m06wctjNc3o7iyBHDMZs2w==", + "m3XYojKO+I6PXlVRUQBC3w==", + "m416yrrAlv+YPClGvGh+qQ==", + "m5JIUETVXcRza4VL4xlJbg==", + "m6get5wjq5j1i5abnpXuZQ==", + "m6srF+pMehggHB1tdoxlPg==", + "m9iuy4UtsjmyPzy6FTTZvw==", + "mAiD16zf+rCc7Qzxjd5buA==", + "mAzsVkijuqihhmhNTTz65g==", + "mDXHuOmI4ayjy2kLSHku1Q==", + "mI0eT4Rlr7QerMIngcu/ng==", + "mMLhjdWNnZ8zts9q+a2v3g==", + "mMfn8OaKBxtetweulho+xQ==", + "mNlYGAOPc6KIMW8ITyBzIg==", + "mNv2Q67zePjk/jbQuvkAFA==", + "mPk1IsU5DmDFA/Ym5+1ojw==", + "mPwCyD0yrIDonVi+fhXyEQ==", + "mS99D+CXhwyfVt8xJ+dJZA==", + "mSJF9dJnxZ15lTC6ilbJ2A==", + "mSstwJq7IkJ0JBJ5T8xDKg==", + "mTAqtg6oi0iytHQCaSVUsA==", + "mTLBkP+yGHsdk5g7zLjVUw==", + "mU4CqbAwpwqegxJaOz9ofQ==", + "mUek9NkXm8HiVhQ6YXiyzA==", + "mVT74Eht+gAowINoMKV7IQ==", + "mW6TCje9Zg2Ep7nzmDjSYQ==", + "mXBfDUt/sBW5OUZs2sihvw==", + "mXPtbPaoNAAlGmUMmJEWBQ==", + "mXZ4JeBwT2WJQL4a/Tm4jQ==", + "mXycPfF5zOvcj1p4hnikWw==", + "mc45FSMtzdw2PTcEBwHWPw==", + "md6zNd7ZBn3qArYqQz7/fw==", + "me61ST+JrXM5k3/a11gRAA==", + "meHzY9dIF7llDpFQo1gyMg==", + "miiOqnhtef1ODjFzMHnxjA==", + "mjFBVRJ7TgnJx+Q74xllPg==", + "mjQS8CpyGnsZIDOIEdYUxg==", + "mk1CKDah7EzDJEdhL22B7w==", + "mmRob7iyTkTLDu8ObmTPow==", + "mnalaO6xJucSiZ0+99r3Cg==", + "mpOtwBvle+nyY6lUBwTemw==", + "mpWNaUH9kn4WY26DWNAh3Q==", + "mr1qjhliRfl87wPOrJbFQg==", + "mrinv7KooPQPrLCNTRWCFg==", + "mrxlFD3FBqpSZr1kuuwxGg==", + "msstzxq++XO0AqNTmA7Bmg==", + "mxug34EekabLz0JynutfBg==", + "myzvc+2MfxGD9uuvZYdnqQ==", + "n+xYzfKmMoB3lWkdZ+D3rg==", + "n1M2dgFPpmaICP+JwxHUug==", + "n1ixvP7SfwYT3L2iWpJg6A==", + "n5GA+pA9mO/f4RN9NL9lNg==", + "n6QVaozMGniCO0PCwGQZ6w==", + "n7Bns42aTungqxKkRfQ5OQ==", + "n7KL1Kv027TSxBVwzt9qeA==", + "n7h9v2N1gOcvMuBEf8uThw==", + "nDAsSla+9XfAlQSPsXtzPA==", + "nE72uQToQFVLOzcu/nMjww==", + "nFBXCPeiwxK9mLXPScXzTA==", + "nFPDZGZowr3XXLmDVpo7hg==", + "nGzPc0kI/EduVjiK7bzM6Q==", + "nHTsDl0xeQPC5zNRnoa0Rw==", + "nHUpYmfV59fe3RWaXhPs3Q==", + "nL4iEd3b5v4Y9fHWDs+Lrw==", + "nMuMtK/Zkb3Xr34oFuX/Lg==", + "nNaGqigseHw30DaAhjBU3g==", + "nOiwBFnXxCBfPCHYITgqNg==", + "nR3ACzeVF5YcLX6Gj6AGyQ==", + "nULSbtw2dXbfVjZh33pDiA==", + "nUgYO7/oVNSX8fJqP2dbdg==", + "nVDxVhaa2o38gd1XJgE3aw==", + "nW3zZshjZEoM8KVJoVfnuQ==", + "nY/H7vThZ+dDxoPRyql+Cg==", + "neQoa8pvETr07blVMN3pgA==", + "nf8x+F03kOpMhsCSUWEhVg==", + "ng1Q0A7ljho3TUWWYl46sw==", + "nhAnHuCGXcYlqzOxrrEe1g==", + "nkbLVLvh3ClKED97+nH+7Q==", + "nkedTagkmf6YE4tEY+0fKw==", + "nknBKPgb7US42v8A0fTl/w==", + "nmD7fEU4u7/4+W/pkC4/0Q==", + "nqpKfidczdgrNaAyPi7BOQ==", + "nqtQI1bSM7DCO9P1jGV97Q==", + "nsnX3tKkN1elr18E31tXDw==", + "nvLEpj6ZZF3LWH3wUB6lKg==", + "nvUKoKfC6j8fz3gEDQrc/w==", + "nvmBgp0YlUrdZ05INsEE8Q==", + "nwtCsN1xEYaHvEOPzBv+qQ==", + "nx/U4Tode5ILux4DSR+QMg==", + "nxDGRpePV3H4NChn4eLwag==", + "nyaekSYTKzfSeSfPrB114Q==", + "nykEOLL/o7h0cs0yvdeT2g==", + "o+areESiXgSO0Lby56cBeg==", + "o+nYS4TqJc6XOiuUzEpC3A==", + "o/Y4U6rWfsUCXJ72p5CUGw==", + "o1uhaQg5/zfne84BFAINUQ==", + "o1zeXHJEKevURAAbUE/Vog==", + "o5XVEpdP4OXH0NEO4Yfc/A==", + "o64LDtKq/Fulf1PkVfFcyg==", + "o7y4zQXQAryST2cak4gVbw==", + "o9tdzmIu+3J/EYU4YWyTkA==", + "oAHVGBSJ2cf4dVnb/KEYmw==", + "oDca3JEdRb4vONT9GUUsaQ==", + "oFNMOKbQXcydxnp8fUNOHw==", + "oFanDWdePmmZN0xqwpUukA==", + "oGH7SMLI2/qjd9Vnhi3s0A==", + "oIU19xAvLJwQSZzIH577aA==", + "oIWwTbkVS5DDL47mY9/1KQ==", + "oKt57TPe4PogmsGssc3Cbg==", + "oLWWIn/2AbKRHnddr2og9g==", + "oMJLQTH1wW7LvOV0KRx/dw==", + "oNOI17POQCAkDwj6lJsYOA==", + "oONlXCW4aAqGczQ/bUllBw==", + "oPcxgoismve6+jXyIKK6AQ==", + "oPlhC4ebXdkIDazeMSn1fQ==", + "oQjugfjraFziga1BcwRLRA==", + "oR8rvIZoeoaZ/ufpo0htfQ==", + "oSnrpW4UmmVXtUGWqLq+tQ==", + "oUqO4HrBvkpSL781qAC9+w==", + "oVlG+0rjrg2tdFImxIeVBA==", + "oad5SwflzN0vfNcyEyF4EA==", + "obW3kzv2KBvuckU7F+tfjA==", + "ocRh5LR1ZIN9Johnht8fhQ==", + "ocpLRASvTgqfkY20YlVFHQ==", + "ocvA1/NbyxM0hanwwY6EiA==", + "odGhKtO4bDW5R8SYiI5yCg==", + "ogcuGHUZJkmv+vCz567a2g==", + "ohK6EftXOqBzIMI+5XnESw==", + "ojZY7Gi2QJXE/fp6Wy31iA==", + "ojf6uL85EuEYgLvHoGhUrw==", + "ojugpLIfzflgU2lonfdGxA==", + "ol9xhVTG9e1wNo50JdZbOA==", + "olTSlmirL9MFhKORiOKYkQ==", + "omAjyj1l6gyQAlBGfdxJTw==", + "onFcHOO1c3pDdfCb5N4WkQ==", + "oqlkgrYe9aCOwHXddxuyag==", + "oxoZP897lgMg/KLcZAtkAg==", + "oyYtf08AkWLR52bXm5+sKw==", + "ozVqYsmUueKifb4lDyVyrg==", + "p+bx+/WQWALXEBCTnIMr4w==", + "p/48hurJ1kh2FFPpyChzJg==", + "p/7qM5+Lwzw1/lIPY91YxQ==", + "p0eNK7zJd7D/HEGaVOrtrQ==", + "p2JPOX8yDQ0agG+tUyyT/g==", + "p3V7NfveB6cNxFW7+XQNeQ==", + "p48i7AfSSAyTdJSyHvOONw==", + "p73gSu4d+4T/ZNNkIv9Nlw==", + "p8W1LgFuW6JSOKjHkx3+aA==", + "pCQmlnn3BxhsV2GwqjRhXg==", + "pFKzcRHSUBqSMtkEJvrR1Q==", + "pGQEWJ38hb/ZYy2P1+FIuw==", + "pHo1O5zrCHCiLvopP2xaWw==", + "pHozgRyMiEmyzThtJnY4MQ==", + "pKaTI+TfcV3p/sxbd2e7YQ==", + "pT1raq2fChffFSIBX3fRiA==", + "pUfWmRXo70yGkUD/x5oIvA==", + "pVG1hL96/+hQ+58rJJy6/A==", + "pVgjGg4TeTNhKimyOu3AAw==", + "pW4gDKtVLj48gNz6V17QdA==", + "pZfn6IiG+V28fN8E2hawDQ==", + "pa8nkpAAzDKUldWjIvYMYg==", + "pcoBh5ic7baSD4TZWb3BSw==", + "pdPwUHauXOowaq9hpL2yFw==", + "pdaY6kZ8+QqkMOInvvACNA==", + "peMW+rpwmXrSwplVuB/gTA==", + "pfGcaa49SM3S6yJIPk/EJQ==", + "plXHHzA8X9QGwWzlJxhLRw==", + "pnJnBzAJlO4j3IRqcfmhkQ==", + "prCOYlboBnzmLEBG/OeVrQ==", + "prOsOG0adI4o+oz50moipw==", + "pulldyBt2sw6QDvTrCh6zw==", + "pv/m2mA/RJiEQu2Qyfv9RA==", + "pvXHwJ3dwf9GDzfDD9JI3g==", + "pw1jplCdTC+b0ThX0FXOjw==", + "pxuSWn1u+bHtRjyh2Z8veA==", + "pyrUqiZ98gVXxlXQNXv5fA==", + "pzC8Y0Vj9MPBy3YXR32z6w==", + "q/siBRjx6wNu+OTvpFKDwA==", + "q4z6A4l3nhX3smTmXr+Sig==", + "q5g3c8tnQTW2EjNfb2sukw==", + "q6LG0VzO1oxiogAAU63hyg==", + "q7m/EtZySBjZNBjQ5m1hKw==", + "q8YF9G2jqydAxSqwyyys5Q==", + "qA0sTaeNPNIiQbjIe1bOgQ==", + "qCPfJTR8ecTw6u6b1yHibA==", + "qE/h/Z+6buZWf+cmPdhxog==", + "qIFpKKwUmztsBpJgMaVvSg==", + "qIUJPanWmGzTD1XxvHp+6w==", + "qNOSm15bdkIDSc/iUr+UTQ==", + "qNyy6Fc0b8oOMWqqaliZ/w==", + "qO4HlyHMK5ygX+6HbwQe8w==", + "qOEIUWtGm5vx/+fg4tuazg==", + "qP1cCE4zsKGTPhjbcpczMw==", + "qQQwJ/aF87BbnLu3okXxaw==", + "qYHdgFAXhF/XcW4lxqfvWQ==", + "qYuo5vY8V3tZx41Kh9/4Dw==", + "qZ2q5j2gH3O56xqxkNhlIA==", + "qaTdVEeZ6S8NMOxfm+wOMA==", + "qcpeZWUlPllQYZU6mHVwUw==", + "qenHZKKlTUiEFv6goKM/Mw==", + "qkvEep4vvXhc2ZJ6R449Mg==", + "qngzBJbiTB4fivrdnE5gOg==", + "qnkFUlJ8QT322JuCI3LQgg==", + "qnsBdl050y9cUaWxbCczRw==", + "qnzWszsyJhYtx8wkMN6b1g==", + "qoK2keBg3hdbn7Q24kkVXg==", + "qpFJZqzkklby+u1UT3c1iA==", + "qt5CsMts2aD4lw/4Q6bHYQ==", + "qxALQrqHoDq9d91nU0DckA==", + "qyRmvxh8p4j4f+61c10ZFQ==", + "r/b5px/UImGNjT/X5sYjuA==", + "r0QffVKB9OD9yGsOtqzlhA==", + "r0hAwlS0mPZVfCSB+2G6uQ==", + "r1VGXWeqGeGbfKjigaAS+Q==", + "r2f2MyT+ww1g9uEBzdYI1w==", + "r36kVMpF+9J+sfI3GeGqow==", + "r3lQAYOYhwlLnDWQIunKqg==", + "r95wJtP5rsTExKMS7QhHcw==", + "rBt6L/KLT7eybxKt5wtFdg==", + "rCxoo4TP/+fupXMuIM0sDA==", + "rHagXw+CkF3uEWPWDKXvog==", + "rIMXaCaozDvrdpvpWvyZOQ==", + "rJ9qVn8/2nOxexWzqIHlcQ==", + "rJCuanCy51ydVD4nInf9IQ==", + "rKAQxu80Q8g1EEhW5Wh8tg==", + "rKb3TBM4EPx/RErFOFVCnQ==", + "rLZII1R6EGus+tYCiUtm6g==", + "rM/BOovNgnvebKMxZQdk7g==", + "rMm9bHK69h0fcMkMdGgeeA==", + "rOYeIcB+Rg5V6JG2k4zS2w==", + "rSvhrHyIlnIBlfNJqemEbw==", + "rTwJggSxTbwIYdp07ly0LA==", + "rUp5Mfc57+A8Q29SPcvH/Q==", + "rWliqgfZ3/uCRBOZ9sMmdA==", + "rXGWY/Gq+ZEsmvBHUfFMmQ==", + "rXSbbRABEf4Ymtda45w8Fw==", + "rXfWkabSPN+23Ei1bdxfmQ==", + "rXtGpN17Onx8LnccJnXwJQ==", + "rZKD8oJnIj5fSNGiccfcvA==", + "raKMXnnX6PFFsbloDqyVzQ==", + "raYifKqev8pASjjuV+UTKQ==", + "rcY4Ot40678ByCfqvGOGdg==", + "rdeftHE7gwAT67wwhCmkYQ==", + "rfPTskbnoh3hRJH6ZAzQRg==", + "rgcXxjx3pDLotH7TTfAoZw==", + "rh7bzsTQ1UZjG7amysr0Gg==", + "rhgtLQh0F9bRA6IllM7AGw==", + "ri4AOITPdB1YHyXV+5S51g==", + "rkeLYwMZ1/pW2EmIibALfA==", + "rlXt6zKE7DswUl0oWGOQUQ==", + "rqHKB91H3qVuQAm+Ym5cUA==", + "rqucO37p86LpzehR/asCSQ==", + "rs2QrN4qzAHCHhkcrAvIfA==", + "rtJdfki8fG6CB36CADp0QA==", + "rtd6mqFgGe98mqO0pFGbSw==", + "rueNryrchijjmWaA3kljYg==", + "rvE64KQGkVkbl07y7JwBqw==", + "rwplpbNJz0ADUHTmzAj15Q==", + "rwtF86ZAbWyKI6kLn4+KBw==", + "rxfACPLtKXbYua18l3WlUw==", + "rzj6mjHCcMEouL66083BAg==", + "s+eHg5K9zZ2Jozu5Oya9ZQ==", + "s/BZAhh1cTV3JCDUQsV8mA==", + "s2AKVTwrY65/SWqQxDGJQg==", + "s5+78jS4hQYrFtxqTW3g1Q==", + "s5RUHVRNAoKMuPR/Jkfc2Q==", + "s7iW1M6gkAMp+D/3jHY58w==", + "s8NpalwgPdHPla7Zi9FJ3w==", + "sBpytpE38xz0zYeT+0qc2A==", + "sC11Rf/mau3FG5SnON4+vQ==", + "sCLMrLjEUQ6P1L8tz90Kxg==", + "sEeblUmISi1HK4omrWuPTA==", + "sGLPmr568+SalaQr8SE/PA==", + "sLJrshdEANp0qk2xOUtTnQ==", + "sLdxIKap0ZfC3GpUk3gjog==", + "sNmW2b2Ud7dZi3qOF8O8EQ==", + "sQAxqWXeiu/Su0pnnXgI9A==", + "sQskMBELEq86o1SJGQqfzg==", + "sQzCwNDlRsSH7iB9cTbBcg==", + "sS6QcitMPdvUBLiMXkWQkw==", + "sWLcS+m4aWk31BiBF+vfJQ==", + "sXlFMSTBFnq0STHj6cS/8w==", + "sa2DECaqYH1z1/AFhpHi+g==", + "saEpnDGBSZWqeXSJm34eOA==", + "scCQPl0em2Zmv/RQYar60g==", + "sfIClgTMtZo9CM9MHaoqhQ==", + "sfowXUMdN2mCoBVrUzulZg==", + "sfte/o9vVNyida/yLvqADA==", + "siHwJx6EgeB1gBT9z/vTyw==", + "skrQRB9xbOsiSA19YgAdIQ==", + "snGTzo540cCqgBjxrfNpKw==", + "soBA65OmZdfBGJkBmY/4Iw==", + "spHVvA/pc7nF9Q4ON020+w==", + "spJI3xFUlpCDqzg0XCxopA==", + "sr3UXbMg5zzkRduFx/as7g==", + "sw+bmpzqsM4gEQtnqocQLQ==", + "swJhrPwllq5JORWiP5EkDA==", + "swsVVsPi/5aPFBGP+jmPIw==", + "syeBfQBUmkXNWCZ1GV8xSA==", + "t+bYn9UqrzKiuxAYGF7RLA==", + "t0WN8TwMLgi8UVEImoFXKg==", + "t2EkpUsLOEOsrnep0nZSmA==", + "t2vWMIh2BvfDSQaz5T1TZw==", + "t3Txxjq43e/CtQmfQTKwWg==", + "t5U+VMsTtlWAAWSW+00SfQ==", + "t5wh9JGSkQO78QoQoEqvXA==", + "t7HaNlXL16fVwjgSXmeOAQ==", + "t8pjhdyNJirkvYgWIO/eKg==", + "tBQDfy48FnIOZI04rxfdcA==", + "tFMJRXfWE9g78O1uBUxeqQ==", + "tFmWYH82I3zb+ymk5dhepA==", + "tG+rpfJBXlyGXxTmkceiKA==", + "tHDbi43e6k6uBgO0hA+Uiw==", + "tIqwBotg052wGBL65DZ+yA==", + "tJt6VDdAPEemBUvnoc4viA==", + "tOdlnsE3L3XCBDJRmb/OqA==", + "tOkYq1BZY152/7IJ6ZYKUg==", + "tU31r8zla146sqczdKXufg==", + "tVhXk9Ff3wAg56FbdNtcFg==", + "tVvWdA+JqH0HR2OlNVRoag==", + "tVw8U1AsslIFmQs4H1xshg==", + "tX8X8KoxUQ8atFSCxgwE1Q==", + "tXVb5f90k9l3e1oK2NGXog==", + "tXuu7YpZOuMLTv87NjKerA==", + "tY916jrSySzrL+YTcVmYKQ==", + "tYeIZjIm0tVEsYxH1iIiUQ==", + "tb5+2dmYALJibez1W4zXgA==", + "td7nDgTDmKPSODRusMcupw==", + "tdgI9v7cqJsgCAeW1Fii1A==", + "tdiTXKrkqxstDasT0D5BPA==", + "tejpAZp7y32SO2+o4OGvwQ==", + "tfgO55QqUyayjDfQh+Zo1Q==", + "tj2rWvF2Fl+XIccctj8Mhw==", + "tnUtJ/DQX9WaVJyTgemsUA==", + "tq5xUJt8GtjDIh1b48SthQ==", + "tr+U/vt+MIGXPRQYYWJfRg==", + "trjM81KANPZrg9iSThWx6Q==", + "tsiqwelcBAMU/HpLGBtMGw==", + "twPn6wTGqI0aR//0wP3xtA==", + "twjiDKJM7528oIu/el4Zbg==", + "tzV7ixFH37ze4zuLILTlfA==", + "u/QxrP1NOM/bOJlJlsi/jQ==", + "u2WQlcMxOACy6VbJXK4FwA==", + "u5cUPxM6/spLIV8VidPrAA==", + "uC2lzm7HaMAoczJO6Z/IhQ==", + "uChFnF0oCwARhAOz/d47eA==", + "uESeJe/nYrHCq4RQbrNpGA==", + "uExgqZkkJnZj252l5dKAGg==", + "uIkVijg7RPi/1j7c18G1qA==", + "uJZGw3IY2nCcdVeWW1geNQ==", + "uMq8cDVWFD+tpn8aeP8Pqg==", + "uNWFZlP7DA96sf+LWiAhtQ==", + "uNzpptKjihEfKRo5A1nWmw==", + "uO+uK1DntCxVRr1KttfUIw==", + "uOHrw37yF9oLLVd16nUpeg==", + "uOkMpYy/7DYYoethJdixfQ==", + "uPdjKJIGzN7pbGZDZdCGaA==", + "uPi8TsGY3vQsMVo/nsbgVQ==", + "uPm+cF4Jq08S5pQhYFjU8A==", + "uPnL9tboMZo0Kl2fe24CmA==", + "uQs79rbD/wEakMUxqMI48A==", + "uSIiF1r9F18avZczmlEuMQ==", + "uT6WRh5UpVdeABssoP2VTg==", + "uTA0XbiH3fTeVV7u5z0b3w==", + "uTHBqApdKOAgdwX3cjrCYQ==", + "uU1TX5DoDg6EcFKgFcn0GA==", + "uXuPA/2KJbb7ZX+NymN3dw==", + "uXvr6vi5kazZ9BCg2PWPJA==", + "uZ2gUA74/7Q33tI2TcGQlg==", + "ucLMWnNDSqE4NOCGWvcGWw==", + "udU65VtsvJspYmamiOsgXw==", + "ueODvMv/f9ZD8O0aIHn4sg==", + "ugY8rTtJkN4CXWMVcRZiZw==", + "uhT12XY79CtbwhcSfAmAXQ==", + "ulLuTZqhEDkX0EJ3xwRP9A==", + "ulpDxLeQnIRPnq6oaah2AA==", + "up2MVDi9ve+s83/nwNtZ7Q==", + "uqe3rFveJ2JIkcZQ3ZMXHQ==", + "uqp92lAqjec8UQYfyjaEZw==", + "ur9JDCVNwzSH4q4ngDlHNQ==", + "uu+ncs63SdQIvG6z4r7Q3Q==", + "uuiJ+yB7JLDh2ulthM0mjg==", + "uvKYnKE01D5r7kR9UQyo5A==", + "uvzmRcvgepW6mZbMfYgcNw==", + "uwA6N5LptSXqIBkTO0Jd7Q==", + "uwGivY3/C9WK+dirRPJZ4A==", + "uzEgwx1iAXAvWPKSVwYSeQ==", + "uzkNhmo2d08tv5AmnyqkoQ==", + "v/PshI6JjkL9nojLlMNfhg==", + "v0Bvws1WYVoEgDt8xmVKew==", + "v1AWe5qb5y3vSKFb7ADeEw==", + "v4xIYrfPGILEbD/LwVDDzA==", + "v6jZicMNM3ysm3U5xu0HoQ==", + "v7BrkRmK0FfWSHunTRHQFQ==", + "vCekQ2nOQKiN/q8Be/qwZg==", + "vFFzkWgGyw6OPADONtEojQ==", + "vFox1d3llOeBeCUZGvTy0A==", + "vFtC0B2oe1gck28JOM1dyg==", + "vGKknndb4j6VTV8DxeT4fQ==", + "vHGjRRSlZHJIliCwIkCAmQ==", + "vHVXsAMQqc0qp7HA5Q+YkA==", + "vHmQUl4WHXs1E/Shh+TeyA==", + "vIORTYSHFIXk5E2NyIvWcQ==", + "vMuaLvAntJB5o7lmt/kVXA==", + "vOJ55zFdgPPauPyFYBf01w==", + "vRgkZZGVN7YZrlml0vxrKA==", + "vSKsa0JhLCe9QFZKkcj58Q==", + "vTAmgfq3GxL4+ubXpzwk5w==", + "vUC0HlTTHj6qNHwfviDtAw==", + "vUE8Iw3NyWXURpXyoNJdaw==", + "vWn9OPnrJgfPavg4D6T/HQ==", + "vX7RIhatQeXAMr1+OjzhZw==", + "vZtL0yWpSIA+9v8i23bZSg==", + "vb6Agwzk4JG0Nn7qRPPFMQ==", + "vbyiKeDCQ4q9dDRI1Q0Ong==", + "vg3jozLXEmAnmJwdfcEN0g==", + "vhdFtKVH4bVatb4n8KzeXw==", + "vjrSYGUpeKOtJ2cNgLFg2g==", + "vljJciS+uuIvL7XXm5688g==", + "vmqfGJE6r4yDahtU/HLrxw==", + "vnOJ3e9Zd4wPx8PX7QgZzQ==", + "voO3krg4sdy4Iu+MZEr8+g==", + "vqYHQ3MnHrAIAr1QHwfIag==", + "vsRNZx4thFFFPneubKq1Fw==", + "vvEH5A39TTe1AOC11rRCLA==", + "vvh9vAIrXjIwLVkuJb5oDQ==", + "vwno3vugCvt6ooT3CD4qIQ==", + "w+jzM0I5DRzoUiLS/9QIMQ==", + "w0PKdssv+Zc5J/BbphoxpA==", + "w1zN28mSrI/gqHsgs4ME3A==", + "w3G+qXXqqKi8F5s+qvkBUg==", + "w5N/aHbtOIKzcvG3GlMjGA==", + "wDiGoFEfIVEDyyc4VpwhWQ==", + "wEJDulZafLuXCvcqBYioFQ==", + "wHA+D5cObfV3kGORCdEknw==", + "wI7JrSPQwYHpv2lRsQu9nQ==", + "wIfvvLKC61gOpsddUFjVog==", + "wJ4uCrl4DPg70ltw1dZO3w==", + "wJKFMqh6MGctWfasjHrPEg==", + "wJpepvmtQQ3sz3tVFDnFqw==", + "wK6Srd83eLigZ11Q20XGrg==", + "wM8tnXO4PDlLVHspZFcjYw==", + "wMOE/pEKVIklE75xjt6b6w==", + "wMum67lfk5E1ohUObJgrOg==", + "wMyJLQJdmrC2TSeFkIuSvQ==", + "wOc4TbwQGUwOC1B3BEZ4OQ==", + "wOhbpTzmFla8R0kI9OiHaA==", + "wPhJcp7U7IVX83szbIOOxQ==", + "wQKL8Ga6JQkpZ7yymDkC3w==", + "wR2Gxb07nkaPcZHlEjr8iA==", + "wRqaDZVHHurp5whOQ1kDbQ==", + "wTO49YX/ePHMWtcoxUAHpw==", + "wUYhs4j3W9nIywu1HIv2JA==", + "wVfSZYjMjbTsD2gaSbwuqQ==", + "wX2URK6eDDHeEOF3cgPgHA==", + "wX70jKLKJApHnhyK0r6t3A==", + "wajwXfWz2J+O+NVaj6j2UQ==", + "wc+8ohFWgOF4VlSYiZIGwQ==", + "wdRyYjaM11VmqkkxV/5bsA==", + "wfwuxn+Vja1DNwiDwL2pcQ==", + "wgH1GlUxWi6/yLLFzE76uQ==", + "who8uUamlHWHXnBf7dwy4A==", + "wlWxtQDJ+siGhN2fJn3qtw==", + "wnfYUctNK+UPwefX5y4/Rw==", + "wpZqFkKafFpLcykN2IISqg==", + "wqUJ1Gq1Yz2cXFkbcCmzHQ==", + "wqWqe0KRjZlUIrGgEOG9Mg==", + "wrewZ0hoHODf7qmoGcOd7g==", + "wsp+vmW8sEqXYVURd/gjHA==", + "wt+qDLU38kzNU75ZYi3Hbw==", + "wtyAZIfhomcHe9dLbYoSvA==", + "wux5Y8AipBnc5tJapTzgEQ==", + "wv4NC9CIpwuGf/nOQYe/oA==", + "wxkb8evGEaGf/rg/1XUWiA==", + "wy/Z8505o4sVovk4UuBp1A==", + "wyqmQGB6vgRVrYtmB2vB7w==", + "wyx5mnUMgP5wjykjAfTO7w==", + "x+8rwkqKCv0juoT5m1A4eg==", + "x/BIDm6TKMhqu/gtb3kGyw==", + "x/MpsQvziUpW40nNUHDS5Q==", + "x0eIHCvQLd2jdDaXwSWTYQ==", + "x1A74vg/hwwjAx6GrkU8zw==", + "x2NpqNnqRihktNzpxmepkQ==", + "x2nSgcTjA3oGgI8mMgiqjw==", + "x5lyMArsv1MuJmEFlWCnNw==", + "x5zMDuW66467ofgL3spLUQ==", + "x6M66krXSi0EhppwmDmsxA==", + "x6lNRGgJcRxgKTlzhc1WPg==", + "x8kRVzohTdhkryvYeMvkMw==", + "x9TIZ9Ua++3BX+MpjgTuWA==", + "x9VwDdFPp/rJ+SF16ooWYg==", + "xAAipGfHTGTjp9Qk1MR8RQ==", + "xJi0T+psHOXMivSOVpMWeQ==", + "xLm/bJBonpTs0PwsF0DvRg==", + "xMIHeno2qj3V8q9H1xezeg==", + "xNilc7UOu1kyP0+nK5MrLw==", + "xPe76nHyHmald6kmMQsKdg==", + "xQpYjaAmrQudWgsdu24J0A==", + "xTizUioizbMQxD0T6fy/EQ==", + "xUXEE7OBBCudsQnuj5ycOA==", + "xWYecfzAtXT9WyQ8NYY/hw==", + "xX6atcCApI08oVLjjLteLg==", + "xYD8jrCDmuQna+p1ebnKDQ==", + "xbBxUP9JyY0wDgHDipBHeg==", + "xdCCdP8SNBOK3IsX6PiPQA==", + "xdmY+qyoxxuRZa9kuNpDEg==", + "xfYZ6qhWNBqqJ0PdWRjOwA==", + "xfjBQk3CrNjhufdPIhr91A==", + "xiFlcSfa/gnPiO+LwbixcQ==", + "xiyRfVG0EfBA+rCk+tgWRQ==", + "xjA21QjNdThLW3VV7SCnrg==", + "xjTMO2mvtpvwQrounD4e8g==", + "xktOghh1S9nIX6fXWnT+Ug==", + "xmGgK3W5y+oCd0K2u8XjZQ==", + "xmsYnsJq78/f9xuKuQ2pBQ==", + "xoPSM86Se+1hHX0y3hhdkw==", + "xs8J3cesq7lDhP/dNltqOw==", + "xsCZVhCk2qJmOqvUjK3Y8Q==", + "xsf0m31Am0W9eLhopAkfnA==", + "xukOAM0QVsA72qEy0yku9A==", + "xvipmmwKdYt4eoKvvRnjEg==", + "xweGAZf+Yb3TtwR/sGmGIA==", + "xzGzN5Hhbh0m/KezjNvXbQ==", + "y+1I05LDAYJ09tKMs3zW6g==", + "y+cl1/Knb9MZPz8nBB0M+w==", + "y/e3HSdg7T19FanRpJ7+7Q==", + "y1J+o6DC2sETFsySgpDZyA==", + "y2JOIoIiT9cV1VxplZPraQ==", + "y2Tn2gmhKs5WKc01ce74rg==", + "y4/HohCJxtt+cT7nLJB08w==", + "y4Y4mSSTw/WrIdRpktc5Hw==", + "y4iBxAMn/KzMmaWShdYiIw==", + "y4mfEDerrhaqApDdhP5vjA==", + "y7yS9x3yshVhMpDbQtfYOQ==", + "yCu+DVU/ceMTOZ5h/7wQTg==", + "yD3Dd4ToRrl53k/2NSCJiw==", + "yDrAd1ot38soBk7zKdnT8A==", + "yKLLiqzxfrCsr6+Rm6kx1Q==", + "yKrsKX4/1B1C0TyvciNz5w==", + "yL1DwlIIREPuyuCFULi0uw==", + "yLAhLNezvqVHmN1SfMRrPw==", + "yOE90OHQdyOfrAgwDvn2gA==", + "yPIeWcW8+3HjDagegrN8bw==", + "yQCLV9IoPyXEOaj3IdFMWw==", + "yQmNZnp/JZywbBiZs3gecA==", + "yS/yMnJDHW0iaOsbj4oPTg==", + "yTVJKBn72RjakMBXDoBKHg==", + "yTgN5xFIdz1MzFS6xMl5uQ==", + "yU3N0HMSP5etuHPNrVkZtg==", + "yV3IbbTWAbHMhMGVvgb/ZQ==", + "yYBIS9PZbKo7Gram7IXWPA==", + "yYVW07lOZHdgtX42xJONIA==", + "yYmnM/WOgi+48Rw7foGyXA==", + "yYp4iuI5f/y/l1AEJxYolQ==", + "ybpTgPr3SjJ12Rj5lC/IMA==", + "ycjv4XkS5O7zcF3sqq9MwQ==", + "yctId8ltkl3+xqi9bj+RqA==", + "ydVj2odhergi+2zGUwK4/A==", + "yf06Slv9l3IZEjVqvxP2aA==", + "yfAaL0MMtSXPQ37pBdmHxQ==", + "yhI5jHlfFJxu4eV5VJO2zQ==", + "yhRi5M9Etuu9HSu4d24i3w==", + "yhexr/OFKfZl0o3lS70e4w==", + "ylA6sU7Kaf9fMNIx1+sIlw==", + "ymtA8EMPMgmMcimWZZ0A1Q==", + "ynaj4XjU27b7XbqPyxI8Ig==", + "yqQPU4jT9XvRABZgNQXjgg==", + "yqtj8GfLaUHYv/BsdjxIVw==", + "ysRQ+7Aq7eVLOp88KnFVMA==", + "ytDXLDBqWiU1w3sTurYmaw==", + "yteeQr3ub2lDXgLziZV+DQ==", + "yxCyBXqGWA735JEyljDP7Q==", + "z+1oDVy8GJ5u/UDF+bIQdA==", + "z/e5M2lE9qh3bzB97jZCKA==", + "z0BU//aSjYHAkGGk3ZSGNg==", + "z20AAnvj7WsfJeOu3vemlA==", + "z3L2BNjQOMOfTVBUxcpnRA==", + "z4Bft++f72QeDh4PWGr/sw==", + "z4oKy2wKH+sbNSgGjbdHGw==", + "z5DveTu377UW8IHnsiUGZg==", + "z920R8eahJPiTsifrPYdxA==", + "z9cd+Qj+ueX34Zf3997MNQ==", + "zCRZgVsHbQZcVMHd9pGD3A==", + "zCpibjrZOA3FQ4lYt0WoVA==", + "zDSQ3NJuUGkVOlvVCATRwA==", + "zDUZCzQesFjO1JI3PwDjfg==", + "zEzWZ6l7EKoVUxvk/l78Mw==", + "zJ7ScHNxr2leCDNNcuDApA==", + "zNLlWGW/aKBhUwQZ4DZWoQ==", + "zVupSPz7cD0v/mD/eUIIjg==", + "zZtYkKU50PPEj6qSbO5/Sw==", + "za4rzveYVMFe3Gw531DQJQ==", + "zaqyy3GaJ7cp8qDoLJWcTw==", + "zbjXhZaeyMfdTb2zxvmRMg==", + "zeELfk015D5krExLKRUYtg==", + "zeHF6fdeqcOId3fRUGscRw==", + "zgEyxj/sCs63O98sZS94Yw==", + "zi04Yc01ZheuFAQc59E45A==", + "zirOtGUXeRL22ezfotZfQg==", + "zm+z+OOyHhljV2TjA3U9zw==", + "zrZWcqQsUE3ocWE0fG+SOA==", + "ztULoqHvCOE6qV7ocqa4/w==", + "zwQ/3MzTJ9rfBmrANIh14w==", + "zwY6tCjjya/bgrYaCncaag==", + "zxsSqovedB3HT99jVblCnQ==", + "zyA9f5J7mw5InjhcfeumAQ==", +]); diff --git a/browser/components/newtab/lib/HighlightsFeed.jsm b/browser/components/newtab/lib/HighlightsFeed.jsm new file mode 100644 index 0000000000..45a30bfa97 --- /dev/null +++ b/browser/components/newtab/lib/HighlightsFeed.jsm @@ -0,0 +1,350 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const { shortURL } = ChromeUtils.import( + "resource://activity-stream/lib/ShortURL.jsm" +); +const { SectionsManager } = ChromeUtils.import( + "resource://activity-stream/lib/SectionsManager.jsm" +); +const { TOP_SITES_DEFAULT_ROWS, TOP_SITES_MAX_SITES_PER_ROW } = + ChromeUtils.importESModule( + "resource://activity-stream/common/Reducers.sys.mjs" + ); +const { Dedupe } = ChromeUtils.importESModule( + "resource://activity-stream/common/Dedupe.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "FilterAdult", + "resource://activity-stream/lib/FilterAdult.jsm" +); +ChromeUtils.defineESModuleGetters(lazy, { + LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", +}); +ChromeUtils.defineModuleGetter( + lazy, + "Screenshots", + "resource://activity-stream/lib/Screenshots.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "DownloadsManager", + "resource://activity-stream/lib/DownloadsManager.jsm" +); + +const HIGHLIGHTS_MAX_LENGTH = 16; +const MANY_EXTRA_LENGTH = + HIGHLIGHTS_MAX_LENGTH * 5 + + TOP_SITES_DEFAULT_ROWS * TOP_SITES_MAX_SITES_PER_ROW; +const SECTION_ID = "highlights"; +const SYNC_BOOKMARKS_FINISHED_EVENT = "weave:engine:sync:applied"; +const BOOKMARKS_RESTORE_SUCCESS_EVENT = "bookmarks-restore-success"; +const BOOKMARKS_RESTORE_FAILED_EVENT = "bookmarks-restore-failed"; +const RECENT_DOWNLOAD_THRESHOLD = 36 * 60 * 60 * 1000; + +class HighlightsFeed { + constructor() { + this.dedupe = new Dedupe(this._dedupeKey); + this.linksCache = new lazy.LinksCache( + lazy.NewTabUtils.activityStreamLinks, + "getHighlights", + ["image"] + ); + lazy.PageThumbs.addExpirationFilter(this); + this.downloadsManager = new lazy.DownloadsManager(); + } + + _dedupeKey(site) { + // Treat bookmarks, pocket, and downloaded items as un-dedupable, otherwise show one of a url + return ( + site && + (site.pocket_id || site.type === "bookmark" || site.type === "download" + ? {} + : site.url) + ); + } + + init() { + Services.obs.addObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT); + Services.obs.addObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT); + Services.obs.addObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT); + SectionsManager.onceInitialized(this.postInit.bind(this)); + } + + postInit() { + SectionsManager.enableSection(SECTION_ID, true /* isStartup */); + this.fetchHighlights({ broadcast: true, isStartup: true }); + this.downloadsManager.init(this.store); + } + + uninit() { + SectionsManager.disableSection(SECTION_ID); + lazy.PageThumbs.removeExpirationFilter(this); + Services.obs.removeObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT); + Services.obs.removeObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT); + Services.obs.removeObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT); + } + + observe(subject, topic, data) { + // When we receive a notification that a sync has happened for bookmarks, + // or Places finished importing or restoring bookmarks, refresh highlights + const manyBookmarksChanged = + (topic === SYNC_BOOKMARKS_FINISHED_EVENT && data === "bookmarks") || + topic === BOOKMARKS_RESTORE_SUCCESS_EVENT || + topic === BOOKMARKS_RESTORE_FAILED_EVENT; + if (manyBookmarksChanged) { + this.fetchHighlights({ broadcast: true }); + } + } + + filterForThumbnailExpiration(callback) { + const state = this.store + .getState() + .Sections.find(section => section.id === SECTION_ID); + + callback( + state && state.initialized + ? state.rows.reduce((acc, site) => { + // Screenshots call in `fetchImage` will search for preview_image_url or + // fallback to URL, so we prevent both from being expired. + acc.push(site.url); + if (site.preview_image_url) { + acc.push(site.preview_image_url); + } + return acc; + }, []) + : [] + ); + } + + /** + * Chronologically sort highlights of all types except 'visited'. Then just append + * the rest at the end of highlights. + * @param {Array} pages The full list of links to order. + * @return {Array} A sorted array of highlights + */ + _orderHighlights(pages) { + const splitHighlights = { chronologicalCandidates: [], visited: [] }; + for (let page of pages) { + if (page.type === "history") { + splitHighlights.visited.push(page); + } else { + splitHighlights.chronologicalCandidates.push(page); + } + } + + return splitHighlights.chronologicalCandidates + .sort((a, b) => a.date_added < b.date_added) + .concat(splitHighlights.visited); + } + + /** + * Refresh the highlights data for content. + * @param {bool} options.broadcast Should the update be broadcasted. + */ + async fetchHighlights(options = {}) { + // If TopSites are enabled we need them for deduping, so wait for + // TOP_SITES_UPDATED. We also need the section to be registered to update + // state, so wait for postInit triggered by SectionsManager initializing. + if ( + (!this.store.getState().TopSites.initialized && + this.store.getState().Prefs.values["feeds.system.topsites"] && + this.store.getState().Prefs.values["feeds.topsites"]) || + !this.store.getState().Sections.length + ) { + return; + } + + // We broadcast when we want to force an update, so get fresh links + if (options.broadcast) { + this.linksCache.expire(); + } + + // Request more than the expected length to allow for items being removed by + // deduping against Top Sites or multiple history from the same domain, etc. + const manyPages = await this.linksCache.request({ + numItems: MANY_EXTRA_LENGTH, + excludeBookmarks: + !this.store.getState().Prefs.values[ + "section.highlights.includeBookmarks" + ], + excludeHistory: + !this.store.getState().Prefs.values[ + "section.highlights.includeVisited" + ], + excludePocket: + !this.store.getState().Prefs.values["section.highlights.includePocket"], + }); + + if ( + this.store.getState().Prefs.values["section.highlights.includeDownloads"] + ) { + // We only want 1 download that is less than 36 hours old, and the file currently exists + let results = await this.downloadsManager.getDownloads( + RECENT_DOWNLOAD_THRESHOLD, + { numItems: 1, onlySucceeded: true, onlyExists: true } + ); + if (results.length) { + // We only want 1 download, the most recent one + manyPages.push({ + ...results[0], + type: "download", + }); + } + } + + const orderedPages = this._orderHighlights(manyPages); + + // Remove adult highlights if we need to + const checkedAdult = lazy.FilterAdult.filter(orderedPages); + + // Remove any Highlights that are in Top Sites already + const [, deduped] = this.dedupe.group( + this.store.getState().TopSites.rows, + checkedAdult + ); + + // Keep all "bookmark"s and at most one (most recent) "history" per host + const highlights = []; + const hosts = new Set(); + for (const page of deduped) { + const hostname = shortURL(page); + // Skip this history page if we already something from the same host + if (page.type === "history" && hosts.has(hostname)) { + continue; + } + + // If we already have the image for the card, use that immediately. Else + // asynchronously fetch the image. NEVER fetch a screenshot for downloads + if (!page.image && page.type !== "download") { + this.fetchImage(page, options.isStartup); + } + + // Adjust the type for 'history' items that are also 'bookmarked' when we + // want to include bookmarks + if ( + page.type === "history" && + page.bookmarkGuid && + this.store.getState().Prefs.values[ + "section.highlights.includeBookmarks" + ] + ) { + page.type = "bookmark"; + } + + // We want the page, so update various fields for UI + Object.assign(page, { + hasImage: page.type !== "download", // Downloads do not have an image - all else types fall back to a screenshot + hostname, + type: page.type, + pocket_id: page.pocket_id, + }); + + // Add the "bookmark", "pocket", or not-skipped "history" + highlights.push(page); + hosts.add(hostname); + + // Remove internal properties that might be updated after dispatch + delete page.__sharedCache; + + // Skip the rest if we have enough items + if (highlights.length === HIGHLIGHTS_MAX_LENGTH) { + break; + } + } + + const { initialized } = this.store + .getState() + .Sections.find(section => section.id === SECTION_ID); + // Broadcast when required or if it is the first update. + const shouldBroadcast = options.broadcast || !initialized; + + SectionsManager.updateSection( + SECTION_ID, + { rows: highlights }, + shouldBroadcast, + options.isStartup + ); + } + + /** + * Fetch an image for a given highlight and update the card with it. If no + * image is available then fallback to fetching a screenshot. + */ + fetchImage(page, isStartup = false) { + // Request a screenshot if we don't already have one pending + const { preview_image_url: imageUrl, url } = page; + return lazy.Screenshots.maybeCacheScreenshot( + page, + imageUrl || url, + "image", + image => { + SectionsManager.updateSectionCard( + SECTION_ID, + url, + { image }, + true, + isStartup + ); + } + ); + } + + onAction(action) { + // Relay the downloads actions to DownloadsManager - it is a child of HighlightsFeed + this.downloadsManager.onAction(action); + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.SYSTEM_TICK: + case at.TOP_SITES_UPDATED: + this.fetchHighlights({ + broadcast: false, + isStartup: !!action.meta?.isStartup, + }); + break; + case at.PREF_CHANGED: + // Update existing pages when the user changes what should be shown + if (action.data.name.startsWith("section.highlights.include")) { + this.fetchHighlights({ broadcast: true }); + } + break; + case at.PLACES_HISTORY_CLEARED: + case at.PLACES_LINK_BLOCKED: + case at.DOWNLOAD_CHANGED: + case at.POCKET_LINK_DELETED_OR_ARCHIVED: + this.fetchHighlights({ broadcast: true }); + break; + case at.PLACES_LINKS_CHANGED: + case at.PLACES_SAVED_TO_POCKET: + this.linksCache.expire(); + this.fetchHighlights({ broadcast: false }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} + +const EXPORTED_SYMBOLS = [ + "HighlightsFeed", + "SECTION_ID", + "MANY_EXTRA_LENGTH", + "SYNC_BOOKMARKS_FINISHED_EVENT", + "BOOKMARKS_RESTORE_SUCCESS_EVENT", + "BOOKMARKS_RESTORE_FAILED_EVENT", +]; diff --git a/browser/components/newtab/lib/InfoBar.jsm b/browser/components/newtab/lib/InfoBar.jsm new file mode 100644 index 0000000000..c75f17f197 --- /dev/null +++ b/browser/components/newtab/lib/InfoBar.jsm @@ -0,0 +1,172 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + RemoteL10n: "resource://activity-stream/lib/RemoteL10n.sys.mjs", +}); + +class InfoBarNotification { + constructor(message, dispatch) { + this._dispatch = dispatch; + this.dispatchUserAction = this.dispatchUserAction.bind(this); + this.buttonCallback = this.buttonCallback.bind(this); + this.infobarCallback = this.infobarCallback.bind(this); + this.message = message; + this.notification = null; + } + + /** + * Show the infobar notification and send an impression ping + * + * @param {object} browser Browser reference for the currently selected tab + */ + showNotification(browser) { + let { content } = this.message; + let { gBrowser } = browser.ownerGlobal; + let doc = gBrowser.ownerDocument; + let notificationContainer; + if (content.type === "global") { + notificationContainer = browser.ownerGlobal.gNotificationBox; + } else { + notificationContainer = gBrowser.getNotificationBox(browser); + } + + let priority = content.priority || notificationContainer.PRIORITY_SYSTEM; + + this.notification = notificationContainer.appendNotification( + this.message.id, + { + label: this.formatMessageConfig(doc, content.text), + image: content.icon || "chrome://branding/content/icon64.png", + priority, + eventCallback: this.infobarCallback, + }, + content.buttons.map(b => this.formatButtonConfig(b)) + ); + + this.addImpression(); + } + + formatMessageConfig(doc, content) { + let docFragment = doc.createDocumentFragment(); + // notificationbox will only `appendChild` for documentFragments + docFragment.appendChild( + lazy.RemoteL10n.createElement(doc, "span", { content }) + ); + + return docFragment; + } + + formatButtonConfig(button) { + let btnConfig = { callback: this.buttonCallback, ...button }; + // notificationbox will set correct data-l10n-id attributes if passed in + // using the l10n-id key. Otherwise the `button.label` text is used. + if (button.label.string_id) { + btnConfig["l10n-id"] = button.label.string_id; + } + + return btnConfig; + } + + addImpression() { + // Record an impression in ASRouter for frequency capping + this._dispatch({ type: "IMPRESSION", data: this.message }); + // Send a user impression telemetry ping + this.sendUserEventTelemetry("IMPRESSION"); + } + + /** + * Called when one of the infobar buttons is clicked + */ + buttonCallback(notificationBox, btnDescription, target) { + this.dispatchUserAction( + btnDescription.action, + target.ownerGlobal.gBrowser.selectedBrowser + ); + let isPrimary = target.classList.contains("primary"); + let eventName = isPrimary + ? "CLICK_PRIMARY_BUTTON" + : "CLICK_SECONDARY_BUTTON"; + this.sendUserEventTelemetry(eventName); + } + + dispatchUserAction(action, selectedBrowser) { + this._dispatch({ type: "USER_ACTION", data: action }, selectedBrowser); + } + + /** + * Called when interacting with the toolbar (but not through the buttons) + */ + infobarCallback(eventType) { + if (eventType === "removed") { + this.notification = null; + // eslint-disable-next-line no-use-before-define + InfoBar._activeInfobar = null; + } else if (this.notification) { + this.sendUserEventTelemetry("DISMISSED"); + this.notification = null; + // eslint-disable-next-line no-use-before-define + InfoBar._activeInfobar = null; + } + } + + sendUserEventTelemetry(event) { + const ping = { + message_id: this.message.id, + event, + }; + this._dispatch({ + type: "INFOBAR_TELEMETRY", + data: { action: "infobar_user_event", ...ping }, + }); + } +} + +const InfoBar = { + _activeInfobar: null, + + maybeLoadCustomElement(win) { + if (!win.customElements.get("remote-text")) { + Services.scriptloader.loadSubScript( + "resource://activity-stream/data/custom-elements/paragraph.js", + win + ); + } + }, + + maybeInsertFTL(win) { + win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl"); + win.MozXULElement.insertFTLIfNeeded( + "browser/defaultBrowserNotification.ftl" + ); + }, + + showInfoBarMessage(browser, message, dispatch) { + // Prevent stacking multiple infobars + if (this._activeInfobar) { + return null; + } + + const win = browser?.ownerGlobal; + + if (!win || lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + return null; + } + + this.maybeLoadCustomElement(win); + this.maybeInsertFTL(win); + + let notification = new InfoBarNotification(message, dispatch); + notification.showNotification(browser); + this._activeInfobar = true; + + return notification; + }, +}; + +const EXPORTED_SYMBOLS = ["InfoBar"]; diff --git a/browser/components/newtab/lib/LinksCache.sys.mjs b/browser/components/newtab/lib/LinksCache.sys.mjs new file mode 100644 index 0000000000..0dfb89e74e --- /dev/null +++ b/browser/components/newtab/lib/LinksCache.sys.mjs @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This should be slightly less than SYSTEM_TICK_INTERVAL as timer +// comparisons are too exact while the async/await functionality will make the +// last recorded time a little bit later. This causes the comparasion to skip +// updates. +// It should be 10% less than SYSTEM_TICK to update at least once every 5 mins. +// https://github.com/mozilla/activity-stream/pull/3695#discussion_r144678214 +const EXPIRATION_TIME = 4.5 * 60 * 1000; // 4.5 minutes + +/** + * Cache link results from a provided object property and refresh after some + * amount of time has passed. Allows for migrating data from previously cached + * links to the new links with the same url. + */ +export class LinksCache { + /** + * Create a links cache for a given object property. + * + * @param {object} linkObject Object containing the link property + * @param {string} linkProperty Name of property on object to access + * @param {array} properties Optional properties list to migrate to new links. + * @param {function} shouldRefresh Optional callback receiving the old and new + * options to refresh even when not expired. + */ + constructor( + linkObject, + linkProperty, + properties = [], + shouldRefresh = () => {} + ) { + this.clear(); + + // Allow getting links from both methods and array properties + this.linkGetter = options => { + const ret = linkObject[linkProperty]; + return typeof ret === "function" ? ret.call(linkObject, options) : ret; + }; + + // Always migrate the shared cache data in addition to any custom properties + this.migrateProperties = ["__sharedCache", ...properties]; + this.shouldRefresh = shouldRefresh; + } + + /** + * Clear the cached data. + */ + clear() { + this.cache = Promise.resolve([]); + this.lastOptions = {}; + this.expire(); + } + + /** + * Force the next request to update the cache. + */ + expire() { + delete this.lastUpdate; + } + + /** + * Request data and update the cache if necessary. + * + * @param {object} options Optional data to pass to the underlying method. + * @returns {promise(array)} Links array with objects that can be modified. + */ + async request(options = {}) { + // Update the cache if the data has been expired + const now = Date.now(); + if ( + this.lastUpdate === undefined || + now > this.lastUpdate + EXPIRATION_TIME || + // Allow custom rules around refreshing based on options + this.shouldRefresh(this.lastOptions, options) + ) { + // Update request state early so concurrent requests can refer to it + this.lastOptions = options; + this.lastUpdate = now; + + // Save a promise before awaits, so other requests wait for correct data + // eslint-disable-next-line no-async-promise-executor + this.cache = new Promise(async (resolve, reject) => { + try { + // Allow fast lookup of old links by url that might need to migrate + const toMigrate = new Map(); + for (const oldLink of await this.cache) { + if (oldLink) { + toMigrate.set(oldLink.url, oldLink); + } + } + + // Update the cache with migrated links without modifying source objects + resolve( + (await this.linkGetter(options)).map(link => { + // Keep original array hole positions + if (!link) { + return link; + } + + // Migrate data to the new link copy if we have an old link + const newLink = Object.assign({}, link); + const oldLink = toMigrate.get(newLink.url); + if (oldLink) { + for (const property of this.migrateProperties) { + const oldValue = oldLink[property]; + if (oldValue !== undefined) { + newLink[property] = oldValue; + } + } + } else { + // Share data among link copies and new links from future requests + newLink.__sharedCache = {}; + } + // Provide a helper to update the cached link + newLink.__sharedCache.updateLink = (property, value) => { + newLink[property] = value; + }; + + return newLink; + }) + ); + } catch (error) { + reject(error); + } + }); + } + + // Provide a shallow copy of the cached link objects for callers to modify + return (await this.cache).map(link => link && Object.assign({}, link)); + } +} diff --git a/browser/components/newtab/lib/MomentsPageHub.jsm b/browser/components/newtab/lib/MomentsPageHub.jsm new file mode 100644 index 0000000000..e37d1df8b1 --- /dev/null +++ b/browser/components/newtab/lib/MomentsPageHub.jsm @@ -0,0 +1,174 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + setInterval: "resource://gre/modules/Timer.sys.mjs", + clearInterval: "resource://gre/modules/Timer.sys.mjs", +}); + +// Frequency at which to check for new messages +const SYSTEM_TICK_INTERVAL = 5 * 60 * 1000; +const HOMEPAGE_OVERRIDE_PREF = "browser.startup.homepage_override.once"; + +// For the "reach" event of Messaging Experiments +const REACH_EVENT_CATEGORY = "messaging_experiments"; +const REACH_EVENT_METHOD = "reach"; +// Note it's not "moments-page" as Telemetry Events only accepts understores +// for the event `object` +const REACH_EVENT_OBJECT = "moments_page"; + +class _MomentsPageHub { + constructor() { + this.id = "moments-page-hub"; + this.state = {}; + this.checkHomepageOverridePref = this.checkHomepageOverridePref.bind(this); + this._initialized = false; + } + + async init( + waitForInitialized, + { handleMessageRequest, addImpression, blockMessageById, sendTelemetry } + ) { + if (this._initialized) { + return; + } + + this._initialized = true; + this._handleMessageRequest = handleMessageRequest; + this._addImpression = addImpression; + this._blockMessageById = blockMessageById; + this._sendTelemetry = sendTelemetry; + + // Need to wait for ASRouter to initialize before trying to fetch messages + await waitForInitialized; + + this.messageRequest({ + triggerId: "momentsUpdate", + template: "update_action", + }); + + const _intervalId = lazy.setInterval( + () => this.checkHomepageOverridePref(), + SYSTEM_TICK_INTERVAL + ); + this.state = { _intervalId }; + } + + _sendPing(ping) { + this._sendTelemetry({ + type: "MOMENTS_PAGE_TELEMETRY", + data: { action: "moments_user_event", ...ping }, + }); + } + + sendUserEventTelemetry(message) { + this._sendPing({ + message_id: message.id, + bucket_id: message.id, + event: "MOMENTS_PAGE_SET", + }); + } + + /** + * If we don't have `expire` defined with the message it could be because + * it depends on user dependent parameters. Since the message matched + * targeting we calculate `expire` based on the current timestamp and the + * `expireDelta` which defines for how long it should be available. + * @param expireDelta {number} - Offset in milliseconds from the current date + */ + getExpirationDate(expireDelta) { + return Date.now() + expireDelta; + } + + executeAction(message) { + const { id, data } = message.content.action; + switch (id) { + case "moments-wnp": + const { url, expireDelta } = data; + let { expire } = data; + if (!expire) { + expire = this.getExpirationDate(expireDelta); + } + // In order to reset this action we can dispatch a new message that + // will overwrite the prev value with an expiration date from the past. + Services.prefs.setStringPref( + HOMEPAGE_OVERRIDE_PREF, + JSON.stringify({ message_id: message.id, url, expire }) + ); + // Add impression and block immediately after taking the action + this.sendUserEventTelemetry(message); + this._addImpression(message); + this._blockMessageById(message.id); + break; + } + } + + _recordReachEvent(message) { + const extra = { branches: message.branchSlug }; + Services.telemetry.recordEvent( + REACH_EVENT_CATEGORY, + REACH_EVENT_METHOD, + REACH_EVENT_OBJECT, + message.experimentSlug, + extra + ); + } + + async messageRequest({ triggerId, template }) { + const telemetryObject = { triggerId }; + TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + const messages = await this._handleMessageRequest({ + triggerId, + template, + returnAll: true, + }); + TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + + // Record the "reach" event for all the messages with `forReachEvent`, + // only execute action for the first message without forReachEvent. + const nonReachMessages = []; + for (const message of messages) { + if (message.forReachEvent) { + if (!message.forReachEvent.sent) { + this._recordReachEvent(message); + message.forReachEvent.sent = true; + } + } else { + nonReachMessages.push(message); + } + } + if (nonReachMessages.length) { + this.executeAction(nonReachMessages[0]); + } + } + + /** + * Pref is set via Remote Settings message. We want to continously + * monitor new messages that come in to ensure the one with the + * highest priority is set. + */ + checkHomepageOverridePref() { + this.messageRequest({ + triggerId: "momentsUpdate", + template: "update_action", + }); + } + + uninit() { + lazy.clearInterval(this.state._intervalId); + this.state = {}; + this._initialized = false; + } +} + +/** + * ToolbarBadgeHub - singleton instance of _ToolbarBadgeHub that can initiate + * message requests and render messages. + */ +const MomentsPageHub = new _MomentsPageHub(); + +const EXPORTED_SYMBOLS = ["_MomentsPageHub", "MomentsPageHub"]; diff --git a/browser/components/newtab/lib/NewTabInit.jsm b/browser/components/newtab/lib/NewTabInit.jsm new file mode 100644 index 0000000000..f1a7f0d142 --- /dev/null +++ b/browser/components/newtab/lib/NewTabInit.jsm @@ -0,0 +1,57 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +/** + * NewTabInit - A placeholder for now. This will send a copy of the state to all + * newly opened tabs. + */ +class NewTabInit { + constructor() { + this._repliedEarlyTabs = new Map(); + } + + reply(target) { + // Skip this reply if we already replied to an early tab + if (this._repliedEarlyTabs.get(target)) { + return; + } + + const action = { + type: at.NEW_TAB_INITIAL_STATE, + data: this.store.getState(), + }; + this.store.dispatch(ac.AlsoToOneContent(action, target)); + + // Remember that this early tab has already gotten a rehydration response in + // case it thought we lost its initial REQUEST and asked again + if (this._repliedEarlyTabs.has(target)) { + this._repliedEarlyTabs.set(target, true); + } + } + + onAction(action) { + switch (action.type) { + case at.NEW_TAB_STATE_REQUEST: + this.reply(action.meta.fromTarget); + break; + case at.NEW_TAB_INIT: + // Initialize data for early tabs that might REQUEST twice + if (action.data.simulated) { + this._repliedEarlyTabs.set(action.data.portID, false); + } + break; + case at.NEW_TAB_UNLOAD: + // Clean up for any tab (no-op if not an early tab) + this._repliedEarlyTabs.delete(action.meta.fromTarget); + break; + } + } +} + +const EXPORTED_SYMBOLS = ["NewTabInit"]; diff --git a/browser/components/newtab/lib/OnboardingMessageProvider.jsm b/browser/components/newtab/lib/OnboardingMessageProvider.jsm new file mode 100644 index 0000000000..bcc02aba5f --- /dev/null +++ b/browser/components/newtab/lib/OnboardingMessageProvider.jsm @@ -0,0 +1,1276 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; +/* globals Localization */ + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { FeatureCalloutMessages } = ChromeUtils.importESModule( + "resource://activity-stream/lib/FeatureCalloutMessages.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BrowserUtils: "resource://gre/modules/BrowserUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + ShellService: "resource:///modules/ShellService.sys.mjs", +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "usesFirefoxSync", + "services.sync.username" +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "mobileDevices", + "services.sync.clients.devices.mobile", + 0 +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "hidePrivatePin", + "browser.startup.upgradeDialog.pinPBM.disabled", + false +); + +const L10N = new Localization([ + "branding/brand.ftl", + "browser/newtab/onboarding.ftl", + "toolkit/branding/brandings.ftl", + "toolkit/branding/accounts.ftl", +]); + +const HOMEPAGE_PREF = "browser.startup.homepage"; +const NEWTAB_PREF = "browser.newtabpage.enabled"; + +const BASE_MESSAGES = () => [ + { + id: "FXA_ACCOUNTS_BADGE", + template: "toolbar_badge", + content: { + delay: 10000, // delay for 10 seconds + target: "fxa-toolbar-menu-button", + }, + targeting: "false", + trigger: { id: "toolbarBadgeUpdate" }, + }, + { + id: "PROTECTIONS_PANEL_1", + template: "protections_panel", + content: { + title: { string_id: "cfr-protections-panel-header" }, + body: { string_id: "cfr-protections-panel-body" }, + link_text: { string_id: "cfr-protections-panel-link-text" }, + cta_url: `${Services.urlFormatter.formatURLPref( + "app.support.baseURL" + )}etp-promotions?as=u&utm_source=inproduct`, + cta_type: "OPEN_URL", + }, + trigger: { id: "protectionsPanelOpen" }, + }, + { + id: "CFR_FIREFOX_VIEW", + groups: ["cfr"], + template: "cfr_doorhanger", + //If Firefox View button has been moved to the overflow menu, we want to change the anchor element + content: { + bucket_id: "CFR_FIREFOX_VIEW", + anchor_id: "firefox-view-button", + alt_anchor_id: "nav-bar-overflow-button", + layout: "icon_and_message", + icon: "chrome://browser/content/cfr-lightning.svg", + icon_dark_theme: "chrome://browser/content/cfr-lightning-dark.svg", + icon_class: "cfr-doorhanger-small-icon", + heading_text: { + string_id: "firefoxview-cfr-header-v2", + }, + text: { + string_id: "firefoxview-cfr-body-v2", + }, + buttons: { + primary: { + label: { + string_id: "firefoxview-cfr-primarybutton", + }, + action: { + type: "OPEN_FIREFOX_VIEW", + navigate: true, + }, + }, + secondary: [ + { + label: { + string_id: "firefoxview-cfr-secondarybutton", + }, + action: { + type: "CANCEL", + }, + }, + ], + }, + skip_address_bar_notifier: true, + }, + frequency: { + lifetime: 1, + }, + trigger: { + id: "nthTabClosed", + }, + // Avoid breaking existing tests that close tabs for now. + targeting: `!inMr2022Holdback && fxViewButtonAreaType != null && (currentDate|date - profileAgeCreated) / 86400000 >= 2 && tabsClosedCount >= 3 && 'browser.firefox-view.view-count'|preferenceValue == 0 && !'browser.newtabpage.activity-stream.asrouter.providers.cfr'|preferenceIsUserSet`, + }, + { + id: "FX_MR_106_UPGRADE", + template: "spotlight", + targeting: "true", + content: { + template: "multistage", + id: "FX_MR_106_UPGRADE", + transitions: true, + modal: "tab", + screens: [ + { + id: "UPGRADE_PIN_FIREFOX", + content: { + position: "split", + split_narrow_bkg_position: "-155px", + image_alt_text: { + string_id: "mr2022-onboarding-pin-image-alt", + }, + progress_bar: "true", + background: + "url('chrome://activity-stream/content/data/content/assets/mr-pintaskbar.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-onboarding-existing-pin-header", + }, + subtitle: { + string_id: "mr2022-onboarding-existing-pin-subtitle", + }, + primary_button: { + label: { + string_id: "mr2022-onboarding-pin-primary-button-label", + }, + action: { + navigate: true, + type: "PIN_FIREFOX_TO_TASKBAR", + }, + }, + checkbox: { + label: { + string_id: "mr2022-onboarding-existing-pin-checkbox-label", + }, + defaultValue: true, + action: { + type: "MULTI_ACTION", + navigate: true, + data: { + actions: [ + { + type: "PIN_FIREFOX_TO_TASKBAR", + data: { + privatePin: true, + }, + }, + { + type: "PIN_FIREFOX_TO_TASKBAR", + }, + ], + }, + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-secondary-skip-button-label", + }, + action: { + navigate: true, + }, + has_arrow_icon: true, + }, + }, + }, + { + id: "UPGRADE_SET_DEFAULT", + content: { + position: "split", + split_narrow_bkg_position: "-60px", + image_alt_text: { + string_id: "mr2022-onboarding-default-image-alt", + }, + progress_bar: "true", + background: + "url('chrome://activity-stream/content/data/content/assets/mr-settodefault.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-onboarding-set-default-title", + }, + subtitle: { + string_id: "mr2022-onboarding-set-default-subtitle", + }, + primary_button: { + label: { + string_id: "mr2022-onboarding-set-default-primary-button-label", + }, + action: { + navigate: true, + type: "SET_DEFAULT_BROWSER", + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-secondary-skip-button-label", + }, + action: { + navigate: true, + }, + has_arrow_icon: true, + }, + }, + }, + { + id: "UPGRADE_IMPORT_SETTINGS", + content: { + position: "split", + split_narrow_bkg_position: "-42px", + image_alt_text: { + string_id: "mr2022-onboarding-import-image-alt", + }, + progress_bar: "true", + background: + "url('chrome://activity-stream/content/data/content/assets/mr-import.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-onboarding-import-header", + }, + subtitle: { + string_id: "mr2022-onboarding-import-subtitle", + }, + primary_button: { + label: { + string_id: + "mr2022-onboarding-import-primary-button-label-no-attribution", + }, + action: { + type: "SHOW_MIGRATION_WIZARD", + data: {}, + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-secondary-skip-button-label", + }, + action: { + navigate: true, + }, + has_arrow_icon: true, + }, + }, + }, + { + id: "UPGRADE_MOBILE_DOWNLOAD", + content: { + position: "split", + split_narrow_bkg_position: "-160px", + image_alt_text: { + string_id: "mr2022-onboarding-mobile-download-image-alt", + }, + background: + "url('chrome://activity-stream/content/data/content/assets/mr-mobilecrosspromo.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + progress_bar: true, + logo: {}, + title: { + string_id: "mr2022-onboarding-mobile-download-title", + }, + subtitle: { + string_id: "mr2022-onboarding-mobile-download-subtitle", + }, + hero_image: { + url: "chrome://activity-stream/content/data/content/assets/mobile-download-qr-existing-user.svg", + }, + cta_paragraph: { + text: { + string_id: "mr2022-onboarding-mobile-download-cta-text", + string_name: "download-label", + }, + action: { + type: "OPEN_URL", + data: { + args: "https://www.mozilla.org/firefox/mobile/get-app/?utm_medium=firefox-desktop&utm_source=onboarding-modal&utm_campaign=mr2022&utm_content=existing-global", + where: "tab", + }, + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-secondary-skip-button-label", + }, + action: { + navigate: true, + }, + has_arrow_icon: true, + }, + }, + }, + { + id: "UPGRADE_PIN_PRIVATE_WINDOW", + content: { + position: "split", + split_narrow_bkg_position: "-100px", + image_alt_text: { + string_id: "mr2022-onboarding-pin-private-image-alt", + }, + progress_bar: "true", + background: + "url('chrome://activity-stream/content/data/content/assets/mr-pinprivate.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-upgrade-onboarding-pin-private-window-header", + }, + subtitle: { + string_id: + "mr2022-upgrade-onboarding-pin-private-window-subtitle", + }, + primary_button: { + label: { + string_id: + "mr2022-upgrade-onboarding-pin-private-window-primary-button-label", + }, + action: { + type: "PIN_FIREFOX_TO_TASKBAR", + data: { + privatePin: true, + }, + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-secondary-skip-button-label", + }, + action: { + navigate: true, + }, + has_arrow_icon: true, + }, + }, + }, + { + id: "UPGRADE_DATA_RECOMMENDATION", + content: { + position: "split", + split_narrow_bkg_position: "-80px", + image_alt_text: { + string_id: "mr2022-onboarding-privacy-segmentation-image-alt", + }, + progress_bar: "true", + background: + "url('chrome://activity-stream/content/data/content/assets/mr-privacysegmentation.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-onboarding-privacy-segmentation-title", + }, + subtitle: { + string_id: "mr2022-onboarding-privacy-segmentation-subtitle", + }, + cta_paragraph: { + text: { + string_id: "mr2022-onboarding-privacy-segmentation-text-cta", + }, + }, + primary_button: { + label: { + string_id: + "mr2022-onboarding-privacy-segmentation-button-primary-label", + }, + action: { + type: "SET_PREF", + data: { + pref: { + name: "browser.dataFeatureRecommendations.enabled", + value: true, + }, + }, + navigate: true, + }, + }, + additional_button: { + label: { + string_id: + "mr2022-onboarding-privacy-segmentation-button-secondary-label", + }, + style: "secondary", + action: { + type: "SET_PREF", + data: { + pref: { + name: "browser.dataFeatureRecommendations.enabled", + value: false, + }, + }, + navigate: true, + }, + }, + }, + }, + { + id: "UPGRADE_GRATITUDE", + content: { + position: "split", + progress_bar: "true", + split_narrow_bkg_position: "-228px", + image_alt_text: { + string_id: "mr2022-onboarding-gratitude-image-alt", + }, + background: + "url('chrome://activity-stream/content/data/content/assets/mr-gratitude.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)", + logo: {}, + title: { + string_id: "mr2022-onboarding-gratitude-title", + }, + subtitle: { + string_id: "mr2022-onboarding-gratitude-subtitle", + }, + primary_button: { + label: { + string_id: "mr2022-onboarding-gratitude-primary-button-label", + }, + action: { + type: "OPEN_FIREFOX_VIEW", + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "mr2022-onboarding-gratitude-secondary-button-label", + }, + action: { + navigate: true, + }, + }, + }, + }, + ], + }, + }, + { + id: "FX_100_UPGRADE", + template: "spotlight", + targeting: "false", + content: { + template: "multistage", + id: "FX_100_UPGRADE", + transitions: true, + screens: [ + { + id: "UPGRADE_PIN_FIREFOX", + content: { + logo: { + imageURL: + "chrome://activity-stream/content/data/content/assets/heart.webp", + height: "73px", + }, + has_noodles: true, + title: { + fontSize: "36px", + string_id: "fx100-upgrade-thanks-header", + }, + title_style: "fancy shine", + background: + "url('chrome://activity-stream/content/data/content/assets/confetti.svg') top / 100% no-repeat var(--in-content-page-background)", + subtitle: { + string_id: "fx100-upgrade-thanks-keep-body", + }, + primary_button: { + label: { + string_id: "fx100-thank-you-pin-primary-button-label", + }, + action: { + navigate: true, + type: "PIN_FIREFOX_TO_TASKBAR", + }, + }, + secondary_button: { + label: { + string_id: "onboarding-not-now-button-label", + }, + action: { + navigate: true, + }, + }, + }, + }, + ], + }, + }, + { + id: "PB_NEWTAB_FOCUS_PROMO", + type: "default", + template: "pb_newtab", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "FOCUS", + promoHeader: "fluent:about-private-browsing-focus-promo-header-c", + promoImageLarge: "chrome://browser/content/assets/focus-promo.png", + promoLinkText: "fluent:about-private-browsing-focus-promo-cta", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: "fluent:about-private-browsing-focus-promo-text-c", + promoTitleEnabled: true, + promoButton: { + action: { + type: "SHOW_SPOTLIGHT", + data: { + content: { + id: "FOCUS_PROMO", + template: "multistage", + modal: "tab", + backdrop: "transparent", + screens: [ + { + id: "DEFAULT_MODAL_UI", + content: { + logo: { + imageURL: + "chrome://browser/content/assets/focus-logo.svg", + height: "48px", + }, + title: { + string_id: "spotlight-focus-promo-title", + }, + subtitle: { + string_id: "spotlight-focus-promo-subtitle", + }, + dismiss_button: { + action: { + navigate: true, + }, + }, + ios: { + action: { + data: { + args: "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fus%2Fapp%2Ffirefox-focus-privacy-browser%2Fid1055677337", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + android: { + action: { + data: { + args: "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.focus", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + tiles: { + type: "mobile_downloads", + data: { + QR_code: { + image_url: + "chrome://browser/content/assets/focus-qr-code.svg", + alt_text: { + string_id: "spotlight-focus-promo-qr-code", + }, + }, + marketplace_buttons: ["ios", "android"], + }, + }, + }, + }, + ], + }, + }, + }, + }, + }, + priority: 2, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + // Exclude the next 2 messages: 1) Klar for en 2) Klar for de + targeting: + "!(region in [ 'DE', 'AT', 'CH'] && localeLanguageCode == 'en') && localeLanguageCode != 'de'", + }, + { + id: "PB_NEWTAB_KLAR_PROMO", + type: "default", + template: "pb_newtab", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "FOCUS", + promoHeader: "fluent:about-private-browsing-focus-promo-header-c", + promoImageLarge: "chrome://browser/content/assets/focus-promo.png", + promoLinkText: "Download Firefox Klar", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: + "Firefox Klar clears your history every time while blocking ads and trackers.", + promoTitleEnabled: true, + promoButton: { + action: { + type: "SHOW_SPOTLIGHT", + data: { + content: { + id: "KLAR_PROMO", + template: "multistage", + modal: "tab", + backdrop: "transparent", + screens: [ + { + id: "DEFAULT_MODAL_UI", + order: 0, + content: { + logo: { + imageURL: + "chrome://browser/content/assets/focus-logo.svg", + height: "48px", + }, + title: "Get Firefox Klar", + subtitle: { + string_id: "spotlight-focus-promo-subtitle", + }, + dismiss_button: { + action: { + navigate: true, + }, + }, + ios: { + action: { + data: { + args: "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fde%2Fapp%2Fklar-by-firefox%2Fid1073435754", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + android: { + action: { + data: { + args: "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.klar", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + tiles: { + type: "mobile_downloads", + data: { + QR_code: { + image_url: + "chrome://browser/content/assets/klar-qr-code.svg", + alt_text: "Scan the QR code to get Firefox Klar", + }, + marketplace_buttons: ["ios", "android"], + }, + }, + }, + }, + ], + }, + }, + }, + }, + }, + priority: 2, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + targeting: "region in [ 'DE', 'AT', 'CH'] && localeLanguageCode == 'en'", + }, + { + id: "PB_NEWTAB_KLAR_PROMO_DE", + type: "default", + template: "pb_newtab", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "FOCUS", + promoHeader: "fluent:about-private-browsing-focus-promo-header-c", + promoImageLarge: "chrome://browser/content/assets/focus-promo.png", + promoLinkText: "fluent:about-private-browsing-focus-promo-cta", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: "fluent:about-private-browsing-focus-promo-text-c", + promoTitleEnabled: true, + promoButton: { + action: { + type: "SHOW_SPOTLIGHT", + data: { + content: { + id: "FOCUS_PROMO", + template: "multistage", + modal: "tab", + backdrop: "transparent", + screens: [ + { + id: "DEFAULT_MODAL_UI", + content: { + logo: { + imageURL: + "chrome://browser/content/assets/focus-logo.svg", + height: "48px", + }, + title: { + string_id: "spotlight-focus-promo-title", + }, + subtitle: { + string_id: "spotlight-focus-promo-subtitle", + }, + dismiss_button: { + action: { + navigate: true, + }, + }, + ios: { + action: { + data: { + args: "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fde%2Fapp%2Fklar-by-firefox%2Fid1073435754", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + android: { + action: { + data: { + args: "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.klar", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + tiles: { + type: "mobile_downloads", + data: { + QR_code: { + image_url: + "chrome://browser/content/assets/klar-qr-code.svg", + alt_text: { + string_id: "spotlight-focus-promo-qr-code", + }, + }, + marketplace_buttons: ["ios", "android"], + }, + }, + }, + }, + ], + }, + }, + }, + }, + }, + priority: 2, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + targeting: "localeLanguageCode == 'de'", + }, + { + id: "PB_NEWTAB_INFO_SECTION", + template: "pb_newtab", + content: { + promoEnabled: false, + infoEnabled: true, + infoIcon: "", + infoTitle: "", + infoBody: "fluent:about-private-browsing-info-description-private-window", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitleEnabled: false, + }, + targeting: "true", + }, + { + id: "PB_NEWTAB_PIN_PROMO", + template: "pb_newtab", + type: "default", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "PIN", + promoHeader: "fluent:about-private-browsing-pin-promo-header", + promoImageLarge: + "chrome://browser/content/assets/private-promo-asset.svg", + promoLinkText: "fluent:about-private-browsing-pin-promo-link-text", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: "fluent:about-private-browsing-pin-promo-title", + promoTitleEnabled: true, + promoButton: { + action: { + type: "MULTI_ACTION", + data: { + actions: [ + { + type: "SET_PREF", + data: { + pref: { + name: "browser.privateWindowSeparation.enabled", + value: true, + }, + }, + }, + { + type: "PIN_FIREFOX_TO_TASKBAR", + data: { + privatePin: true, + }, + }, + { + type: "BLOCK_MESSAGE", + data: { + id: "PB_NEWTAB_PIN_PROMO", + }, + }, + { + type: "OPEN_ABOUT_PAGE", + data: { args: "privatebrowsing", where: "current" }, + }, + ], + }, + }, + }, + }, + priority: 3, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + targeting: "!inMr2022Holdback && doesAppNeedPrivatePin", + }, + { + id: "PB_NEWTAB_COOKIE_BANNERS_PROMO", + template: "pb_newtab", + type: "default", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "COOKIE_BANNERS", + promoHeader: "fluent:about-private-browsing-cookie-banners-promo-header", + promoImageLarge: + "chrome://browser/content/assets/cookie-banners-begone.svg", + promoLinkText: + "fluent:about-private-browsing-cookie-banners-promo-button", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: "fluent:about-private-browsing-cookie-banners-promo-message", + promoTitleEnabled: true, + promoButton: { + action: { + type: "MULTI_ACTION", + data: { + actions: [ + { + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.mode", + value: Ci.nsICookieBannerService.MODE_REJECT, + }, + }, + }, + { + // This pref may be removed (with the normal pref controlling + // both modes), at which time we should remove this action. + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.mode.privateBrowsing", + value: Ci.nsICookieBannerService.MODE_REJECT, + }, + }, + }, + { + // Reset this pref to default + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.detectOnly", + }, + }, + }, + { + type: "BLOCK_MESSAGE", + data: { + id: "PB_NEWTAB_COOKIE_BANNERS_PROMO", + }, + }, + { + type: "OPEN_ABOUT_PAGE", + data: { args: "privatebrowsing", where: "current" }, + }, + ], + }, + }, + }, + }, + priority: 4, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + targeting: `!'cookiebanners.service.mode'|preferenceIsUserSet`, + }, + { + id: "CFR_COOKIEBANNER", + groups: ["cfr"], + template: "cfr_doorhanger", + content: { + bucket_id: "CFR_COOKIEBANNER", + anchor_id: "tracking-protection-icon-container", + layout: "icon_and_message", + icon: "chrome://browser/skin/controlcenter/3rdpartycookies.svg", + icon_class: "cfr-doorhanger-small-icon", + persistent_doorhanger: true, + show_in_private_browsing: true, + heading_text: { + string_id: "cfr-cbh-header", + }, + text: { + string_id: "cfr-cbh-body", + }, + buttons: { + primary: { + label: { + string_id: "cfr-cbh-confirm-button", + }, + action: { + type: "MULTI_ACTION", + data: { + actions: [ + { + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.mode", + value: 1, + }, + }, + }, + { + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.mode.privateBrowsing", + value: 1, + }, + }, + }, + { + type: "SET_PREF", + data: { + pref: { + name: "cookiebanners.service.detectOnly", + value: false, + }, + }, + }, + { + type: "RELOAD_BROWSER", + }, + ], + }, + }, + }, + secondary: [ + { + label: { + string_id: "cfr-cbh-dismiss-button", + }, + action: { + type: "CANCEL", + }, + }, + ], + }, + skip_address_bar_notifier: true, + }, + frequency: { + custom: [{ period: 24 * 60 * 60 * 1000 * 2, cap: 1 }], + lifetime: 2, + }, + trigger: { + id: "cookieBannerDetected", + }, + targeting: `'cookiebanners.ui.desktop.enabled'|preferenceValue == true && 'cookiebanners.service.detectOnly'|preferenceValue == true`, + }, +]; + +// Eventually, move Feature Callout messages to their own provider +const ONBOARDING_MESSAGES = () => + BASE_MESSAGES().concat(FeatureCalloutMessages.getMessages()); + +const OnboardingMessageProvider = { + async getExtraAttributes() { + const [header, button_label] = await L10N.formatMessages([ + { id: "onboarding-welcome-header" }, + { id: "onboarding-start-browsing-button-label" }, + ]); + return { header: header.value, button_label: button_label.value }; + }, + async getMessages() { + const messages = await this.translateMessages(await ONBOARDING_MESSAGES()); + return messages; + }, + async getUntranslatedMessages() { + // This is helpful for jsonSchema testing - since we are localizing in the provider + const messages = await ONBOARDING_MESSAGES(); + return messages; + }, + async translateMessages(messages) { + let translatedMessages = []; + for (const msg of messages) { + let translatedMessage = { ...msg }; + + // If the message has no content, do not attempt to translate it + if (!translatedMessage.content) { + translatedMessages.push(translatedMessage); + continue; + } + + // Translate any secondary buttons separately + if (msg.content.secondary_button) { + const [secondary_button_string] = await L10N.formatMessages([ + { id: msg.content.secondary_button.label.string_id }, + ]); + translatedMessage.content.secondary_button.label = + secondary_button_string.value; + } + if (msg.content.header) { + const [header_string] = await L10N.formatMessages([ + { id: msg.content.header.string_id }, + ]); + translatedMessage.content.header = header_string.value; + } + translatedMessages.push(translatedMessage); + } + return translatedMessages; + }, + async _doesAppNeedPin(privateBrowsing = false) { + const needPin = await lazy.ShellService.doesAppNeedPin(privateBrowsing); + return needPin; + }, + async _doesAppNeedDefault() { + let checkDefault = Services.prefs.getBoolPref( + "browser.shell.checkDefaultBrowser", + false + ); + let isDefault = await lazy.ShellService.isDefaultBrowser(); + return checkDefault && !isDefault; + }, + _shouldShowPrivacySegmentationScreen() { + // Fall back to pref: browser.privacySegmentation.preferences.show + return lazy.NimbusFeatures.majorRelease2022.getVariable( + "feltPrivacyShowPreferencesSection" + ); + }, + _doesHomepageNeedReset() { + return ( + Services.prefs.prefHasUserValue(HOMEPAGE_PREF) || + Services.prefs.prefHasUserValue(NEWTAB_PREF) + ); + }, + + async getUpgradeMessage() { + let message = (await OnboardingMessageProvider.getMessages()).find( + ({ id }) => id === "FX_MR_106_UPGRADE" + ); + + let { content } = message; + // Helper to find screens and remove them where applicable. + function removeScreens(check) { + const { screens } = content; + for (let i = 0; i < screens?.length; i++) { + if (check(screens[i])) { + screens.splice(i--, 1); + } + } + } + + // Helper to prepare mobile download screen content + function prepareMobileDownload() { + let mobileContent = content.screens.find( + screen => screen.id === "UPGRADE_MOBILE_DOWNLOAD" + )?.content; + + if (!mobileContent) { + return; + } + if (!lazy.BrowserUtils.sendToDeviceEmailsSupported()) { + // If send to device emails are not supported for a user's locale, + // remove the send to device link and update the screen text + delete mobileContent.cta_paragraph.action; + mobileContent.cta_paragraph.text = { + string_id: "mr2022-onboarding-no-mobile-download-cta-text", + }; + } + // Update CN specific QRCode url + if (AppConstants.isChinaRepack()) { + mobileContent.hero_image.url = `${mobileContent.hero_image.url.slice( + 0, + mobileContent.hero_image.url.indexOf(".svg") + )}-cn.svg`; + } + } + + let pinScreen = content.screens?.find( + screen => screen.id === "UPGRADE_PIN_FIREFOX" + ); + const needPin = await this._doesAppNeedPin(); + const needDefault = await this._doesAppNeedDefault(); + const needPrivatePin = + !lazy.hidePrivatePin && (await this._doesAppNeedPin(true)); + const showSegmentation = this._shouldShowPrivacySegmentationScreen(); + + //If a user has Firefox as default remove import screen + if (!needDefault) { + removeScreens(screen => screen.id?.startsWith("UPGRADE_IMPORT_SETTINGS")); + } + + // If already pinned, convert "pin" screen to "welcome" with desired action. + let removeDefault = !needDefault; + // If user doesn't need pin, update screen to set "default" or "get started" configuration + if (!needPin && pinScreen) { + // don't need to show the checkbox + delete pinScreen.content.checkbox; + + removeDefault = true; + let primary = pinScreen.content.primary_button; + if (needDefault) { + pinScreen.id = "UPGRADE_ONLY_DEFAULT"; + pinScreen.content.subtitle = { + string_id: "mr2022-onboarding-existing-set-default-only-subtitle", + }; + primary.label.string_id = + "mr2022-onboarding-set-default-primary-button-label"; + + // The "pin" screen will now handle "default" so remove other "default." + primary.action.type = "SET_DEFAULT_BROWSER"; + } else { + pinScreen.id = "UPGRADE_GET_STARTED"; + pinScreen.content.subtitle = { + string_id: "mr2022-onboarding-get-started-primary-subtitle", + }; + primary.label = { + string_id: "mr2022-onboarding-get-started-primary-button-label", + }; + delete primary.action.type; + } + } + + // If a user has Firefox private pinned remove pin private window screen + // We also remove standalone pin private window screen if a user doesn't have + // Firefox pinned in which case the option is shown as checkbox with UPGRADE_PIN_FIREFOX screen + if (!needPrivatePin || needPin) { + removeScreens(screen => + screen.id?.startsWith("UPGRADE_PIN_PRIVATE_WINDOW") + ); + } + + if (!showSegmentation) { + removeScreens(screen => + screen.id?.startsWith("UPGRADE_DATA_RECOMMENDATION") + ); + } + + //If privatePin, remove checkbox from pinscreen + if (!needPrivatePin) { + delete content.screens?.find( + screen => screen.id === "UPGRADE_PIN_FIREFOX" + )?.content?.checkbox; + } + + if (removeDefault) { + removeScreens(screen => screen.id?.startsWith("UPGRADE_SET_DEFAULT")); + } + + // Remove mobile download screen if user has sync enabled + if (lazy.usesFirefoxSync && lazy.mobileDevices > 0) { + removeScreens(screen => screen.id === "UPGRADE_MOBILE_DOWNLOAD"); + } else { + prepareMobileDownload(); + } + + return message; + }, +}; + +const EXPORTED_SYMBOLS = ["OnboardingMessageProvider"]; diff --git a/browser/components/newtab/lib/PageEventManager.sys.mjs b/browser/components/newtab/lib/PageEventManager.sys.mjs new file mode 100644 index 0000000000..7b45a5e070 --- /dev/null +++ b/browser/components/newtab/lib/PageEventManager.sys.mjs @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Methods for setting up and tearing down page event listeners. These are used + * to dismiss Feature Callouts when the callout's anchor element is clicked. + */ +export class PageEventManager { + /** + * A set of parameters defining a page event listener. + * @typedef {Object} PageEventListenerParams + * @property {String} type Event type string e.g. `click` + * @property {String} selectors Target selector, e.g. `tag.class, #id[attr]` + * @property {PageEventListenerOptions} [options] addEventListener options + * + * @typedef {Object} PageEventListenerOptions + * @property {Boolean} [capture] Use event capturing phase? + * @property {Boolean} [once] Remove listener after first event? + * @property {Boolean} [preventDefault] Inverted value for `passive` option + */ + + /** + * Maps event listener params to their abort controllers. + * @type {Map<PageEventListenerParams, AbortController>} + */ + _listeners = new Map(); + + /** + * @param {Document} doc The document to look for event targets in + */ + constructor(doc) { + this.doc = doc; + } + + /** + * Adds a page event listener. + * @param {PageEventListenerParams} params + * @param {Function} callback Function to call when event is triggered + */ + on(params, callback) { + if (this._listeners.has(params)) { + return; + } + const { type, selectors, options = {} } = params; + const controller = new AbortController(); + const opt = { + capture: !!options.capture, + passive: !options.preventDefault, + signal: controller.signal, + }; + const targets = this.doc.querySelectorAll(selectors); + for (const target of targets) { + target.addEventListener(type, callback, opt); + } + this._listeners.set(params, controller); + } + + /** + * Removes a page event listener. + * @param {PageEventListenerParams} params + */ + off(params) { + const controller = this._listeners.get(params); + if (!controller) { + return; + } + controller.abort(); + this._listeners.delete(params); + } + + /** + * Adds a page event listener that is removed after the first event. + * @param {PageEventListenerParams} params + * @param {Function} callback Function to call when event is triggered + */ + once(params, callback) { + const wrappedCallback = (...args) => { + this.off(params); + callback(...args); + }; + this.on(params, wrappedCallback); + } + + /** + * Removes all page event listeners. + */ + clear() { + for (const controller of this._listeners.values()) { + controller.abort(); + } + this._listeners.clear(); + } +} diff --git a/browser/components/newtab/lib/PanelTestProvider.sys.mjs b/browser/components/newtab/lib/PanelTestProvider.sys.mjs new file mode 100644 index 0000000000..b5ff59290a --- /dev/null +++ b/browser/components/newtab/lib/PanelTestProvider.sys.mjs @@ -0,0 +1,626 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const TWO_DAYS = 2 * 24 * 3600 * 1000; + +const MESSAGES = () => [ + { + id: "WNP_THANK_YOU", + template: "update_action", + content: { + action: { + id: "moments-wnp", + data: { + url: "https://www.mozilla.org/%LOCALE%/etc/firefox/retention/thank-you-a/", + expireDelta: TWO_DAYS, + }, + }, + }, + trigger: { id: "momentsUpdate" }, + }, + { + id: "WHATS_NEW_FINGERPRINTER_COUNTER_ALT", + template: "whatsnew_panel_message", + order: 6, + content: { + bucket_id: "WHATS_NEW_72", + published_date: 1574776601000, + title: "Title", + icon_url: + "chrome://activity-stream/content/data/content/assets/protection-report-icon.png", + icon_alt: { string_id: "cfr-badge-reader-label-newfeature" }, + body: "Message body", + link_text: "Click here", + cta_url: "about:blank", + cta_type: "OPEN_PROTECTION_REPORT", + }, + targeting: `firefoxVersion >= 72`, + trigger: { id: "whatsNewPanelOpened" }, + }, + { + id: "WHATS_NEW_70_1", + template: "whatsnew_panel_message", + order: 3, + content: { + bucket_id: "WHATS_NEW_70_1", + published_date: 1560969794394, + title: "Protection Is Our Focus", + icon_url: + "chrome://activity-stream/content/data/content/assets/whatsnew-send-icon.png", + icon_alt: "Firefox Send Logo", + body: "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.", + cta_url: "https://blog.mozilla.org/", + cta_type: "OPEN_URL", + }, + targeting: `firefoxVersion > 69`, + trigger: { id: "whatsNewPanelOpened" }, + }, + { + id: "WHATS_NEW_70_2", + template: "whatsnew_panel_message", + order: 1, + content: { + bucket_id: "WHATS_NEW_70_1", + published_date: 1560969794394, + title: "Another thing new in Firefox 70", + body: "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.", + link_text: "Learn more on our blog", + cta_url: "https://blog.mozilla.org/", + cta_type: "OPEN_URL", + }, + targeting: `firefoxVersion > 69`, + trigger: { id: "whatsNewPanelOpened" }, + }, + { + id: "WHATS_NEW_SEARCH_SHORTCUTS_84", + template: "whatsnew_panel_message", + order: 2, + content: { + bucket_id: "WHATS_NEW_SEARCH_SHORTCUTS_84", + published_date: 1560969794394, + title: "Title", + icon_url: "chrome://global/skin/icons/check.svg", + icon_alt: "", + body: "Message content", + cta_url: + "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/search-shortcuts", + cta_type: "OPEN_URL", + link_text: "Click here", + }, + targeting: "firefoxVersion >= 84", + trigger: { + id: "whatsNewPanelOpened", + }, + }, + { + id: "WHATS_NEW_PIONEER_82", + template: "whatsnew_panel_message", + order: 1, + content: { + bucket_id: "WHATS_NEW_PIONEER_82", + published_date: 1603152000000, + title: "Put your data to work for a better internet", + body: "Contribute your data to Mozilla's Pioneer program to help researchers understand pressing technology issues like misinformation, data privacy, and ethical AI.", + cta_url: "about:blank", + cta_where: "tab", + cta_type: "OPEN_ABOUT_PAGE", + link_text: "Join Pioneer", + }, + targeting: "firefoxVersion >= 82", + trigger: { + id: "whatsNewPanelOpened", + }, + }, + { + id: "WHATS_NEW_MEDIA_SESSION_82", + template: "whatsnew_panel_message", + order: 3, + content: { + bucket_id: "WHATS_NEW_MEDIA_SESSION_82", + published_date: 1603152000000, + title: "Title", + body: "Message content", + cta_url: + "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/media-keyboard-control", + cta_type: "OPEN_URL", + link_text: "Click here", + }, + targeting: "firefoxVersion >= 82", + trigger: { + id: "whatsNewPanelOpened", + }, + }, + { + id: "WHATS_NEW_69_1", + template: "whatsnew_panel_message", + order: 1, + content: { + bucket_id: "WHATS_NEW_69_1", + published_date: 1557346235089, + title: "Something new in Firefox 69", + body: "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.", + link_text: "Learn more on our blog", + cta_url: "https://blog.mozilla.org/", + cta_type: "OPEN_URL", + }, + targeting: `firefoxVersion > 68`, + trigger: { id: "whatsNewPanelOpened" }, + }, + { + id: "PERSONALIZED_CFR_MESSAGE", + template: "cfr_doorhanger", + groups: ["cfr"], + content: { + layout: "icon_and_message", + category: "cfrFeatures", + bucket_id: "PERSONALIZED_CFR_MESSAGE", + notification_text: "Personalized CFR Recommendation", + heading_text: { string_id: "cfr-doorhanger-bookmark-fxa-header" }, + info_icon: { + label: { + attributes: { + tooltiptext: { string_id: "cfr-doorhanger-fxa-close-btn-tooltip" }, + }, + }, + sumo_path: "https://example.com", + }, + text: { string_id: "cfr-doorhanger-bookmark-fxa-body" }, + icon: "chrome://branding/content/icon64.png", + icon_class: "cfr-doorhanger-large-icon", + persistent_doorhanger: true, + buttons: { + primary: { + label: { string_id: "cfr-doorhanger-milestone-ok-button" }, + action: { + type: "OPEN_URL", + data: { + args: "https://send.firefox.com/login/?utm_source=activity-stream&entrypoint=activity-stream-cfr-pdf", + where: "tabshifted", + }, + }, + }, + secondary: [ + { + label: { string_id: "cfr-doorhanger-extension-cancel-button" }, + action: { type: "CANCEL" }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-never-show-recommendation", + }, + }, + { + label: { + string_id: "cfr-doorhanger-extension-manage-settings-button", + }, + action: { + type: "OPEN_PREFERENCES_PAGE", + data: { category: "general-cfrfeatures" }, + }, + }, + ], + }, + }, + targeting: "scores.PERSONALIZED_CFR_MESSAGE.score > scoreThreshold", + trigger: { + id: "openURL", + patterns: ["*://*/*.pdf"], + }, + }, + { + id: "MULTISTAGE_SPOTLIGHT_MESSAGE", + groups: ["panel-test-provider"], + template: "spotlight", + content: { + id: "MULTISTAGE_SPOTLIGHT_MESSAGE", + template: "multistage", + backdrop: "transparent", + transitions: true, + screens: [ + { + id: "AW_PIN_FIREFOX", + content: { + has_noodles: true, + title: { + string_id: "mr2022-onboarding-welcome-pin-header", + }, + logo: { + imageURL: "chrome://browser/content/callout-tab-pickup.svg", + darkModeImageURL: + "chrome://browser/content/callout-tab-pickup-dark.svg", + reducedMotionImageURL: + "chrome://activity-stream/content/data/content/assets/glyph-pin-16.svg", + darkModeReducedMotionImageURL: + "chrome://activity-stream/content/data/content/assets/firefox.svg", + alt: "sample alt text", + }, + hero_text: { + string_id: "fx100-thank-you-hero-text", + }, + help_text: { + text: "Some sample help text", + }, + primary_button: { + label: { + string_id: "mr2022-onboarding-pin-primary-button-label", + }, + action: { + navigate: true, + type: "PIN_FIREFOX_TO_TASKBAR", + }, + }, + secondary_button: { + label: { + string_id: "onboarding-not-now-button-label", + }, + action: { + navigate: true, + }, + }, + dismiss_button: { + action: { + dismiss: true, + }, + }, + }, + }, + { + id: "AW_SET_DEFAULT", + content: { + has_noodles: true, + logo: { + imageURL: "chrome://browser/content/logos/vpn-promo-logo.svg", + height: "100px", + }, + title: { + fontSize: "36px", + fontWeight: 276, + string_id: "mr2022-onboarding-set-default-title", + }, + subtitle: { + string_id: "mr2022-onboarding-set-default-subtitle", + }, + primary_button: { + label: { + string_id: "mr2022-onboarding-set-default-primary-button-label", + }, + action: { + navigate: true, + type: "SET_DEFAULT_BROWSER", + }, + }, + secondary_button: { + label: { + string_id: "onboarding-not-now-button-label", + }, + action: { + navigate: true, + }, + }, + }, + }, + { + id: "BACKGROUND_IMAGE", + content: { + background: "#000", + text_color: "light", + progress_bar: true, + logo: { + imageURL: + "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/a3c640c8-7594-4bb2-bc18-8b4744f3aaf2.gif", + }, + title: "A dialog with a background", + subtitle: + "The text color is configurable and a progress bar style step indicator is used", + primary_button: { + label: "Continue", + action: { + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "onboarding-not-now-button-label", + }, + action: { + navigate: true, + }, + }, + }, + }, + { + id: "BACKGROUND_COLOR", + content: { + background: "white", + progress_bar: true, + logo: { + height: "200px", + imageURL: "", + }, + title: { + fontSize: "36px", + fontWeight: 276, + raw: "Peace of mind.", + }, + title_style: "fancy shine", + text_color: "dark", + subtitle: "Using progress bar style step indicator", + primary_button: { + label: "Continue", + action: { + navigate: true, + }, + }, + secondary_button: { + label: { + string_id: "onboarding-not-now-button-label", + }, + action: { + navigate: true, + }, + }, + }, + }, + ], + }, + frequency: { lifetime: 3 }, + trigger: { id: "defaultBrowserCheck" }, + }, + { + id: "PB_FOCUS_PROMO", + groups: ["panel-test-provider"], + template: "spotlight", + content: { + template: "multistage", + backdrop: "transparent", + screens: [ + { + id: "PBM_FIREFOX_FOCUS", + order: 0, + content: { + logo: { + imageURL: "chrome://browser/content/assets/focus-logo.svg", + height: "48px", + }, + title: { + string_id: "spotlight-focus-promo-title", + }, + subtitle: { + string_id: "spotlight-focus-promo-subtitle", + }, + dismiss_button: { + action: { + dismiss: true, + }, + }, + ios: { + action: { + data: { + args: "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fus%2Fapp%2Ffirefox-focus-privacy-browser%2Fid1055677337", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + android: { + action: { + data: { + args: "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.focus", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + email_link: { + action: { + data: { + args: "https://mozilla.org", + where: "tabshifted", + }, + type: "OPEN_URL", + navigate: true, + }, + }, + tiles: { + type: "mobile_downloads", + data: { + QR_code: { + image_url: + "chrome://browser/content/assets/focus-qr-code.svg", + alt_text: { + string_id: "spotlight-focus-promo-qr-code", + }, + }, + email: { + link_text: "Email yourself a link", + }, + marketplace_buttons: ["ios", "android"], + }, + }, + }, + }, + ], + }, + trigger: { id: "defaultBrowserCheck" }, + }, + { + id: "PB_NEWTAB_VPN_PROMO", + template: "pb_newtab", + content: { + promoEnabled: true, + promoType: "VPN", + infoEnabled: true, + infoBody: "fluent:about-private-browsing-info-description-private-window", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitleEnabled: false, + promoLinkType: "button", + promoLinkText: "fluent:about-private-browsing-prominent-cta", + promoSectionStyle: "below-search", + promoHeader: "fluent:about-private-browsing-get-privacy", + promoTitle: "fluent:about-private-browsing-hide-activity-1", + promoTitleEnabled: true, + promoImageLarge: "chrome://browser/content/assets/moz-vpn.svg", + promoButton: { + action: { + type: "OPEN_URL", + data: { + args: "https://vpn.mozilla.org/", + }, + }, + }, + }, + groups: ["panel-test-provider"], + targeting: "region != 'CN' && !hasActiveEnterprisePolicies", + frequency: { lifetime: 3 }, + }, + { + id: "PB_PIN_PROMO", + template: "pb_newtab", + groups: ["pbNewtab"], + content: { + infoBody: "fluent:about-private-browsing-info-description-simplified", + infoEnabled: true, + infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg", + infoLinkText: "fluent:about-private-browsing-learn-more-link", + infoTitle: "", + infoTitleEnabled: false, + promoEnabled: true, + promoType: "PIN", + promoHeader: "Private browsing freedom in one click", + promoImageLarge: + "chrome://browser/content/assets/private-promo-asset.svg", + promoLinkText: "Pin To Taskbar", + promoLinkType: "button", + promoSectionStyle: "below-search", + promoTitle: + "No saved cookies or history, right from your desktop. Browse like no one’s watching.", + promoTitleEnabled: true, + promoButton: { + action: { + type: "MULTI_ACTION", + data: { + actions: [ + { + type: "SET_PREF", + data: { + pref: { + name: "browser.privateWindowSeparation.enabled", + value: true, + }, + }, + }, + { + type: "PIN_FIREFOX_TO_TASKBAR", + }, + { + type: "BLOCK_MESSAGE", + data: { + id: "PB_PIN_PROMO", + }, + }, + { + type: "OPEN_ABOUT_PAGE", + data: { args: "privatebrowsing", where: "current" }, + }, + ], + }, + }, + }, + }, + priority: 3, + frequency: { + custom: [ + { + cap: 3, + period: 604800000, // Max 3 per week + }, + ], + lifetime: 12, + }, + targeting: + "region != 'CN' && !hasActiveEnterprisePolicies && doesAppNeedPin", + }, + { + id: "TEST_TOAST_NOTIFICATION1", + weight: 100, + template: "toast_notification", + content: { + title: { + string_id: "cfr-doorhanger-bookmark-fxa-header", + }, + body: "Body", + image_url: + "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/a3c640c8-7594-4bb2-bc18-8b4744f3aaf2.gif", + launch_url: "https://mozilla.org", + requireInteraction: true, + actions: [ + { + action: "dismiss", + title: "Dismiss", + windowsSystemActivationType: true, + }, + { + action: "snooze", + title: "Snooze", + windowsSystemActivationType: true, + }, + { action: "callback", title: "Callback" }, + ], + tag: "test_toast_notification", + }, + groups: ["panel-test-provider"], + targeting: "!hasActiveEnterprisePolicies", + trigger: { id: "backgroundTaskMessage" }, + frequency: { lifetime: 3 }, + }, + { + id: "MR2022_BACKGROUND_UPDATE_TOAST_NOTIFICATION", + weight: 100, + template: "toast_notification", + content: { + title: { + string_id: "mr2022-background-update-toast-title", + }, + body: { + string_id: "mr2022-background-update-toast-text", + }, + image_url: + "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/673d2808-e5d8-41b9-957e-f60d53233b97.png", + requireInteraction: true, + actions: [ + { + action: "open", + title: { + string_id: "mr2022-background-update-toast-primary-button-label", + }, + }, + { + action: "snooze", + windowsSystemActivationType: true, + title: { + string_id: "mr2022-background-update-toast-secondary-button-label", + }, + }, + ], + tag: "mr2022_background_update", + }, + groups: ["panel-test-provider"], + targeting: "!hasActiveEnterprisePolicies", + trigger: { id: "backgroundTaskMessage" }, + frequency: { lifetime: 3 }, + }, +]; + +export const PanelTestProvider = { + getMessages() { + return Promise.resolve( + MESSAGES().map(message => ({ + ...message, + targeting: `providerCohorts.panel_local_testing == "SHOW_TEST"`, + })) + ); + }, +}; diff --git a/browser/components/newtab/lib/PersistentCache.sys.mjs b/browser/components/newtab/lib/PersistentCache.sys.mjs new file mode 100644 index 0000000000..4a27f26a30 --- /dev/null +++ b/browser/components/newtab/lib/PersistentCache.sys.mjs @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * A file (disk) based persistent cache of a JSON serializable object. + */ +export class PersistentCache { + /** + * Create a cache object based on a name. + * + * @param {string} name Name of the cache. It will be used to create the filename. + * @param {boolean} preload (optional). Whether the cache should be preloaded from file. Defaults to false. + */ + constructor(name, preload = false) { + this.name = name; + this._filename = `activity-stream.${name}.json`; + if (preload) { + this._load(); + } + } + + /** + * Set a value to be cached with the specified key. + * + * @param {string} key The cache key. + * @param {object} value The data to be cached. + */ + async set(key, value) { + const data = await this._load(); + data[key] = value; + await this._persist(data); + } + + /** + * Get a value from the cache. + * + * @param {string} key (optional) The cache key. If not provided, we return the full cache. + * @returns {object} The cached data. + */ + async get(key) { + const data = await this._load(); + return key ? data[key] : data; + } + + /** + * Load the cache into memory if it isn't already. + */ + _load() { + return ( + this._cache || + // eslint-disable-next-line no-async-promise-executor + (this._cache = new Promise(async (resolve, reject) => { + let filepath; + try { + filepath = PathUtils.join(PathUtils.localProfileDir, this._filename); + } catch (error) { + reject(error); + return; + } + + let data = {}; + try { + data = await IOUtils.readJSON(filepath); + } catch (error) { + if ( + // isInstance() is not available in node unit test. It should be safe to use instanceof as it's directly from IOUtils. + // eslint-disable-next-line mozilla/use-isInstance + !(error instanceof DOMException) || + error.name !== "NotFoundError" + ) { + console.error( + `Failed to parse ${this._filename}: ${error.message}` + ); + } + } + + resolve(data); + })) + ); + } + + /** + * Persist the cache to file. + */ + async _persist(data) { + const filepath = PathUtils.join(PathUtils.localProfileDir, this._filename); + await IOUtils.writeJSON(filepath, data, { + tmpPath: `${filepath}.tmp`, + }); + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm new file mode 100644 index 0000000000..cc625076ba --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +// We load this into a worker using importScripts, and in tests using import. +// We use var to avoid name collision errors. +// eslint-disable-next-line no-var +var EXPORTED_SYMBOLS = ["NaiveBayesTextTagger"]; + +const NaiveBayesTextTagger = class NaiveBayesTextTagger { + constructor(model, toksToTfIdfVector) { + this.model = model; + this.toksToTfIdfVector = toksToTfIdfVector; + } + + /** + * Determines if the tokenized text belongs to class according to binary naive Bayes + * classifier. Returns an object containing the class label ("label"), and + * the log probability ("logProb") that the text belongs to that class. If + * the positive class is more likely, then "label" is the positive class + * label. If the negative class is matched, then "label" is set to null. + */ + tagTokens(tokens) { + let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs); + + let bestLogProb = null; + let bestClassId = -1; + let bestClassLabel = null; + let logSumExp = 0.0; // will be P(x). Used to create a proper probability + for (let classId = 0; classId < this.model.classes.length; classId++) { + let classModel = this.model.classes[classId]; + let classLogProb = classModel.log_prior; + + // dot fv with the class model + for (let pair of Object.values(fv)) { + let [termId, tfidf] = pair; + classLogProb += tfidf * classModel.feature_log_probs[termId]; + } + + if (bestLogProb === null || classLogProb > bestLogProb) { + bestLogProb = classLogProb; + bestClassId = classId; + } + logSumExp += Math.exp(classLogProb); + } + + // now normalize the probability by dividing by P(x) + logSumExp = Math.log(logSumExp); + bestLogProb -= logSumExp; + if (bestClassId === this.model.positive_class_id) { + bestClassLabel = this.model.positive_class_label; + } else { + bestClassLabel = null; + } + + let confident = + bestClassId === this.model.positive_class_id && + bestLogProb > this.model.positive_class_threshold_log_prob; + return { + label: bestClassLabel, + logProb: bestLogProb, + confident, + }; + } +}; diff --git a/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm new file mode 100644 index 0000000000..639c92b6e4 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm @@ -0,0 +1,65 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +// We load this into a worker using importScripts, and in tests using import. +// We use var to avoid name collision errors. +// eslint-disable-next-line no-var +var EXPORTED_SYMBOLS = ["NmfTextTagger"]; + +const NmfTextTagger = class NmfTextTagger { + constructor(model, toksToTfIdfVector) { + this.model = model; + this.toksToTfIdfVector = toksToTfIdfVector; + } + + /** + * A multiclass classifier that scores tokenized text against several classes through + * inference of a nonnegative matrix factorization of TF-IDF vectors and + * class labels. Returns a map of class labels as string keys to scores. + * (Higher is more confident.) All classes get scored, so it is up to + * consumer of this data determine what classes are most valuable. + */ + tagTokens(tokens) { + let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs); + let fve = Object.values(fv); + + // normalize by the sum of the vector + let sum = 0.0; + for (let pair of fve) { + // eslint-disable-next-line prefer-destructuring + sum += pair[1]; + } + for (let i = 0; i < fve.length; i++) { + // eslint-disable-next-line prefer-destructuring + fve[i][1] /= sum; + } + + // dot the document with each topic vector so that we can transform it into + // the latent space + let toksInLatentSpace = []; + for (let topicVect of this.model.topic_word) { + let fvDotTwv = 0; + // dot fv with each topic word vector + for (let pair of fve) { + let [termId, tfidf] = pair; + fvDotTwv += tfidf * topicVect[termId]; + } + toksInLatentSpace.push(fvDotTwv); + } + + // now project toksInLatentSpace back into class space + let predictions = {}; + Object.keys(this.model.document_topic).forEach(topic => { + let score = 0; + for (let i = 0; i < toksInLatentSpace.length; i++) { + score += toksInLatentSpace[i] * this.model.document_topic[topic][i]; + } + predictions[topic] = score; + }); + + return predictions; + } +}; diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm new file mode 100644 index 0000000000..c1f54408f2 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm @@ -0,0 +1,282 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + Utils: "resource://services-settings/Utils.sys.mjs", +}); + +const { BasePromiseWorker } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseWorker.sys.mjs" +); + +const RECIPE_NAME = "personality-provider-recipe"; +const MODELS_NAME = "personality-provider-models"; + +class PersonalityProvider { + constructor(modelKeys) { + this.modelKeys = modelKeys; + this.onSync = this.onSync.bind(this); + this.setup(); + } + + setScores(scores) { + this.scores = scores || {}; + this.interestConfig = this.scores.interestConfig; + this.interestVector = this.scores.interestVector; + } + + get personalityProviderWorker() { + if (this._personalityProviderWorker) { + return this._personalityProviderWorker; + } + + this._personalityProviderWorker = new BasePromiseWorker( + "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorker.js" + ); + + return this._personalityProviderWorker; + } + + get baseAttachmentsURL() { + // Returning a promise, so we can have an async getter. + return this._getBaseAttachmentsURL(); + } + + async _getBaseAttachmentsURL() { + if (this._baseAttachmentsURL) { + return this._baseAttachmentsURL; + } + const server = lazy.Utils.SERVER_URL; + const serverInfo = await ( + await fetch(`${server}/`, { + credentials: "omit", + }) + ).json(); + const { + capabilities: { + attachments: { base_url }, + }, + } = serverInfo; + this._baseAttachmentsURL = base_url; + return this._baseAttachmentsURL; + } + + setup() { + this.setupSyncAttachment(RECIPE_NAME); + this.setupSyncAttachment(MODELS_NAME); + } + + teardown() { + this.teardownSyncAttachment(RECIPE_NAME); + this.teardownSyncAttachment(MODELS_NAME); + if (this._personalityProviderWorker) { + this._personalityProviderWorker.terminate(); + } + } + + setupSyncAttachment(collection) { + lazy.RemoteSettings(collection).on("sync", this.onSync); + } + + teardownSyncAttachment(collection) { + lazy.RemoteSettings(collection).off("sync", this.onSync); + } + + onSync(event) { + this.personalityProviderWorker.post("onSync", [event]); + } + + /** + * Gets contents of the attachment if it already exists on file, + * and if not attempts to download it. + */ + getAttachment(record) { + return this.personalityProviderWorker.post("getAttachment", [record]); + } + + /** + * Returns a Recipe from remote settings to be consumed by a RecipeExecutor. + * A Recipe is a set of instructions on how to processes a RecipeExecutor. + */ + async getRecipe() { + if (!this.recipes || !this.recipes.length) { + const result = await lazy.RemoteSettings(RECIPE_NAME).get(); + this.recipes = await Promise.all( + result.map(async record => ({ + ...(await this.getAttachment(record)), + recordKey: record.key, + })) + ); + } + return this.recipes[0]; + } + + /** + * Grabs a slice of browse history for building a interest vector + */ + async fetchHistory(columns, beginTimeSecs, endTimeSecs) { + let sql = `SELECT url, title, visit_count, frecency, last_visit_date, description + FROM moz_places + WHERE last_visit_date >= ${beginTimeSecs * 1000000} + AND last_visit_date < ${endTimeSecs * 1000000}`; + columns.forEach(requiredColumn => { + sql += ` AND IFNULL(${requiredColumn}, '') <> ''`; + }); + sql += " LIMIT 30000"; + + const { activityStreamProvider } = lazy.NewTabUtils; + const history = await activityStreamProvider.executePlacesQuery(sql, { + columns, + params: {}, + }); + + return history; + } + + /** + * Handles setup and metrics of history fetch. + */ + async getHistory() { + let endTimeSecs = new Date().getTime() / 1000; + let beginTimeSecs = endTimeSecs - this.interestConfig.history_limit_secs; + if ( + !this.interestConfig || + !this.interestConfig.history_required_fields || + !this.interestConfig.history_required_fields.length + ) { + return []; + } + let history = await this.fetchHistory( + this.interestConfig.history_required_fields, + beginTimeSecs, + endTimeSecs + ); + + return history; + } + + async setBaseAttachmentsURL() { + await this.personalityProviderWorker.post("setBaseAttachmentsURL", [ + await this.baseAttachmentsURL, + ]); + } + + async setInterestConfig() { + this.interestConfig = this.interestConfig || (await this.getRecipe()); + await this.personalityProviderWorker.post("setInterestConfig", [ + this.interestConfig, + ]); + } + + async setInterestVector() { + await this.personalityProviderWorker.post("setInterestVector", [ + this.interestVector, + ]); + } + + async fetchModels() { + const models = await lazy.RemoteSettings(MODELS_NAME).get(); + return this.personalityProviderWorker.post("fetchModels", [models]); + } + + async generateTaggers() { + await this.personalityProviderWorker.post("generateTaggers", [ + this.modelKeys, + ]); + } + + async generateRecipeExecutor() { + await this.personalityProviderWorker.post("generateRecipeExecutor"); + } + + async createInterestVector() { + const history = await this.getHistory(); + + const interestVectorResult = await this.personalityProviderWorker.post( + "createInterestVector", + [history] + ); + + return interestVectorResult; + } + + async init(callback) { + await this.setBaseAttachmentsURL(); + await this.setInterestConfig(); + if (!this.interestConfig) { + return; + } + + // We always generate a recipe executor, no cache used here. + // This is because the result of this is an object with + // functions (taggers) so storing it in cache is not possible. + // Thus we cannot use it to rehydrate anything. + const fetchModelsResult = await this.fetchModels(); + // If this fails, log an error and return. + if (!fetchModelsResult.ok) { + return; + } + await this.generateTaggers(); + await this.generateRecipeExecutor(); + + // If we don't have a cached vector, create a new one. + if (!this.interestVector) { + const interestVectorResult = await this.createInterestVector(); + // If that failed, log an error and return. + if (!interestVectorResult.ok) { + return; + } + this.interestVector = interestVectorResult.interestVector; + } + + // This happens outside the createInterestVector call above, + // because create can be skipped if rehydrating from cache. + // In that case, the interest vector is provided and not created, so we just set it. + await this.setInterestVector(); + + this.initialized = true; + if (callback) { + callback(); + } + } + + async calculateItemRelevanceScore(pocketItem) { + if (!this.initialized) { + return pocketItem.item_score || 1; + } + const itemRelevanceScore = await this.personalityProviderWorker.post( + "calculateItemRelevanceScore", + [pocketItem] + ); + if (!itemRelevanceScore) { + return -1; + } + const { scorableItem, rankingVector } = itemRelevanceScore; + // Put the results on the item for debugging purposes. + pocketItem.scorableItem = scorableItem; + pocketItem.rankingVector = rankingVector; + return rankingVector.score; + } + + /** + * Returns an object holding the personalization scores of this provider instance. + */ + getScores() { + return { + // We cannot return taggers here. + // What we return here goes into persistent cache, and taggers have functions on it. + // If we attempted to save taggers into persistent cache, it would store it to disk, + // and the next time we load it, it would start thowing function is not defined. + interestConfig: this.interestConfig, + interestVector: this.interestVector, + }; + } +} + +const EXPORTED_SYMBOLS = ["PersonalityProvider"]; diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js new file mode 100644 index 0000000000..3ed118857e --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env mozilla/chrome-worker */ + +"use strict"; + +// Order of these are important. +/* import-globals-from /toolkit/components/workerloader/require.js */ +/* import-globals-from Tokenize.jsm */ +/* import-globals-from NaiveBayesTextTagger.jsm */ +/* import-globals-from NmfTextTagger.jsm */ +/* import-globals-from RecipeExecutor.jsm */ +/* import-globals-from PersonalityProviderWorkerClass.jsm */ +importScripts( + "resource://gre/modules/workers/require.js", + "resource://activity-stream/lib/PersonalityProvider/Tokenize.jsm", + "resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.jsm", + "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm" +); + +const PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js"); + +const personalityProviderWorker = new PersonalityProviderWorker(); + +// This is boiler plate worker stuff that connects it to the main thread PromiseWorker. +const worker = new PromiseWorker.AbstractWorker(); +worker.dispatch = function (method, args = []) { + return personalityProviderWorker[method](...args); +}; +worker.postMessage = function (message, ...transfers) { + self.postMessage(message, ...transfers); +}; +worker.close = function () { + self.close(); +}; + +self.addEventListener("message", msg => worker.handleMessage(msg)); +self.addEventListener("unhandledrejection", function (error) { + throw error.reason; +}); diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm new file mode 100644 index 0000000000..e761f827d2 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm @@ -0,0 +1,311 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +// PersonalityProviderWorker.js imports the following scripts before this. +/* import-globals-from Tokenize.jsm */ +/* import-globals-from NaiveBayesTextTagger.jsm */ +/* import-globals-from NmfTextTagger.jsm */ +/* import-globals-from RecipeExecutor.jsm */ + +// We load this into a worker using importScripts, and in tests using import. +// We use var to avoid name collision errors. +// eslint-disable-next-line no-var +var EXPORTED_SYMBOLS = ["PersonalityProviderWorker"]; + +// A helper function to create a hash out of a file. +async function _getFileHash(filepath) { + const data = await IOUtils.read(filepath); + // File is an instance of Uint8Array + const digest = await crypto.subtle.digest("SHA-256", data); + const uint8 = new Uint8Array(digest); + // return the two-digit hexadecimal code for a byte + const toHex = b => b.toString(16).padStart(2, "0"); + return Array.from(uint8, toHex).join(""); +} + +/** + * V2 provider builds and ranks an interest profile (also called an “interest vector”) off the browse history. + * This allows Firefox to classify pages into topics, by examining the text found on the page. + * It does this by looking at the history text content, title, and description. + */ +const PersonalityProviderWorker = class PersonalityProviderWorker { + async getPersonalityProviderDir() { + const personalityProviderDir = PathUtils.join( + await PathUtils.getLocalProfileDir(), + "personality-provider" + ); + + // Cache this so we don't need to await again. + this.getPersonalityProviderDir = () => + Promise.resolve(personalityProviderDir); + return personalityProviderDir; + } + + setBaseAttachmentsURL(url) { + this.baseAttachmentsURL = url; + } + + setInterestConfig(interestConfig) { + this.interestConfig = interestConfig; + } + + setInterestVector(interestVector) { + this.interestVector = interestVector; + } + + onSync(event) { + const { + data: { created, updated, deleted }, + } = event; + // Remove every removed attachment. + const toRemove = deleted.concat(updated.map(u => u.old)); + toRemove.forEach(record => this.deleteAttachment(record)); + + // Download every new/updated attachment. + const toDownload = created.concat(updated.map(u => u.new)); + // maybeDownloadAttachment is async but we don't care inside onSync. + toDownload.forEach(record => this.maybeDownloadAttachment(record)); + } + + /** + * Attempts to download the attachment, but only if it doesn't already exist. + */ + async maybeDownloadAttachment(record, retries = 3) { + const { + attachment: { filename, hash, size }, + } = record; + await IOUtils.makeDirectory(await this.getPersonalityProviderDir()); + const localFilePath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + let retry = 0; + while ( + retry++ < retries && + // exists is an issue for perf because I might not need to call it. + (!(await IOUtils.exists(localFilePath)) || + (await IOUtils.stat(localFilePath)).size !== size || + (await _getFileHash(localFilePath)) !== hash) + ) { + await this._downloadAttachment(record); + } + } + + /** + * Downloads the attachment to disk assuming the dir already exists + * and any existing files matching the filename are clobbered. + */ + async _downloadAttachment(record) { + const { + attachment: { location, filename }, + } = record; + const remoteFilePath = this.baseAttachmentsURL + location; + const localFilePath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + const xhr = new XMLHttpRequest(); + // Set false here for a synchronous request, because we're in a worker. + xhr.open("GET", remoteFilePath, false); + xhr.setRequestHeader("Accept-Encoding", "gzip"); + xhr.responseType = "arraybuffer"; + xhr.withCredentials = false; + xhr.send(null); + + if (xhr.status !== 200) { + console.error(`Failed to fetch ${remoteFilePath}: ${xhr.statusText}`); + return; + } + + const buffer = xhr.response; + const bytes = new Uint8Array(buffer); + + await IOUtils.write(localFilePath, bytes, { + tmpPath: `${localFilePath}.tmp`, + }); + } + + async deleteAttachment(record) { + const { + attachment: { filename }, + } = record; + await IOUtils.makeDirectory(await this.getPersonalityProviderDir()); + const path = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + await IOUtils.remove(path, { ignoreAbsent: true }); + // Cleanup the directory if it is empty, do nothing if it is not empty. + try { + await IOUtils.remove(await this.getPersonalityProviderDir(), { + ignoreAbsent: true, + }); + } catch (e) { + // This is likely because the directory is not empty, so we don't care. + } + } + + /** + * Gets contents of the attachment if it already exists on file, + * and if not attempts to download it. + */ + async getAttachment(record) { + const { + attachment: { filename }, + } = record; + const filepath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + try { + await this.maybeDownloadAttachment(record); + return await IOUtils.readJSON(filepath); + } catch (error) { + console.error(`Failed to load ${filepath}: ${error.message}`); + } + return {}; + } + + async fetchModels(models) { + this.models = await Promise.all( + models.map(async record => ({ + ...(await this.getAttachment(record)), + recordKey: record.key, + })) + ); + if (!this.models.length) { + return { + ok: false, + }; + } + return { + ok: true, + }; + } + + generateTaggers(modelKeys) { + if (!this.taggers) { + let nbTaggers = []; + let nmfTaggers = {}; + + for (let model of this.models) { + if (!modelKeys.includes(model.recordKey)) { + continue; + } + if (model.model_type === "nb") { + nbTaggers.push(new NaiveBayesTextTagger(model, toksToTfIdfVector)); + } else if (model.model_type === "nmf") { + nmfTaggers[model.parent_tag] = new NmfTextTagger( + model, + toksToTfIdfVector + ); + } + } + this.taggers = { nbTaggers, nmfTaggers }; + } + } + + /** + * Sets and generates a Recipe Executor. + * A Recipe Executor is a set of actions that can be consumed by a Recipe. + * The Recipe determines the order and specifics of which the actions are called. + */ + generateRecipeExecutor() { + const recipeExecutor = new RecipeExecutor( + this.taggers.nbTaggers, + this.taggers.nmfTaggers, + tokenize + ); + this.recipeExecutor = recipeExecutor; + } + + /** + * Examines the user's browse history and returns an interest vector that + * describes the topics the user frequently browses. + */ + createInterestVector(history) { + let interestVector = {}; + + for (let historyRec of history) { + let ivItem = this.recipeExecutor.executeRecipe( + historyRec, + this.interestConfig.history_item_builder + ); + if (ivItem === null) { + continue; + } + interestVector = this.recipeExecutor.executeCombinerRecipe( + interestVector, + ivItem, + this.interestConfig.interest_combiner + ); + if (interestVector === null) { + return null; + } + } + + const finalResult = this.recipeExecutor.executeRecipe( + interestVector, + this.interestConfig.interest_finalizer + ); + + return { + ok: true, + interestVector: finalResult, + }; + } + + /** + * Calculates a score of a Pocket item when compared to the user's interest + * vector. Returns the score. Higher scores are better. Assumes this.interestVector + * is populated. + */ + calculateItemRelevanceScore(pocketItem) { + const { personalization_models } = pocketItem; + let scorableItem; + + // If the server provides some models, we can just use them, + // and skip generating them. + if (personalization_models && Object.keys(personalization_models).length) { + scorableItem = { + id: pocketItem.id, + item_tags: personalization_models, + item_score: pocketItem.item_score, + item_sort_id: 1, + }; + } else { + scorableItem = this.recipeExecutor.executeRecipe( + pocketItem, + this.interestConfig.item_to_rank_builder + ); + if (scorableItem === null) { + return null; + } + } + + // We're doing a deep copy on an object. + let rankingVector = JSON.parse(JSON.stringify(this.interestVector)); + + Object.keys(scorableItem).forEach(key => { + rankingVector[key] = scorableItem[key]; + }); + + rankingVector = this.recipeExecutor.executeRecipe( + rankingVector, + this.interestConfig.item_ranker + ); + + if (rankingVector === null) { + return null; + } + + return { scorableItem, rankingVector }; + } +}; diff --git a/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm new file mode 100644 index 0000000000..9dbf8b802d --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm @@ -0,0 +1,1126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +// We load this into a worker using importScripts, and in tests using import. +// We use var to avoid name collision errors. +// eslint-disable-next-line no-var +var EXPORTED_SYMBOLS = ["RecipeExecutor"]; + +/** + * RecipeExecutor is the core feature engineering pipeline for the in-browser + * personalization work. These pipelines are called "recipes". A recipe is an + * array of objects that define a "step" in the recipe. A step is simply an + * object with a field "function" that specifies what is being done in the step + * along with other fields that are semantically defined for that step. + * + * There are two types of recipes "builder" recipes and "combiner" recipes. Builder + * recipes mutate an object until it matches some set of critera. Combiner + * recipes take two objects, (a "left" and a "right"), and specify the steps + * to merge the right object into the left object. + * + * A short nonsense example recipe is: + * [ {"function": "get_url_domain", "path_length": 1, "field": "url", "dest": "url_domain"}, + * {"function": "nb_tag", "fields": ["title", "description"]}, + * {"function": "conditionally_nmf_tag", "fields": ["title", "description"]} ] + * + * Recipes are sandboxed by the fact that the step functions must be explicitly + * allowed. Functions allowed for builder recipes are specifed in the + * RecipeExecutor.ITEM_BUILDER_REGISTRY, while combiner functions are allowed + * in RecipeExecutor.ITEM_COMBINER_REGISTRY . + */ +const RecipeExecutor = class RecipeExecutor { + constructor(nbTaggers, nmfTaggers, tokenize) { + this.ITEM_BUILDER_REGISTRY = { + nb_tag: this.naiveBayesTag, + conditionally_nmf_tag: this.conditionallyNmfTag, + accept_item_by_field_value: this.acceptItemByFieldValue, + tokenize_url: this.tokenizeUrl, + get_url_domain: this.getUrlDomain, + tokenize_field: this.tokenizeField, + copy_value: this.copyValue, + keep_top_k: this.keepTopK, + scalar_multiply: this.scalarMultiply, + elementwise_multiply: this.elementwiseMultiply, + vector_multiply: this.vectorMultiply, + scalar_add: this.scalarAdd, + vector_add: this.vectorAdd, + make_boolean: this.makeBoolean, + allow_fields: this.allowFields, + filter_by_value: this.filterByValue, + l2_normalize: this.l2Normalize, + prob_normalize: this.probNormalize, + set_default: this.setDefault, + lookup_value: this.lookupValue, + copy_to_map: this.copyToMap, + scalar_multiply_tag: this.scalarMultiplyTag, + apply_softmax_tags: this.applySoftmaxTags, + }; + this.ITEM_COMBINER_REGISTRY = { + combiner_add: this.combinerAdd, + combiner_max: this.combinerMax, + combiner_collect_values: this.combinerCollectValues, + }; + this.nbTaggers = nbTaggers; + this.nmfTaggers = nmfTaggers; + this.tokenize = tokenize; + } + + /** + * Determines the type of a field. Valid types are: + * string + * number + * array + * map (strings to anything) + */ + _typeOf(data) { + let t = typeof data; + if (t === "object") { + if (data === null) { + return "null"; + } + if (Array.isArray(data)) { + return "array"; + } + return "map"; + } + return t; + } + + /** + * Returns a scalar, either because it was a constant, or by + * looking it up from the item. Allows for a default value if the lookup + * fails. + */ + _lookupScalar(item, k, dfault) { + if (this._typeOf(k) === "number") { + return k; + } else if ( + this._typeOf(k) === "string" && + k in item && + this._typeOf(item[k]) === "number" + ) { + return item[k]; + } + return dfault; + } + + /** + * Simply appends all the strings from a set fields together. If the field + * is a list, then the cells of the list are append. + */ + _assembleText(item, fields) { + let textArr = []; + for (let field of fields) { + if (field in item) { + let type = this._typeOf(item[field]); + if (type === "string") { + textArr.push(item[field]); + } else if (type === "array") { + for (let ele of item[field]) { + textArr.push(String(ele)); + } + } else { + textArr.push(String(item[field])); + } + } + } + return textArr.join(" "); + } + + /** + * Runs the naive bayes text taggers over a set of text fields. Stores the + * results in new fields: + * nb_tags: a map of text strings to probabilites + * nb_tokens: the tokenized text that was tagged + * + * Config: + * fields: an array containing a list of fields to concatenate and tag + */ + naiveBayesTag(item, config) { + let text = this._assembleText(item, config.fields); + let tokens = this.tokenize(text); + let tags = {}; + let extended_tags = {}; + + for (let nbTagger of this.nbTaggers) { + let result = nbTagger.tagTokens(tokens); + if (result.label !== null && result.confident) { + extended_tags[result.label] = result; + tags[result.label] = Math.exp(result.logProb); + } + } + item.nb_tags = tags; + item.nb_tags_extended = extended_tags; + item.nb_tokens = tokens; + return item; + } + + /** + * Selectively runs NMF text taggers depending on which tags were found + * by the naive bayes taggers. Writes the results in into new fields: + * nmf_tags_parent_weights: map of pareent tags to probabilites of those parent tags + * nmf_tags: map of strings to maps of strings to probabilities + * nmf_tags_parent map of child tags to parent tags + * + * Config: + * Not configurable + */ + conditionallyNmfTag(item, config) { + let nestedNmfTags = {}; + let parentTags = {}; + let parentWeights = {}; + + if (!("nb_tags" in item) || !("nb_tokens" in item)) { + return null; + } + + Object.keys(item.nb_tags).forEach(parentTag => { + let nmfTagger = this.nmfTaggers[parentTag]; + if (nmfTagger !== undefined) { + nestedNmfTags[parentTag] = {}; + parentWeights[parentTag] = item.nb_tags[parentTag]; + let nmfTags = nmfTagger.tagTokens(item.nb_tokens); + Object.keys(nmfTags).forEach(nmfTag => { + nestedNmfTags[parentTag][nmfTag] = nmfTags[nmfTag]; + parentTags[nmfTag] = parentTag; + }); + } + }); + + item.nmf_tags = nestedNmfTags; + item.nmf_tags_parent = parentTags; + item.nmf_tags_parent_weights = parentWeights; + + return item; + } + + /** + * Checks a field's value against another value (either from another field + * or a constant). If the test passes, then the item is emitted, otherwise + * the pipeline is aborted. + * + * Config: + * field Field to read the value to test. Left side of operator. + * op one of ==, !=, <, <=, >, >= + * rhsValue Constant value to compare against. Right side of operator. + * rhsField Field to read value to compare against. Right side of operator. + * + * NOTE: rhsValue takes precidence over rhsField. + */ + acceptItemByFieldValue(item, config) { + if (!(config.field in item)) { + return null; + } + let rhs = null; + if ("rhsValue" in config) { + rhs = config.rhsValue; + } else if ("rhsField" in config && config.rhsField in item) { + rhs = item[config.rhsField]; + } + if (rhs === null) { + return null; + } + + if ( + // eslint-disable-next-line eqeqeq + (config.op === "==" && item[config.field] == rhs) || + // eslint-disable-next-line eqeqeq + (config.op === "!=" && item[config.field] != rhs) || + (config.op === "<" && item[config.field] < rhs) || + (config.op === "<=" && item[config.field] <= rhs) || + (config.op === ">" && item[config.field] > rhs) || + (config.op === ">=" && item[config.field] >= rhs) + ) { + return item; + } + + return null; + } + + /** + * Splits a URL into text-like tokens. + * + * Config: + * field Field containing a URL + * dest Field to write the tokens to as an array of strings + * + * NOTE: Any initial 'www' on the hostname is removed. + */ + tokenizeUrl(item, config) { + if (!(config.field in item)) { + return null; + } + + let url = new URL(item[config.field]); + let domain = url.hostname; + if (domain.startsWith("www.")) { + domain = domain.substring(4); + } + let toks = this.tokenize(domain); + let pathToks = this.tokenize( + decodeURIComponent(url.pathname.replace(/\+/g, " ")) + ); + for (let tok of pathToks) { + toks.push(tok); + } + for (let pair of url.searchParams.entries()) { + let k = this.tokenize(decodeURIComponent(pair[0].replace(/\+/g, " "))); + for (let tok of k) { + toks.push(tok); + } + if (pair[1] !== null && pair[1] !== "") { + let v = this.tokenize(decodeURIComponent(pair[1].replace(/\+/g, " "))); + for (let tok of v) { + toks.push(tok); + } + } + } + item[config.dest] = toks; + + return item; + } + + /** + * Gets the hostname (minus any initial "www." along with the left most + * directories on the path. + * + * Config: + * field Field containing the URL + * dest Field to write the array of strings to + * path_length OPTIONAL (DEFAULT: 0) Number of leftmost subdirectories to include + */ + getUrlDomain(item, config) { + if (!(config.field in item)) { + return null; + } + + let url = new URL(item[config.field]); + let domain = url.hostname.toLocaleLowerCase(); + if (domain.startsWith("www.")) { + domain = domain.substring(4); + } + item[config.dest] = domain; + let pathLength = 0; + if ("path_length" in config) { + pathLength = config.path_length; + } + if (pathLength > 0) { + item[config.dest] += url.pathname + .toLocaleLowerCase() + .split("/") + .slice(0, pathLength + 1) + .join("/"); + } + + return item; + } + + /** + * Splits a field into tokens. + * Config: + * field Field containing a string to tokenize + * dest Field to write the array of strings to + */ + tokenizeField(item, config) { + if (!(config.field in item)) { + return null; + } + + item[config.dest] = this.tokenize(item[config.field]); + + return item; + } + + /** + * Deep copy from one field to another. + * Config: + * src Field to read from + * dest Field to write to + */ + copyValue(item, config) { + if (!(config.src in item)) { + return null; + } + + item[config.dest] = JSON.parse(JSON.stringify(item[config.src])); + + return item; + } + + /** + * Converts a field containing a map of strings to a map of strings + * to numbers, to a map of strings to numbers containing at most k elements. + * This operation is performed by first, promoting all the subkeys up one + * level, and then taking the top (or bottom) k values. + * + * Config: + * field Points to a map of strings to a map of strings to numbers + * k Maximum number of items to keep + * descending OPTIONAL (DEFAULT: True) Sorts score in descending order + * (i.e. keeps maximum) + */ + keepTopK(item, config) { + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, 1048576); + let descending = !("descending" in config) || config.descending !== false; + + // we can't sort by the values in the map, so we have to convert this + // to an array, and then sort. + let sortable = []; + Object.keys(item[config.field]).forEach(outerKey => { + let innerType = this._typeOf(item[config.field][outerKey]); + if (innerType === "map") { + Object.keys(item[config.field][outerKey]).forEach(innerKey => { + sortable.push({ + key: innerKey, + value: item[config.field][outerKey][innerKey], + }); + }); + } else { + sortable.push({ key: outerKey, value: item[config.field][outerKey] }); + } + }); + + sortable.sort((a, b) => { + if (descending) { + return b.value - a.value; + } + return a.value - b.value; + }); + + // now take the top k + let newMap = {}; + let i = 0; + for (let pair of sortable) { + if (i >= k) { + break; + } + newMap[pair.key] = pair.value; + i++; + } + item[config.field] = newMap; + + return item; + } + + /** + * Scalar multiplies a vector by some constant + * + * Config: + * field Points to: + * a map of strings to numbers + * an array of numbers + * a number + * k Either a number, or a string. If it's a number then This + * is the scalar value to multiply by. If it's a string, + * the value in the pointed to field is used. + * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric + * value is found, then use this value. + */ + scalarMultiply(item, config) { + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, config.dfault); + + let fieldType = this._typeOf(item[config.field]); + if (fieldType === "number") { + item[config.field] *= k; + } else if (fieldType === "array") { + for (let i = 0; i < item[config.field].length; i++) { + item[config.field][i] *= k; + } + } else if (fieldType === "map") { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] *= k; + }); + } else { + return null; + } + + return item; + } + + /** + * Elementwise multiplies either two maps or two arrays together, storing + * the result in left. If left and right are of the same type, results in an + * error. + * + * Maps are special case. For maps the left must be a nested map such as: + * { k1: { k11: 1, k12: 2}, k2: { k21: 3, k22: 4 } } and right needs to be + * simple map such as: { k1: 5, k2: 6} . The operation is then to mulitply + * every value of every right key, to every value every subkey where the + * parent keys match. Using the previous examples, the result would be: + * { k1: { k11: 5, k12: 10 }, k2: { k21: 18, k22: 24 } } . + * + * Config: + * left + * right + */ + elementwiseMultiply(item, config) { + if (!(config.left in item) || !(config.right in item)) { + return null; + } + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + item[config.left][i] *= item[config.right][i]; + } + } else if (leftType === "map") { + Object.keys(item[config.left]).forEach(outerKey => { + let r = 0.0; + if (outerKey in item[config.right]) { + r = item[config.right][outerKey]; + } + Object.keys(item[config.left][outerKey]).forEach(innerKey => { + item[config.left][outerKey][innerKey] *= r; + }); + }); + } else if (leftType === "number") { + item[config.left] *= item[config.right]; + } else { + return null; + } + + return item; + } + + /** + * Vector multiplies (i.e. dot products) two vectors and stores the result in + * third field. Both vectors must either by maps, or arrays of numbers with + * the same length. + * + * Config: + * left A field pointing to either a map of strings to numbers, + * or an array of numbers + * right A field pointing to either a map of strings to numbers, + * or an array of numbers + * dest The field to store the dot product. + */ + vectorMultiply(item, config) { + if (!(config.left in item) || !(config.right in item)) { + return null; + } + + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + + let destVal = 0.0; + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + destVal += item[config.left][i] * item[config.right][i]; + } + } else if (leftType === "map") { + Object.keys(item[config.left]).forEach(key => { + if (key in item[config.right]) { + destVal += item[config.left][key] * item[config.right][key]; + } + }); + } else { + return null; + } + + item[config.dest] = destVal; + return item; + } + + /** + * Adds a constant value to all elements in the field. Mathematically, + * this is the same as taking a 1-vector, scalar multiplying it by k, + * and then vector adding it to a field. + * + * Config: + * field A field pointing to either a map of strings to numbers, + * or an array of numbers + * k Either a number, or a string. If it's a number then This + * is the scalar value to multiply by. If it's a string, + * the value in the pointed to field is used. + * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric + * value is found, then use this value. + */ + scalarAdd(item, config) { + let k = this._lookupScalar(item, config.k, config.dfault); + if (!(config.field in item)) { + return null; + } + + let fieldType = this._typeOf(item[config.field]); + if (fieldType === "array") { + for (let i = 0; i < item[config.field].length; i++) { + item[config.field][i] += k; + } + } else if (fieldType === "map") { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] += k; + }); + } else if (fieldType === "number") { + item[config.field] += k; + } else { + return null; + } + + return item; + } + + /** + * Adds two vectors together and stores the result in left. + * + * Config: + * left A field pointing to either a map of strings to numbers, + * or an array of numbers + * right A field pointing to either a map of strings to numbers, + * or an array of numbers + */ + vectorAdd(item, config) { + if (!(config.left in item)) { + return this.copyValue(item, { src: config.right, dest: config.left }); + } + if (!(config.right in item)) { + return null; + } + + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + item[config.left][i] += item[config.right][i]; + } + return item; + } else if (leftType === "map") { + Object.keys(item[config.right]).forEach(key => { + let v = 0; + if (key in item[config.left]) { + v = item[config.left][key]; + } + item[config.left][key] = v + item[config.right][key]; + }); + return item; + } + + return null; + } + + /** + * Converts a vector from real values to boolean integers. (i.e. either 1/0 + * or 1/-1). + * + * Config: + * field Field containing either a map of strings to numbers or + * an array of numbers to convert. + * threshold OPTIONAL (DEFAULT: 0) Values above this will be replaced + * with 1.0. Those below will be converted to 0. + * keep_negative OPTIONAL (DEFAULT: False) If true, values below the + * threshold will be converted to -1 instead of 0. + */ + makeBoolean(item, config) { + if (!(config.field in item)) { + return null; + } + let threshold = this._lookupScalar(item, config.threshold, 0.0); + let type = this._typeOf(item[config.field]); + if (type === "array") { + for (let i = 0; i < item[config.field].length; i++) { + if (item[config.field][i] > threshold) { + item[config.field][i] = 1.0; + } else if (config.keep_negative) { + item[config.field][i] = -1.0; + } else { + item[config.field][i] = 0.0; + } + } + } else if (type === "map") { + Object.keys(item[config.field]).forEach(key => { + let value = item[config.field][key]; + if (value > threshold) { + item[config.field][key] = 1.0; + } else if (config.keep_negative) { + item[config.field][key] = -1.0; + } else { + item[config.field][key] = 0.0; + } + }); + } else if (type === "number") { + let value = item[config.field]; + if (value > threshold) { + item[config.field] = 1.0; + } else if (config.keep_negative) { + item[config.field] = -1.0; + } else { + item[config.field] = 0.0; + } + } else { + return null; + } + + return item; + } + + /** + * Removes all keys from the item except for the ones specified. + * + * fields An array of strings indicating the fields to keep + */ + allowFields(item, config) { + let newItem = {}; + for (let ele of config.fields) { + if (ele in item) { + newItem[ele] = item[ele]; + } + } + return newItem; + } + + /** + * Removes all keys whose value does not exceed some threshold. + * + * Config: + * field Points to a map of strings to numbers + * threshold Values must exceed this value, otherwise they are removed. + */ + filterByValue(item, config) { + if (!(config.field in item)) { + return null; + } + let threshold = this._lookupScalar(item, config.threshold, 0.0); + let filtered = {}; + Object.keys(item[config.field]).forEach(key => { + let value = item[config.field][key]; + if (value > threshold) { + filtered[key] = value; + } + }); + item[config.field] = filtered; + + return item; + } + + /** + * Rewrites a field so that its values are now L2 normed. + * + * Config: + * field Points to a map of strings to numbers, or an array of numbers + */ + l2Normalize(item, config) { + if (!(config.field in item)) { + return null; + } + let data = item[config.field]; + let type = this._typeOf(data); + if (type === "array") { + let norm = 0.0; + for (let datum of data) { + norm += datum * datum; + } + norm = Math.sqrt(norm); + if (norm !== 0) { + for (let i = 0; i < data.length; i++) { + data[i] /= norm; + } + } + } else if (type === "map") { + let norm = 0.0; + Object.keys(data).forEach(key => { + norm += data[key] * data[key]; + }); + norm = Math.sqrt(norm); + if (norm !== 0) { + Object.keys(data).forEach(key => { + data[key] /= norm; + }); + } + } else { + return null; + } + + item[config.field] = data; + + return item; + } + + /** + * Rewrites a field so that all of its values sum to 1.0 + * + * Config: + * field Points to a map of strings to numbers, or an array of numbers + */ + probNormalize(item, config) { + if (!(config.field in item)) { + return null; + } + let data = item[config.field]; + let type = this._typeOf(data); + if (type === "array") { + let norm = 0.0; + for (let datum of data) { + norm += datum; + } + if (norm !== 0) { + for (let i = 0; i < data.length; i++) { + data[i] /= norm; + } + } + } else if (type === "map") { + let norm = 0.0; + Object.keys(item[config.field]).forEach(key => { + norm += item[config.field][key]; + }); + if (norm !== 0) { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] /= norm; + }); + } + } else { + return null; + } + + return item; + } + + /** + * Stores a value, if it is not already present + * + * Config: + * field field to write to if it is missing + * value value to store in that field + */ + setDefault(item, config) { + let val = this._lookupScalar(item, config.value, config.value); + if (!(config.field in item)) { + item[config.field] = val; + } + + return item; + } + + /** + * Selctively promotes an value from an inner map up to the outer map + * + * Config: + * haystack Points to a map of strings to values + * needle Key inside the map we should promote up + * dest Where we should write the value of haystack[needle] + */ + lookupValue(item, config) { + if (config.haystack in item && config.needle in item[config.haystack]) { + item[config.dest] = item[config.haystack][config.needle]; + } + + return item; + } + + /** + * Demotes a field into a map + * + * Config: + * src Field to copy + * dest_map Points to a map + * dest_key Key inside dest_map to copy src to + */ + copyToMap(item, config) { + if (config.src in item) { + if (!(config.dest_map in item)) { + item[config.dest_map] = {}; + } + item[config.dest_map][config.dest_key] = item[config.src]; + } + + return item; + } + + /** + * Config: + * field Points to a string to number map + * k Scalar to multiply the values by + * log_scale Boolean, if true, then the values will be transformed + * by a logrithm prior to multiplications + */ + scalarMultiplyTag(item, config) { + let EPSILON = 0.000001; + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, 1); + let type = this._typeOf(item[config.field]); + if (type === "map") { + Object.keys(item[config.field]).forEach(parentKey => { + Object.keys(item[config.field][parentKey]).forEach(key => { + let v = item[config.field][parentKey][key]; + if (config.log_scale) { + v = Math.log(v + EPSILON); + } + item[config.field][parentKey][key] = v * k; + }); + }); + } else { + return null; + } + + return item; + } + + /** + * Independently applies softmax across all subtags. + * + * Config: + * field Points to a map of strings with values being another map of strings + */ + applySoftmaxTags(item, config) { + let type = this._typeOf(item[config.field]); + if (type !== "map") { + return null; + } + + let abort = false; + let softmaxSum = {}; + Object.keys(item[config.field]).forEach(tag => { + if (this._typeOf(item[config.field][tag]) !== "map") { + abort = true; + return; + } + if (abort) { + return; + } + softmaxSum[tag] = 0; + Object.keys(item[config.field][tag]).forEach(subtag => { + if (this._typeOf(item[config.field][tag][subtag]) !== "number") { + abort = true; + return; + } + let score = item[config.field][tag][subtag]; + softmaxSum[tag] += Math.exp(score); + }); + }); + if (abort) { + return null; + } + + Object.keys(item[config.field]).forEach(tag => { + Object.keys(item[config.field][tag]).forEach(subtag => { + item[config.field][tag][subtag] = + Math.exp(item[config.field][tag][subtag]) / softmaxSum[tag]; + }); + }); + + return item; + } + + /** + * Vector adds a field and stores the result in left. + * + * Config: + * field The field to vector add + */ + combinerAdd(left, right, config) { + if (!(config.field in right)) { + return left; + } + let type = this._typeOf(right[config.field]); + if (!(config.field in left)) { + if (type === "map") { + left[config.field] = {}; + } else if (type === "array") { + left[config.field] = []; + } else if (type === "number") { + left[config.field] = 0; + } else { + return null; + } + } + if (type !== this._typeOf(left[config.field])) { + return null; + } + if (type === "map") { + Object.keys(right[config.field]).forEach(key => { + if (!(key in left[config.field])) { + left[config.field][key] = 0; + } + left[config.field][key] += right[config.field][key]; + }); + } else if (type === "array") { + for (let i = 0; i < right[config.field].length; i++) { + if (i < left[config.field].length) { + left[config.field][i] += right[config.field][i]; + } else { + left[config.field].push(right[config.field][i]); + } + } + } else if (type === "number") { + left[config.field] += right[config.field]; + } else { + return null; + } + + return left; + } + + /** + * Stores the maximum value of the field in left. + * + * Config: + * field The field to vector add + */ + combinerMax(left, right, config) { + if (!(config.field in right)) { + return left; + } + let type = this._typeOf(right[config.field]); + if (!(config.field in left)) { + if (type === "map") { + left[config.field] = {}; + } else if (type === "array") { + left[config.field] = []; + } else if (type === "number") { + left[config.field] = 0; + } else { + return null; + } + } + if (type !== this._typeOf(left[config.field])) { + return null; + } + if (type === "map") { + Object.keys(right[config.field]).forEach(key => { + if ( + !(key in left[config.field]) || + right[config.field][key] > left[config.field][key] + ) { + left[config.field][key] = right[config.field][key]; + } + }); + } else if (type === "array") { + for (let i = 0; i < right[config.field].length; i++) { + if (i < left[config.field].length) { + if (left[config.field][i] < right[config.field][i]) { + left[config.field][i] = right[config.field][i]; + } + } else { + left[config.field].push(right[config.field][i]); + } + } + } else if (type === "number") { + if (left[config.field] < right[config.field]) { + left[config.field] = right[config.field]; + } + } else { + return null; + } + + return left; + } + + /** + * Associates a value in right with another value in right. This association + * is then stored in a map in left. + * + * For example: If a sequence of rights is: + * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 41 } + * { 'tags': {}, 'url_domain': 'mbusa.com/mercedes', 'time': 21 } + * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 34 } + * + * Then assuming a 'sum' operation, left can build a map that would look like: + * { + * 'maseratiusa.com/maserati': 75, + * 'mbusa.com/mercedes': 21, + * } + * + * Fields: + * left_field field in the left to store / update the map + * right_key_field Field in the right to use as a key + * right_value_field Field in the right to use as a value + * operation One of "sum", "max", "overwrite", "count" + */ + combinerCollectValues(left, right, config) { + let op; + if (config.operation === "sum") { + op = (a, b) => a + b; + } else if (config.operation === "max") { + op = (a, b) => (a > b ? a : b); + } else if (config.operation === "overwrite") { + op = (a, b) => b; + } else if (config.operation === "count") { + op = (a, b) => a + 1; + } else { + return null; + } + if (!(config.left_field in left)) { + left[config.left_field] = {}; + } + if ( + !(config.right_key_field in right) || + !(config.right_value_field in right) + ) { + return left; + } + + let key = right[config.right_key_field]; + let rightValue = right[config.right_value_field]; + let leftValue = 0.0; + if (key in left[config.left_field]) { + leftValue = left[config.left_field][key]; + } + + left[config.left_field][key] = op(leftValue, rightValue); + + return left; + } + + /** + * Executes a recipe. Returns an object on success, or null on failure. + */ + executeRecipe(item, recipe) { + let newItem = item; + if (recipe) { + for (let step of recipe) { + let op = this.ITEM_BUILDER_REGISTRY[step.function]; + if (op === undefined) { + return null; + } + newItem = op.call(this, newItem, step); + if (newItem === null) { + break; + } + } + } + return newItem; + } + + /** + * Executes a recipe. Returns an object on success, or null on failure. + */ + executeCombinerRecipe(item1, item2, recipe) { + let newItem1 = item1; + for (let step of recipe) { + let op = this.ITEM_COMBINER_REGISTRY[step.function]; + if (op === undefined) { + return null; + } + newItem1 = op.call(this, newItem1, item2, step); + if (newItem1 === null) { + break; + } + } + + return newItem1; + } +}; diff --git a/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm b/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm new file mode 100644 index 0000000000..94835557a6 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm @@ -0,0 +1,89 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +// We load this into a worker using importScripts, and in tests using import. +// We use var to avoid name collision errors. +// eslint-disable-next-line no-var +var EXPORTED_SYMBOLS = ["tokenize", "toksToTfIdfVector"]; + +// Unicode specifies certain mnemonics for code pages and character classes. +// They call them "character properties" https://en.wikipedia.org/wiki/Unicode_character_property . +// These mnemonics are have been adopted by many regular expression libraries, +// however the standard Javascript regexp system doesn't support unicode +// character properties, so we have to define these ourself. +// +// Each of these sections contains the characters values / ranges for specific +// character property: Whitespace, Symbol (S), Punctuation (P), Number (N), +// Mark (M), and Letter (L). +const UNICODE_SPACE = + "\x20\xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000"; +const UNICODE_SYMBOL = + "\\x24\\x2B\x3C-\x3E\\x5E\x60\\x7C\x7E\xA2-\xA6\xA8\xA9\xAC\xAE-\xB1\xB4\xB8\xD7\xF7\u02C2-\u02C5\u02D2-\u02DF\u02E5-\u02EB\u02ED\u02EF-\u02FF\u0375\u0384\u0385\u03F6\u0482\u058D-\u058F\u0606-\u0608\u060B\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09F2\u09F3\u09FA\u09FB\u0AF1\u0B70\u0BF3-\u0BFA\u0C7F\u0D4F\u0D79\u0E3F\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u17DB\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u1FBD\u1FBF-\u1FC1\u1FCD-\u1FCF\u1FDD-\u1FDF\u1FED-\u1FEF\u1FFD\u1FFE\u2044\u2052\u207A-\u207C\u208A-\u208C\u20A0-\u20BE\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116-\u2118\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u2140-\u2144\u214A-\u214D\u214F\u218A\u218B\u2190-\u2307\u230C-\u2328\u232B-\u23FE\u2400-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u2767\u2794-\u27C4\u27C7-\u27E5\u27F0-\u2982\u2999-\u29D7\u29DC-\u29FB\u29FE-\u2B73\u2B76-\u2B95\u2B98-\u2BB9\u2BBD-\u2BC8\u2BCA-\u2BD1\u2BEC-\u2BEF\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u309B\u309C\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA700-\uA716\uA720\uA721\uA789\uA78A\uA828-\uA82B\uA836-\uA839\uAA77-\uAA79\uAB5B\uFB29\uFBB2-\uFBC1\uFDFC\uFDFD\uFE62\uFE64-\uFE66\uFE69\uFF04\uFF0B\uFF1C-\uFF1E\uFF3E\uFF40\uFF5C\uFF5E\uFFE0-\uFFE6\uFFE8-\uFFEE\uFFFC\uFFFD"; +const UNICODE_PUNCT = + "\x21-\x23\x25-\\x2A\x2C-\x2F\x3A\x3B\\x3F\x40\\x5B-\\x5D\x5F\\x7B\x7D\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E44\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65"; + +const UNICODE_NUMBER = + "0-9\xB2\xB3\xB9\xBC-\xBE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D58-\u0D5E\u0D66-\u0D78\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19"; +const UNICODE_MARK = + "\u0300-\u036F\u0483-\u0489\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED\u0711\u0730-\u074A\u07A6-\u07B0\u07EB-\u07F3\u0816-\u0819\u081B-\u0823\u0825-\u0827\u0829-\u082D\u0859-\u085B\u08D4-\u08E1\u08E3-\u0903\u093A-\u093C\u093E-\u094F\u0951-\u0957\u0962\u0963\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7\u09C8\u09CB-\u09CD\u09D7\u09E2\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A70\u0A71\u0A75\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AE2\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B56\u0B57\u0B62\u0B63\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7\u0C00-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C62\u0C63\u0C81-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CE2\u0CE3\u0D01-\u0D03\u0D3E-\u0D44\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D62\u0D63\u0D82\u0D83\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2\u0DF3\u0E31\u0E34-\u0E3A\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB\u0EBC\u0EC8-\u0ECD\u0F18\u0F19\u0F35\u0F37\u0F39\u0F3E\u0F3F\u0F71-\u0F84\u0F86\u0F87\u0F8D-\u0F97\u0F99-\u0FBC\u0FC6\u102B-\u103E\u1056-\u1059\u105E-\u1060\u1062-\u1064\u1067-\u106D\u1071-\u1074\u1082-\u108D\u108F\u109A-\u109D\u135D-\u135F\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17B4-\u17D3\u17DD\u180B-\u180D\u1885\u1886\u18A9\u1920-\u192B\u1930-\u193B\u1A17-\u1A1B\u1A55-\u1A5E\u1A60-\u1A7C\u1A7F\u1AB0-\u1ABE\u1B00-\u1B04\u1B34-\u1B44\u1B6B-\u1B73\u1B80-\u1B82\u1BA1-\u1BAD\u1BE6-\u1BF3\u1C24-\u1C37\u1CD0-\u1CD2\u1CD4-\u1CE8\u1CED\u1CF2-\u1CF4\u1CF8\u1CF9\u1DC0-\u1DF5\u1DFB-\u1DFF\u20D0-\u20F0\u2CEF-\u2CF1\u2D7F\u2DE0-\u2DFF\u302A-\u302F\u3099\u309A\uA66F-\uA672\uA674-\uA67D\uA69E\uA69F\uA6F0\uA6F1\uA802\uA806\uA80B\uA823-\uA827\uA880\uA881\uA8B4-\uA8C5\uA8E0-\uA8F1\uA926-\uA92D\uA947-\uA953\uA980-\uA983\uA9B3-\uA9C0\uA9E5\uAA29-\uAA36\uAA43\uAA4C\uAA4D\uAA7B-\uAA7D\uAAB0\uAAB2-\uAAB4\uAAB7\uAAB8\uAABE\uAABF\uAAC1\uAAEB-\uAAEF\uAAF5\uAAF6\uABE3-\uABEA\uABEC\uABED\uFB1E\uFE00-\uFE0F\uFE20-\uFE2F"; +const UNICODE_LETTER = + "A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u08B6-\u08BD\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C80-\u1C88\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AE\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC"; + +const REGEXP_SPLITS = new RegExp( + `[${UNICODE_SPACE}${UNICODE_SYMBOL}${UNICODE_PUNCT}]+` +); +// Match all token characters, so okay for regex to split multiple code points +// eslint-disable-next-line no-misleading-character-class +const REGEXP_ALPHANUMS = new RegExp( + `^[${UNICODE_NUMBER}${UNICODE_MARK}${UNICODE_LETTER}]+$` +); + +/** + * Downcases the text, and splits it into consecutive alphanumeric characters. + * This is locale aware, and so will not strip accents. This uses "word + * breaks", and os is not appropriate for languages without them + * (e.g. Chinese). + */ +function tokenize(text) { + return text + .toLocaleLowerCase() + .split(REGEXP_SPLITS) + .filter(tok => tok.match(REGEXP_ALPHANUMS)); +} + +/** + * Converts a sequence of tokens into an L2 normed TF-IDF. Any terms that are + * not preindexed (i.e. does have a computed inverse document frequency) will + * be dropped. + */ +function toksToTfIdfVector(tokens, vocab_idfs) { + let tfidfs = {}; + + // calcualte the term frequencies + for (let tok of tokens) { + if (!(tok in vocab_idfs)) { + continue; + } + if (!(tok in tfidfs)) { + tfidfs[tok] = [vocab_idfs[tok][0], 1]; + } else { + tfidfs[tok][1]++; + } + } + + // now multiply by the log inverse document frequencies, then take + // the L2 norm of this. + let l2Norm = 0.0; + Object.keys(tfidfs).forEach(tok => { + tfidfs[tok][1] *= vocab_idfs[tok][1]; + l2Norm += tfidfs[tok][1] * tfidfs[tok][1]; + }); + l2Norm = Math.sqrt(l2Norm); + Object.keys(tfidfs).forEach(tok => { + tfidfs[tok][1] /= l2Norm; + }); + + return tfidfs; +} diff --git a/browser/components/newtab/lib/PlacesFeed.jsm b/browser/components/newtab/lib/PlacesFeed.jsm new file mode 100644 index 0000000000..19dcfbdfde --- /dev/null +++ b/browser/components/newtab/lib/PlacesFeed.jsm @@ -0,0 +1,566 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { + actionCreators: ac, + actionTypes: at, + actionUtils: au, +} = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { shortURL } = ChromeUtils.import( + "resource://activity-stream/lib/ShortURL.jsm" +); +const { AboutNewTab } = ChromeUtils.import( + "resource:///modules/AboutNewTab.jsm" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PartnerLinkAttribution: "resource:///modules/PartnerLinkAttribution.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", +}); + +const LINK_BLOCKED_EVENT = "newtab-linkBlocked"; +const PLACES_LINKS_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for places links changed events + +// The pref to store the blocked sponsors of the sponsored Top Sites. +// The value of this pref is an array (JSON serialized) of hostnames of the +// blocked sponsors. +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; + +/** + * PlacesObserver - observes events from PlacesUtils.observers + */ +class PlacesObserver { + constructor(dispatch) { + this.dispatch = dispatch; + this.QueryInterface = ChromeUtils.generateQI(["nsISupportsWeakReference"]); + this.handlePlacesEvent = this.handlePlacesEvent.bind(this); + } + + handlePlacesEvent(events) { + const removedPages = []; + const removedBookmarks = []; + + for (const { + itemType, + source, + dateAdded, + guid, + title, + url, + isRemovedFromStore, + isTagging, + type, + } of events) { + switch (type) { + case "history-cleared": + this.dispatch({ type: at.PLACES_HISTORY_CLEARED }); + break; + case "page-removed": + if (isRemovedFromStore) { + removedPages.push(url); + } + break; + case "bookmark-added": + // Skips items that are not bookmarks (like folders), about:* pages or + // default bookmarks, added when the profile is created. + if ( + isTagging || + itemType !== lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK || + source === lazy.PlacesUtils.bookmarks.SOURCES.IMPORT || + source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE || + source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP || + source === lazy.PlacesUtils.bookmarks.SOURCES.SYNC || + (!url.startsWith("http://") && !url.startsWith("https://")) + ) { + return; + } + + this.dispatch({ type: at.PLACES_LINKS_CHANGED }); + this.dispatch({ + type: at.PLACES_BOOKMARK_ADDED, + data: { + bookmarkGuid: guid, + bookmarkTitle: title, + dateAdded: dateAdded * 1000, + url, + }, + }); + break; + case "bookmark-removed": + if ( + isTagging || + (itemType === lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK && + source !== lazy.PlacesUtils.bookmarks.SOURCES.IMPORT && + source !== lazy.PlacesUtils.bookmarks.SOURCES.RESTORE && + source !== + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP && + source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC) + ) { + removedBookmarks.push(url); + } + break; + } + } + + if (removedPages.length || removedBookmarks.length) { + this.dispatch({ type: at.PLACES_LINKS_CHANGED }); + } + + if (removedPages.length) { + this.dispatch({ + type: at.PLACES_LINKS_DELETED, + data: { urls: removedPages }, + }); + } + + if (removedBookmarks.length) { + this.dispatch({ + type: at.PLACES_BOOKMARKS_REMOVED, + data: { urls: removedBookmarks }, + }); + } + } +} + +class PlacesFeed { + constructor() { + this.placesChangedTimer = null; + this.customDispatch = this.customDispatch.bind(this); + this.placesObserver = new PlacesObserver(this.customDispatch); + } + + addObservers() { + lazy.PlacesUtils.observers.addListener( + ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"], + this.placesObserver.handlePlacesEvent + ); + + Services.obs.addObserver(this, LINK_BLOCKED_EVENT); + } + + /** + * setTimeout - A custom function that creates an nsITimer that can be cancelled + * + * @param {func} callback A function to be executed after the timer expires + * @param {int} delay The time (in ms) the timer should wait before the function is executed + */ + setTimeout(callback, delay) { + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT); + return timer; + } + + customDispatch(action) { + // If we are changing many links at once, delay this action and only dispatch + // one action at the end + if (action.type === at.PLACES_LINKS_CHANGED) { + if (this.placesChangedTimer) { + this.placesChangedTimer.delay = PLACES_LINKS_CHANGED_DELAY_TIME; + } else { + this.placesChangedTimer = this.setTimeout(() => { + this.placesChangedTimer = null; + this.store.dispatch(ac.OnlyToMain(action)); + }, PLACES_LINKS_CHANGED_DELAY_TIME); + } + } else { + this.store.dispatch(ac.BroadcastToContent(action)); + } + } + + removeObservers() { + if (this.placesChangedTimer) { + this.placesChangedTimer.cancel(); + this.placesChangedTimer = null; + } + lazy.PlacesUtils.observers.removeListener( + ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"], + this.placesObserver.handlePlacesEvent + ); + Services.obs.removeObserver(this, LINK_BLOCKED_EVENT); + } + + /** + * observe - An observer for the LINK_BLOCKED_EVENT. + * Called when a link is blocked. + * Links can be blocked outside of newtab, + * which is why we need to listen to this + * on such a generic level. + * + * @param {null} subject + * @param {str} topic The name of the event + * @param {str} value The data associated with the event + */ + observe(subject, topic, value) { + if (topic === LINK_BLOCKED_EVENT) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PLACES_LINK_BLOCKED, + data: { url: value }, + }) + ); + } + } + + /** + * Open a link in a desired destination defaulting to action's event. + */ + openLink(action, where = "", isPrivate = false) { + const params = { + private: isPrivate, + targetBrowser: action._target.browser, + forceForeground: false, // This ensure we maintain user preference for how to open new tabs. + globalHistoryOptions: { + triggeringSponsoredURL: action.data.sponsored_tile_id + ? action.data.url + : undefined, + }, + }; + + // Always include the referrer (even for http links) if we have one + const { event, referrer, typedBonus } = action.data; + if (referrer) { + const ReferrerInfo = Components.Constructor( + "@mozilla.org/referrer-info;1", + "nsIReferrerInfo", + "init" + ); + params.referrerInfo = new ReferrerInfo( + Ci.nsIReferrerInfo.UNSAFE_URL, + true, + Services.io.newURI(referrer) + ); + } + + // Pocket gives us a special reader URL to open their stories in + const urlToOpen = + action.data.type === "pocket" ? action.data.open_url : action.data.url; + + try { + let uri = Services.io.newURI(urlToOpen); + if (!["http", "https"].includes(uri.scheme)) { + throw new Error( + `Can't open link using ${uri.scheme} protocol from the new tab page.` + ); + } + } catch (e) { + console.error(e); + return; + } + + // Mark the page as typed for frecency bonus before opening the link + if (typedBonus) { + lazy.PlacesUtils.history.markPageAsTyped(Services.io.newURI(urlToOpen)); + } + + const win = action._target.browser.ownerGlobal; + win.openTrustedLinkIn( + urlToOpen, + where || win.whereToOpenLink(event), + params + ); + + // If there's an original URL e.g. using the unprocessed %YYYYMMDDHH% tag, + // add a visit for that so it may become a frecent top site. + if (action.data.original_url) { + lazy.PlacesUtils.history.insert({ + url: action.data.original_url, + visits: [{ transition: lazy.PlacesUtils.history.TRANSITION_TYPED }], + }); + } + } + + async saveToPocket(site, browser) { + const sendToPocket = + lazy.NimbusFeatures.pocketNewtab.getVariable("sendToPocket"); + // An experiment to send the user directly to Pocket's signup page. + if (sendToPocket && !lazy.pktApi.isUserLoggedIn()) { + const pocketNewtabExperiment = lazy.ExperimentAPI.getExperiment({ + featureId: "pocketNewtab", + }); + const pocketSiteHost = Services.prefs.getStringPref( + "extensions.pocket.site" + ); // getpocket.com + let utmSource = "firefox_newtab_save_button"; + // We want to know if the user is in a Pocket newtab related experiment. + let utmCampaign = pocketNewtabExperiment?.slug; + let utmContent = pocketNewtabExperiment?.branch?.slug; + + const url = new URL(`https://${pocketSiteHost}/signup`); + url.searchParams.append("utm_source", utmSource); + if (utmCampaign && utmContent) { + url.searchParams.append("utm_campaign", utmCampaign); + url.searchParams.append("utm_content", utmContent); + } + + const win = browser.ownerGlobal; + win.openTrustedLinkIn(url.href, "tab"); + return; + } + + const { url, title } = site; + try { + let data = await lazy.NewTabUtils.activityStreamLinks.addPocketEntry( + url, + title, + browser + ); + if (data) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PLACES_SAVED_TO_POCKET, + data: { + url, + open_url: data.item.open_url, + title, + pocket_id: data.item.item_id, + }, + }) + ); + } + } catch (err) { + console.error(err); + } + } + + /** + * Deletes an item from a user's saved to Pocket feed + * @param {int} itemID + * The unique ID given by Pocket for that item; used to look the item up when deleting + */ + async deleteFromPocket(itemID) { + try { + await lazy.NewTabUtils.activityStreamLinks.deletePocketEntry(itemID); + this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED }); + } catch (err) { + console.error(err); + } + } + + /** + * Archives an item from a user's saved to Pocket feed + * @param {int} itemID + * The unique ID given by Pocket for that item; used to look the item up when archiving + */ + async archiveFromPocket(itemID) { + try { + await lazy.NewTabUtils.activityStreamLinks.archivePocketEntry(itemID); + this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED }); + } catch (err) { + console.error(err); + } + } + + /** + * Sends an attribution request for Top Sites interactions. + * @param {object} data + * Attribution paramters from a Top Site. + */ + makeAttributionRequest(data) { + let args = Object.assign( + { + campaignID: Services.prefs.getStringPref( + "browser.partnerlink.campaign.topsites" + ), + }, + data + ); + lazy.PartnerLinkAttribution.makeRequest(args); + } + + async fillSearchTopSiteTerm({ _target, data }) { + const searchEngine = await Services.search.getEngineByAlias(data.label); + _target.browser.ownerGlobal.gURLBar.search(data.label, { + searchEngine, + searchModeEntry: "topsites_newtab", + }); + } + + _getDefaultSearchEngine(isPrivateWindow) { + return Services.search[ + isPrivateWindow ? "defaultPrivateEngine" : "defaultEngine" + ]; + } + + handoffSearchToAwesomebar(action) { + const { _target, data, meta } = action; + const searchEngine = this._getDefaultSearchEngine( + lazy.PrivateBrowsingUtils.isBrowserPrivate(_target.browser) + ); + const urlBar = _target.browser.ownerGlobal.gURLBar; + let isFirstChange = true; + + const newtabSession = AboutNewTab.activityStream.store.feeds + .get("feeds.telemetry") + ?.sessions.get(au.getPortIdOfSender(action)); + if (!data || !data.text) { + urlBar.setHiddenFocus(); + } else { + urlBar.handoff(data.text, searchEngine, newtabSession?.session_id); + isFirstChange = false; + } + + const checkFirstChange = () => { + // Check if this is the first change since we hidden focused. If it is, + // remove hidden focus styles, prepend the search alias and hide the + // in-content search. + if (isFirstChange) { + isFirstChange = false; + urlBar.removeHiddenFocus(true); + urlBar.handoff("", searchEngine, newtabSession?.session_id); + this.store.dispatch( + ac.OnlyToOneContent({ type: at.DISABLE_SEARCH }, meta.fromTarget) + ); + urlBar.removeEventListener("compositionstart", checkFirstChange); + urlBar.removeEventListener("paste", checkFirstChange); + } + }; + + const onKeydown = ev => { + // Check if the keydown will cause a value change. + if (ev.key.length === 1 && !ev.altKey && !ev.ctrlKey && !ev.metaKey) { + checkFirstChange(); + } + // If the Esc button is pressed, we are done. Show in-content search and cleanup. + if (ev.key === "Escape") { + onDone(); // eslint-disable-line no-use-before-define + } + }; + + const onDone = ev => { + // We are done. Show in-content search again and cleanup. + this.store.dispatch( + ac.OnlyToOneContent({ type: at.SHOW_SEARCH }, meta.fromTarget) + ); + + const forceSuppressFocusBorder = ev?.type === "mousedown"; + urlBar.removeHiddenFocus(forceSuppressFocusBorder); + + urlBar.removeEventListener("keydown", onKeydown); + urlBar.removeEventListener("mousedown", onDone); + urlBar.removeEventListener("blur", onDone); + urlBar.removeEventListener("compositionstart", checkFirstChange); + urlBar.removeEventListener("paste", checkFirstChange); + }; + + urlBar.addEventListener("keydown", onKeydown); + urlBar.addEventListener("mousedown", onDone); + urlBar.addEventListener("blur", onDone); + urlBar.addEventListener("compositionstart", checkFirstChange); + urlBar.addEventListener("paste", checkFirstChange); + } + + /** + * Add the hostnames of the given urls to the Top Sites sponsor blocklist. + * + * @param {array} urls + * An array of the objects structured as `{ url }` + */ + addToBlockedTopSitesSponsors(urls) { + const blockedPref = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + const merged = new Set([...blockedPref, ...urls.map(url => shortURL(url))]); + + Services.prefs.setStringPref( + TOP_SITES_BLOCKED_SPONSORS_PREF, + JSON.stringify([...merged]) + ); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + // Briefly avoid loading services for observing for better startup timing + Services.tm.dispatchToMainThread(() => this.addObservers()); + break; + case at.UNINIT: + this.removeObservers(); + break; + case at.ABOUT_SPONSORED_TOP_SITES: { + const url = `${Services.urlFormatter.formatURLPref( + "app.support.baseURL" + )}sponsor-privacy`; + const win = action._target.browser.ownerGlobal; + win.openTrustedLinkIn(url, "tab"); + break; + } + case at.BLOCK_URL: { + if (action.data) { + let sponsoredTopSites = []; + action.data.forEach(site => { + const { url, pocket_id, isSponsoredTopSite } = site; + lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id }); + if (isSponsoredTopSite) { + sponsoredTopSites.push({ url }); + } + }); + if (sponsoredTopSites.length) { + this.addToBlockedTopSitesSponsors(sponsoredTopSites); + } + } + break; + } + case at.BOOKMARK_URL: + lazy.NewTabUtils.activityStreamLinks.addBookmark( + action.data, + action._target.browser.ownerGlobal + ); + break; + case at.DELETE_BOOKMARK_BY_ID: + lazy.NewTabUtils.activityStreamLinks.deleteBookmark(action.data); + break; + case at.DELETE_HISTORY_URL: { + const { url, forceBlock, pocket_id } = action.data; + lazy.NewTabUtils.activityStreamLinks.deleteHistoryEntry(url); + if (forceBlock) { + lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id }); + } + break; + } + case at.OPEN_NEW_WINDOW: + this.openLink(action, "window"); + break; + case at.OPEN_PRIVATE_WINDOW: + this.openLink(action, "window", true); + break; + case at.SAVE_TO_POCKET: + this.saveToPocket(action.data.site, action._target.browser); + break; + case at.DELETE_FROM_POCKET: + this.deleteFromPocket(action.data.pocket_id); + break; + case at.ARCHIVE_FROM_POCKET: + this.archiveFromPocket(action.data.pocket_id); + break; + case at.FILL_SEARCH_TERM: + this.fillSearchTopSiteTerm(action); + break; + case at.HANDOFF_SEARCH_TO_AWESOMEBAR: + this.handoffSearchToAwesomebar(action); + break; + case at.OPEN_LINK: { + this.openLink(action); + break; + } + case at.PARTNER_LINK_ATTRIBUTION: + this.makeAttributionRequest(action.data); + break; + } + } +} + +// Exported for testing only +PlacesFeed.PlacesObserver = PlacesObserver; + +const EXPORTED_SYMBOLS = ["PlacesFeed"]; diff --git a/browser/components/newtab/lib/PrefsFeed.jsm b/browser/components/newtab/lib/PrefsFeed.jsm new file mode 100644 index 0000000000..f6f46b069d --- /dev/null +++ b/browser/components/newtab/lib/PrefsFeed.jsm @@ -0,0 +1,280 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", +}); + +class PrefsFeed { + constructor(prefMap) { + this._prefMap = prefMap; + this._prefs = new Prefs(); + this.onExperimentUpdated = this.onExperimentUpdated.bind(this); + this.onPocketExperimentUpdated = this.onPocketExperimentUpdated.bind(this); + } + + onPrefChanged(name, value) { + const prefItem = this._prefMap.get(name); + if (prefItem) { + let action = "BroadcastToContent"; + if (prefItem.skipBroadcast) { + action = "OnlyToMain"; + if (prefItem.alsoToPreloaded) { + action = "AlsoToPreloaded"; + } + } + + this.store.dispatch( + ac[action]({ + type: at.PREF_CHANGED, + data: { name, value }, + }) + ); + } + } + + _setStringPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getStringPref); + } + + _setBoolPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getBoolPref); + } + + _setIntPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getIntPref); + } + + _setPref(values, key, defaultValue, getPrefFunction) { + let value = getPrefFunction( + `browser.newtabpage.activity-stream.${key}`, + defaultValue + ); + values[key] = value; + this._prefMap.set(key, { value }); + } + + /** + * Handler for when experiment data updates. + */ + onExperimentUpdated(event, reason) { + const value = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { + name: "featureConfig", + value, + }, + }) + ); + } + + /** + * Handler for Pocket specific experiment data updates. + */ + onPocketExperimentUpdated(event, reason) { + const value = lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}; + // Loaded experiments are set up inside init() + if ( + reason !== "feature-experiment-loaded" && + reason !== "feature-rollout-loaded" + ) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { + name: "pocketConfig", + value, + }, + }) + ); + } + } + + init() { + this._prefs.observeBranch(this); + lazy.NimbusFeatures.newtab.onUpdate(this.onExperimentUpdated); + lazy.NimbusFeatures.pocketNewtab.onUpdate(this.onPocketExperimentUpdated); + + this._storage = this.store.dbStorage.getDbTable("sectionPrefs"); + + // Get the initial value of each activity stream pref + const values = {}; + for (const name of this._prefMap.keys()) { + values[name] = this._prefs.get(name); + } + + // These are not prefs, but are needed to determine stuff in content that can only be + // computed in main process + values.isPrivateBrowsingEnabled = lazy.PrivateBrowsingUtils.enabled; + values.platform = AppConstants.platform; + + // Save the geo pref if we have it + if (lazy.Region.home) { + values.region = lazy.Region.home; + this.geo = values.region; + } else if (this.geo !== "") { + // Watch for geo changes and use a dummy value for now + Services.obs.addObserver(this, lazy.Region.REGION_TOPIC); + this.geo = ""; + } + + // Get the firefox accounts url for links and to send firstrun metrics to. + values.fxa_endpoint = Services.prefs.getStringPref( + "browser.newtabpage.activity-stream.fxaccounts.endpoint", + "https://accounts.firefox.com" + ); + + // Get the firefox update channel with values as default, nightly, beta or release + values.appUpdateChannel = Services.prefs.getStringPref( + "app.update.channel", + "" + ); + + // Read the pref for search shortcuts top sites experiment from firefox.js and store it + // in our internal list of prefs to watch + let searchTopSiteExperimentPrefValue = Services.prefs.getBoolPref( + "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts" + ); + values["improvesearch.topSiteSearchShortcuts"] = + searchTopSiteExperimentPrefValue; + this._prefMap.set("improvesearch.topSiteSearchShortcuts", { + value: searchTopSiteExperimentPrefValue, + }); + + values.mayHaveSponsoredTopSites = Services.prefs.getBoolPref( + "browser.topsites.useRemoteSetting" + ); + + // Read the pref for search hand-off from firefox.js and store it + // in our internal list of prefs to watch + let handoffToAwesomebarPrefValue = Services.prefs.getBoolPref( + "browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar" + ); + values["improvesearch.handoffToAwesomebar"] = handoffToAwesomebarPrefValue; + this._prefMap.set("improvesearch.handoffToAwesomebar", { + value: handoffToAwesomebarPrefValue, + }); + + // Read the pref for the cached default engine name from firefox.js and + // store it in our internal list of prefs to watch + let placeholderPrefValue = Services.prefs.getStringPref( + "browser.urlbar.placeholderName", + "" + ); + values["urlbar.placeholderName"] = placeholderPrefValue; + this._prefMap.set("urlbar.placeholderName", { + value: placeholderPrefValue, + }); + + // Add experiment values and default values + values.featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + values.pocketConfig = + lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}; + this._setBoolPref(values, "logowordmark.alwaysVisible", false); + this._setBoolPref(values, "feeds.section.topstories", false); + this._setBoolPref(values, "discoverystream.enabled", false); + this._setBoolPref( + values, + "discoverystream.sponsored-collections.enabled", + false + ); + this._setBoolPref(values, "discoverystream.isCollectionDismissible", false); + this._setBoolPref(values, "discoverystream.hardcoded-basic-layout", false); + this._setBoolPref(values, "discoverystream.personalization.enabled", false); + this._setBoolPref(values, "discoverystream.personalization.override"); + this._setStringPref( + values, + "discoverystream.personalization.modelKeys", + "" + ); + this._setStringPref(values, "discoverystream.spocs-endpoint", ""); + this._setStringPref(values, "discoverystream.spocs-endpoint-query", ""); + this._setStringPref(values, "newNewtabExperience.colors", ""); + + // Set the initial state of all prefs in redux + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREFS_INITIAL_VALUES, + data: values, + meta: { + isStartup: true, + }, + }) + ); + } + + uninit() { + this.removeListeners(); + } + + removeListeners() { + this._prefs.ignoreBranch(this); + lazy.NimbusFeatures.newtab.offUpdate(this.onExperimentUpdated); + lazy.NimbusFeatures.pocketNewtab.offUpdate(this.onPocketExperimentUpdated); + if (this.geo === "") { + Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC); + } + } + + async _setIndexedDBPref(id, value) { + const name = id === "topsites" ? id : `feeds.section.${id}`; + try { + await this._storage.set(name, value); + } catch (e) { + console.error("Could not set section preferences."); + } + } + + observe(subject, topic, data) { + switch (topic) { + case lazy.Region.REGION_TOPIC: + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { name: "region", value: lazy.Region.home }, + }) + ); + break; + } + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.uninit(); + break; + case at.CLEAR_PREF: + Services.prefs.clearUserPref(this._prefs._branchStr + action.data.name); + break; + case at.SET_PREF: + this._prefs.set(action.data.name, action.data.value); + break; + case at.UPDATE_SECTION_PREFS: + this._setIndexedDBPref(action.data.id, action.data.value); + break; + } + } +} + +const EXPORTED_SYMBOLS = ["PrefsFeed"]; diff --git a/browser/components/newtab/lib/RecommendationProvider.jsm b/browser/components/newtab/lib/RecommendationProvider.jsm new file mode 100644 index 0000000000..2438dfe542 --- /dev/null +++ b/browser/components/newtab/lib/RecommendationProvider.jsm @@ -0,0 +1,132 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +// Use XPCOMUtils.defineLazyModuleGetters to make the test harness keeps working +// after bug 1608279. +// +// The test harness's workaround for "lazy getter on a plain object" is to +// set the `lazy` object's prototype to the global object, inside the lazy +// getter API. +// +// ChromeUtils.defineModuleGetter is converted into a static import declaration +// by babel-plugin-jsm-to-esmodules, and it doesn't work for the following +// 2 reasons: +// +// * There's no other lazy getter API call in this file, and the workaround +// above stops working +// * babel-plugin-jsm-to-esmodules ignores the first parameter of the lazy +// getter API, and the result is wrong +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const lazy = {}; +XPCOMUtils.defineLazyModuleGetters(lazy, { + PersonalityProvider: + "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm", +}); + +const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const PREF_PERSONALIZATION_MODEL_KEYS = + "discoverystream.personalization.modelKeys"; +const PREF_PERSONALIZATION = "discoverystream.personalization.enabled"; + +// The main purpose of this class is to handle interactions with the recommendation provider. +// A recommendation provider scores a list of stories, currently this is a personality provider. +// So all calls to the provider, anything involved with the setup of the provider, +// accessing prefs for the provider, or updaing devtools with provider state, is contained in here. +class RecommendationProvider { + setProvider(scores) { + // A provider is already set. This can happen when new stories come in + // and we need to update their scores. + // We can use the existing one, a fresh one is created after startup. + // Using the existing one might be a bit out of date, + // but it's fine for now. We can rely on restarts for updates. + // See bug 1629931 for improvements to this. + if (this.provider) { + return; + } + // At this point we've determined we can successfully create a v2 personalization provider. + this.provider = new lazy.PersonalityProvider(this.modelKeys); + this.provider.setScores(scores); + } + + /* + * This calls any async initialization that's required, + * and then signals to devtools when that's done. + */ + async init() { + if (this.provider && this.provider.init) { + await this.provider.init(); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_INIT, + }) + ); + } + } + + get modelKeys() { + if (!this._modelKeys) { + this._modelKeys = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_MODEL_KEYS]; + } + + return this._modelKeys; + } + + getScores() { + return this.provider.getScores(); + } + + async calculateItemRelevanceScore(item) { + if (this.provider) { + const scoreResult = await this.provider.calculateItemRelevanceScore(item); + if (scoreResult === 0 || scoreResult) { + item.score = scoreResult; + } + } + } + + teardown() { + if (this.provider && this.provider.teardown) { + // This removes any in memory listeners if available. + this.provider.teardown(); + } + } + + resetState() { + this._modelKeys = null; + this.provider = null; + } + + onAction(action) { + switch (action.type) { + case at.DISCOVERY_STREAM_CONFIG_CHANGE: + this.teardown(); + this.resetState(); + break; + case at.PREF_CHANGED: + switch (action.data.name) { + case PREF_PERSONALIZATION_MODEL_KEYS: + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_RESET, + }) + ); + break; + } + break; + case at.DISCOVERY_STREAM_PERSONALIZATION_TOGGLE: + let enabled = this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + + this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION, !enabled)); + break; + } + } +} + +const EXPORTED_SYMBOLS = ["RecommendationProvider"]; diff --git a/browser/components/newtab/lib/RemoteL10n.sys.mjs b/browser/components/newtab/lib/RemoteL10n.sys.mjs new file mode 100644 index 0000000000..fe25c56a79 --- /dev/null +++ b/browser/components/newtab/lib/RemoteL10n.sys.mjs @@ -0,0 +1,249 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The downloaded Fluent file is located in this sub-directory of the local + * profile directory. + */ +const USE_REMOTE_L10N_PREF = + "browser.newtabpage.activity-stream.asrouter.useRemoteL10n"; + +/** + * All supported locales for remote l10n + * + * This is used by ASRouter.jsm to check if the locale is supported before + * issuing the request for remote fluent files to RemoteSettings. + * + * Note: + * * this is generated based on "browser/locales/all-locales" as l10n doesn't + * provide an API to fetch that list + * + * * this list doesn't include "en-US", though "en-US" is well supported and + * `_RemoteL10n.isLocaleSupported()` will handle it properly + */ +const ALL_LOCALES = new Set([ + "ach", + "af", + "an", + "ar", + "ast", + "az", + "be", + "bg", + "bn", + "bo", + "br", + "brx", + "bs", + "ca", + "ca-valencia", + "cak", + "ckb", + "cs", + "cy", + "da", + "de", + "dsb", + "el", + "en-CA", + "en-GB", + "eo", + "es-AR", + "es-CL", + "es-ES", + "es-MX", + "et", + "eu", + "fa", + "ff", + "fi", + "fr", + "fy-NL", + "ga-IE", + "gd", + "gl", + "gn", + "gu-IN", + "he", + "hi-IN", + "hr", + "hsb", + "hu", + "hy-AM", + "hye", + "ia", + "id", + "is", + "it", + "ja", + "ja-JP-mac", + "ka", + "kab", + "kk", + "km", + "kn", + "ko", + "lij", + "lo", + "lt", + "ltg", + "lv", + "meh", + "mk", + "mr", + "ms", + "my", + "nb-NO", + "ne-NP", + "nl", + "nn-NO", + "oc", + "pa-IN", + "pl", + "pt-BR", + "pt-PT", + "rm", + "ro", + "ru", + "scn", + "si", + "sk", + "sl", + "son", + "sq", + "sr", + "sv-SE", + "szl", + "ta", + "te", + "th", + "tl", + "tr", + "trs", + "uk", + "ur", + "uz", + "vi", + "wo", + "xh", + "zh-CN", + "zh-TW", +]); + +export class _RemoteL10n { + constructor() { + this._l10n = null; + } + + createElement(doc, elem, options = {}) { + let node; + if (options.content && options.content.string_id) { + node = doc.createElement("remote-text"); + } else { + node = doc.createElementNS("http://www.w3.org/1999/xhtml", elem); + } + if (options.classList) { + node.classList.add(options.classList); + } + this.setString(node, options); + + return node; + } + + // If `string_id` is present it means we are relying on fluent for translations. + // Otherwise, we have a vanilla string. + setString(el, { content, attributes = {} }) { + if (content && content.string_id) { + for (let [fluentId, value] of Object.entries(attributes)) { + el.setAttribute(`fluent-variable-${fluentId}`, value); + } + el.setAttribute("fluent-remote-id", content.string_id); + } else { + el.textContent = content; + } + } + + /** + * Creates a new DOMLocalization instance with the Fluent file from Remote Settings. + * + * Note: it will use the local Fluent file in any of following cases: + * * the remote Fluent file is not available + * * it was told to use the local Fluent file + */ + _createDOML10n() { + /* istanbul ignore next */ + let useRemoteL10n = Services.prefs.getBoolPref(USE_REMOTE_L10N_PREF, true); + if (useRemoteL10n && !L10nRegistry.getInstance().hasSource("cfr")) { + const appLocale = Services.locale.appLocaleAsBCP47; + const l10nFluentDir = PathUtils.join( + Services.dirsvc.get("ProfLD", Ci.nsIFile).path, + "settings", + "main", + "ms-language-packs" + ); + let cfrIndexedFileSource = new L10nFileSource( + "cfr", + "app", + [appLocale], + `file://${l10nFluentDir}/`, + { + addResourceOptions: { + allowOverrides: true, + }, + }, + [`file://${l10nFluentDir}/browser/newtab/asrouter.ftl`] + ); + L10nRegistry.getInstance().registerSources([cfrIndexedFileSource]); + } else if (!useRemoteL10n && L10nRegistry.getInstance().hasSource("cfr")) { + L10nRegistry.getInstance().removeSources(["cfr"]); + } + + return new DOMLocalization( + [ + "branding/brand.ftl", + "browser/defaultBrowserNotification.ftl", + "browser/newtab/asrouter.ftl", + "toolkit/branding/accounts.ftl", + "toolkit/branding/brandings.ftl", + ], + false + ); + } + + get l10n() { + if (!this._l10n) { + this._l10n = this._createDOML10n(); + } + return this._l10n; + } + + reloadL10n() { + this._l10n = null; + } + + isLocaleSupported(locale) { + return locale === "en-US" || ALL_LOCALES.has(locale); + } + + /** + * Format given `localizableText`. + * + * Format `localizableText` if it is an object using any `string_id` field, + * otherwise return `localizableText` unmodified. + * + * @param {object|string} `localizableText` to format. + * @return {string} formatted text. + */ + async formatLocalizableText(localizableText) { + if (typeof localizableText !== "string") { + // It's more useful to get an error than passing through an object without + // a `string_id` field. + let value = await this.l10n.formatValue(localizableText.string_id); + return value; + } + return localizableText; + } +} + +export const RemoteL10n = new _RemoteL10n(); diff --git a/browser/components/newtab/lib/Screenshots.jsm b/browser/components/newtab/lib/Screenshots.jsm new file mode 100644 index 0000000000..b454702265 --- /dev/null +++ b/browser/components/newtab/lib/Screenshots.jsm @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const EXPORTED_SYMBOLS = ["Screenshots"]; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BackgroundPageThumbs: "resource://gre/modules/BackgroundPageThumbs.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", +}); + +const GREY_10 = "#F9F9FA"; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "gPrivilegedAboutProcessEnabled", + "browser.tabs.remote.separatePrivilegedContentProcess", + false +); + +const Screenshots = { + /** + * Get a screenshot / thumbnail for a url. Either returns the disk cached + * image or initiates a background request for the url. + * + * @param url {string} The url to get a thumbnail + * @return {Promise} Resolves a custom object or null if failed + */ + async getScreenshotForURL(url) { + try { + await lazy.BackgroundPageThumbs.captureIfMissing(url, { + backgroundColor: GREY_10, + }); + + // The privileged about content process is able to use the moz-page-thumb + // protocol, so if it's enabled, send that down. + if (lazy.gPrivilegedAboutProcessEnabled) { + return lazy.PageThumbs.getThumbnailURL(url); + } + + // Otherwise, for normal content processes, we fallback to using + // Blob URIs for the screenshots. + const imgPath = lazy.PageThumbs.getThumbnailPath(url); + + const filePathResponse = await fetch(`file://${imgPath}`); + const fileContents = await filePathResponse.blob(); + + // Check if the file is empty, which indicates there isn't actually a + // thumbnail, so callers can show a failure state. + if (fileContents.size === 0) { + return null; + } + + return { path: imgPath, data: fileContents }; + } catch (err) { + console.error(`getScreenshot(${url}) failed: ${err}`); + } + + // We must have failed to get the screenshot, so persist the failure by + // storing an empty file. Future calls will then skip requesting and return + // failure, so do the same thing here. The empty file should not expire with + // the usual filtering process to avoid repeated background requests, which + // can cause unwanted high CPU, network and memory usage - Bug 1384094 + try { + await lazy.PageThumbs._store(url, url, null, true); + } catch (err) { + // Probably failed to create the empty file, but not much more we can do. + } + return null; + }, + + /** + * Checks if all the open windows are private browsing windows. If so, we do not + * want to collect screenshots. If there exists at least 1 non-private window, + * we are ok to collect screenshots. + */ + _shouldGetScreenshots() { + for (let win of Services.wm.getEnumerator("navigator:browser")) { + if (!lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + // As soon as we encounter 1 non-private window, screenshots are fair game. + return true; + } + } + return false; + }, + + /** + * Conditionally get a screenshot for a link if there's no existing pending + * screenshot. Updates the cached link's desired property with the result. + * + * @param link {object} Link object to update + * @param url {string} Url to get a screenshot of + * @param property {string} Name of property on object to set + @ @param onScreenshot {function} Callback for when the screenshot loads + */ + async maybeCacheScreenshot(link, url, property, onScreenshot) { + // If there are only private windows open, do not collect screenshots + if (!this._shouldGetScreenshots()) { + return; + } + // __sharedCache may not exist yet for links from default top sites that + // don't have a default tippy top icon. + if (!link.__sharedCache) { + link.__sharedCache = { + updateLink(prop, val) { + link[prop] = val; + }, + }; + } + const cache = link.__sharedCache; + // Nothing to do if we already have a pending screenshot or + // if a previous request failed and returned null. + if (cache.fetchingScreenshot || link[property] !== undefined) { + return; + } + + // Save the promise to the cache so other links get it immediately + cache.fetchingScreenshot = this.getScreenshotForURL(url); + + // Clean up now that we got the screenshot + const screenshot = await cache.fetchingScreenshot; + delete cache.fetchingScreenshot; + + // Update the cache for future links and call back for existing content + cache.updateLink(property, screenshot); + onScreenshot(screenshot); + }, +}; diff --git a/browser/components/newtab/lib/SearchShortcuts.sys.mjs b/browser/components/newtab/lib/SearchShortcuts.sys.mjs new file mode 100644 index 0000000000..8b8edc18c2 --- /dev/null +++ b/browser/components/newtab/lib/SearchShortcuts.sys.mjs @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// List of sites we match against Topsites in order to identify sites +// that should be converted to search Topsites +export const SEARCH_SHORTCUTS = [ + { keyword: "@amazon", shortURL: "amazon", url: "https://amazon.com" }, + { keyword: "@\u767E\u5EA6", shortURL: "baidu", url: "https://baidu.com" }, + { keyword: "@google", shortURL: "google", url: "https://google.com" }, + { + keyword: "@\u044F\u043D\u0434\u0435\u043A\u0441", + shortURL: "yandex", + url: "https://yandex.com", + }, +]; + +// These can be added via the editor but will not be added organically +export const CUSTOM_SEARCH_SHORTCUTS = [ + ...SEARCH_SHORTCUTS, + { keyword: "@bing", shortURL: "bing", url: "https://bing.com" }, + { + keyword: "@duckduckgo", + shortURL: "duckduckgo", + url: "https://duckduckgo.com", + }, + { keyword: "@ebay", shortURL: "ebay", url: "https://ebay.com" }, + { keyword: "@twitter", shortURL: "twitter", url: "https://twitter.com" }, + { + keyword: "@wikipedia", + shortURL: "wikipedia", + url: "https://wikipedia.org", + }, +]; + +// Note: you must add the activity stream branch to the beginning of this if using outside activity stream +export const SEARCH_SHORTCUTS_EXPERIMENT = + "improvesearch.topSiteSearchShortcuts"; + +export const SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF = + "improvesearch.topSiteSearchShortcuts.searchEngines"; + +export const SEARCH_SHORTCUTS_HAVE_PINNED_PREF = + "improvesearch.topSiteSearchShortcuts.havePinned"; + +export function getSearchProvider(candidateShortURL) { + return ( + SEARCH_SHORTCUTS.filter(match => candidateShortURL === match.shortURL)[0] || + null + ); +} + +// Get the search form URL for a given search keyword. This allows us to pick +// different tippytop icons for the different variants. Sush as yandex.com vs. yandex.ru. +// See more details in bug 1643523. +export async function getSearchFormURL(keyword) { + const engine = await Services.search.getEngineByAlias(keyword); + return engine?.wrappedJSObject._searchForm; +} + +// Check topsite against predefined list of valid search engines +// https://searchfox.org/mozilla-central/rev/ca869724246f4230b272ed1c8b9944596e80d920/toolkit/components/search/nsSearchService.js#939 +export async function checkHasSearchEngine(keyword) { + return (await Services.search.getAppProvidedEngines()).find( + e => e.aliases.includes(keyword) && !e.hidden + ); +} diff --git a/browser/components/newtab/lib/SectionsManager.jsm b/browser/components/newtab/lib/SectionsManager.jsm new file mode 100644 index 0000000000..a9b2c4eeae --- /dev/null +++ b/browser/components/newtab/lib/SectionsManager.jsm @@ -0,0 +1,712 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { EventEmitter } = ChromeUtils.importESModule( + "resource://gre/modules/EventEmitter.sys.mjs" +); +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { getDefaultOptions } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamStorage.jsm" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +/* + * Generators for built in sections, keyed by the pref name for their feed. + * Built in sections may depend on options stored as serialised JSON in the pref + * `${feed_pref_name}.options`. + */ +const BUILT_IN_SECTIONS = ({ newtab, pocketNewtab }) => ({ + "feeds.section.topstories": options => ({ + id: "topstories", + pref: { + titleString: { + id: "home-prefs-recommended-by-header", + values: { provider: options.provider_name }, + }, + descString: { + id: "home-prefs-recommended-by-description-new", + values: { provider: options.provider_name }, + }, + nestedPrefs: [ + ...(options.show_spocs + ? [ + { + name: "showSponsored", + titleString: + "home-prefs-recommended-by-option-sponsored-stories", + icon: "icon-info", + eventSource: "POCKET_SPOCS", + }, + ] + : []), + ...(pocketNewtab.recentSavesEnabled + ? [ + { + name: "showRecentSaves", + titleString: "home-prefs-recommended-by-option-recent-saves", + icon: "icon-info", + eventSource: "POCKET_RECENT_SAVES", + }, + ] + : []), + ], + learnMore: { + link: { + href: "https://getpocket.com/firefox/new_tab_learn_more", + id: "home-prefs-recommended-by-learn-more", + }, + }, + }, + shouldHidePref: options.hidden, + eventSource: "TOP_STORIES", + icon: options.provider_icon, + title: { + id: "newtab-section-header-pocket", + values: { provider: options.provider_name }, + }, + learnMore: { + link: { + href: "https://getpocket.com/firefox/new_tab_learn_more", + message: { id: "newtab-pocket-learn-more" }, + }, + }, + compactCards: false, + rowsPref: "section.topstories.rows", + maxRows: 4, + availableLinkMenuOptions: [ + "CheckBookmarkOrArchive", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + ], + emptyState: { + message: { + id: "newtab-empty-section-topstories", + values: { provider: options.provider_name }, + }, + icon: "check", + }, + shouldSendImpressionStats: true, + dedupeFrom: ["highlights"], + }), + "feeds.section.highlights": options => ({ + id: "highlights", + pref: { + titleString: { + id: "home-prefs-recent-activity-header", + }, + descString: { + id: "home-prefs-recent-activity-description", + }, + nestedPrefs: [ + { + name: "section.highlights.includeVisited", + titleString: "home-prefs-highlights-option-visited-pages", + }, + { + name: "section.highlights.includeBookmarks", + titleString: "home-prefs-highlights-options-bookmarks", + }, + { + name: "section.highlights.includeDownloads", + titleString: "home-prefs-highlights-option-most-recent-download", + }, + { + name: "section.highlights.includePocket", + titleString: "home-prefs-highlights-option-saved-to-pocket", + hidden: !Services.prefs.getBoolPref( + "extensions.pocket.enabled", + true + ), + }, + ], + }, + shouldHidePref: false, + eventSource: "HIGHLIGHTS", + icon: "chrome://global/skin/icons/highlights.svg", + title: { + id: "newtab-section-header-recent-activity", + }, + compactCards: true, + rowsPref: "section.highlights.rows", + maxRows: 4, + emptyState: { + message: { id: "newtab-empty-section-highlights" }, + icon: "chrome://global/skin/icons/highlights.svg", + }, + shouldSendImpressionStats: false, + }), +}); + +const SectionsManager = { + ACTIONS_TO_PROXY: ["WEBEXT_CLICK", "WEBEXT_DISMISS"], + CONTEXT_MENU_PREFS: { CheckSavedToPocket: "extensions.pocket.enabled" }, + CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES: { + history: [ + "CheckBookmark", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + "DeleteUrl", + ], + bookmark: [ + "CheckBookmark", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + "DeleteUrl", + ], + pocket: [ + "ArchiveFromPocket", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + ], + download: [ + "OpenFile", + "ShowFile", + "Separator", + "GoToDownloadPage", + "CopyDownloadLink", + "Separator", + "RemoveDownload", + "BlockUrl", + ], + }, + initialized: false, + sections: new Map(), + async init(prefs = {}, storage) { + this._storage = storage; + const featureConfig = { + newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {}, + pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}, + }; + + for (const feedPrefName of Object.keys(BUILT_IN_SECTIONS(featureConfig))) { + const optionsPrefName = `${feedPrefName}.options`; + await this.addBuiltInSection(feedPrefName, prefs[optionsPrefName]); + + this._dedupeConfiguration = []; + this.sections.forEach(section => { + if (section.dedupeFrom) { + this._dedupeConfiguration.push({ + id: section.id, + dedupeFrom: section.dedupeFrom, + }); + } + }); + } + + Object.keys(this.CONTEXT_MENU_PREFS).forEach(k => + Services.prefs.addObserver(this.CONTEXT_MENU_PREFS[k], this) + ); + + this.initialized = true; + this.emit(this.INIT); + }, + observe(subject, topic, data) { + switch (topic) { + case "nsPref:changed": + for (const pref of Object.keys(this.CONTEXT_MENU_PREFS)) { + if (data === this.CONTEXT_MENU_PREFS[pref]) { + this.updateSections(); + } + } + break; + } + }, + updateSectionPrefs(id, collapsed) { + const section = this.sections.get(id); + if (!section) { + return; + } + + const updatedSection = Object.assign({}, section, { + pref: Object.assign({}, section.pref, collapsed), + }); + this.updateSection(id, updatedSection, true); + }, + async addBuiltInSection(feedPrefName, optionsPrefValue = "{}") { + let options; + let storedPrefs; + const featureConfig = { + newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {}, + pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}, + }; + try { + options = JSON.parse(optionsPrefValue); + } catch (e) { + options = {}; + console.error(`Problem parsing options pref for ${feedPrefName}`); + } + try { + storedPrefs = (await this._storage.get(feedPrefName)) || {}; + } catch (e) { + storedPrefs = {}; + console.error(`Problem getting stored prefs for ${feedPrefName}`); + } + const defaultSection = + BUILT_IN_SECTIONS(featureConfig)[feedPrefName](options); + const section = Object.assign({}, defaultSection, { + pref: Object.assign( + {}, + defaultSection.pref, + getDefaultOptions(storedPrefs) + ), + }); + section.pref.feed = feedPrefName; + this.addSection(section.id, Object.assign(section, { options })); + }, + addSection(id, options) { + this.updateLinkMenuOptions(options, id); + this.sections.set(id, options); + this.emit(this.ADD_SECTION, id, options); + }, + removeSection(id) { + this.emit(this.REMOVE_SECTION, id); + this.sections.delete(id); + }, + enableSection(id, isStartup = false) { + this.updateSection(id, { enabled: true }, true, isStartup); + this.emit(this.ENABLE_SECTION, id); + }, + disableSection(id) { + this.updateSection( + id, + { enabled: false, rows: [], initialized: false }, + true + ); + this.emit(this.DISABLE_SECTION, id); + }, + updateSections() { + this.sections.forEach((section, id) => + this.updateSection(id, section, true) + ); + }, + updateSection(id, options, shouldBroadcast, isStartup = false) { + this.updateLinkMenuOptions(options, id); + if (this.sections.has(id)) { + const optionsWithDedupe = Object.assign({}, options, { + dedupeConfigurations: this._dedupeConfiguration, + }); + this.sections.set(id, Object.assign(this.sections.get(id), options)); + this.emit( + this.UPDATE_SECTION, + id, + optionsWithDedupe, + shouldBroadcast, + isStartup + ); + } + }, + + /** + * Save metadata to places db and add a visit for that URL. + */ + updateBookmarkMetadata({ url }) { + this.sections.forEach((section, id) => { + if (id === "highlights") { + // Skip Highlights cards, we already have that metadata. + return; + } + if (section.rows) { + section.rows.forEach(card => { + if ( + card.url === url && + card.description && + card.title && + card.image + ) { + lazy.PlacesUtils.history.update({ + url: card.url, + title: card.title, + description: card.description, + previewImageURL: card.image, + }); + // Highlights query skips bookmarks with no visits. + lazy.PlacesUtils.history.insert({ + url, + title: card.title, + visits: [{}], + }); + } + }); + } + }); + }, + + /** + * Sets the section's context menu options. These are all available context menu + * options minus the ones that are tied to a pref (see CONTEXT_MENU_PREFS) set + * to false. + * + * @param options section options + * @param id section ID + */ + updateLinkMenuOptions(options, id) { + if (options.availableLinkMenuOptions) { + options.contextMenuOptions = options.availableLinkMenuOptions.filter( + o => + !this.CONTEXT_MENU_PREFS[o] || + Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o]) + ); + } + + // Once we have rows, we can give each card it's own context menu based on it's type. + // We only want to do this for highlights because those have different data types. + // All other sections (built by the web extension API) will have the same context menu per section + if (options.rows && id === "highlights") { + this._addCardTypeLinkMenuOptions(options.rows); + } + }, + + /** + * Sets each card in highlights' context menu options based on the card's type. + * (See types.js for a list of types) + * + * @param rows section rows containing a type for each card + */ + _addCardTypeLinkMenuOptions(rows) { + for (let card of rows) { + if (!this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[card.type]) { + console.error( + `No context menu for highlight type ${card.type} is configured` + ); + } else { + card.contextMenuOptions = + this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[card.type]; + + // Remove any options that shouldn't be there based on CONTEXT_MENU_PREFS. + // For example: If the Pocket extension is disabled, we should remove the CheckSavedToPocket option + // for each card that has it + card.contextMenuOptions = card.contextMenuOptions.filter( + o => + !this.CONTEXT_MENU_PREFS[o] || + Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o]) + ); + } + } + }, + + /** + * Update a specific section card by its url. This allows an action to be + * broadcast to all existing pages to update a specific card without having to + * also force-update the rest of the section's cards and state on those pages. + * + * @param id The id of the section with the card to be updated + * @param url The url of the card to update + * @param options The options to update for the card + * @param shouldBroadcast Whether or not to broadcast the update + * @param isStartup If this update is during startup. + */ + updateSectionCard(id, url, options, shouldBroadcast, isStartup = false) { + if (this.sections.has(id)) { + const card = this.sections.get(id).rows.find(elem => elem.url === url); + if (card) { + Object.assign(card, options); + } + this.emit( + this.UPDATE_SECTION_CARD, + id, + url, + options, + shouldBroadcast, + isStartup + ); + } + }, + removeSectionCard(sectionId, url) { + if (!this.sections.has(sectionId)) { + return; + } + const rows = this.sections + .get(sectionId) + .rows.filter(row => row.url !== url); + this.updateSection(sectionId, { rows }, true); + }, + onceInitialized(callback) { + if (this.initialized) { + callback(); + } else { + this.once(this.INIT, callback); + } + }, + uninit() { + Object.keys(this.CONTEXT_MENU_PREFS).forEach(k => + Services.prefs.removeObserver(this.CONTEXT_MENU_PREFS[k], this) + ); + SectionsManager.initialized = false; + }, +}; + +for (const action of [ + "ACTION_DISPATCHED", + "ADD_SECTION", + "REMOVE_SECTION", + "ENABLE_SECTION", + "DISABLE_SECTION", + "UPDATE_SECTION", + "UPDATE_SECTION_CARD", + "INIT", + "UNINIT", +]) { + SectionsManager[action] = action; +} + +EventEmitter.decorate(SectionsManager); + +class SectionsFeed { + constructor() { + this.init = this.init.bind(this); + this.onAddSection = this.onAddSection.bind(this); + this.onRemoveSection = this.onRemoveSection.bind(this); + this.onUpdateSection = this.onUpdateSection.bind(this); + this.onUpdateSectionCard = this.onUpdateSectionCard.bind(this); + } + + init() { + SectionsManager.on(SectionsManager.ADD_SECTION, this.onAddSection); + SectionsManager.on(SectionsManager.REMOVE_SECTION, this.onRemoveSection); + SectionsManager.on(SectionsManager.UPDATE_SECTION, this.onUpdateSection); + SectionsManager.on( + SectionsManager.UPDATE_SECTION_CARD, + this.onUpdateSectionCard + ); + // Catch any sections that have already been added + SectionsManager.sections.forEach((section, id) => + this.onAddSection( + SectionsManager.ADD_SECTION, + id, + section, + true /* isStartup */ + ) + ); + } + + uninit() { + SectionsManager.uninit(); + SectionsManager.emit(SectionsManager.UNINIT); + SectionsManager.off(SectionsManager.ADD_SECTION, this.onAddSection); + SectionsManager.off(SectionsManager.REMOVE_SECTION, this.onRemoveSection); + SectionsManager.off(SectionsManager.UPDATE_SECTION, this.onUpdateSection); + SectionsManager.off( + SectionsManager.UPDATE_SECTION_CARD, + this.onUpdateSectionCard + ); + } + + onAddSection(event, id, options, isStartup = false) { + if (options) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.SECTION_REGISTER, + data: Object.assign({ id }, options), + meta: { + isStartup, + }, + }) + ); + + // Make sure the section is in sectionOrder pref. Otherwise, prepend it. + const orderedSections = this.orderedSectionIds; + if (!orderedSections.includes(id)) { + orderedSections.unshift(id); + this.store.dispatch( + ac.SetPref("sectionOrder", orderedSections.join(",")) + ); + } + } + } + + onRemoveSection(event, id) { + this.store.dispatch( + ac.BroadcastToContent({ type: at.SECTION_DEREGISTER, data: id }) + ); + } + + onUpdateSection( + event, + id, + options, + shouldBroadcast = false, + isStartup = false + ) { + if (options) { + const action = { + type: at.SECTION_UPDATE, + data: Object.assign(options, { id }), + meta: { + isStartup, + }, + }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + } + + onUpdateSectionCard( + event, + id, + url, + options, + shouldBroadcast = false, + isStartup = false + ) { + if (options) { + const action = { + type: at.SECTION_UPDATE_CARD, + data: { id, url, options }, + meta: { + isStartup, + }, + }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + } + + get orderedSectionIds() { + return this.store.getState().Prefs.values.sectionOrder.split(","); + } + + get enabledSectionIds() { + let sections = this.store + .getState() + .Sections.filter(section => section.enabled) + .map(s => s.id); + // Top Sites is a special case. Append if the feed is enabled. + if (this.store.getState().Prefs.values["feeds.topsites"]) { + sections.push("topsites"); + } + return sections; + } + + moveSection(id, direction) { + const orderedSections = this.orderedSectionIds; + const enabledSections = this.enabledSectionIds; + let index = orderedSections.indexOf(id); + orderedSections.splice(index, 1); + if (direction > 0) { + // "Move Down" + while (index < orderedSections.length) { + // If the section at the index is enabled/visible, insert moved section after. + // Otherwise, move on to the next spot and check it. + if (enabledSections.includes(orderedSections[index++])) { + break; + } + } + } else { + // "Move Up" + while (index > 0) { + // If the section at the previous index is enabled/visible, insert moved section there. + // Otherwise, move on to the previous spot and check it. + index--; + if (enabledSections.includes(orderedSections[index])) { + break; + } + } + } + + orderedSections.splice(index, 0, id); + this.store.dispatch(ac.SetPref("sectionOrder", orderedSections.join(","))); + } + + async onAction(action) { + switch (action.type) { + case at.INIT: + SectionsManager.onceInitialized(this.init); + break; + // Wait for pref values, as some sections have options stored in prefs + case at.PREFS_INITIAL_VALUES: + SectionsManager.init( + action.data, + this.store.dbStorage.getDbTable("sectionPrefs") + ); + break; + case at.PREF_CHANGED: { + if (action.data) { + const matched = action.data.name.match( + /^(feeds.section.(\S+)).options$/i + ); + if (matched) { + await SectionsManager.addBuiltInSection( + matched[1], + action.data.value + ); + this.store.dispatch({ + type: at.SECTION_OPTIONS_CHANGED, + data: matched[2], + }); + } + } + break; + } + case at.UPDATE_SECTION_PREFS: + SectionsManager.updateSectionPrefs(action.data.id, action.data.value); + break; + case at.PLACES_BOOKMARK_ADDED: + SectionsManager.updateBookmarkMetadata(action.data); + break; + case at.WEBEXT_DISMISS: + if (action.data) { + SectionsManager.removeSectionCard( + action.data.source, + action.data.url + ); + } + break; + case at.SECTION_DISABLE: + SectionsManager.disableSection(action.data); + break; + case at.SECTION_ENABLE: + SectionsManager.enableSection(action.data); + break; + case at.SECTION_MOVE: + this.moveSection(action.data.id, action.data.direction); + break; + case at.UNINIT: + this.uninit(); + break; + } + if ( + SectionsManager.ACTIONS_TO_PROXY.includes(action.type) && + SectionsManager.sections.size > 0 + ) { + SectionsManager.emit( + SectionsManager.ACTION_DISPATCHED, + action.type, + action.data + ); + } + } +} + +const EXPORTED_SYMBOLS = ["SectionsFeed", "SectionsManager"]; diff --git a/browser/components/newtab/lib/ShortURL.jsm b/browser/components/newtab/lib/ShortURL.jsm new file mode 100644 index 0000000000..19416904f0 --- /dev/null +++ b/browser/components/newtab/lib/ShortURL.jsm @@ -0,0 +1,83 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "IDNService", + "@mozilla.org/network/idn-service;1", + "nsIIDNService" +); + +/** + * Properly convert internationalized domain names. + * @param {string} host Domain hostname. + * @returns {string} Hostname suitable to be displayed. + */ +function handleIDNHost(hostname) { + try { + return lazy.IDNService.convertToDisplayIDN(hostname, {}); + } catch (e) { + // If something goes wrong (e.g. host is an IP address) just fail back + // to the full domain. + return hostname; + } +} + +/** + * Get the effective top level domain of a host. + * @param {string} host The host to be analyzed. + * @return {str} The suffix or empty string if there's no suffix. + */ +function getETLD(host) { + try { + return Services.eTLD.getPublicSuffixFromHost(host); + } catch (err) { + return ""; + } +} + +/** + * shortURL - Creates a short version of a link's url, used for display purposes + * e.g. {url: http://www.foosite.com} => "foosite" + * + * @param {obj} link A link object + * {str} link.url (required)- The url of the link + * @return {str} A short url + */ +function shortURL({ url }) { + if (!url) { + return ""; + } + + // Make sure we have a valid / parseable url + let parsed; + try { + parsed = new URL(url); + } catch (ex) { + // Not entirely sure what we have, but just give it back + return url; + } + + // Clean up the url (lowercase hostname via URL and remove www.) + const hostname = parsed.hostname.replace(/^www\./i, ""); + + // Remove the eTLD (e.g., com, net) and the preceding period from the hostname + const eTLD = getETLD(hostname); + const eTLDExtra = eTLD.length ? -(eTLD.length + 1) : Infinity; + + // Ideally get the short eTLD-less host but fall back to longer url parts + return ( + handleIDNHost(hostname.slice(0, eTLDExtra) || hostname) || + parsed.pathname || + parsed.href + ); +} + +const EXPORTED_SYMBOLS = ["shortURL", "getETLD"]; diff --git a/browser/components/newtab/lib/SiteClassifier.jsm b/browser/components/newtab/lib/SiteClassifier.jsm new file mode 100644 index 0000000000..0b174676b9 --- /dev/null +++ b/browser/components/newtab/lib/SiteClassifier.jsm @@ -0,0 +1,99 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +// Returns whether the passed in params match the criteria. +// To match, they must contain all the params specified in criteria and the values +// must match if a value is provided in criteria. +function _hasParams(criteria, params) { + for (let param of criteria) { + const val = params.get(param.key); + if ( + val === null || + (param.value && param.value !== val) || + (param.prefix && !val.startsWith(param.prefix)) + ) { + return false; + } + } + return true; +} + +/** + * classifySite + * Classifies a given URL into a category based on classification data from RemoteSettings. + * The data from remote settings can match a category by one of the following: + * - match the exact URL + * - match the hostname or second level domain (sld) + * - match query parameter(s), and optionally their values or prefixes + * - match both (hostname or sld) and query parameter(s) + * + * The data looks like: + * [{ + * "type": "hostname-and-params-match", + * "criteria": [ + * { + * "url": "https://matchurl.com", + * "hostname": "matchhostname.com", + * "sld": "secondleveldomain", + * "params": [ + * { + * "key": "matchparam", + * "value": "matchvalue", + * "prefix": "matchpPrefix", + * }, + * ], + * }, + * ], + * "weight": 300, + * },...] + */ +async function classifySite(url, RS = RemoteSettings) { + let category = "other"; + let parsedURL; + + // Try to parse the url. + for (let _url of [url, `https://${url}`]) { + try { + parsedURL = new URL(_url); + break; + } catch (e) {} + } + + if (parsedURL) { + // If we parsed successfully, find a match. + const hostname = parsedURL.hostname.replace(/^www\./i, ""); + const params = parsedURL.searchParams; + // NOTE: there will be an initial/default local copy of the data in m-c. + // Therefore, this should never return an empty list []. + const siteTypes = await RS("sites-classification").get(); + const sortedSiteTypes = siteTypes.sort( + (x, y) => (y.weight || 0) - (x.weight || 0) + ); + for (let type of sortedSiteTypes) { + for (let criteria of type.criteria) { + if (criteria.url && criteria.url !== url) { + continue; + } + if (criteria.hostname && criteria.hostname !== hostname) { + continue; + } + if (criteria.sld && criteria.sld !== hostname.split(".")[0]) { + continue; + } + if (criteria.params && !_hasParams(criteria.params, params)) { + continue; + } + return type.type; + } + } + } + return category; +} + +const EXPORTED_SYMBOLS = ["classifySite"]; diff --git a/browser/components/newtab/lib/SnippetsTestMessageProvider.sys.mjs b/browser/components/newtab/lib/SnippetsTestMessageProvider.sys.mjs new file mode 100644 index 0000000000..7f28572f32 --- /dev/null +++ b/browser/components/newtab/lib/SnippetsTestMessageProvider.sys.mjs @@ -0,0 +1,688 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const TEST_ICON = "chrome://branding/content/icon64.png"; +const TEST_ICON_16 = "chrome://branding/content/icon16.png"; +const TEST_ICON_BW = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAQAAAC1+jfqAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQfjBQ8QDifrKGc/AAABf0lEQVQoz4WRO08UUQCFvztzd1AgG9jRgGwkhEoMIYGSygYt+A00tpZGY0jYxAJKEwkNjX9AK2xACx4dhFiQQCiMMRr2kYXdnQcz7L0z91qAMVac6hTfSU7OgVsk/prtyfSNfRb7ge2cd7dmVucP/wM2lwqVqoyICahRx9Nz71+8AnAAvlTct+dSYDBYcgJ+Fj68XFu/AfamnIoWFoHFYrAUuYMSn55/fAIOxIs1t4MhQpNxRYsUD0ld7r8DCfZph4QecrqkhCREgMLSeISQkAy0UBgE0CYgIkeRA9HdsCQhpEGCxichpItHigEcPH4XJLRbTf8STY0iiiuu60Ifxexx04F0N+aCgJCAhPQmD/cp/RC5A79WvUyhUHSIidAIoESv9VfAhW9n8+XqTCoyMsz1cviMMrGz9BrjAuboYHZajyXCInEocI8yvccbC+0muABanR4/tONjQz3DzgNKtj9sfv66XD9B/3tT9g/akb7h0bJwzxqqmlRHLr4rLPwBlYWoYj77l2AAAAAldEVYdGRhdGU6Y3JlYXRlADIwMTktMDUtMTVUMTY6MTQ6MzkrMDA6MDD5/4XBAAAAJXRFWHRkYXRlOm1vZGlmeQAyMDE5LTA1LTE1VDE2OjE0OjM5KzAwOjAwiKI9fQAAAABJRU5ErkJggg=="; + +const MESSAGES = () => [ + { + template: "simple_snippet", + template_version: "1.1.2", + content: { + text: "This is for <link0>preferences</link0> and <link1>about</link1>", + icon: "https://snippets.cdn.mozilla.net/media/icons/1a8bb10e-8166-4e14-9e41-c1f85a41bcd2.png", + button_label: "Button Label", + section_title_icon: + "https://snippets.cdn.mozilla.net/media/icons/5878847e-a1fb-4204-aad9-09f6cf7f99ee.png", + section_title_text: "Messages from Firefox", + section_title_url: + "https://support.mozilla.org/kb/snippets-firefox-faq?utm_source=desktop-snippet&utm_medium=snippet&utm_campaign=&utm_term=&utm_content=", + tall: false, + block_button_text: "Remove this", + do_not_autoblock: true, + links: { + link0: { + action: "OPEN_PREFERENCES_PAGE", + entrypoint_value: "snippet", + args: "sync", + }, + link1: { + action: "OPEN_ABOUT_PAGE", + args: "about", + entrypoint_name: "entryPoint", + entrypoint_value: "snippet", + }, + }, + button_action: "OPEN_PREFERENCES_PAGE", + button_entrypoint_value: "snippet", + }, + id: "preview-13516_button_preferences", + }, + { + template: "simple_snippet", + template_version: "1.1.2", + content: { + text: "This is for <link0>preferences</link0> and <link1>about</link1>", + icon: "https://snippets.cdn.mozilla.net/media/icons/1a8bb10e-8166-4e14-9e41-c1f85a41bcd2.png", + button_label: "Button Label", + section_title_icon: + "https://snippets.cdn.mozilla.net/media/icons/5878847e-a1fb-4204-aad9-09f6cf7f99ee.png", + section_title_text: "Messages from Firefox", + section_title_url: + "https://support.mozilla.org/kb/snippets-firefox-faq?utm_source=desktop-snippet&utm_medium=snippet&utm_campaign=&utm_term=&utm_content=", + tall: false, + block_button_text: "Remove this", + do_not_autoblock: true, + links: { + link0: { + action: "OPEN_PREFERENCES_PAGE", + entrypoint_value: "snippet", + }, + link1: { + action: "OPEN_ABOUT_PAGE", + args: "about", + entrypoint_name: "entryPoint", + entrypoint_value: "snippet", + }, + }, + button_action: "OPEN_ABOUT_PAGE", + button_action_args: "logins", + button_entrypoint_name: "entryPoint", + button_entrypoint_value: "snippet", + }, + id: "preview-13517_button_about", + }, + { + id: "SIMPLE_TEST_1", + template: "simple_snippet", + campaign: "test_campaign_blocking", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + title: "Firefox Account!", + title_icon: TEST_ICON_16, + title_icon_dark_theme: TEST_ICON_BW, + text: "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_TEST_1_NO_DARK_THEME", + template: "simple_snippet", + campaign: "test_campaign_blocking", + content: { + icon: TEST_ICON, + icon_dark_theme: "", + title: "Firefox Account!", + title_icon: TEST_ICON_16, + title_icon_dark_theme: "", + text: "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_TEST_1_SAME_CAMPAIGN", + template: "simple_snippet", + campaign: "test_campaign_blocking", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + text: "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_TEST_TALL", + template: "simple_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + text: "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + button_label: "Get one now!", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + block_button_text: "Block", + tall: true, + }, + }, + { + id: "SIMPLE_TEST_BUTTON_URL_1", + template: "simple_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Get one now!", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + text: "Sync it, link it, take it with you. All this and more with a Firefox Account.", + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_TEST_BUTTON_ACTION_1", + template: "simple_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Open about:config", + button_action: "OPEN_ABOUT_PAGE", + button_action_args: "config", + text: "Testing the OPEN_ABOUT_PAGE action", + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_WITH_TITLE_TEST_1", + template: "simple_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + title: "Ready to sync?", + text: "Get connected with a <syncLink>Firefox account</syncLink>.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "NEWSLETTER_TEST_DEFAULTS", + template: "newsletter_snippet", + content: { + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_title: "Be a part of a movement.", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: TEST_ICON_BW, + scene1_text: + "Internet shutdowns, hackers, harassment – the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.", + scene1_button_label: "Continue", + scene1_button_color: "#712b00", + scene1_button_background_color: "#ff9400", + scene2_title: "Let's do this!", + locale: "en-CA", + scene2_dismiss_button_text: "Dismiss", + scene2_text: + "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.", + scene2_privacy_html: + "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.", + scene2_newsletter: "mozilla-foundation", + success_text: "Check your inbox for the confirmation!", + error_text: "Error!", + retry_button_label: "Try again?", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "NEWSLETTER_TEST_1", + template: "newsletter_snippet", + content: { + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_title: "Be a part of a movement.", + scene1_title_icon: "", + scene1_text: + "Internet shutdowns, hackers, harassment – the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.", + scene1_button_label: "Continue", + scene1_button_color: "#712b00", + scene1_button_background_color: "#ff9400", + scene2_title: "Let's do this!", + locale: "en-CA", + scene2_dismiss_button_text: "Dismiss", + scene2_text: + "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.", + scene2_privacy_html: + "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.", + scene2_button_label: "Sign Me up", + scene2_email_placeholder_text: "Your email here", + scene2_newsletter: "mozilla-foundation", + success_text: "Check your inbox for the confirmation!", + error_text: "Error!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "NEWSLETTER_TEST_SCENE1_SECTION_TITLE_ICON", + template: "newsletter_snippet", + content: { + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_title: "Be a part of a movement.", + scene1_title_icon: "", + scene1_text: + "Internet shutdowns, hackers, harassment – the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.", + scene1_button_label: "Continue", + scene1_button_color: "#712b00", + scene1_button_background_color: "#ff9400", + scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg", + scene1_section_title_text: + "All the Firefox news that's fit to Firefox print!", + scene2_title: "Let's do this!", + locale: "en-CA", + scene2_dismiss_button_text: "Dismiss", + scene2_text: + "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.", + scene2_privacy_html: + "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.", + scene2_button_label: "Sign Me up", + scene2_email_placeholder_text: "Your email here", + scene2_newsletter: "mozilla-foundation", + success_text: "Check your inbox for the confirmation!", + error_text: "Error!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "FXA_SNIPPET_TEST_1", + template: "fxa_signup_snippet", + content: { + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_button_label: "Get connected with sync!", + scene1_button_color: "#712b00", + scene1_button_background_color: "#ff9400", + + scene1_text: + "Connect to Firefox by securely syncing passwords, bookmarks, and open tabs.", + scene1_title: "Browser better.", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: TEST_ICON_BW, + + scene2_text: + "Connect to your Firefox account to securely sync passwords, bookmarks, and open tabs.", + scene2_title: "Title 123", + scene2_email_placeholder_text: "Your email", + scene2_button_label: "Continue", + scene2_dismiss_button_text: "Dismiss", + }, + }, + { + id: "FXA_SNIPPET_TEST_TITLE_ICON", + template: "fxa_signup_snippet", + content: { + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_button_label: "Get connected with sync!", + scene1_button_color: "#712b00", + scene1_button_background_color: "#ff9400", + + scene1_text: + "Connect to Firefox by securely syncing passwords, bookmarks, and open tabs.", + scene1_title: "Browser better.", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: TEST_ICON_BW, + + scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg", + scene1_section_title_text: "Firefox Accounts: Receivable benefits", + + scene2_text: + "Connect to your Firefox account to securely sync passwords, bookmarks, and open tabs.", + scene2_title: "Title 123", + scene2_email_placeholder_text: "Your email", + scene2_button_label: "Continue", + scene2_dismiss_button_text: "Dismiss", + }, + }, + { + id: "SNIPPETS_SEND_TO_DEVICE_TEST", + template: "send_to_device_snippet", + content: { + include_sms: true, + locale: "en-CA", + country: "us", + message_id_sms: "ff-mobilesn-download", + message_id_email: "download-firefox-mobile", + + scene1_button_background_color: "#6200a4", + scene1_button_color: "#FFFFFF", + scene1_button_label: "Install now", + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_text: "Browse without compromise with Firefox Mobile.", + scene1_title: "Full-featured. Customizable. Lightning fast", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: TEST_ICON_BW, + + scene2_button_label: "Send", + scene2_disclaimer_html: + "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.", + scene2_dismiss_button_text: "Dismiss", + scene2_icon: TEST_ICON, + scene2_icon_dark_theme: TEST_ICON_BW, + scene2_input_placeholder: "Your email address or phone number", + scene2_text: + "Send Firefox to your phone and take a powerful independent browser with you.", + scene2_title: "Let's do this!", + + error_text: "Oops, there was a problem.", + success_title: "Your download link was sent.", + success_text: "Check your device for the email message!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "SNIPPETS_SCENE2_SEND_TO_DEVICE_TEST", + template: "send_to_device_scene2_snippet", + content: { + include_sms: true, + locale: "en-CA", + country: "us", + message_id_sms: "ff-mobilesn-download", + message_id_email: "download-firefox-mobile", + scene2_icon: TEST_ICON, + section_title_icon: + "https://snippets.cdn.mozilla.net/media/icons/094b0707-ab65-4b2e-99a1-a84122b6ab26.png", + section_title_text: "Messages from Firefox", + section_title_url: "https://support.mozilla.org/kb", + scene2_button_label: "Send", + scene2_disclaimer_html: + "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.", + scene2_input_placeholder: "Your email address or phone number", + scene2_text: + "Send Firefox to your phone and take a powerful independent browser with you.", + error_text: "Oops, there was a problem.", + success_title: "Your download link was sent.", + success_text: "Check your device for the email message!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "SNIPPETS_SEND_TO_DEVICE_TEST_NO_DARK_THEME", + template: "send_to_device_snippet", + content: { + include_sms: true, + locale: "en-CA", + country: "us", + message_id_sms: "ff-mobilesn-download", + message_id_email: "download-firefox-mobile", + + scene1_button_background_color: "#6200a4", + scene1_button_color: "#FFFFFF", + scene1_button_label: "Install now", + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: "", + scene1_text: "Browse without compromise with Firefox Mobile.", + scene1_title: "Full-featured. Customizable. Lightning fast", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: "", + + scene2_button_label: "Send", + scene2_disclaimer_html: + "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.", + scene2_dismiss_button_text: "Dismiss", + scene2_icon: TEST_ICON, + scene2_icon_dark_theme: "", + scene2_input_placeholder: "Your email address or phone number", + scene2_text: + "Send Firefox to your phone and take a powerful independent browser with you.", + scene2_title: "Let's do this!", + + error_text: "Oops, there was a problem.", + success_title: "Your download link was sent.", + success_text: "Check your device for the email message!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "SNIPPETS_SEND_TO_DEVICE_TEST_SECTION_TITLE_ICON", + template: "send_to_device_snippet", + content: { + include_sms: true, + locale: "en-CA", + country: "us", + message_id_sms: "ff-mobilesn-download", + message_id_email: "download-firefox-mobile", + + scene1_button_background_color: "#6200a4", + scene1_button_color: "#FFFFFF", + scene1_button_label: "Install now", + scene1_icon: TEST_ICON, + scene1_icon_dark_theme: TEST_ICON_BW, + scene1_text: "Browse without compromise with Firefox Mobile.", + scene1_title: "Full-featured. Customizable. Lightning fast", + scene1_title_icon: TEST_ICON_16, + scene1_title_icon_dark_theme: TEST_ICON_BW, + scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg", + scene1_section_title_text: "Send Firefox to your mobile device!", + + scene2_button_label: "Send", + scene2_disclaimer_html: + "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.", + scene2_dismiss_button_text: "Dismiss", + scene2_icon: TEST_ICON, + scene2_icon_dark_theme: TEST_ICON_BW, + scene2_input_placeholder: "Your email address or phone number", + scene2_text: + "Send Firefox to your phone and take a powerful independent browser with you.", + scene2_title: "Let's do this!", + + error_text: "Oops, there was a problem.", + success_title: "Your download link was sent.", + success_text: "Check your device for the email message!", + links: { + privacyLink: { + url: "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894", + }, + }, + }, + }, + { + id: "EOY_TEST_1", + template: "eoy_snippet", + content: { + highlight_color: "#f05", + background_color: "#ddd", + text_color: "yellow", + selected_button: "donation_amount_first", + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Donate", + monthly_checkbox_label_text: "Make my donation monthly", + currency_code: "usd", + donation_amount_first: 50, + donation_amount_second: 25, + donation_amount_third: 10, + donation_amount_fourth: 5, + donation_form_url: + "https://donate.mozilla.org/pl/?utm_source=desktop-snippet&utm_medium=snippet&utm_campaign=donate&utm_term=7556", + text: "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?", + }, + }, + { + id: "EOY_BOLD_TEST_1", + template: "eoy_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + selected_button: "donation_amount_second", + button_label: "Donate", + monthly_checkbox_label_text: "Make my donation monthly", + currency_code: "usd", + donation_amount_first: 50, + donation_amount_second: 25, + donation_amount_third: 10, + donation_amount_fourth: 5, + donation_form_url: "https://donate.mozilla.org", + text: "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?", + test: "bold", + }, + }, + { + id: "EOY_TAKEOVER_TEST_1", + template: "eoy_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Donate", + monthly_checkbox_label_text: "Make my donation monthly", + currency_code: "usd", + donation_amount_first: 50, + donation_amount_second: 25, + donation_amount_third: 10, + donation_amount_fourth: 5, + donation_form_url: "https://donate.mozilla.org", + text: "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?", + test: "takeover", + }, + }, + { + id: "SIMPLE_TEST_WITH_SECTION_HEADING", + template: "simple_snippet", + content: { + button_label: "Get one now!", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + title: "Firefox Account!", + text: "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + section_title_icon: "chrome://global/skin/icons/pocket.svg", + section_title_text: "Messages from Mozilla", + }, + }, + { + id: "SIMPLE_TEST_WITH_SECTION_HEADING_AND_LINK", + template: "simple_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + title: "Firefox Account!", + text: "Sync it, link it, take it with you. All this and more with a Firefox Account.", + block_button_text: "Block", + section_title_icon: "chrome://global/skin/icons/pocket.svg", + section_title_text: "Messages from Mozilla (click for info)", + section_title_url: "https://www.mozilla.org/about", + }, + }, + { + id: "SIMPLE_BELOW_SEARCH_TEST_1", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + text: "Securely store passwords, bookmarks, and more with a Firefox Account. <syncLink>Sign up</syncLink>", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_BELOW_SEARCH_TEST_2", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + text: "<syncLink>Connect your Firefox Account to Sync</syncLink> your protected passwords, open tabs and bookmarks, and they'll always be available to you - on all of your devices.", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SIMPLE_BELOW_SEARCH_TEST_TITLE", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + title: "See if you've been part of an online data breach.", + text: "Securely store passwords, bookmarks, and more with a Firefox Account. <syncLink>Sign up</syncLink>", + links: { + syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" }, + }, + block_button_text: "Block", + }, + }, + { + id: "SPECIAL_SNIPPET_BUTTON_1", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Find Out Now", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + title: "See if you've been part of an online data breach.", + text: "Firefox Monitor tells you what hackers already know about you.", + block_button_text: "Block", + }, + }, + { + id: "SPECIAL_SNIPPET_LONG_CONTENT", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Find Out Now", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + title: "See if you've been part of an online data breach.", + text: "Firefox Monitor tells you what hackers already know about you. Here's some extra text to make the content really long.", + block_button_text: "Block", + }, + }, + { + id: "SPECIAL_SNIPPET_NO_TITLE", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + icon_dark_theme: TEST_ICON_BW, + button_label: "Find Out Now", + button_url: "https://www.mozilla.org/en-US/firefox/accounts", + text: "Firefox Monitor tells you what hackers already know about you.", + block_button_text: "Block", + }, + }, + { + id: "SPECIAL_SNIPPET_MONITOR", + template: "simple_below_search_snippet", + content: { + icon: TEST_ICON, + title: "See if you've been part of an online data breach.", + text: "Firefox Monitor tells you what hackers already know about you.", + button_label: "Get monitor", + button_action: "ENABLE_FIREFOX_MONITOR", + button_action_args: { + url: "https://monitor.firefox.com/oauth/init?utm_source=snippets&utm_campaign=monitor-snippet-test&form_type=email&entrypoint=newtab", + flowRequestParams: { + entrypoint: "snippets", + utm_term: "monitor", + form_type: "email", + }, + }, + block_button_text: "Block", + }, + }, +]; + +export const SnippetsTestMessageProvider = { + getMessages() { + return Promise.resolve( + MESSAGES() + // Ensures we never actually show test except when triggered by debug tools + .map(message => ({ + ...message, + targeting: `providerCohorts.snippets_local_testing == "SHOW_TEST"`, + })) + ); + }, +}; diff --git a/browser/components/newtab/lib/Spotlight.jsm b/browser/components/newtab/lib/Spotlight.jsm new file mode 100644 index 0000000000..a40af7f97a --- /dev/null +++ b/browser/components/newtab/lib/Spotlight.jsm @@ -0,0 +1,85 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyModuleGetters(lazy, { + AboutWelcomeTelemetry: + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm", +}); + +XPCOMUtils.defineLazyGetter( + lazy, + "AWTelemetry", + () => new lazy.AboutWelcomeTelemetry() +); + +const Spotlight = { + sendUserEventTelemetry(event, message, dispatch) { + const ping = { + message_id: message.content.id, + event, + }; + dispatch({ + type: "SPOTLIGHT_TELEMETRY", + data: { action: "spotlight_user_event", ...ping }, + }); + }, + + defaultDispatch(message) { + if (message.type === "SPOTLIGHT_TELEMETRY") { + const { message_id, event } = message.data; + lazy.AWTelemetry.sendTelemetry({ message_id, event }); + } + }, + + /** + * Shows spotlight tab or window modal specific to the given browser + * @param browser The browser for spotlight display + * @param message Message containing content to show + * @param dispatchCFRAction A function to dispatch resulting actions + * @return boolean value capturing if spotlight was displayed + */ + async showSpotlightDialog(browser, message, dispatch = this.defaultDispatch) { + const win = browser?.ownerGlobal; + if (!win || win.gDialogBox.isOpen) { + return false; + } + const spotlight_url = "chrome://browser/content/spotlight.html"; + + const dispatchCFRAction = + // This also blocks CFR impressions, which is fine for current use cases. + message.content?.metrics === "block" ? () => {} : dispatch; + + // This handles `IMPRESSION` events used by ASRouter for frequency caps. + // AboutWelcome handles `IMPRESSION` events for telemetry. + this.sendUserEventTelemetry("IMPRESSION", message, dispatchCFRAction); + dispatchCFRAction({ type: "IMPRESSION", data: message }); + + if (message.content?.modal === "tab") { + let { closedPromise } = win.gBrowser.getTabDialogBox(browser).open( + spotlight_url, + { + features: "resizable=no", + allowDuplicateDialogs: false, + }, + message.content + ); + await closedPromise; + } else { + await win.gDialogBox.open(spotlight_url, message.content); + } + + // If dismissed report telemetry and exit + this.sendUserEventTelemetry("DISMISS", message, dispatchCFRAction); + return true; + }, +}; + +const EXPORTED_SYMBOLS = ["Spotlight"]; diff --git a/browser/components/newtab/lib/Store.jsm b/browser/components/newtab/lib/Store.jsm new file mode 100644 index 0000000000..36c2e03498 --- /dev/null +++ b/browser/components/newtab/lib/Store.jsm @@ -0,0 +1,195 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { ActivityStreamMessageChannel } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamMessageChannel.jsm" +); +const { ActivityStreamStorage } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamStorage.jsm" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { reducers } = ChromeUtils.importESModule( + "resource://activity-stream/common/Reducers.sys.mjs" +); +const { redux } = ChromeUtils.import( + "resource://activity-stream/vendor/Redux.jsm" +); + +/** + * Store - This has a similar structure to a redux store, but includes some extra + * functionality to allow for routing of actions between the Main processes + * and child processes via a ActivityStreamMessageChannel. + * It also accepts an array of "Feeds" on inititalization, which + * can listen for any action that is dispatched through the store. + */ +class Store { + /** + * constructor - The redux store and message manager are created here, + * but no listeners are added until "init" is called. + */ + constructor() { + this._middleware = this._middleware.bind(this); + // Bind each redux method so we can call it directly from the Store. E.g., + // store.dispatch() will call store._store.dispatch(); + for (const method of ["dispatch", "getState", "subscribe"]) { + this[method] = (...args) => this._store[method](...args); + } + this.feeds = new Map(); + this._prefs = new Prefs(); + this._messageChannel = new ActivityStreamMessageChannel({ + dispatch: this.dispatch, + }); + this._store = redux.createStore( + redux.combineReducers(reducers), + redux.applyMiddleware(this._middleware, this._messageChannel.middleware) + ); + this.storage = null; + } + + /** + * _middleware - This is redux middleware consumed by redux.createStore. + * it calls each feed's .onAction method, if one + * is defined. + */ + _middleware() { + return next => action => { + next(action); + for (const store of this.feeds.values()) { + if (store.onAction) { + store.onAction(action); + } + } + }; + } + + /** + * initFeed - Initializes a feed by calling its constructor function + * + * @param {string} feedName The name of a feed, as defined in the object + * passed to Store.init + * @param {Action} initAction An optional action to initialize the feed + */ + initFeed(feedName, initAction) { + const feed = this._feedFactories.get(feedName)(); + feed.store = this; + this.feeds.set(feedName, feed); + if (initAction && feed.onAction) { + feed.onAction(initAction); + } + } + + /** + * uninitFeed - Removes a feed and calls its uninit function if defined + * + * @param {string} feedName The name of a feed, as defined in the object + * passed to Store.init + * @param {Action} uninitAction An optional action to uninitialize the feed + */ + uninitFeed(feedName, uninitAction) { + const feed = this.feeds.get(feedName); + if (!feed) { + return; + } + if (uninitAction && feed.onAction) { + feed.onAction(uninitAction); + } + this.feeds.delete(feedName); + } + + /** + * onPrefChanged - Listener for handling feed changes. + */ + onPrefChanged(name, value) { + if (this._feedFactories.has(name)) { + if (value) { + this.initFeed(name, this._initAction); + } else { + this.uninitFeed(name, this._uninitAction); + } + } + } + + /** + * init - Initializes the ActivityStreamMessageChannel channel, and adds feeds. + * + * Note that it intentionally initializes the TelemetryFeed first so that the + * addon is able to report the init errors from other feeds. + * + * @param {Map} feedFactories A Map of feeds with the name of the pref for + * the feed as the key and a function that + * constructs an instance of the feed. + * @param {Action} initAction An optional action that will be dispatched + * to feeds when they're created. + * @param {Action} uninitAction An optional action for when feeds uninit. + */ + async init(feedFactories, initAction, uninitAction) { + this._feedFactories = feedFactories; + this._initAction = initAction; + this._uninitAction = uninitAction; + + const telemetryKey = "feeds.telemetry"; + if (feedFactories.has(telemetryKey) && this._prefs.get(telemetryKey)) { + this.initFeed(telemetryKey); + } + + await this._initIndexedDB(telemetryKey); + + for (const pref of feedFactories.keys()) { + if (pref !== telemetryKey && this._prefs.get(pref)) { + this.initFeed(pref); + } + } + + this._prefs.observeBranch(this); + + // Dispatch an initial action after all enabled feeds are ready + if (initAction) { + this.dispatch(initAction); + } + + // Dispatch NEW_TAB_INIT/NEW_TAB_LOAD events after INIT event. + this._messageChannel.simulateMessagesForExistingTabs(); + } + + async _initIndexedDB(telemetryKey) { + this.dbStorage = new ActivityStreamStorage({ + storeNames: ["sectionPrefs", "snippets"], + }); + // Accessing the db causes the object stores to be created / migrated. + // This needs to happen before other instances try to access the db, which + // would update only a subset of the stores to the latest version. + try { + await this.dbStorage.db; // eslint-disable-line no-unused-expressions + } catch (e) { + this.dbStorage.telemetry = null; + } + } + + /** + * uninit - Uninitalizes each feed, clears them, and destroys the message + * manager channel. + * + * @return {type} description + */ + uninit() { + if (this._uninitAction) { + this.dispatch(this._uninitAction); + } + this._prefs.ignoreBranch(this); + this.feeds.clear(); + this._feedFactories = null; + } + + /** + * getMessageChannel - Used by the AboutNewTabParent actor to get the message channel. + */ + getMessageChannel() { + return this._messageChannel; + } +} + +const EXPORTED_SYMBOLS = ["Store"]; diff --git a/browser/components/newtab/lib/SystemTickFeed.jsm b/browser/components/newtab/lib/SystemTickFeed.jsm new file mode 100644 index 0000000000..d55cedc45d --- /dev/null +++ b/browser/components/newtab/lib/SystemTickFeed.jsm @@ -0,0 +1,75 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + clearInterval: "resource://gre/modules/Timer.sys.mjs", + setInterval: "resource://gre/modules/Timer.sys.mjs", +}); + +// Frequency at which SYSTEM_TICK events are fired +const SYSTEM_TICK_INTERVAL = 5 * 60 * 1000; + +class SystemTickFeed { + init() { + this._idleService = Cc["@mozilla.org/widget/useridleservice;1"].getService( + Ci.nsIUserIdleService + ); + this._hasObserver = false; + this.setTimer(); + } + + setTimer() { + this.intervalId = lazy.setInterval(() => { + if (this._idleService.idleTime > SYSTEM_TICK_INTERVAL) { + this.cancelTimer(); + Services.obs.addObserver(this, "user-interaction-active"); + this._hasObserver = true; + return; + } + this.dispatchTick(); + }, SYSTEM_TICK_INTERVAL); + } + + cancelTimer() { + lazy.clearInterval(this.intervalId); + this.intervalId = null; + } + + observe() { + this.dispatchTick(); + Services.obs.removeObserver(this, "user-interaction-active"); + this._hasObserver = false; + this.setTimer(); + } + + dispatchTick() { + ChromeUtils.idleDispatch(() => + this.store.dispatch({ type: at.SYSTEM_TICK }) + ); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.cancelTimer(); + if (this._hasObserver) { + Services.obs.removeObserver(this, "user-interaction-active"); + this._hasObserver = false; + } + break; + } + } +} + +const EXPORTED_SYMBOLS = ["SystemTickFeed", "SYSTEM_TICK_INTERVAL"]; diff --git a/browser/components/newtab/lib/TelemetryFeed.jsm b/browser/components/newtab/lib/TelemetryFeed.jsm new file mode 100644 index 0000000000..653603d809 --- /dev/null +++ b/browser/components/newtab/lib/TelemetryFeed.jsm @@ -0,0 +1,1403 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule( + "resource://activity-stream/common/ActorConstants.sys.mjs" +); + +const { actionTypes: at, actionUtils: au } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { classifySite } = ChromeUtils.import( + "resource://activity-stream/lib/SiteClassifier.jsm" +); + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "AboutNewTab", + "resource:///modules/AboutNewTab.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "PingCentre", + "resource:///modules/PingCentre.jsm" +); +ChromeUtils.defineESModuleGetters(lazy, { + ClientID: "resource://gre/modules/ClientID.sys.mjs", + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + ExtensionSettingsStore: + "resource://gre/modules/ExtensionSettingsStore.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs", + UTEventReporting: "resource://activity-stream/lib/UTEventReporting.sys.mjs", + UpdateUtils: "resource://gre/modules/UpdateUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); +ChromeUtils.defineModuleGetter( + lazy, + "HomePage", + "resource:///modules/HomePage.jsm" +); +XPCOMUtils.defineLazyModuleGetters(lazy, { + AboutWelcomeTelemetry: + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm", +}); +XPCOMUtils.defineLazyGetter( + lazy, + "Telemetry", + () => new lazy.AboutWelcomeTelemetry() +); + +const ACTIVITY_STREAM_ID = "activity-stream"; +const DOMWINDOW_OPENED_TOPIC = "domwindowopened"; +const DOMWINDOW_UNLOAD_TOPIC = "unload"; +const TAB_PINNED_EVENT = "TabPinned"; + +// This is a mapping table between the user preferences and its encoding code +const USER_PREFS_ENCODING = { + showSearch: 1 << 0, + "feeds.topsites": 1 << 1, + "feeds.section.topstories": 1 << 2, + "feeds.section.highlights": 1 << 3, + "feeds.snippets": 1 << 4, + showSponsored: 1 << 5, + "asrouter.userprefs.cfr.addons": 1 << 6, + "asrouter.userprefs.cfr.features": 1 << 7, + showSponsoredTopSites: 1 << 8, +}; + +const PREF_IMPRESSION_ID = "impressionId"; +const TELEMETRY_PREF = "telemetry"; +const EVENTS_TELEMETRY_PREF = "telemetry.ut.events"; +const STRUCTURED_INGESTION_ENDPOINT_PREF = + "telemetry.structuredIngestion.endpoint"; +// List of namespaces for the structured ingestion system. +// They are defined in https://github.com/mozilla-services/mozilla-pipeline-schemas +const STRUCTURED_INGESTION_NAMESPACE_AS = "activity-stream"; +const STRUCTURED_INGESTION_NAMESPACE_MS = "messaging-system"; +const STRUCTURED_INGESTION_NAMESPACE_CS = "contextual-services"; + +// Used as the missing value for timestamps in the session ping +const TIMESTAMP_MISSING_VALUE = -1; + +// Page filter for onboarding telemetry, any value other than these will +// be set as "other" +const ONBOARDING_ALLOWED_PAGE_VALUES = [ + "about:welcome", + "about:home", + "about:newtab", +]; + +XPCOMUtils.defineLazyGetter( + lazy, + "browserSessionId", + () => lazy.TelemetrySession.getMetadata("").sessionId +); + +// The scalar category for TopSites of Contextual Services +const SCALAR_CATEGORY_TOPSITES = "contextual.services.topsites"; +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +XPCOMUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +class TelemetryFeed { + constructor() { + this.sessions = new Map(); + this._prefs = new Prefs(); + this._impressionId = this.getOrCreateImpressionId(); + this._aboutHomeSeen = false; + this._classifySite = classifySite; + this._addWindowListeners = this._addWindowListeners.bind(this); + this._browserOpenNewtabStart = null; + this.handleEvent = this.handleEvent.bind(this); + } + + get telemetryEnabled() { + return this._prefs.get(TELEMETRY_PREF); + } + + get eventTelemetryEnabled() { + return this._prefs.get(EVENTS_TELEMETRY_PREF); + } + + get structuredIngestionEndpointBase() { + return this._prefs.get(STRUCTURED_INGESTION_ENDPOINT_PREF); + } + + get telemetryClientId() { + Object.defineProperty(this, "telemetryClientId", { + value: lazy.ClientID.getClientID(), + }); + return this.telemetryClientId; + } + + get processStartTs() { + let startupInfo = Services.startup.getStartupInfo(); + let processStartTs = startupInfo.process.getTime(); + + Object.defineProperty(this, "processStartTs", { + value: processStartTs, + }); + return this.processStartTs; + } + + init() { + this._beginObservingNewtabPingPrefs(); + Services.obs.addObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + // Add pin tab event listeners on future windows + Services.obs.addObserver(this._addWindowListeners, DOMWINDOW_OPENED_TOPIC); + // Listen for pin tab events on all open windows + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._addWindowListeners(win); + } + // Set two scalars for the "deletion-request" ping (See bug 1602064 and 1729474) + Services.telemetry.scalarSet( + "deletion.request.impression_id", + this._impressionId + ); + Services.telemetry.scalarSet("deletion.request.context_id", lazy.contextId); + Glean.newtab.locale.set(Services.locale.appLocaleAsBCP47); + } + + handleEvent(event) { + switch (event.type) { + case TAB_PINNED_EVENT: + this.countPinnedTab(event.target); + break; + case DOMWINDOW_UNLOAD_TOPIC: + this._removeWindowListeners(event.target); + break; + } + } + + _removeWindowListeners(win) { + win.removeEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent); + win.removeEventListener(TAB_PINNED_EVENT, this.handleEvent); + } + + _addWindowListeners(win) { + win.addEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent); + win.addEventListener(TAB_PINNED_EVENT, this.handleEvent); + } + + countPinnedTab(target, source = "TAB_CONTEXT_MENU") { + const win = target.ownerGlobal; + if (lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + return; + } + const event = Object.assign(this.createPing(), { + action: "activity_stream_user_event", + event: TAB_PINNED_EVENT.toUpperCase(), + value: { total_pinned_tabs: this.countTotalPinnedTabs() }, + source, + // These fields are required but not relevant for this ping + page: "n/a", + session_id: "n/a", + }); + this.sendEvent(event); + } + + countTotalPinnedTabs() { + let pinnedTabs = 0; + for (let win of Services.wm.getEnumerator("navigator:browser")) { + if (win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + continue; + } + for (let tab of win.gBrowser.tabs) { + pinnedTabs += tab.pinned ? 1 : 0; + } + } + + return pinnedTabs; + } + + getOrCreateImpressionId() { + let impressionId = this._prefs.get(PREF_IMPRESSION_ID); + if (!impressionId) { + impressionId = String(Services.uuid.generateUUID()); + this._prefs.set(PREF_IMPRESSION_ID, impressionId); + } + return impressionId; + } + + browserOpenNewtabStart() { + let now = Cu.now(); + this._browserOpenNewtabStart = Math.round(this.processStartTs + now); + + ChromeUtils.addProfilerMarker( + "UserTiming", + now, + "browser-open-newtab-start" + ); + } + + setLoadTriggerInfo(port) { + // XXX note that there is a race condition here; we're assuming that no + // other tab will be interleaving calls to browserOpenNewtabStart and + // when at.NEW_TAB_INIT gets triggered by RemotePages and calls this + // method. For manually created windows, it's hard to imagine us hitting + // this race condition. + // + // However, for session restore, where multiple windows with multiple tabs + // might be restored much closer together in time, it's somewhat less hard, + // though it should still be pretty rare. + // + // The fix to this would be making all of the load-trigger notifications + // return some data with their notifications, and somehow propagate that + // data through closures into the tab itself so that we could match them + // + // As of this writing (very early days of system add-on perf telemetry), + // the hypothesis is that hitting this race should be so rare that makes + // more sense to live with the slight data inaccuracy that it would + // introduce, rather than doing the correct but complicated thing. It may + // well be worth reexamining this hypothesis after we have more experience + // with the data. + + let data_to_save; + try { + if (!this._browserOpenNewtabStart) { + throw new Error("No browser-open-newtab-start recorded."); + } + data_to_save = { + load_trigger_ts: this._browserOpenNewtabStart, + load_trigger_type: "menu_plus_or_keyboard", + }; + } catch (e) { + // if no mark was returned, we have nothing to save + return; + } + this.saveSessionPerfData(port, data_to_save); + } + + /** + * Lazily initialize PingCentre for Activity Stream to send pings + */ + get pingCentre() { + Object.defineProperty(this, "pingCentre", { + value: new lazy.PingCentre({ topic: ACTIVITY_STREAM_ID }), + }); + return this.pingCentre; + } + + /** + * Lazily initialize UTEventReporting to send pings + */ + get utEvents() { + Object.defineProperty(this, "utEvents", { + value: new lazy.UTEventReporting(), + }); + return this.utEvents; + } + + /** + * Get encoded user preferences, multiple prefs will be combined via bitwise OR operator + */ + get userPreferences() { + let prefs = 0; + + for (const pref of Object.keys(USER_PREFS_ENCODING)) { + if (this._prefs.get(pref)) { + prefs |= USER_PREFS_ENCODING[pref]; + } + } + return prefs; + } + + /** + * Check if it is in the CFR experiment cohort by querying against the + * experiment manager of Messaging System + * + * @return {bool} + */ + get isInCFRCohort() { + const experimentData = lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "cfr", + }); + if (experimentData && experimentData.slug) { + return true; + } + + return false; + } + + /** + * addSession - Start tracking a new session + * + * @param {string} id the portID of the open session + * @param {string} the URL being loaded for this session (optional) + * @return {obj} Session object + */ + addSession(id, url) { + // XXX refactor to use setLoadTriggerInfo or saveSessionPerfData + + // "unexpected" will be overwritten when appropriate + let load_trigger_type = "unexpected"; + let load_trigger_ts; + + if (!this._aboutHomeSeen && url === "about:home") { + this._aboutHomeSeen = true; + + // XXX note that this will be incorrectly set in the following cases: + // session_restore following by clicking on the toolbar button, + // or someone who has changed their default home page preference to + // something else and later clicks the toolbar. It will also be + // incorrectly unset if someone changes their "Home Page" preference to + // about:newtab. + // + // That said, the ratio of these mistakes to correct cases should + // be very small, and these issues should follow away as we implement + // the remaining load_trigger_type values for about:home in issue 3556. + // + // XXX file a bug to implement remaining about:home cases so this + // problem will go away and link to it here. + load_trigger_type = "first_window_opened"; + + // The real perceived trigger of first_window_opened is the OS-level + // clicking of the icon. We express this by using the process start + // absolute timestamp. + load_trigger_ts = this.processStartTs; + } + + const session = { + session_id: String(Services.uuid.generateUUID()), + // "unknown" will be overwritten when appropriate + page: url ? url : "unknown", + perf: { + load_trigger_type, + is_preloaded: false, + }, + }; + + if (load_trigger_ts) { + session.perf.load_trigger_ts = load_trigger_ts; + } + + this.sessions.set(id, session); + return session; + } + + /** + * endSession - Stop tracking a session + * + * @param {string} portID the portID of the session that just closed + */ + endSession(portID) { + const session = this.sessions.get(portID); + + if (!session) { + // It's possible the tab was never visible – in which case, there was no user session. + return; + } + + this.sendDiscoveryStreamLoadedContent(portID, session); + this.sendDiscoveryStreamImpressions(portID, session); + + Glean.newtab.closed.record({ newtab_visit_id: session.session_id }); + if ( + this.telemetryEnabled && + (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) + ) { + GleanPings.newtab.submit("newtab_session_end"); + } + + if (session.perf.visibility_event_rcvd_ts) { + let absNow = this.processStartTs + Cu.now(); + session.session_duration = Math.round( + absNow - session.perf.visibility_event_rcvd_ts + ); + + // Rounding all timestamps in perf to ease the data processing on the backend. + // NB: use `TIMESTAMP_MISSING_VALUE` if the value is missing. + session.perf.visibility_event_rcvd_ts = Math.round( + session.perf.visibility_event_rcvd_ts + ); + session.perf.load_trigger_ts = Math.round( + session.perf.load_trigger_ts || TIMESTAMP_MISSING_VALUE + ); + session.perf.topsites_first_painted_ts = Math.round( + session.perf.topsites_first_painted_ts || TIMESTAMP_MISSING_VALUE + ); + } else { + // This session was never shown (i.e. the hidden preloaded newtab), there was no user session either. + this.sessions.delete(portID); + return; + } + + let sessionEndEvent = this.createSessionEndEvent(session); + this.sendEvent(sessionEndEvent); + this.sendUTEvent(sessionEndEvent, this.utEvents.sendSessionEndEvent); + this.sessions.delete(portID); + } + + /** + * Send impression pings for Discovery Stream for a given session. + * + * @note the impression reports are stored in session.impressionSets for different + * sources, and will be sent separately accordingly. + * + * @param {String} port The session port with which this is associated + * @param {Object} session The session object + */ + sendDiscoveryStreamImpressions(port, session) { + const { impressionSets } = session; + + if (!impressionSets) { + return; + } + + Object.keys(impressionSets).forEach(source => { + const { tiles, window_inner_width, window_inner_height } = + impressionSets[source]; + const payload = this.createImpressionStats(port, { + source, + tiles, + window_inner_width, + window_inner_height, + }); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + }); + } + + /** + * Send loaded content pings for Discovery Stream for a given session. + * + * @note the loaded content reports are stored in session.loadedContentSets for different + * sources, and will be sent separately accordingly. + * + * @param {String} port The session port with which this is associated + * @param {Object} session The session object + */ + sendDiscoveryStreamLoadedContent(port, session) { + const { loadedContentSets } = session; + + if (!loadedContentSets) { + return; + } + + Object.keys(loadedContentSets).forEach(source => { + const tiles = loadedContentSets[source]; + const payload = this.createImpressionStats(port, { + source, + tiles, + loaded: tiles.length, + }); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + }); + } + + /** + * handleNewTabInit - Handle NEW_TAB_INIT, which creates a new session and sets the a flag + * for session.perf based on whether or not this new tab is preloaded + * + * @param {obj} action the Action object + */ + handleNewTabInit(action) { + const session = this.addSession( + au.getPortIdOfSender(action), + action.data.url + ); + session.perf.is_preloaded = + action.data.browser.getAttribute("preloadedState") === "preloaded"; + } + + /** + * createPing - Create a ping with common properties + * + * @param {string} id The portID of the session, if a session is relevant (optional) + * @return {obj} A telemetry ping + */ + createPing(portID) { + const ping = { + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + user_prefs: this.userPreferences, + }; + + // If the ping is part of a user session, add session-related info + if (portID) { + const session = this.sessions.get(portID) || this.addSession(portID); + Object.assign(ping, { session_id: session.session_id }); + + if (session.page) { + Object.assign(ping, { page: session.page }); + } + } + return ping; + } + + /** + * createImpressionStats - Create a ping for an impression stats + * + * @param {string} portID The portID of the open session + * @param {ob} data The data object to be included in the ping. + * @return {obj} A telemetry ping + */ + createImpressionStats(portID, data) { + let ping = Object.assign(this.createPing(portID), data, { + impression_id: this._impressionId, + }); + // Make sure `session_id` and `client_id` are not in the ping. + delete ping.session_id; + delete ping.client_id; + return ping; + } + + createUserEvent(action) { + return Object.assign( + this.createPing(au.getPortIdOfSender(action)), + action.data, + { action: "activity_stream_user_event" } + ); + } + + createSessionEndEvent(session) { + return Object.assign(this.createPing(), { + session_id: session.session_id, + page: session.page, + session_duration: session.session_duration, + action: "activity_stream_session", + perf: session.perf, + profile_creation_date: + lazy.TelemetryEnvironment.currentEnvironment.profile.resetDate || + lazy.TelemetryEnvironment.currentEnvironment.profile.creationDate, + }); + } + + /** + * Create a ping for AS router event. The client_id is set to "n/a" by default, + * different component can override this by its own telemetry collection policy. + */ + async createASRouterEvent(action) { + let event = { + ...action.data, + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + }; + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (event.event_context && typeof event.event_context === "object") { + event.event_context = JSON.stringify(event.event_context); + } + switch (event.action) { + case "cfr_user_event": + event = await this.applyCFRPolicy(event); + break; + case "snippets_local_testing_user_event": + case "snippets_user_event": + event = await this.applySnippetsPolicy(event); + break; + case "badge_user_event": + case "whats-new-panel_user_event": + event = await this.applyWhatsNewPolicy(event); + break; + case "infobar_user_event": + event = await this.applyInfoBarPolicy(event); + break; + case "spotlight_user_event": + event = await this.applySpotlightPolicy(event); + break; + case "toast_notification_user_event": + event = await this.applyToastNotificationPolicy(event); + break; + case "moments_user_event": + event = await this.applyMomentsPolicy(event); + break; + case "onboarding_user_event": + event = await this.applyOnboardingPolicy(event, session); + break; + case "asrouter_undesired_event": + event = this.applyUndesiredEventPolicy(event); + break; + default: + event = { ping: event }; + break; + } + return event; + } + + /** + * Per Bug 1484035, CFR metrics comply with following policies: + * 1). In release, it collects impression_id and bucket_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + * 4). In Private Browsing windows, unless in experiment, collects impression_id and bucket_id + */ + async applyCFRPolicy(ping) { + if ( + (lazy.UpdateUtils.getUpdateChannel(true) === "release" || + ping.is_private) && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + delete ping.is_private; + return { ping, pingType: "cfr" }; + } + + /** + * Per Bug 1482134, all the metrics for What's New panel use client_id in + * all the release channels + */ + async applyWhatsNewPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + delete ping.action; + return { ping, pingType: "whats-new-panel" }; + } + + async applyInfoBarPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "infobar" }; + } + + async applySpotlightPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "spotlight" }; + } + + async applyToastNotificationPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "toast_notification" }; + } + + /** + * Per Bug 1484035, Moments metrics comply with following policies: + * 1). In release, it collects impression_id, and treats bucket_id as message_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + */ + async applyMomentsPolicy(ping) { + if ( + lazy.UpdateUtils.getUpdateChannel(true) === "release" && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + return { ping, pingType: "moments" }; + } + + /** + * Per Bug 1485069, all the metrics for Snippets in AS router use client_id in + * all the release channels + */ + async applySnippetsPolicy(ping) { + ping.client_id = await this.telemetryClientId; + delete ping.action; + return { ping, pingType: "snippets" }; + } + + /** + * Per Bug 1482134, all the metrics for Onboarding in AS router use client_id in + * all the release channels + */ + async applyOnboardingPolicy(ping, session) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + if (ping.action === "onboarding_user_event" && session && session.page) { + let event_context; + + try { + event_context = ping.event_context + ? JSON.parse(ping.event_context) + : {}; + } catch (e) { + // If `ping.event_context` is not a JSON serialized string, then we create a `value` + // key for it + event_context = { value: ping.event_context }; + } + + if (ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page)) { + event_context.page = session.page; + } else { + console.error(`Invalid 'page' for Onboarding event: ${session.page}`); + } + ping.event_context = JSON.stringify(event_context); + } + delete ping.action; + return { ping, pingType: "onboarding" }; + } + + applyUndesiredEventPolicy(ping) { + ping.impression_id = this._impressionId; + delete ping.action; + return { ping, pingType: "undesired-events" }; + } + + sendEvent(event_object) { + switch (event_object.action) { + case "activity_stream_user_event": + this.sendEventPing(event_object); + break; + case "activity_stream_session": + this.sendSessionPing(event_object); + break; + } + } + + async sendEventPing(ping) { + delete ping.action; + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + if (ping.value && typeof ping.value === "object") { + ping.value = JSON.stringify(ping.value); + } + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_AS, + "events", + 1 + ); + } + + async sendSessionPing(ping) { + delete ping.action; + ping.client_id = await this.telemetryClientId; + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_AS, + "sessions", + 1 + ); + } + + sendUTEvent(event_object, eventFunction) { + if (this.telemetryEnabled && this.eventTelemetryEnabled) { + eventFunction(event_object); + } + } + + /** + * Generates an endpoint for Structured Ingestion telemetry pipeline. Note that + * Structured Ingestion requires a different endpoint for each ping. See more + * details about endpoint schema at: + * https://github.com/mozilla/gcp-ingestion/blob/master/docs/edge.md#postput-request + * + * @param {String} namespace Namespace of the ping, such as "activity-stream" or "messaging-system". + * @param {String} pingType Type of the ping, such as "impression-stats". + * @param {String} version Endpoint version for this ping type. + */ + _generateStructuredIngestionEndpoint(namespace, pingType, version) { + const uuid = Services.uuid.generateUUID().toString(); + // Structured Ingestion does not support the UUID generated by Services.uuid, + // because it contains leading and trailing braces. Need to trim them first. + const docID = uuid.slice(1, -1); + const extension = `${namespace}/${pingType}/${version}/${docID}`; + return `${this.structuredIngestionEndpointBase}/${extension}`; + } + + sendStructuredIngestionEvent(eventObject, namespace, pingType, version) { + if (this.telemetryEnabled) { + this.pingCentre.sendStructuredIngestionPing( + eventObject, + this._generateStructuredIngestionEndpoint(namespace, pingType, version), + namespace + ); + } + } + + handleImpressionStats(action) { + const payload = this.createImpressionStats( + au.getPortIdOfSender(action), + action.data + ); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + } + + handleTopSitesSponsoredImpressionStats(action) { + const { data } = action; + const { + type, + position, + source, + advertiser: advertiser_name, + tile_id, + } = data; + // Legacy telemetry (scalars and PingCentre payloads) expects 1-based tile + // positions. + const legacyTelemetryPosition = position + 1; + + let pingType; + + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (type === "impression") { + pingType = "topsites-impression"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.impression`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.impression.record({ + advertiser_name, + tile_id: tile_id.toString(), + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else if (type === "click") { + pingType = "topsites-click"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.click`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.click.record({ + advertiser_name, + tile_id: tile_id.toString(), + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else { + console.error("Unknown ping type for sponsored TopSites impression"); + return; + } + + let payload = { + ...data, + position: legacyTelemetryPosition, + context_id: lazy.contextId, + }; + delete payload.type; + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_CS, + pingType, + "1" + ); + } + + handleTopSitesOrganicImpressionStats(action) { + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (!session) { + return; + } + + switch (action.data?.type) { + case "impression": + Glean.topsites.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + case "click": + Glean.topsites.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + default: + break; + } + } + + handleUserEvent(action) { + let userEvent = this.createUserEvent(action); + this.sendEvent(userEvent); + this.sendUTEvent(userEvent, this.utEvents.sendUserEvent); + } + + handleDiscoveryStreamUserEvent(action) { + const pocket_logged_in_status = lazy.pktApi.isUserLoggedIn(); + Glean.pocket.isSignedIn.set(pocket_logged_in_status); + this.handleUserEvent({ + ...action, + data: { + ...(action.data || {}), + value: { + ...(action.data?.value || {}), + pocket_logged_in_status, + }, + }, + }); + const session = this.sessions.get(au.getPortIdOfSender(action)); + switch (action.data?.event) { + case "CLICK": + if ( + action.data.source === "POPULAR_TOPICS" || + action.data.value?.card_type === "topics_widget" + ) { + Glean.pocket.topicClick.record({ + newtab_visit_id: session.session_id, + topic: action.data.value?.topic, + }); + } else if (["spoc", "organic"].includes(action.data.value?.card_type)) { + Glean.pocket.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: action.data.value?.card_type === "spoc", + position: action.data.action_position, + }); + } + break; + case "SAVE_TO_POCKET": + Glean.pocket.save.record({ + newtab_visit_id: session.session_id, + is_sponsored: action.data.value?.card_type === "spoc", + position: action.data.action_position, + }); + break; + } + } + + async handleASRouterUserEvent(action) { + const { ping, pingType } = await this.createASRouterEvent(action); + if (!pingType) { + console.error("Unknown ping type for ASRouter telemetry"); + return; + } + + // Now that the action has become a ping, we can echo it to Glean. + if (this.telemetryEnabled) { + lazy.Telemetry.submitGleanPingForPing({ ...ping, pingType }); + } + + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_MS, + pingType, + "1" + ); + } + + /** + * This function is used by ActivityStreamStorage to report errors + * trying to access IndexedDB. + */ + SendASRouterUndesiredEvent(data) { + this.handleASRouterUserEvent({ + data: { ...data, action: "asrouter_undesired_event" }, + }); + } + + async sendPageTakeoverData() { + if (this.telemetryEnabled) { + const value = {}; + let newtabAffected = false; + let homeAffected = false; + let newtabCategory = "disabled"; + let homePageCategory = "disabled"; + + // Check whether or not about:home and about:newtab are set to a custom URL. + // If so, classify them. + if (Services.prefs.getBoolPref("browser.newtabpage.enabled")) { + newtabCategory = "enabled"; + if ( + lazy.AboutNewTab.newTabURLOverridden && + !lazy.AboutNewTab.newTabURL.startsWith("moz-extension://") + ) { + value.newtab_url_category = await this._classifySite( + lazy.AboutNewTab.newTabURL + ); + newtabAffected = true; + newtabCategory = value.newtab_url_category; + } + } + // Check if the newtab page setting is controlled by an extension. + await lazy.ExtensionSettingsStore.initialize(); + const newtabExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "url_overrides", + "newTabURL" + ); + if (newtabExtensionInfo && newtabExtensionInfo.id) { + value.newtab_extension_id = newtabExtensionInfo.id; + newtabAffected = true; + newtabCategory = "extension"; + } + + const homePageURL = lazy.HomePage.get(); + if ( + !["about:home", "about:blank"].includes(homePageURL) && + !homePageURL.startsWith("moz-extension://") + ) { + value.home_url_category = await this._classifySite(homePageURL); + homeAffected = true; + homePageCategory = value.home_url_category; + } + const homeExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "prefs", + "homepage_override" + ); + if (homeExtensionInfo && homeExtensionInfo.id) { + value.home_extension_id = homeExtensionInfo.id; + homeAffected = true; + homePageCategory = "extension"; + } + if (!homeAffected && !lazy.HomePage.overridden) { + homePageCategory = "enabled"; + } + + let page; + if (newtabAffected && homeAffected) { + page = "both"; + } else if (newtabAffected) { + page = "about:newtab"; + } else if (homeAffected) { + page = "about:home"; + } + + if (page) { + const event = Object.assign(this.createPing(), { + action: "activity_stream_user_event", + event: "PAGE_TAKEOVER_DATA", + value, + page, + session_id: "n/a", + }); + this.sendEvent(event); + } + Glean.newtab.newtabCategory.set(newtabCategory); + Glean.newtab.homepageCategory.set(homePageCategory); + if (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) { + GleanPings.newtab.submit("component_init"); + } + } + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.sendPageTakeoverData(); + break; + case at.NEW_TAB_INIT: + this.handleNewTabInit(action); + break; + case at.NEW_TAB_UNLOAD: + this.endSession(au.getPortIdOfSender(action)); + break; + case at.SAVE_SESSION_PERF_DATA: + this.saveSessionPerfData(au.getPortIdOfSender(action), action.data); + break; + case at.TELEMETRY_IMPRESSION_STATS: + this.handleImpressionStats(action); + break; + case at.DISCOVERY_STREAM_IMPRESSION_STATS: + this.handleDiscoveryStreamImpressionStats( + au.getPortIdOfSender(action), + action.data + ); + break; + case at.DISCOVERY_STREAM_LOADED_CONTENT: + this.handleDiscoveryStreamLoadedContent( + au.getPortIdOfSender(action), + action.data + ); + break; + case at.DISCOVERY_STREAM_USER_EVENT: + this.handleDiscoveryStreamUserEvent(action); + break; + case at.TELEMETRY_USER_EVENT: + this.handleUserEvent(action); + break; + // The next few action types come from ASRouter, which doesn't use + // Actions from Actions.jsm, but uses these other custom strings. + case msg.TOOLBAR_BADGE_TELEMETRY: + // Intentional fall-through + case msg.TOOLBAR_PANEL_TELEMETRY: + // Intentional fall-through + case msg.MOMENTS_PAGE_TELEMETRY: + // Intentional fall-through + case msg.DOORHANGER_TELEMETRY: + // Intentional fall-through + case msg.INFOBAR_TELEMETRY: + // Intentional fall-through + case msg.SPOTLIGHT_TELEMETRY: + // Intentional fall-through + case msg.TOAST_NOTIFICATION_TELEMETRY: + // Intentional fall-through + case at.AS_ROUTER_TELEMETRY_USER_EVENT: + this.handleASRouterUserEvent(action); + break; + case at.TOP_SITES_SPONSORED_IMPRESSION_STATS: + this.handleTopSitesSponsoredImpressionStats(action); + break; + case at.TOP_SITES_ORGANIC_IMPRESSION_STATS: + this.handleTopSitesOrganicImpressionStats(action); + break; + case at.UNINIT: + this.uninit(); + break; + } + } + + /** + * Handle impression stats actions from Discovery Stream. The data will be + * stored into the session.impressionSets object for the given port, so that + * it is sent to the server when the session ends. + * + * @note session.impressionSets will be keyed on `source` of the `data`, + * all the data will be appended to an array for the same source. + * + * @param {String} port The session port with which this is associated + * @param {Object} data The impression data structured as {source: "SOURCE", tiles: [{id: 123}]} + * + */ + handleDiscoveryStreamImpressionStats(port, data) { + let session = this.sessions.get(port); + + if (!session) { + throw new Error("Session does not exist."); + } + + const { window_inner_width, window_inner_height, source, tiles } = data; + const impressionSets = session.impressionSets || {}; + const impressions = impressionSets[source] || { + tiles: [], + window_inner_width, + window_inner_height, + }; + // The payload might contain other properties, we need `id`, `pos` and potentially `shim` here. + tiles.forEach(tile => { + impressions.tiles.push({ + id: tile.id, + pos: tile.pos, + ...(tile.shim ? { shim: tile.shim } : {}), + }); + Glean.pocket.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: tile.type === "spoc", + position: tile.pos, + }); + }); + impressionSets[source] = impressions; + session.impressionSets = impressionSets; + } + + /** + * Handle loaded content actions from Discovery Stream. The data will be + * stored into the session.loadedContentSets object for the given port, so that + * it is sent to the server when the session ends. + * + * @note session.loadedContentSets will be keyed on `source` of the `data`, + * all the data will be appended to an array for the same source. + * + * @param {String} port The session port with which this is associated + * @param {Object} data The loaded content structured as {source: "SOURCE", tiles: [{id: 123}]} + * + */ + handleDiscoveryStreamLoadedContent(port, data) { + let session = this.sessions.get(port); + + if (!session) { + throw new Error("Session does not exist."); + } + + const loadedContentSets = session.loadedContentSets || {}; + const loadedContents = loadedContentSets[data.source] || []; + // The payload might contain other properties, we need `id` and `pos` here. + data.tiles.forEach(tile => + loadedContents.push({ id: tile.id, pos: tile.pos }) + ); + loadedContentSets[data.source] = loadedContents; + session.loadedContentSets = loadedContentSets; + } + + /** + * Take all enumerable members of the data object and merge them into + * the session.perf object for the given port, so that it is sent to the + * server when the session ends. All members of the data object should + * be valid values of the perf object, as defined in pings.js and the + * data*.md documentation. + * + * @note Any existing keys with the same names already in the + * session perf object will be overwritten by values passed in here. + * + * @param {String} port The session with which this is associated + * @param {Object} data The perf data to be + */ + saveSessionPerfData(port, data) { + // XXX should use try/catch and send a bad state indicator if this + // get blows up. + let session = this.sessions.get(port); + + // XXX Partial workaround for #3118; avoids the worst incorrect associations + // of times with browsers, by associating the load trigger with the + // visibility event as the user is most likely associating the trigger to + // the tab just shown. This helps avoid associating with a preloaded + // browser as those don't get the event until shown. Better fix for more + // cases forthcoming. + // + // XXX the about:home check (and the corresponding test) should go away + // once the load_trigger stuff in addSession is refactored into + // setLoadTriggerInfo. + // + if (data.visibility_event_rcvd_ts && session.page !== "about:home") { + this.setLoadTriggerInfo(port); + } + + let timestamp = data.topsites_first_painted_ts; + + if ( + timestamp && + session.page === "about:home" && + !lazy.HomePage.overridden && + Services.prefs.getIntPref("browser.startup.page") === 1 + ) { + lazy.AboutNewTab.maybeRecordTopsitesPainted(timestamp); + } + + Object.assign(session.perf, data); + + if (data.visibility_event_rcvd_ts && !session.newtabOpened) { + session.newtabOpened = true; + const source = ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page) + ? session.page + : "other"; + Glean.newtab.opened.record({ + newtab_visit_id: session.session_id, + source, + }); + } + } + + _beginObservingNewtabPingPrefs() { + const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; + const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream."; + const NEWTAB_PING_PREFS = { + showSearch: Glean.newtabSearch.enabled, + "feeds.topsites": Glean.topsites.enabled, + showSponsoredTopSites: Glean.topsites.sponsoredEnabled, + "feeds.section.topstories": Glean.pocket.enabled, + showSponsored: Glean.pocket.sponsoredStoriesEnabled, + topSitesRows: Glean.topsites.rows, + }; + const setNewtabPrefMetrics = fullPrefName => { + const pref = fullPrefName.slice(ACTIVITY_STREAM_PREF_BRANCH.length); + if (!Object.hasOwn(NEWTAB_PING_PREFS, pref)) { + return; + } + const metric = NEWTAB_PING_PREFS[pref]; + switch (Services.prefs.getPrefType(fullPrefName)) { + case Services.prefs.PREF_BOOL: + metric.set(Services.prefs.getBoolPref(fullPrefName)); + break; + + case Services.prefs.PREF_INT: + metric.set(Services.prefs.getIntPref(fullPrefName)); + break; + } + }; + Services.prefs.addObserver( + ACTIVITY_STREAM_PREF_BRANCH, + (subject, topic, data) => setNewtabPrefMetrics(data) + ); + for (const pref of Object.keys(NEWTAB_PING_PREFS)) { + const fullPrefName = ACTIVITY_STREAM_PREF_BRANCH + pref; + setNewtabPrefMetrics(fullPrefName); + } + Glean.pocket.isSignedIn.set(lazy.pktApi.isUserLoggedIn()); + + const setBlockedSponsorsMetrics = () => { + let blocklist; + try { + blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + } catch (e) {} + if (blocklist) { + Glean.newtab.blockedSponsors.set(blocklist); + } + }; + + Services.prefs.addObserver( + TOP_SITES_BLOCKED_SPONSORS_PREF, + setBlockedSponsorsMetrics + ); + setBlockedSponsorsMetrics(); + } + + uninit() { + try { + Services.obs.removeObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + Services.obs.removeObserver( + this._addWindowListeners, + DOMWINDOW_OPENED_TOPIC + ); + } catch (e) { + // Operation can fail when uninit is called before + // init has finished setting up the observer + } + + // Only uninit if the getter has initialized it + if (Object.prototype.hasOwnProperty.call(this, "pingCentre")) { + this.pingCentre.uninit(); + } + if (Object.prototype.hasOwnProperty.call(this, "utEvents")) { + this.utEvents.uninit(); + } + + // TODO: Send any unfinished sessions + } +} + +const EXPORTED_SYMBOLS = [ + "TelemetryFeed", + "USER_PREFS_ENCODING", + "PREF_IMPRESSION_ID", + "TELEMETRY_PREF", + "EVENTS_TELEMETRY_PREF", + "STRUCTURED_INGESTION_ENDPOINT_PREF", +]; diff --git a/browser/components/newtab/lib/TippyTopProvider.sys.mjs b/browser/components/newtab/lib/TippyTopProvider.sys.mjs new file mode 100644 index 0000000000..8f32516119 --- /dev/null +++ b/browser/components/newtab/lib/TippyTopProvider.sys.mjs @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const TIPPYTOP_PATH = "chrome://activity-stream/content/data/content/tippytop/"; +const TIPPYTOP_JSON_PATH = + "chrome://activity-stream/content/data/content/tippytop/top_sites.json"; + +/* + * Get a domain from a url optionally stripping subdomains. + */ +export function getDomain(url, strip = "www.") { + let domain = ""; + try { + domain = new URL(url).hostname; + } catch (ex) {} + if (strip === "*") { + try { + domain = Services.eTLD.getBaseDomainFromHost(domain); + } catch (ex) {} + } else if (domain.startsWith(strip)) { + domain = domain.slice(strip.length); + } + return domain; +} + +export class TippyTopProvider { + constructor() { + this._sitesByDomain = new Map(); + this.initialized = false; + } + + async init() { + // Load the Tippy Top sites from the json manifest. + try { + for (const site of await ( + await fetch(TIPPYTOP_JSON_PATH, { + credentials: "omit", + }) + ).json()) { + for (const domain of site.domains) { + this._sitesByDomain.set(domain, site); + } + } + this.initialized = true; + } catch (error) { + console.error("Failed to load tippy top manifest."); + } + } + + processSite(site, strip) { + const tippyTop = this._sitesByDomain.get(getDomain(site.url, strip)); + if (tippyTop) { + site.tippyTopIcon = TIPPYTOP_PATH + tippyTop.image_url; + site.smallFavicon = TIPPYTOP_PATH + tippyTop.favicon_url; + site.backgroundColor = tippyTop.background_color; + } + return site; + } +} diff --git a/browser/components/newtab/lib/ToastNotification.jsm b/browser/components/newtab/lib/ToastNotification.jsm new file mode 100644 index 0000000000..0b2d756415 --- /dev/null +++ b/browser/components/newtab/lib/ToastNotification.jsm @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + RemoteL10n: "resource://activity-stream/lib/RemoteL10n.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetters(lazy, { + AlertsService: ["@mozilla.org/alerts-service;1", "nsIAlertsService"], +}); + +const ToastNotification = { + // Allow testing to stub the alerts service. + get AlertsService() { + return lazy.AlertsService; + }, + + sendUserEventTelemetry(event, message, dispatch) { + const ping = { + message_id: message.id, + event, + }; + dispatch({ + type: "TOAST_NOTIFICATION_TELEMETRY", + data: { action: "toast_notification_user_event", ...ping }, + }); + }, + + /** + * Show a toast notification. + * @param message Message containing content to show. + * @param dispatch A function to dispatch resulting actions. + * @return boolean value capturing if toast notification was displayed. + */ + async showToastNotification(message, dispatch) { + let { content } = message; + let title = await lazy.RemoteL10n.formatLocalizableText(content.title); + let body = await lazy.RemoteL10n.formatLocalizableText(content.body); + + // The only link between background task message experiment and user + // re-engagement via the notification is the associated "tag". Said tag is + // usually controlled by the message content, but for message experiments, + // we want to avoid a missing tag and to ensure a deterministic tag for + // easier analysis, including across branches. + let { tag } = content; + + let experimentMetadata = + lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "backgroundTaskMessage", + }) || {}; + + if ( + experimentMetadata?.active && + experimentMetadata?.slug && + experimentMetadata?.branch?.slug + ) { + // Like `my-experiment:my-branch`. + tag = `${experimentMetadata?.slug}:${experimentMetadata?.branch?.slug}`; + } + + // There are two events named `IMPRESSION` the first one refers to telemetry + // while the other refers to ASRouter impressions used for the frequency cap + this.sendUserEventTelemetry("IMPRESSION", message, dispatch); + dispatch({ type: "IMPRESSION", data: message }); + + let alert = Cc["@mozilla.org/alert-notification;1"].createInstance( + Ci.nsIAlertNotification + ); + let systemPrincipal = Services.scriptSecurityManager.getSystemPrincipal(); + alert.init( + tag, + content.image_url + ? Services.urlFormatter.formatURL(content.image_url) + : content.image_url, + title, + body, + true /* aTextClickable */, + content.data, + null /* aDir */, + null /* aLang */, + null /* aData */, + systemPrincipal, + null /* aInPrivateBrowsing */, + content.requireInteraction + ); + + if (content.actions) { + let actions = Cu.cloneInto(content.actions, {}); + for (let action of actions) { + if (action.title) { + action.title = await lazy.RemoteL10n.formatLocalizableText( + action.title + ); + } + } + alert.actions = actions; + } + + if (content.launch_url) { + alert.launchURL = Services.urlFormatter.formatURL(content.launch_url); + } + + this.AlertsService.showAlert(alert); + + return true; + }, +}; + +const EXPORTED_SYMBOLS = ["ToastNotification"]; diff --git a/browser/components/newtab/lib/ToolbarBadgeHub.jsm b/browser/components/newtab/lib/ToolbarBadgeHub.jsm new file mode 100644 index 0000000000..f403ca9186 --- /dev/null +++ b/browser/components/newtab/lib/ToolbarBadgeHub.jsm @@ -0,0 +1,318 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + clearTimeout: "resource://gre/modules/Timer.sys.mjs", + requestIdleCallback: "resource://gre/modules/Timer.sys.mjs", + setTimeout: "resource://gre/modules/Timer.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + EveryWindow: "resource:///modules/EveryWindow.jsm", + ToolbarPanelHub: "resource://activity-stream/lib/ToolbarPanelHub.jsm", +}); + +let notificationsByWindow = new WeakMap(); + +class _ToolbarBadgeHub { + constructor() { + this.id = "toolbar-badge-hub"; + this.state = {}; + this.prefs = { + WHATSNEW_TOOLBAR_PANEL: "browser.messaging-system.whatsNewPanel.enabled", + }; + this.removeAllNotifications = this.removeAllNotifications.bind(this); + this.removeToolbarNotification = this.removeToolbarNotification.bind(this); + this.addToolbarNotification = this.addToolbarNotification.bind(this); + this.registerBadgeToAllWindows = this.registerBadgeToAllWindows.bind(this); + this._sendPing = this._sendPing.bind(this); + this.sendUserEventTelemetry = this.sendUserEventTelemetry.bind(this); + + this._handleMessageRequest = null; + this._addImpression = null; + this._blockMessageById = null; + this._sendTelemetry = null; + this._initialized = false; + } + + async init( + waitForInitialized, + { + handleMessageRequest, + addImpression, + blockMessageById, + unblockMessageById, + sendTelemetry, + } + ) { + if (this._initialized) { + return; + } + + this._initialized = true; + this._handleMessageRequest = handleMessageRequest; + this._blockMessageById = blockMessageById; + this._unblockMessageById = unblockMessageById; + this._addImpression = addImpression; + this._sendTelemetry = sendTelemetry; + // Need to wait for ASRouter to initialize before trying to fetch messages + await waitForInitialized; + this.messageRequest({ + triggerId: "toolbarBadgeUpdate", + template: "toolbar_badge", + }); + // Listen for pref changes that could trigger new badges + Services.prefs.addObserver(this.prefs.WHATSNEW_TOOLBAR_PANEL, this); + } + + observe(aSubject, aTopic, aPrefName) { + switch (aPrefName) { + case this.prefs.WHATSNEW_TOOLBAR_PANEL: + this.messageRequest({ + triggerId: "toolbarBadgeUpdate", + template: "toolbar_badge", + }); + break; + } + } + + maybeInsertFTL(win) { + win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl"); + } + + executeAction({ id, data, message_id }) { + switch (id) { + case "show-whatsnew-button": + lazy.ToolbarPanelHub.enableToolbarButton(); + lazy.ToolbarPanelHub.enableAppmenuButton(); + break; + } + } + + _clearBadgeTimeout() { + if (this.state.showBadgeTimeoutId) { + lazy.clearTimeout(this.state.showBadgeTimeoutId); + } + } + + removeAllNotifications(event) { + if (event) { + // ignore right clicks + if ( + (event.type === "mousedown" || event.type === "click") && + event.button !== 0 + ) { + return; + } + // ignore keyboard access that is not one of the usual accessor keys + if ( + event.type === "keypress" && + event.key !== " " && + event.key !== "Enter" + ) { + return; + } + + event.target.removeEventListener( + "mousedown", + this.removeAllNotifications + ); + event.target.removeEventListener("keypress", this.removeAllNotifications); + // If we have an event it means the user interacted with the badge + // we should send telemetry + if (this.state.notification) { + this.sendUserEventTelemetry("CLICK", this.state.notification); + } + } + // Will call uninit on every window + lazy.EveryWindow.unregisterCallback(this.id); + if (this.state.notification) { + this._blockMessageById(this.state.notification.id); + } + this._clearBadgeTimeout(); + this.state = {}; + } + + removeToolbarNotification(toolbarButton) { + // Remove it from the element that displays the badge + toolbarButton + .querySelector(".toolbarbutton-badge") + .classList.remove("feature-callout"); + toolbarButton.removeAttribute("badged"); + // Remove id used for for aria-label badge description + const notificationDescription = toolbarButton.querySelector( + "#toolbarbutton-notification-description" + ); + if (notificationDescription) { + notificationDescription.remove(); + toolbarButton.removeAttribute("aria-labelledby"); + toolbarButton.removeAttribute("aria-describedby"); + } + } + + addToolbarNotification(win, message) { + const document = win.browser.ownerDocument; + if (message.content.action) { + this.executeAction({ ...message.content.action, message_id: message.id }); + } + let toolbarbutton = document.getElementById(message.content.target); + if (toolbarbutton) { + const badge = toolbarbutton.querySelector(".toolbarbutton-badge"); + badge.classList.add("feature-callout"); + toolbarbutton.setAttribute("badged", true); + // If we have additional aria-label information for the notification + // we add this content to the hidden `toolbarbutton-text` node. + // We then use `aria-labelledby` to link this description to the button + // that received the notification badge. + if (message.content.badgeDescription) { + // Insert strings as soon as we know we're showing them + this.maybeInsertFTL(win); + toolbarbutton.setAttribute( + "aria-labelledby", + `toolbarbutton-notification-description ${message.content.target}` + ); + // Because tooltiptext is different to the label, it gets duplicated as + // the description. Setting `describedby` to the same value as + // `labelledby` will be detected by the a11y code and the description + // will be removed. + toolbarbutton.setAttribute( + "aria-describedby", + `toolbarbutton-notification-description ${message.content.target}` + ); + const descriptionEl = document.createElement("span"); + descriptionEl.setAttribute( + "id", + "toolbarbutton-notification-description" + ); + descriptionEl.hidden = true; + document.l10n.setAttributes( + descriptionEl, + message.content.badgeDescription.string_id + ); + toolbarbutton.appendChild(descriptionEl); + } + // `mousedown` event required because of the `onmousedown` defined on + // the button that prevents `click` events from firing + toolbarbutton.addEventListener("mousedown", this.removeAllNotifications); + // `keypress` event required for keyboard accessibility + toolbarbutton.addEventListener("keypress", this.removeAllNotifications); + this.state = { notification: { id: message.id } }; + + // Impression should be added when the badge becomes visible + this._addImpression(message); + // Send a telemetry ping when adding the notification badge + this.sendUserEventTelemetry("IMPRESSION", message); + + return toolbarbutton; + } + + return null; + } + + registerBadgeToAllWindows(message) { + if (message.template === "update_action") { + this.executeAction({ ...message.content.action, message_id: message.id }); + // No badge to set only an action to execute + return; + } + + lazy.EveryWindow.registerCallback( + this.id, + win => { + if (notificationsByWindow.has(win)) { + // nothing to do + return; + } + const el = this.addToolbarNotification(win, message); + notificationsByWindow.set(win, el); + }, + win => { + const el = notificationsByWindow.get(win); + if (el) { + this.removeToolbarNotification(el); + } + notificationsByWindow.delete(win); + } + ); + } + + registerBadgeNotificationListener(message, options = {}) { + // We need to clear any existing notifications and only show + // the one set by devtools + if (options.force) { + this.removeAllNotifications(); + // When debugging immediately show the badge + this.registerBadgeToAllWindows(message); + return; + } + + if (message.content.delay) { + this.state.showBadgeTimeoutId = lazy.setTimeout(() => { + lazy.requestIdleCallback(() => this.registerBadgeToAllWindows(message)); + }, message.content.delay); + } else { + this.registerBadgeToAllWindows(message); + } + } + + async messageRequest({ triggerId, template }) { + const telemetryObject = { triggerId }; + TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + const message = await this._handleMessageRequest({ + triggerId, + template, + }); + TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject); + if (message) { + this.registerBadgeNotificationListener(message); + } + } + + _sendPing(ping) { + this._sendTelemetry({ + type: "TOOLBAR_BADGE_TELEMETRY", + data: { action: "badge_user_event", ...ping }, + }); + } + + sendUserEventTelemetry(event, message) { + const win = Services.wm.getMostRecentWindow("navigator:browser"); + // Only send pings for non private browsing windows + if ( + win && + !lazy.PrivateBrowsingUtils.isBrowserPrivate( + win.ownerGlobal.gBrowser.selectedBrowser + ) + ) { + this._sendPing({ + message_id: message.id, + event, + }); + } + } + + uninit() { + this._clearBadgeTimeout(); + this.state = {}; + this._initialized = false; + notificationsByWindow = new WeakMap(); + Services.prefs.removeObserver(this.prefs.WHATSNEW_TOOLBAR_PANEL, this); + } +} + +/** + * ToolbarBadgeHub - singleton instance of _ToolbarBadgeHub that can initiate + * message requests and render messages. + */ +const ToolbarBadgeHub = new _ToolbarBadgeHub(); + +const EXPORTED_SYMBOLS = ["ToolbarBadgeHub", "_ToolbarBadgeHub"]; diff --git a/browser/components/newtab/lib/ToolbarPanelHub.jsm b/browser/components/newtab/lib/ToolbarPanelHub.jsm new file mode 100644 index 0000000000..53e08c026b --- /dev/null +++ b/browser/components/newtab/lib/ToolbarPanelHub.jsm @@ -0,0 +1,612 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PanelMultiView: "resource:///modules/PanelMultiView.sys.mjs", + Preferences: "resource://gre/modules/Preferences.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + RemoteL10n: "resource://activity-stream/lib/RemoteL10n.sys.mjs", + + SpecialMessageActions: + "resource://messaging-system/lib/SpecialMessageActions.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + EveryWindow: "resource:///modules/EveryWindow.jsm", +}); +XPCOMUtils.defineLazyServiceGetter( + lazy, + "TrackingDBService", + "@mozilla.org/tracking-db-service;1", + "nsITrackingDBService" +); + +const idToTextMap = new Map([ + [Ci.nsITrackingDBService.TRACKERS_ID, "trackerCount"], + [Ci.nsITrackingDBService.TRACKING_COOKIES_ID, "cookieCount"], + [Ci.nsITrackingDBService.CRYPTOMINERS_ID, "cryptominerCount"], + [Ci.nsITrackingDBService.FINGERPRINTERS_ID, "fingerprinterCount"], + [Ci.nsITrackingDBService.SOCIAL_ID, "socialCount"], +]); + +const WHATSNEW_ENABLED_PREF = "browser.messaging-system.whatsNewPanel.enabled"; +const PROTECTIONS_PANEL_INFOMSG_PREF = + "browser.protections_panel.infoMessage.seen"; + +const TOOLBAR_BUTTON_ID = "whats-new-menu-button"; +const APPMENU_BUTTON_ID = "appMenu-whatsnew-button"; + +const BUTTON_STRING_ID = "cfr-whatsnew-button"; +const WHATS_NEW_PANEL_SELECTOR = "PanelUI-whatsNew-message-container"; + +class _ToolbarPanelHub { + constructor() { + this.triggerId = "whatsNewPanelOpened"; + this._showAppmenuButton = this._showAppmenuButton.bind(this); + this._hideAppmenuButton = this._hideAppmenuButton.bind(this); + this._showToolbarButton = this._showToolbarButton.bind(this); + this._hideToolbarButton = this._hideToolbarButton.bind(this); + this.insertProtectionPanelMessage = + this.insertProtectionPanelMessage.bind(this); + + this.state = {}; + this._initialized = false; + } + + async init(waitForInitialized, { getMessages, sendTelemetry }) { + if (this._initialized) { + return; + } + + this._initialized = true; + this._getMessages = getMessages; + this._sendTelemetry = sendTelemetry; + // Wait for ASRouter messages to become available in order to know + // if we can show the What's New panel + await waitForInitialized; + // Enable the application menu button so that the user can access + // the panel outside of the toolbar button + await this.enableAppmenuButton(); + + this.state = { + protectionPanelMessageSeen: Services.prefs.getBoolPref( + PROTECTIONS_PANEL_INFOMSG_PREF, + false + ), + }; + } + + uninit() { + this._initialized = false; + lazy.EveryWindow.unregisterCallback(TOOLBAR_BUTTON_ID); + lazy.EveryWindow.unregisterCallback(APPMENU_BUTTON_ID); + } + + get messages() { + return this._getMessages({ + template: "whatsnew_panel_message", + triggerId: "whatsNewPanelOpened", + returnAll: true, + }); + } + + toggleWhatsNewPref(event) { + // Checkbox onclick handler gets called before the checkbox state gets toggled, + // so we have to call it with the opposite value. + let newValue = !event.target.checked; + lazy.Preferences.set(WHATSNEW_ENABLED_PREF, newValue); + + this.sendUserEventTelemetry( + event.target.ownerGlobal, + "WNP_PREF_TOGGLE", + // Message id is not applicable in this case, the notification state + // is not related to a particular message + { id: "n/a" }, + { value: { prefValue: newValue } } + ); + } + + maybeInsertFTL(win) { + win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl"); + win.MozXULElement.insertFTLIfNeeded("toolkit/branding/brandings.ftl"); + win.MozXULElement.insertFTLIfNeeded("toolkit/branding/accounts.ftl"); + } + + maybeLoadCustomElement(win) { + if (!win.customElements.get("remote-text")) { + Services.scriptloader.loadSubScript( + "resource://activity-stream/data/custom-elements/paragraph.js", + win + ); + } + } + + // Turns on the Appmenu (hamburger menu) button for all open windows and future windows. + async enableAppmenuButton() { + if ((await this.messages).length) { + lazy.EveryWindow.registerCallback( + APPMENU_BUTTON_ID, + this._showAppmenuButton, + this._hideAppmenuButton + ); + } + } + + // Removes the button from the Appmenu. + // Only used in tests. + disableAppmenuButton() { + lazy.EveryWindow.unregisterCallback(APPMENU_BUTTON_ID); + } + + // Turns on the Toolbar button for all open windows and future windows. + async enableToolbarButton() { + if ((await this.messages).length) { + lazy.EveryWindow.registerCallback( + TOOLBAR_BUTTON_ID, + this._showToolbarButton, + this._hideToolbarButton + ); + } + } + + // When the panel is hidden we want to run some cleanup + _onPanelHidden(win) { + const panelContainer = win.document.getElementById( + "customizationui-widget-panel" + ); + // When the panel is hidden we want to remove any toolbar buttons that + // might have been added as an entry point to the panel + const removeToolbarButton = () => { + lazy.EveryWindow.unregisterCallback(TOOLBAR_BUTTON_ID); + }; + if (!panelContainer) { + return; + } + panelContainer.addEventListener("popuphidden", removeToolbarButton, { + once: true, + }); + } + + // Newer messages first and use `order` field to decide between messages + // with the same timestamp + _sortWhatsNewMessages(m1, m2) { + // Sort by published_date in descending order. + if (m1.content.published_date === m2.content.published_date) { + // Ascending order + return m1.order - m2.order; + } + if (m1.content.published_date > m2.content.published_date) { + return -1; + } + return 1; + } + + // Render what's new messages into the panel. + async renderMessages(win, doc, containerId, options = {}) { + // Set the checked status of the footer checkbox + let value = lazy.Preferences.get(WHATSNEW_ENABLED_PREF); + let checkbox = win.document.getElementById("panelMenu-toggleWhatsNew"); + + checkbox.checked = value; + + this.maybeLoadCustomElement(win); + const messages = + (options.force && options.messages) || + (await this.messages).sort(this._sortWhatsNewMessages); + const container = lazy.PanelMultiView.getViewNode(doc, containerId); + + if (messages) { + // Targeting attribute state might have changed making new messages + // available and old messages invalid, we need to refresh + this.removeMessages(win, containerId); + let previousDate = 0; + // Get and store any variable part of the message content + this.state.contentArguments = await this._contentArguments(); + for (let message of messages) { + container.appendChild( + this._createMessageElements(win, doc, message, previousDate) + ); + previousDate = message.content.published_date; + } + } + + this._onPanelHidden(win); + + // Panel impressions are not associated with one particular message + // but with a set of messages. We concatenate message ids and send them + // back for every impression. + const eventId = { + id: messages + .map(({ id }) => id) + .sort() + .join(","), + }; + // Check `mainview` attribute to determine if the panel is shown as a + // subview (inside the application menu) or as a toolbar dropdown. + // https://searchfox.org/mozilla-central/rev/07f7390618692fa4f2a674a96b9b677df3a13450/browser/components/customizableui/PanelMultiView.jsm#1268 + const mainview = win.PanelUI.whatsNewPanel.hasAttribute("mainview"); + this.sendUserEventTelemetry(win, "IMPRESSION", eventId, { + value: { view: mainview ? "toolbar_dropdown" : "application_menu" }, + }); + } + + removeMessages(win, containerId) { + const doc = win.document; + const messageNodes = lazy.PanelMultiView.getViewNode( + doc, + containerId + ).querySelectorAll(".whatsNew-message"); + for (const messageNode of messageNodes) { + messageNode.remove(); + } + } + + /** + * Dispatch the action defined in the message and user telemetry event. + */ + _dispatchUserAction(win, message) { + let url; + try { + // Set platform specific path variables for SUMO articles + url = Services.urlFormatter.formatURL(message.content.cta_url); + } catch (e) { + console.error(e); + url = message.content.cta_url; + } + lazy.SpecialMessageActions.handleAction( + { + type: message.content.cta_type, + data: { + args: url, + where: message.content.cta_where || "tabshifted", + }, + }, + win.browser + ); + + this.sendUserEventTelemetry(win, "CLICK", message); + } + + /** + * Attach event listener to dispatch message defined action. + */ + _attachCommandListener(win, element, message) { + // Add event listener for `mouseup` not to overlap with the + // `mousedown` & `click` events dispatched from PanelMultiView.sys.mjs + // https://searchfox.org/mozilla-central/rev/7531325c8660cfa61bf71725f83501028178cbb9/browser/components/customizableui/PanelMultiView.jsm#1830-1837 + element.addEventListener("mouseup", () => { + this._dispatchUserAction(win, message); + }); + element.addEventListener("keyup", e => { + if (e.key === "Enter" || e.key === " ") { + this._dispatchUserAction(win, message); + } + }); + } + + _createMessageElements(win, doc, message, previousDate) { + const { content } = message; + const messageEl = lazy.RemoteL10n.createElement(doc, "div"); + messageEl.classList.add("whatsNew-message"); + + // Only render date if it is different from the one rendered before. + if (content.published_date !== previousDate) { + messageEl.appendChild( + lazy.RemoteL10n.createElement(doc, "p", { + classList: "whatsNew-message-date", + content: new Date(content.published_date).toLocaleDateString( + "default", + { + month: "long", + day: "numeric", + year: "numeric", + } + ), + }) + ); + } + + const wrapperEl = lazy.RemoteL10n.createElement(doc, "div"); + wrapperEl.doCommand = () => this._dispatchUserAction(win, message); + wrapperEl.classList.add("whatsNew-message-body"); + messageEl.appendChild(wrapperEl); + + if (content.icon_url) { + wrapperEl.classList.add("has-icon"); + const iconEl = lazy.RemoteL10n.createElement(doc, "img"); + iconEl.src = content.icon_url; + iconEl.classList.add("whatsNew-message-icon"); + if (content.icon_alt && content.icon_alt.string_id) { + doc.l10n.setAttributes(iconEl, content.icon_alt.string_id); + } else { + iconEl.setAttribute("alt", content.icon_alt); + } + wrapperEl.appendChild(iconEl); + } + + wrapperEl.appendChild(this._createMessageContent(win, doc, content)); + + if (content.link_text) { + const anchorEl = lazy.RemoteL10n.createElement(doc, "a", { + classList: "text-link", + content: content.link_text, + }); + anchorEl.doCommand = () => this._dispatchUserAction(win, message); + wrapperEl.appendChild(anchorEl); + } + + // Attach event listener on entire message container + this._attachCommandListener(win, messageEl, message); + + return messageEl; + } + + /** + * Return message title (optional subtitle) and body + */ + _createMessageContent(win, doc, content) { + const wrapperEl = new win.DocumentFragment(); + + wrapperEl.appendChild( + lazy.RemoteL10n.createElement(doc, "h2", { + classList: "whatsNew-message-title", + content: content.title, + attributes: this.state.contentArguments, + }) + ); + + wrapperEl.appendChild( + lazy.RemoteL10n.createElement(doc, "p", { + content: content.body, + classList: "whatsNew-message-content", + attributes: this.state.contentArguments, + }) + ); + + return wrapperEl; + } + + _createHeroElement(win, doc, message) { + this.maybeLoadCustomElement(win); + + const messageEl = lazy.RemoteL10n.createElement(doc, "div"); + messageEl.setAttribute("id", "protections-popup-message"); + messageEl.classList.add("whatsNew-hero-message"); + const wrapperEl = lazy.RemoteL10n.createElement(doc, "div"); + wrapperEl.classList.add("whatsNew-message-body"); + messageEl.appendChild(wrapperEl); + + wrapperEl.appendChild( + lazy.RemoteL10n.createElement(doc, "h2", { + classList: "whatsNew-message-title", + content: message.content.title, + }) + ); + wrapperEl.appendChild( + lazy.RemoteL10n.createElement(doc, "p", { + classList: "protections-popup-content", + content: message.content.body, + }) + ); + + if (message.content.link_text) { + let linkEl = lazy.RemoteL10n.createElement(doc, "a", { + classList: "text-link", + content: message.content.link_text, + }); + linkEl.disabled = true; + wrapperEl.appendChild(linkEl); + this._attachCommandListener(win, linkEl, message); + } else { + this._attachCommandListener(win, wrapperEl, message); + } + + return messageEl; + } + + async _contentArguments() { + const { defaultEngine } = Services.search; + // Between now and 6 weeks ago + const dateTo = new Date(); + const dateFrom = new Date(dateTo.getTime() - 42 * 24 * 60 * 60 * 1000); + const eventsByDate = await lazy.TrackingDBService.getEventsByDateRange( + dateFrom, + dateTo + ); + // Make sure we set all types of possible values to 0 because they might + // be referenced by fluent strings + let totalEvents = { blockedCount: 0 }; + for (let blockedType of idToTextMap.values()) { + totalEvents[blockedType] = 0; + } + // Count all events in the past 6 weeks. Returns an object with: + // `blockedCount` total number of blocked resources + // {tracker|cookie|social...} breakdown by event type as defined by `idToTextMap` + totalEvents = eventsByDate.reduce((acc, day) => { + const type = day.getResultByName("type"); + const count = day.getResultByName("count"); + acc[idToTextMap.get(type)] = (acc[idToTextMap.get(type)] || 0) + count; + acc.blockedCount += count; + return acc; + }, totalEvents); + return { + // Keys need to match variable names used in asrouter.ftl + // `earliestDate` will be either 6 weeks ago or when tracking recording + // started. Whichever is more recent. + earliestDate: Math.max( + new Date(await lazy.TrackingDBService.getEarliestRecordedDate()), + dateFrom + ), + ...totalEvents, + // Passing in `undefined` as string for the Fluent variable name + // in order to match and select the message that does not require + // the variable. + searchEngineName: defaultEngine ? defaultEngine.name : "undefined", + }; + } + + async _showAppmenuButton(win) { + this.maybeInsertFTL(win); + await this._showElement( + win.browser.ownerDocument, + APPMENU_BUTTON_ID, + BUTTON_STRING_ID + ); + } + + _hideAppmenuButton(win, windowClosed) { + // No need to do something if the window is going away + if (!windowClosed) { + this._hideElement(win.browser.ownerDocument, APPMENU_BUTTON_ID); + } + } + + _showToolbarButton(win) { + const document = win.browser.ownerDocument; + this.maybeInsertFTL(win); + return this._showElement(document, TOOLBAR_BUTTON_ID, BUTTON_STRING_ID); + } + + _hideToolbarButton(win) { + this._hideElement(win.browser.ownerDocument, TOOLBAR_BUTTON_ID); + } + + _showElement(document, id, string_id) { + const el = lazy.PanelMultiView.getViewNode(document, id); + document.l10n.setAttributes(el, string_id); + el.hidden = false; + } + + _hideElement(document, id) { + const el = lazy.PanelMultiView.getViewNode(document, id); + if (el) { + el.hidden = true; + } + } + + _sendPing(ping) { + this._sendTelemetry({ + type: "TOOLBAR_PANEL_TELEMETRY", + data: { action: "whats-new-panel_user_event", ...ping }, + }); + } + + sendUserEventTelemetry(win, event, message, options = {}) { + // Only send pings for non private browsing windows + if ( + win && + !lazy.PrivateBrowsingUtils.isBrowserPrivate( + win.ownerGlobal.gBrowser.selectedBrowser + ) + ) { + this._sendPing({ + message_id: message.id, + event, + event_context: options.value, + }); + } + } + + /** + * Inserts a message into the Protections Panel. The message is visible once + * and afterwards set in a collapsed state. It can be shown again using the + * info button in the panel header. + */ + async insertProtectionPanelMessage(event) { + const win = event.target.ownerGlobal; + this.maybeInsertFTL(win); + + const doc = event.target.ownerDocument; + const container = doc.getElementById("messaging-system-message-container"); + const infoButton = doc.getElementById("protections-popup-info-button"); + const panelContainer = doc.getElementById("protections-popup"); + const toggleMessage = () => { + const learnMoreLink = doc.querySelector( + "#messaging-system-message-container .text-link" + ); + if (learnMoreLink) { + container.toggleAttribute("disabled"); + infoButton.toggleAttribute("checked"); + panelContainer.toggleAttribute("infoMessageShowing"); + learnMoreLink.disabled = !learnMoreLink.disabled; + } + }; + if (!container.childElementCount) { + const message = await this._getMessages({ + template: "protections_panel", + triggerId: "protectionsPanelOpen", + }); + if (message) { + const messageEl = this._createHeroElement(win, doc, message); + container.appendChild(messageEl); + infoButton.addEventListener("click", toggleMessage); + this.sendUserEventTelemetry(win, "IMPRESSION", message); + } + } + // Message is collapsed by default. If it was never shown before we want + // to expand it + if ( + !this.state.protectionPanelMessageSeen && + container.hasAttribute("disabled") + ) { + toggleMessage(); + } + // Save state that we displayed the message + if (!this.state.protectionPanelMessageSeen) { + Services.prefs.setBoolPref(PROTECTIONS_PANEL_INFOMSG_PREF, true); + this.state.protectionPanelMessageSeen = true; + } + // Collapse the message after the panel is hidden so we don't get the + // animation when opening the panel + panelContainer.addEventListener( + "popuphidden", + () => { + if ( + this.state.protectionPanelMessageSeen && + !container.hasAttribute("disabled") + ) { + toggleMessage(); + } + }, + { + once: true, + } + ); + } + + /** + * @param {object} [browser] MessageChannel target argument as a response to a + * user action. No message is shown if undefined. + * @param {object[]} messages Messages selected from devtools page + */ + forceShowMessage(browser, messages) { + if (!browser) { + return; + } + const win = browser.ownerGlobal; + const doc = browser.ownerDocument; + this.removeMessages(win, WHATS_NEW_PANEL_SELECTOR); + this.renderMessages(win, doc, WHATS_NEW_PANEL_SELECTOR, { + force: true, + messages: Array.isArray(messages) ? messages : [messages], + }); + win.PanelUI.panel.addEventListener("popuphidden", event => + this.removeMessages(event.target.ownerGlobal, WHATS_NEW_PANEL_SELECTOR) + ); + } +} + +/** + * ToolbarPanelHub - singleton instance of _ToolbarPanelHub that can initiate + * message requests and render messages. + */ +const ToolbarPanelHub = new _ToolbarPanelHub(); + +const EXPORTED_SYMBOLS = ["ToolbarPanelHub", "_ToolbarPanelHub"]; diff --git a/browser/components/newtab/lib/TopSitesFeed.jsm b/browser/components/newtab/lib/TopSitesFeed.jsm new file mode 100644 index 0000000000..9b516cb9ba --- /dev/null +++ b/browser/components/newtab/lib/TopSitesFeed.jsm @@ -0,0 +1,1715 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { TippyTopProvider } = ChromeUtils.importESModule( + "resource://activity-stream/lib/TippyTopProvider.sys.mjs" +); +const { insertPinned, TOP_SITES_MAX_SITES_PER_ROW } = + ChromeUtils.importESModule( + "resource://activity-stream/common/Reducers.sys.mjs" + ); +const { Dedupe } = ChromeUtils.importESModule( + "resource://activity-stream/common/Dedupe.sys.mjs" +); +const { shortURL } = ChromeUtils.import( + "resource://activity-stream/lib/ShortURL.jsm" +); +const { getDefaultOptions } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamStorage.jsm" +); +const { + CUSTOM_SEARCH_SHORTCUTS, + SEARCH_SHORTCUTS_EXPERIMENT, + SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF, + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + checkHasSearchEngine, + getSearchProvider, + getSearchFormURL, +} = ChromeUtils.importESModule( + "resource://activity-stream/lib/SearchShortcuts.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "FilterAdult", + "resource://activity-stream/lib/FilterAdult.jsm" +); +ChromeUtils.defineESModuleGetters(lazy, { + LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs", +}); +ChromeUtils.defineModuleGetter( + lazy, + "Screenshots", + "resource://activity-stream/lib/Screenshots.jsm" +); + +XPCOMUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("TopSitesFeed"); +}); + +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +XPCOMUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +const DEFAULT_SITES_PREF = "default.sites"; +const SHOWN_ON_NEWTAB_PREF = "feeds.topsites"; +const DEFAULT_TOP_SITES = []; +const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages) +const MIN_FAVICON_SIZE = 96; +const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"]; +const PINNED_FAVICON_PROPS_TO_MIGRATE = [ + "favicon", + "faviconRef", + "faviconSize", +]; +const SECTION_ID = "topsites"; +const ROWS_PREF = "topSitesRows"; +const SHOW_SPONSORED_PREF = "showSponsoredTopSites"; +// The default total number of sponsored top sites to fetch from Contile +// and Pocket. +const MAX_NUM_SPONSORED = 2; +// Nimbus variable for the total number of sponsored top sites including +// both Contile and Pocket sources. +// The default will be `MAX_NUM_SPONSORED` if this variable is unspecified. +const NIMBUS_VARIABLE_MAX_SPONSORED = "topSitesMaxSponsored"; +// Nimbus variable to allow more than two sponsored tiles from Contile to be +//considered for Top Sites. +const NIMBUS_VARIABLE_ADDITIONAL_TILES = + "topSitesUseAdditionalTilesFromContile"; +// Nimbus variable to enable the SOV feature for sponsored tiles. +const NIMBUS_VARIABLE_CONTILE_SOV_ENABLED = "topSitesContileSovEnabled"; +// Nimbu variable for the total number of sponsor topsite that come from Contile +// The default will be `CONTILE_MAX_NUM_SPONSORED` if variable is unspecified. +const NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED = "topSitesContileMaxSponsored"; + +// Search experiment stuff +const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile"; +const SEARCH_FILTERS = [ + "google", + "search.yahoo", + "yahoo", + "bing", + "ask", + "duckduckgo", +]; + +const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting"; +const DEFAULT_SITES_OVERRIDE_PREF = + "browser.newtabpage.activity-stream.default.sites"; +const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment."; + +// Mozilla Tiles Service (Contile) prefs +// Nimbus variable for the Contile integration. It falls back to the pref: +// `browser.topsites.contile.enabled`. +const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled"; +const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint"; +const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes +// The maximum number of sponsored top sites to fetch from Contile. +const CONTILE_MAX_NUM_SPONSORED = 2; +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; +const CONTILE_CACHE_PREF = "browser.topsites.contile.cachedTiles"; +const CONTILE_CACHE_VALID_FOR_PREF = "browser.topsites.contile.cacheValidFor"; +const CONTILE_CACHE_LAST_FETCH_PREF = "browser.topsites.contile.lastFetch"; + +// Partners of sponsored tiles. +const SPONSORED_TILE_PARTNER_AMP = "amp"; +const SPONSORED_TILE_PARTNER_MOZ_SALES = "moz-sales"; +const SPONSORED_TILE_PARTNERS = new Set([ + SPONSORED_TILE_PARTNER_AMP, + SPONSORED_TILE_PARTNER_MOZ_SALES, +]); + +function getShortURLForCurrentSearch() { + const url = shortURL({ url: Services.search.defaultEngine.searchForm }); + return url; +} + +class ContileIntegration { + constructor(topSitesFeed) { + this._topSitesFeed = topSitesFeed; + this._lastPeriodicUpdate = 0; + this._sites = []; + // The Share-of-Voice object managed by Shepherd and sent via Contile. + this._sov = null; + } + + get sites() { + return this._sites; + } + + get sov() { + return this._sov; + } + + periodicUpdate() { + let now = Date.now(); + if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) { + this._lastPeriodicUpdate = now; + this.refresh(); + } + } + + async refresh() { + let updateDefaultSites = await this._fetchSites(); + if (updateDefaultSites) { + this._topSitesFeed._readDefaults(); + } + } + + /** + * Clear Contile Cache Prefs. + */ + _resetContileCachePrefs() { + Services.prefs.clearUserPref(CONTILE_CACHE_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_LAST_FETCH_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_VALID_FOR_PREF); + } + + /** + * Filter the tiles whose sponsor is on the Top Sites sponsor blocklist. + * + * @param {array} tiles + * An array of the tile objects + */ + _filterBlockedSponsors(tiles) { + const blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + return tiles.filter(tile => !blocklist.includes(shortURL(tile))); + } + + /** + * Calculate the time Contile response is valid for based on cache-control header + * + * @param {string} cacheHeader + * string value of the Contile resposne cache-control header + */ + _extractCacheValidFor(cacheHeader) { + if (cacheHeader === undefined) { + lazy.log.warn("Contile response cache control header is undefined"); + return 0; + } + const [, staleIfError] = cacheHeader.match(/stale-if-error=\s*([0-9]+)/i); + const [, maxAge] = cacheHeader.match(/max-age=\s*([0-9]+)/i); + const validFor = + Number.parseInt(staleIfError, 10) + Number.parseInt(maxAge, 10); + return isNaN(validFor) ? 0 : validFor; + } + + /** + * Load Tiles from Contile Cache Prefs + */ + _loadTilesFromCache() { + lazy.log.info("Contile client is trying to load tiles from local cache."); + const now = Math.round(Date.now() / 1000); + const lastFetch = Services.prefs.getIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + 0 + ); + const validFor = Services.prefs.getIntPref(CONTILE_CACHE_VALID_FOR_PREF, 0); + if (now <= lastFetch + validFor) { + try { + let cachedTiles = JSON.parse( + Services.prefs.getStringPref(CONTILE_CACHE_PREF) + ); + cachedTiles = this._filterBlockedSponsors(cachedTiles); + this._sites = cachedTiles; + lazy.log.info("Local cache loaded."); + return true; + } catch (error) { + lazy.log.warn(`Failed to load tiles from local cache: ${error}.`); + return false; + } + } + + return false; + } + + /** + * Determine number of Tiles to get from Contile + */ + _getMaxNumFromContile() { + return ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) ?? CONTILE_MAX_NUM_SPONSORED + ); + } + + async _fetchSites() { + if ( + !lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) || + !this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF] + ) { + if (this._sites.length) { + this._sites = []; + return true; + } + return false; + } + try { + let url = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF); + const response = await fetch(url, { credentials: "omit" }); + if (!response.ok) { + lazy.log.warn( + `Contile endpoint returned unexpected status: ${response.status}` + ); + if (response.status === 304 || response.status >= 500) { + return this._loadTilesFromCache(); + } + } + + const lastFetch = Math.round(Date.now() / 1000); + Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, lastFetch); + + // Contile returns 204 indicating there is no content at the moment. + // If this happens, it will clear `this._sites` reset the cached tiles + // to an empty array. + if (response.status === 204) { + if (this._sites.length) { + this._sites = []; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + return true; + } + return false; + } + const body = await response.json(); + + if (body?.sov) { + this._sov = JSON.parse(atob(body.sov)); + } + if (body?.tiles && Array.isArray(body.tiles)) { + const useAdditionalTiles = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_ADDITIONAL_TILES + ); + + const maxNumFromContile = this._getMaxNumFromContile(); + + let { tiles } = body; + if ( + useAdditionalTiles !== undefined && + !useAdditionalTiles && + tiles.length > maxNumFromContile + ) { + tiles.length = maxNumFromContile; + } + tiles = this._filterBlockedSponsors(tiles); + if (tiles.length > maxNumFromContile) { + lazy.log.info("Remove unused links from Contile"); + tiles.length = maxNumFromContile; + } + this._sites = tiles; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + Services.prefs.setIntPref( + CONTILE_CACHE_VALID_FOR_PREF, + this._extractCacheValidFor( + response.headers.get("cache-control") || + response.headers.get("Cache-Control") + ) + ); + + return true; + } + } catch (error) { + lazy.log.warn( + `Failed to fetch data from Contile server: ${error.message}` + ); + return this._loadTilesFromCache(); + } + return false; + } +} + +class TopSitesFeed { + constructor() { + this._contile = new ContileIntegration(this); + this._tippyTopProvider = new TippyTopProvider(); + XPCOMUtils.defineLazyGetter( + this, + "_currentSearchHostname", + getShortURLForCurrentSearch + ); + this.dedupe = new Dedupe(this._dedupeKey); + this.frecentCache = new lazy.LinksCache( + lazy.NewTabUtils.activityStreamLinks, + "getTopSites", + CACHED_LINK_PROPS_TO_MIGRATE, + (oldOptions, newOptions) => + // Refresh if no old options or requesting more items + !(oldOptions.numItems >= newOptions.numItems) + ); + this.pinnedCache = new lazy.LinksCache( + lazy.NewTabUtils.pinnedLinks, + "links", + [...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE] + ); + lazy.PageThumbs.addExpirationFilter(this); + this._nimbusChangeListener = this._nimbusChangeListener.bind(this); + } + + _nimbusChangeListener(event, reason) { + // The Nimbus API current doesn't specify the changed variable(s) in the + // listener callback, so we have to refresh unconditionally on every change + // of the `newtab` feature. It should be a manageable overhead given the + // current update cadence (6 hours) of Nimbus. + // + // Skip the experiment and rollout loading reasons since this feature has + // `isEarlyStartup` enabled, the feature variables are already available + // before the experiment or rollout loads. + if ( + !["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason) + ) { + this._contile.refresh(); + } + } + + init() { + // If the feed was previously disabled PREFS_INITIAL_VALUES was never received + this._readDefaults({ isStartup: true }); + this._storage = this.store.dbStorage.getDbTable("sectionPrefs"); + this._contile.refresh(); + Services.obs.addObserver(this, "browser-search-engine-modified"); + Services.obs.addObserver(this, "browser-region-updated"); + Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener); + } + + uninit() { + lazy.PageThumbs.removeExpirationFilter(this); + Services.obs.removeObserver(this, "browser-search-engine-modified"); + Services.obs.removeObserver(this, "browser-region-updated"); + Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.offUpdate(this._nimbusChangeListener); + } + + observe(subj, topic, data) { + switch (topic) { + case "browser-search-engine-modified": + // We should update the current top sites if the search engine has been changed since + // the search engine that gets filtered out of top sites has changed. + // We also need to drop search shortcuts when their engine gets removed / hidden. + if ( + data === "engine-default" && + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] + ) { + delete this._currentSearchHostname; + this._currentSearchHostname = getShortURLForCurrentSearch(); + } + this.refresh({ broadcast: true }); + break; + case "browser-region-updated": + this._readDefaults(); + break; + case "nsPref:changed": + if ( + data === REMOTE_SETTING_DEFAULTS_PREF || + data === DEFAULT_SITES_OVERRIDE_PREF || + data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH) + ) { + this._readDefaults(); + } + break; + } + } + + _dedupeKey(site) { + return site && site.hostname; + } + + /** + * _readDefaults - sets DEFAULT_TOP_SITES + */ + async _readDefaults({ isStartup = false } = {}) { + this._useRemoteSetting = false; + + if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) { + this.refreshDefaults( + this.store.getState().Prefs.values[DEFAULT_SITES_PREF], + { isStartup } + ); + return; + } + + // Try using default top sites from enterprise policies or tests. The pref + // is locked when set via enterprise policy. Tests have no default sites + // unless they set them via this pref. + if ( + Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) || + Cu.isInAutomation + ) { + let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, ""); + this.refreshDefaults(sites, { isStartup }); + return; + } + + // Clear out the array of any previous defaults. + DEFAULT_TOP_SITES.length = 0; + + // Read defaults from contile. + const contileEnabled = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ); + let hasContileTiles = false; + if (contileEnabled) { + let sponsoredPosition = 1; + for (let site of this._contile.sites) { + let hostname = shortURL(site); + let link = { + isDefault: true, + url: site.url, + hostname, + sendAttributionRequest: false, + label: site.name, + show_sponsored_label: hostname !== "yandex", + sponsored_position: sponsoredPosition++, + sponsored_click_url: site.click_url, + sponsored_impression_url: site.impression_url, + sponsored_tile_id: site.id, + partner: SPONSORED_TILE_PARTNER_AMP, + }; + if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) { + // Only use the image from Contile if it's hi-res, otherwise, fallback + // to the built-in favicons. + link.favicon = site.image_url; + link.faviconSize = site.image_size; + } + DEFAULT_TOP_SITES.push(link); + } + hasContileTiles = sponsoredPosition > 1; + } + + // Read defaults from remote settings. + this._useRemoteSetting = true; + let remoteSettingData = await this._getRemoteConfig(); + + const sponsoredBlocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + + for (let siteData of remoteSettingData) { + let hostname = shortURL(siteData); + // Drop default sites when Contile already provided a sponsored one with + // the same host name. + if ( + contileEnabled && + DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1 + ) { + continue; + } + // Also drop those sponsored sites that were blocked by the user before + // with the same hostname. + if ( + siteData.sponsored_position && + sponsoredBlocklist.includes(hostname) + ) { + continue; + } + let link = { + isDefault: true, + url: siteData.url, + hostname, + sendAttributionRequest: !!siteData.send_attribution_request, + }; + if (siteData.url_urlbar_override) { + link.url_urlbar = siteData.url_urlbar_override; + } + if (siteData.title) { + link.label = siteData.title; + } + if (siteData.search_shortcut) { + link = await this.topSiteToSearchTopSite(link); + } else if (siteData.sponsored_position) { + if (contileEnabled && hasContileTiles) { + continue; + } + const { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + } = siteData; + link = { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + show_sponsored_label: link.hostname !== "yandex", + ...link, + }; + } + DEFAULT_TOP_SITES.push(link); + } + + this.refresh({ broadcast: true, isStartup }); + } + + refreshDefaults(sites, { isStartup = false } = {}) { + // Clear out the array of any previous defaults + DEFAULT_TOP_SITES.length = 0; + + // Add default sites if any based on the pref + if (sites) { + for (const url of sites.split(",")) { + const site = { + isDefault: true, + url, + }; + site.hostname = shortURL(site); + DEFAULT_TOP_SITES.push(site); + } + } + + this.refresh({ broadcast: true, isStartup }); + } + + async _getRemoteConfig(firstTime = true) { + if (!this._remoteConfig) { + this._remoteConfig = await lazy.RemoteSettings("top-sites"); + this._remoteConfig.on("sync", () => { + this._readDefaults(); + }); + } + + let result = []; + let failed = false; + try { + result = await this._remoteConfig.get(); + } catch (ex) { + console.error(ex); + failed = true; + } + if (!result.length) { + console.error("Received empty top sites configuration!"); + failed = true; + } + // If we failed, or the result is empty, try loading from the local dump. + if (firstTime && failed) { + await this._remoteConfig.db.clear(); + // Now call this again. + return this._getRemoteConfig(false); + } + + // Sort sites based on the "order" attribute. + result.sort((a, b) => a.order - b.order); + + result = result.filter(topsite => { + // Filter by region. + if (topsite.exclude_regions?.includes(lazy.Region.home)) { + return false; + } + if ( + topsite.include_regions?.length && + !topsite.include_regions.includes(lazy.Region.home) + ) { + return false; + } + + // Filter by locale. + if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) { + return false; + } + if ( + topsite.include_locales?.length && + !topsite.include_locales.includes(Services.locale.appLocaleAsBCP47) + ) { + return false; + } + + // Filter by experiment. + // Exclude this top site if any of the specified experiments are running. + if ( + topsite.exclude_experiments?.some(experimentID => + Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + // Exclude this top site if none of the specified experiments are running. + if ( + topsite.include_experiments?.length && + topsite.include_experiments.every( + experimentID => + !Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + + return true; + }); + + return result; + } + + filterForThumbnailExpiration(callback) { + const { rows } = this.store.getState().TopSites; + callback( + rows.reduce((acc, site) => { + acc.push(site.url); + if (site.customScreenshotURL) { + acc.push(site.customScreenshotURL); + } + return acc; + }, []) + ); + } + + /** + * shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine? + * + * @param {string} hostname a top site hostname, such as "amazon" or "foo" + * @returns {bool} + */ + shouldFilterSearchTile(hostname) { + if ( + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] && + (SEARCH_FILTERS.includes(hostname) || + hostname === this._currentSearchHostname) + ) { + return true; + } + return false; + } + + /** + * _maybeInsertSearchShortcuts - if the search shortcuts experiment is running, + * insert search shortcuts if needed + * @param {Array} plainPinnedSites (from the pinnedSitesCache) + * @returns {Boolean} Did we insert any search shortcuts? + */ + async _maybeInsertSearchShortcuts(plainPinnedSites) { + // Only insert shortcuts if the experiment is running + if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + // We don't want to insert shortcuts we've previously inserted + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",") + .filter(s => s); // Filter out empty strings + const newInsertedShortcuts = []; + + let shouldPin = this._useRemoteSetting + ? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname) + : this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(","); + shouldPin = shouldPin + .map(getSearchProvider) + .filter(s => s && s.shortURL !== this._currentSearchHostname); + + // If we've previously inserted all search shortcuts return early + if ( + shouldPin.every(shortcut => + prevInsertedShortcuts.includes(shortcut.shortURL) + ) + ) { + return false; + } + + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + + // The plainPinnedSites array is populated with pinned sites at their + // respective indices, and null everywhere else, but is not always the + // right length + const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0); + const pinnedSites = [...plainPinnedSites].concat( + Array(emptySlots).fill(null) + ); + + const tryToInsertSearchShortcut = async shortcut => { + const nextAvailable = pinnedSites.indexOf(null); + // Only add a search shortcut if the site isn't already pinned, we + // haven't previously inserted it, there's space to pin it, and the + // search engine is available in Firefox + if ( + !pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) && + !prevInsertedShortcuts.includes(shortcut.shortURL) && + nextAvailable > -1 && + (await checkHasSearchEngine(shortcut.keyword)) + ) { + const site = await this.topSiteToSearchTopSite({ url: shortcut.url }); + this._pinSiteAt(site, nextAvailable); + pinnedSites[nextAvailable] = site; + newInsertedShortcuts.push(shortcut.shortURL); + } + }; + + for (let shortcut of shouldPin) { + await tryToInsertSearchShortcut(shortcut); + } + + if (newInsertedShortcuts.length) { + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.concat(newInsertedShortcuts).join(",") + ) + ); + return true; + } + } + + return false; + } + + /** + * Fetch topsites spocs from the DiscoveryStream feed. + * + * @returns {Array} An array of sponsored tile objects. + */ + fetchDiscoveryStreamSpocs() { + let sponsored = []; + const { DiscoveryStream } = this.store.getState(); + if (DiscoveryStream) { + const discoveryStreamSpocs = + DiscoveryStream.spocs.data["sponsored-topsites"]?.items || []; + // Find the first component of a type and remove it from layout + const findSponsoredTopsitesPositions = name => { + for (const row of DiscoveryStream.layout) { + for (const component of row.components) { + if (component.placement?.name === name) { + return component.spocs.positions; + } + } + } + return null; + }; + + // Get positions from layout for now. This could be improved if we store position data in state. + const discoveryStreamSpocPositions = + findSponsoredTopsitesPositions("sponsored-topsites"); + + if (discoveryStreamSpocPositions?.length) { + function reformatImageURL(url, width, height) { + // Change the image URL to request a size tailored for the parent container width + // Also: force JPEG, quality 60, no upscaling, no EXIF data + // Uses Thumbor: https://thumbor.readthedocs.io/en/latest/usage.html + // For now we wrap this in single quotes because this is being used in a url() css rule, and otherwise would cause a parsing error. + return `'https://img-getpocket.cdn.mozilla.net/${width}x${height}/filters:format(jpeg):quality(60):no_upscale():strip_exif()/${encodeURIComponent( + url + )}'`; + } + + // We need to loop through potential spocs and set their positions. + // If we run out of spocs or positions, we stop. + // First, we need to know which array is shortest. This is our exit condition. + const minLength = Math.min( + discoveryStreamSpocPositions.length, + discoveryStreamSpocs.length + ); + // Loop until we run out of spocs or positions. + for (let i = 0; i < minLength; i++) { + const positionIndex = discoveryStreamSpocPositions[i].index; + const spoc = discoveryStreamSpocs[i]; + const link = { + favicon: reformatImageURL(spoc.raw_image_src, 96, 96), + faviconSize: 96, + type: "SPOC", + label: spoc.title || spoc.sponsor, + title: spoc.title || spoc.sponsor, + url: spoc.url, + flightId: spoc.flight_id, + id: spoc.id, + guid: spoc.id, + shim: spoc.shim, + // For now we are assuming position based on intended position. + // Actual position can shift based on other content. + // We send the intended position in the ping. + pos: positionIndex, + // Set this so that SPOC topsites won't be shown in the URL bar. + // See Bug 1822027. Note that `sponsored_position` is 1-based. + sponsored_position: positionIndex + 1, + // This is used for topsites deduping. + hostname: shortURL({ url: spoc.url }), + partner: SPONSORED_TILE_PARTNER_MOZ_SALES, + }; + sponsored.push(link); + } + } + } + return sponsored; + } + + // eslint-disable-next-line max-statements + async getLinksWithDefaults(isStartup = false) { + const prefValues = this.store.getState().Prefs.values; + const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW; + const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT]; + // We must wait for search services to initialize in order to access default + // search engine properties without triggering a synchronous initialization + await Services.search.init(); + + // Get all frecent sites from history. + let frecent = []; + const cache = await this.frecentCache.request({ + // We need to overquery due to the top 5 alexa search + default search possibly being removed + numItems: numItems + SEARCH_FILTERS.length + 1, + topsiteFrecency: FRECENCY_THRESHOLD, + }); + for (let link of cache) { + const hostname = shortURL(link); + if (!this.shouldFilterSearchTile(hostname)) { + frecent.push({ + ...(searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link), + hostname, + }); + } + } + + // Get defaults. + let contileSponsored = []; + let notBlockedDefaultSites = []; + for (let link of DEFAULT_TOP_SITES) { + // For sponsored Yandex links, default filtering is reversed: we only + // show them if Yandex is the default search engine. + if (link.sponsored_position && link.hostname === "yandex") { + if (link.hostname !== this._currentSearchHostname) { + continue; + } + } else if (this.shouldFilterSearchTile(link.hostname)) { + continue; + } + // Drop blocked default sites. + if ( + lazy.NewTabUtils.blockedLinks.isBlocked({ + url: link.url, + }) + ) { + continue; + } + // If we've previously blocked a search shortcut, remove the default top site + // that matches the hostname + const searchProvider = getSearchProvider(shortURL(link)); + if ( + searchProvider && + lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url }) + ) { + continue; + } + if (link.sponsored_position) { + if (!prefValues[SHOW_SPONSORED_PREF]) { + continue; + } + contileSponsored[link.sponsored_position - 1] = link; + + // Unpin search shortcut if present for the sponsored link to be shown + // instead. + this._unpinSearchShortcut(link.hostname); + } else { + notBlockedDefaultSites.push( + searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link + ); + } + } + + const discoverySponsored = this.fetchDiscoveryStreamSpocs(); + + const sponsored = await this._mergeSponsoredLinks({ + [SPONSORED_TILE_PARTNER_AMP]: contileSponsored, + [SPONSORED_TILE_PARTNER_MOZ_SALES]: discoverySponsored, + }); + + this._maybeCapSponsoredLinks(sponsored); + + // Get pinned links augmented with desired properties + let plainPinned = await this.pinnedCache.request(); + + // Insert search shortcuts if we need to. + // _maybeInsertSearchShortcuts returns true if any search shortcuts are + // inserted, meaning we need to expire and refresh the pinnedCache + if (await this._maybeInsertSearchShortcuts(plainPinned)) { + this.pinnedCache.expire(); + plainPinned = await this.pinnedCache.request(); + } + + const pinned = await Promise.all( + plainPinned.map(async link => { + if (!link) { + return link; + } + + // Drop pinned search shortcuts when their engine has been removed / hidden. + if (link.searchTopSite) { + const searchProvider = getSearchProvider(shortURL(link)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return null; + } + } + + // Copy all properties from a frecent link and add more + const finder = other => other.url === link.url; + + // Remove frecent link's screenshot if pinned link has a custom one + const frecentSite = frecent.find(finder); + if (frecentSite && link.customScreenshotURL) { + delete frecentSite.screenshot; + } + // If the link is a frecent site, do not copy over 'isDefault', else check + // if the site is a default site + const copy = Object.assign( + {}, + frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) }, + link, + { hostname: shortURL(link) }, + { searchTopSite: !!link.searchTopSite } + ); + + // Add in favicons if we don't already have it + if (!copy.favicon) { + try { + lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI( + await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy]) + ); + + for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) { + copy.__sharedCache.updateLink(prop, copy[prop]); + } + } catch (e) { + // Some issue with favicon, so just continue without one + } + } + + return copy; + }) + ); + + // Remove any duplicates from frecent and default sites + const [, dedupedSponsored, dedupedFrecent, dedupedDefaults] = + this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites); + const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults]; + + // Remove adult sites if we need to + const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned); + + // Insert the original pinned sites into the deduped frecent and defaults. + let withPinned = insertPinned(checkedAdult, pinned); + // Insert sponsored sites at their desired position. + dedupedSponsored.forEach(link => { + if (!link) { + return; + } + let index = link.sponsored_position - 1; + if (index >= withPinned.length) { + withPinned[index] = link; + } else if (withPinned[index]?.sponsored_position) { + // We currently want DiscoveryStream spocs to replace existing spocs. + withPinned[index] = link; + } else { + withPinned.splice(index, 0, link); + } + }); + + // Remove excess items after we inserted sponsored ones. + withPinned = withPinned.slice(0, numItems); + + // Now, get a tippy top icon, a rich icon, or screenshot for every item + for (const link of withPinned) { + if (link) { + // If there is a custom screenshot this is the only image we display + if (link.customScreenshotURL) { + this._fetchScreenshot(link, link.customScreenshotURL, isStartup); + } else if (link.searchTopSite && !link.isDefault) { + await this._attachTippyTopIconForSearchShortcut(link, link.label); + } else { + this._fetchIcon(link, isStartup); + } + + // Remove internal properties that might be updated after dispatch + delete link.__sharedCache; + + // Indicate that these links should get a frecency bonus when clicked + link.typedBonus = true; + } + } + + this._linksWithDefaults = withPinned; + + return withPinned; + } + + /** + * Cap sponsored links if they're more than the specified maximum. + * + * @param {Array} links An array of sponsored links. Capping will be performed in-place. + */ + _maybeCapSponsoredLinks(links) { + // Set maximum sponsored top sites + const maxSponsored = + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_MAX_SPONSORED + ) ?? MAX_NUM_SPONSORED; + if (links.length > maxSponsored) { + links.length = maxSponsored; + } + } + + /** + * Merge sponsored links from all the partners using SOV if present. + * For each tile position, the user is assigned to one partner via stable sampling. + * If the chosen partner doesn't have a tile to serve, another tile from a different + * partner is used as the replacement. + * + * @param {Object} sponsoredLinks An object with sponsored links from all the partners. + * @returns {Array} An array of merged sponsored links. + */ + async _mergeSponsoredLinks(sponsoredLinks) { + if ( + !this._contile.sov || + !lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_SOV_ENABLED + ) + ) { + return Object.values(sponsoredLinks).flat(); + } + + const sampleInput = `${lazy.contextId}-${this._contile.sov.name}`; + let sponsored = []; + + for (const allocation of this._contile.sov.allocations) { + let link = null; + let chosenPartner = null; + const ratios = allocation.allocation.map(alloc => alloc.percentage); + if (ratios.length) { + const index = await lazy.Sampling.ratioSample(sampleInput, ratios); + chosenPartner = allocation.allocation[index].partner; + // Unknown partners are allowed so that new parters can be added to Shepherd + // sooner without waiting for client changes. + link = sponsoredLinks[chosenPartner]?.shift(); + } + + if (!link) { + // If the chosen partner doesn't have a tile for this postion, choose any + // one from another group. For simplicity, we do _not_ do resampling here + // against the remaining partners. + for (const partner of SPONSORED_TILE_PARTNERS) { + if ( + partner === chosenPartner || + sponsoredLinks[partner].length === 0 + ) { + continue; + } + link = sponsoredLinks[partner].shift(); + break; + } + + if (!link) { + // No more links to be added across all the partners, just return. + return sponsored; + } + } + + // Update the position fields. Note that postion is also 1-based in SOV. + link.sponsored_position = allocation.position; + if (link.pos !== undefined) { + // Pocket `pos` is 0-based. + link.pos = allocation.position - 1; + } + sponsored.push(link); + } + // add the remaining contile sponsoredLinks when nimbus variable present + if ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) + ) { + return sponsored.concat(sponsoredLinks[SPONSORED_TILE_PARTNER_AMP]); + } + + return sponsored; + } + + /** + * Attach TippyTop icon to the given search shortcut + * + * Note that it queries the search form URL from search service For Yandex, + * and uses it to choose the best icon for its shortcut variants. + * + * @param {Object} link A link object with a `url` property + * @param {string} keyword Search keyword + */ + async _attachTippyTopIconForSearchShortcut(link, keyword) { + if ( + ["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"].includes(keyword) + ) { + let site = { url: link.url }; + site.url = (await getSearchFormURL(keyword)) || site.url; + this._tippyTopProvider.processSite(site); + link.tippyTopIcon = site.tippyTopIcon; + link.smallFavicon = site.smallFavicon; + link.backgroundColor = site.backgroundColor; + } else { + this._tippyTopProvider.processSite(link); + } + } + + /** + * Refresh the top sites data for content. + * @param {bool} options.broadcast Should the update be broadcasted. + * @param {bool} options.isStartup Being called while TopSitesFeed is initting. + */ + async refresh(options = {}) { + if (!this._startedUp && !options.isStartup) { + // Initial refresh still pending. + return; + } + this._startedUp = true; + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + const links = await this.getLinksWithDefaults({ + isStartup: options.isStartup, + }); + const newAction = { type: at.TOP_SITES_UPDATED, data: { links } }; + let storedPrefs; + try { + storedPrefs = (await this._storage.get(SECTION_ID)) || {}; + } catch (e) { + storedPrefs = {}; + console.error("Problem getting stored prefs for TopSites"); + } + newAction.data.pref = getDefaultOptions(storedPrefs); + + if (options.isStartup) { + newAction.meta = { + isStartup: true, + }; + } + + if (options.broadcast) { + // Broadcast an update to all open content pages + this.store.dispatch(ac.BroadcastToContent(newAction)); + } else { + // Don't broadcast only update the state and update the preloaded tab. + this.store.dispatch(ac.AlsoToPreloaded(newAction)); + } + } + + async updateCustomSearchShortcuts(isStartup = false) { + if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + return; + } + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + // Populate the state with available search shortcuts + let searchShortcuts = []; + for (const engine of await Services.search.getAppProvidedEngines()) { + const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s => + engine.aliases.includes(s.keyword) + ); + if (shortcut) { + let clone = { ...shortcut }; + await this._attachTippyTopIconForSearchShortcut(clone, clone.keyword); + searchShortcuts.push(clone); + } + } + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.UPDATE_SEARCH_SHORTCUTS, + data: { searchShortcuts }, + meta: { + isStartup, + }, + }) + ); + } + + async topSiteToSearchTopSite(site) { + const searchProvider = getSearchProvider(shortURL(site)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return site; + } + return { + ...site, + searchTopSite: true, + label: searchProvider.keyword, + }; + } + + /** + * Get an image for the link preferring tippy top, rich favicon, screenshots. + */ + async _fetchIcon(link, isStartup = false) { + // Nothing to do if we already have a rich icon from the page + if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) { + return; + } + + // Nothing more to do if we can use a default tippy top icon + this._tippyTopProvider.processSite(link); + if (link.tippyTopIcon) { + return; + } + + // Make a request for a better icon + this._requestRichIcon(link.url); + + // Also request a screenshot if we don't have one yet + await this._fetchScreenshot(link, link.url, isStartup); + } + + /** + * Fetch, cache and broadcast a screenshot for a specific topsite. + * @param link cached topsite object + * @param url where to fetch the image from + * @param isStartup Whether the screenshot is fetched while initting TopSitesFeed. + */ + async _fetchScreenshot(link, url, isStartup = false) { + // We shouldn't bother caching screenshots if they won't be shown. + if ( + link.screenshot || + !this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF] + ) { + return; + } + await lazy.Screenshots.maybeCacheScreenshot( + link, + url, + "screenshot", + screenshot => + this.store.dispatch( + ac.BroadcastToContent({ + data: { screenshot, url: link.url }, + type: at.SCREENSHOT_UPDATED, + meta: { + isStartup, + }, + }) + ) + ); + } + + /** + * Dispatch screenshot preview to target or notify if request failed. + * @param customScreenshotURL {string} The URL used to capture the screenshot + * @param target {string} Id of content process where to dispatch the result + */ + async getScreenshotPreview(url, target) { + const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || ""; + this.store.dispatch( + ac.OnlyToOneContent( + { + data: { url, preview }, + type: at.PREVIEW_RESPONSE, + }, + target + ) + ); + } + + _requestRichIcon(url) { + this.store.dispatch({ + type: at.RICH_ICON_MISSING, + data: { url }, + }); + } + + updateSectionPrefs(collapsed) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.TOP_SITES_PREFS_UPDATED, + data: { pref: collapsed }, + }) + ); + } + + /** + * Inform others that top sites data has been updated due to pinned changes. + */ + _broadcastPinnedSitesUpdated() { + // Pinned data changed, so make sure we get latest + this.pinnedCache.expire(); + + // Refresh to update pinned sites with screenshots, trigger deduping, etc. + this.refresh({ broadcast: true }); + } + + /** + * Pin a site at a specific position saving only the desired keys. + * @param customScreenshotURL {string} User set URL of preview image for site + * @param label {string} User set string of custom site name + */ + async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) { + const toPin = { url }; + if (label) { + toPin.label = label; + } + if (customScreenshotURL) { + toPin.customScreenshotURL = customScreenshotURL; + } + if (searchTopSite) { + toPin.searchTopSite = searchTopSite; + } + lazy.NewTabUtils.pinnedLinks.pin(toPin, index); + + await this._clearLinkCustomScreenshot({ customScreenshotURL, url }); + } + + async _clearLinkCustomScreenshot(site) { + // If screenshot url changed or was removed we need to update the cached link obj + if (site.customScreenshotURL !== undefined) { + const pinned = await this.pinnedCache.request(); + const link = pinned.find(pin => pin && pin.url === site.url); + if (link && link.customScreenshotURL !== site.customScreenshotURL) { + link.__sharedCache.updateLink("screenshot", undefined); + } + } + } + + /** + * Handle a pin action of a site to a position. + */ + async pin(action) { + let { site, index } = action.data; + index = this._adjustPinIndexForSponsoredLinks(site, index); + // If valid index provided, pin at that position + if (index >= 0) { + await this._pinSiteAt(site, index); + this._broadcastPinnedSitesUpdated(); + } else { + // Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option, + // then we want to make sure to unblock that link if it has previously been + // blocked. We know if the site has been added because the index will be -1. + if (index === -1) { + lazy.NewTabUtils.blockedLinks.unblock({ url: site.url }); + this.frecentCache.expire(); + } + this.insert(action); + } + } + + /** + * Handle an unpin action of a site. + */ + unpin(action) { + const { site } = action.data; + lazy.NewTabUtils.pinnedLinks.unpin(site); + this._broadcastPinnedSitesUpdated(); + } + + unpinAllSearchShortcuts() { + Services.prefs.clearUserPref( + `browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}` + ); + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if (pinnedLink && pinnedLink.searchTopSite) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + } + } + this.pinnedCache.expire(); + } + + _unpinSearchShortcut(vendor) { + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if ( + pinnedLink && + pinnedLink.searchTopSite && + shortURL(pinnedLink) === vendor + ) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + this.pinnedCache.expire(); + + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(","); + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.filter(s => s !== vendor).join(",") + ) + ); + break; + } + } + } + + /** + * Reduces the given pinning index by the number of preceding sponsored + * sites, to accomodate for sponsored sites pushing pinned ones to the side, + * effectively increasing their index again. + */ + _adjustPinIndexForSponsoredLinks(site, index) { + if (!this._linksWithDefaults) { + return index; + } + // Adjust insertion index for sponsored sites since their position is + // fixed. + let adjustedIndex = index; + for (let i = 0; i < index; i++) { + const link = this._linksWithDefaults[i]; + if ( + link && + link.sponsored_position && + this._linksWithDefaults[i]?.url !== site.url + ) { + adjustedIndex--; + } + } + return adjustedIndex; + } + + /** + * Insert a site to pin at a position shifting over any other pinned sites. + */ + _insertPin(site, originalIndex, draggedFromIndex) { + let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex); + + // Don't insert any pins past the end of the visible top sites. Otherwise, + // we can end up with a bunch of pinned sites that can never be unpinned again + // from the UI. + const topSitesCount = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + if (index >= topSitesCount) { + return; + } + + let pinned = lazy.NewTabUtils.pinnedLinks.links; + if (!pinned[index]) { + this._pinSiteAt(site, index); + } else { + pinned[draggedFromIndex] = null; + // Find the hole to shift the pinned site(s) towards. We shift towards the + // hole left by the site being dragged. + let holeIndex = index; + const indexStep = index > draggedFromIndex ? -1 : 1; + while (pinned[holeIndex]) { + holeIndex += indexStep; + } + if (holeIndex >= topSitesCount || holeIndex < 0) { + // There are no holes, so we will effectively unpin the last slot and shifting + // towards it. This only happens when adding a new top site to an already + // fully pinned grid. + holeIndex = topSitesCount - 1; + } + + // Shift towards the hole. + const shiftingStep = holeIndex > index ? -1 : 1; + while (holeIndex !== index) { + const nextIndex = holeIndex + shiftingStep; + this._pinSiteAt(pinned[nextIndex], holeIndex); + holeIndex = nextIndex; + } + this._pinSiteAt(site, index); + } + } + + /** + * Handle an insert (drop/add) action of a site. + */ + async insert(action) { + let { index } = action.data; + // Treat invalid pin index values (e.g., -1, undefined) as insert in the first position + if (!(index > 0)) { + index = 0; + } + + // Inserting a top site pins it in the specified slot, pushing over any link already + // pinned in the slot (unless it's the last slot, then it replaces). + this._insertPin( + action.data.site, + index, + action.data.draggedFromIndex !== undefined + ? action.data.draggedFromIndex + : this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW + ); + + await this._clearLinkCustomScreenshot(action.data.site); + this._broadcastPinnedSitesUpdated(); + } + + updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) { + // Unpin the deletedShortcuts. + deletedShortcuts.forEach(({ url }) => { + lazy.NewTabUtils.pinnedLinks.unpin({ url }); + }); + + // Pin the addedShortcuts. + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + addedShortcuts.forEach(shortcut => { + // Find first hole in pinnedLinks. + let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link); + if ( + index < 0 && + lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots + ) { + // pinnedLinks can have less slots than the total available. + index = lazy.NewTabUtils.pinnedLinks.links.length; + } + if (index >= 0) { + lazy.NewTabUtils.pinnedLinks.pin(shortcut, index); + } else { + // No slots available, we need to do an insert in first slot and push over other pinned links. + this._insertPin(shortcut, 0, numberOfSlots); + } + }); + + this._broadcastPinnedSitesUpdated(); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.updateCustomSearchShortcuts(true /* isStartup */); + break; + case at.SYSTEM_TICK: + this.refresh({ broadcast: false }); + this._contile.periodicUpdate(); + break; + // All these actions mean we need new top sites + case at.PLACES_HISTORY_CLEARED: + case at.PLACES_LINKS_DELETED: + this.frecentCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PLACES_LINKS_CHANGED: + this.frecentCache.expire(); + this.refresh({ broadcast: false }); + break; + case at.PLACES_LINK_BLOCKED: + this.frecentCache.expire(); + this.pinnedCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PREF_CHANGED: + switch (action.data.name) { + case DEFAULT_SITES_PREF: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data.value); + } + break; + case ROWS_PREF: + case FILTER_DEFAULT_SEARCH_PREF: + case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF: + this.refresh({ broadcast: true }); + break; + case SHOW_SPONSORED_PREF: + if ( + lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) + ) { + this._contile.refresh(); + } else { + this.refresh({ broadcast: true }); + } + if (!action.data.value) { + this._contile._resetContileCachePrefs(); + } + + break; + case SEARCH_SHORTCUTS_EXPERIMENT: + if (action.data.value) { + this.updateCustomSearchShortcuts(); + } else { + this.unpinAllSearchShortcuts(); + } + this.refresh({ broadcast: true }); + } + break; + case at.UPDATE_SECTION_PREFS: + if (action.data.id === SECTION_ID) { + this.updateSectionPrefs(action.data.value); + } + break; + case at.PREFS_INITIAL_VALUES: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data[DEFAULT_SITES_PREF]); + } + break; + case at.TOP_SITES_PIN: + this.pin(action); + break; + case at.TOP_SITES_UNPIN: + this.unpin(action); + break; + case at.TOP_SITES_INSERT: + this.insert(action); + break; + case at.PREVIEW_REQUEST: + this.getScreenshotPreview(action.data.url, action.meta.fromTarget); + break; + case at.UPDATE_PINNED_SEARCH_SHORTCUTS: + this.updatePinnedSearchShortcuts(action.data); + break; + case at.DISCOVERY_STREAM_SPOCS_UPDATE: + // Refresh to update sponsored topsites. + this.refresh({ broadcast: true, isStartup: action.meta.isStartup }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} + +const EXPORTED_SYMBOLS = [ + "TopSitesFeed", + "DEFAULT_TOP_SITES", + "ContileIntegration", +]; diff --git a/browser/components/newtab/lib/TopStoriesFeed.jsm b/browser/components/newtab/lib/TopStoriesFeed.jsm new file mode 100644 index 0000000000..aa51b2a4f8 --- /dev/null +++ b/browser/components/newtab/lib/TopStoriesFeed.jsm @@ -0,0 +1,746 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { shortURL } = ChromeUtils.import( + "resource://activity-stream/lib/ShortURL.jsm" +); +const { SectionsManager } = ChromeUtils.import( + "resource://activity-stream/lib/SectionsManager.jsm" +); +const { PersistentCache } = ChromeUtils.importESModule( + "resource://activity-stream/lib/PersistentCache.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); + +const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours +const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours +const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour +const SECTION_ID = "topstories"; +const IMPRESSION_SOURCE = "TOP_STORIES"; +const SPOC_IMPRESSION_TRACKING_PREF = + "feeds.section.topstories.spoc.impressions"; +const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled"; +const DISCOVERY_STREAM_PREF_ENABLED_PATH = + "browser.newtabpage.activity-stream.discoverystream.enabled"; +const REC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.rec.impressions"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server +const DISCOVERY_STREAM_PREF = "discoverystream.config"; + +class TopStoriesFeed { + constructor(ds) { + // Use discoverystream config pref default values for fast path and + // if needed lazy load activity stream top stories feed based on + // actual user preference when INIT and PREF_CHANGED is invoked + this.discoveryStreamEnabled = + ds && + ds.value && + JSON.parse(ds.value).enabled && + Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false); + if (!this.discoveryStreamEnabled) { + this.initializeProperties(); + } + } + + initializeProperties() { + this.contentUpdateQueue = []; + this.spocCampaignMap = new Map(); + this.cache = new PersistentCache(SECTION_ID, true); + this._prefs = new Prefs(); + this.propertiesInitialized = true; + } + + async onInit() { + SectionsManager.enableSection(SECTION_ID, true /* isStartup */); + if (this.discoveryStreamEnabled) { + return; + } + + try { + const { options } = SectionsManager.sections.get(SECTION_ID); + const apiKey = this.getApiKeyFromPref(options.api_key_pref); + this.stories_endpoint = this.produceFinalEndpointUrl( + options.stories_endpoint, + apiKey + ); + this.topics_endpoint = this.produceFinalEndpointUrl( + options.topics_endpoint, + apiKey + ); + this.read_more_endpoint = options.read_more_endpoint; + this.stories_referrer = options.stories_referrer; + this.show_spocs = options.show_spocs; + this.storiesLastUpdated = 0; + this.topicsLastUpdated = 0; + this.storiesLoaded = false; + this.dispatchPocketCta(this._prefs.get("pocketCta"), false); + + // Cache is used for new page loads, which shouldn't have changed data. + // If we have changed data, cache should be cleared, + // and last updated should be 0, and we can fetch. + let { stories, topics } = await this.loadCachedData(); + if (this.storiesLastUpdated === 0) { + stories = await this.fetchStories(); + } + if (this.topicsLastUpdated === 0) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, true); + this.storiesLoaded = true; + + // This is filtered so an update function can return true to retry on the next run + this.contentUpdateQueue = this.contentUpdateQueue.filter(update => + update() + ); + } catch (e) { + console.error(`Problem initializing top stories feed: ${e.message}`); + } + } + + init() { + SectionsManager.onceInitialized(this.onInit.bind(this)); + } + + async clearCache() { + await this.cache.set("stories", {}); + await this.cache.set("topics", {}); + await this.cache.set("spocs", {}); + } + + uninit() { + this.storiesLoaded = false; + SectionsManager.disableSection(SECTION_ID); + } + + getPocketState(target) { + const action = { + type: at.POCKET_LOGGED_IN, + data: lazy.pktApi.isUserLoggedIn(), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + dispatchPocketCta(data, shouldBroadcast) { + const action = { type: at.POCKET_CTA, data: JSON.parse(data) }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + + /** + * doContentUpdate - Updates topics and stories in the topstories section. + * + * Sections have one update action for the whole section. + * Redux creates a state race condition if you call the same action, + * twice, concurrently. Because of this, doContentUpdate is + * one place to update both topics and stories in a single action. + * + * Section updates used old topics if none are available, + * but clear stories if none are available. Because of this, if no + * stories are passed, we instead use the existing stories in state. + * + * @param {Object} This is an object with potential new stories or topics. + * @param {Boolean} shouldBroadcast If we should update existing tabs or not. For first page + * loads or pref changes, we want to update existing tabs, + * for system tick or other updates we do not. + */ + doContentUpdate({ stories, topics }, shouldBroadcast) { + let updateProps = {}; + if (stories) { + updateProps.rows = stories; + } else { + const { Sections } = this.store.getState(); + if (Sections && Sections.find) { + updateProps.rows = Sections.find(s => s.id === SECTION_ID).rows; + } + } + if (topics) { + Object.assign(updateProps, { + topics, + read_more_endpoint: this.read_more_endpoint, + }); + } + + // We should only be calling this once per init. + this.dispatchUpdateEvent(shouldBroadcast, updateProps); + } + + async fetchStories() { + if (!this.stories_endpoint) { + return null; + } + try { + const response = await fetch(this.stories_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Stories endpoint returned unexpected status: ${response.status}` + ); + } + + const body = await response.json(); + this.updateSettings(body.settings); + this.stories = this.rotate(this.transform(body.recommendations)); + this.cleanUpTopRecImpressionPref(); + + if (this.show_spocs && body.spocs) { + this.spocCampaignMap = new Map( + body.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(body.spocs); + this.cleanUpCampaignImpressionPref(); + } + this.storiesLastUpdated = Date.now(); + body._timestamp = this.storiesLastUpdated; + this.cache.set("stories", body); + } catch (error) { + console.error(`Failed to fetch content: ${error.message}`); + } + return this.stories; + } + + async loadCachedData() { + const data = await this.cache.get(); + let stories = data.stories && data.stories.recommendations; + let topics = data.topics && data.topics.topics; + + if (stories && !!stories.length && this.storiesLastUpdated === 0) { + this.updateSettings(data.stories.settings); + this.stories = this.rotate(this.transform(stories)); + this.storiesLastUpdated = data.stories._timestamp; + if (data.stories.spocs && data.stories.spocs.length) { + this.spocCampaignMap = new Map( + data.stories.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(data.stories.spocs); + this.cleanUpCampaignImpressionPref(); + } + } + if (topics && !!topics.length && this.topicsLastUpdated === 0) { + this.topics = topics; + this.topicsLastUpdated = data.topics._timestamp; + } + + return { topics: this.topics, stories: this.stories }; + } + + transform(items) { + if (!items) { + return []; + } + + const calcResult = items + .filter(s => !lazy.NewTabUtils.blockedLinks.isBlocked({ url: s.url })) + .map(s => { + let mapped = { + guid: s.id, + hostname: s.domain || shortURL(Object.assign({}, s, { url: s.url })), + type: + Date.now() - s.published_timestamp * 1000 <= STORIES_NOW_THRESHOLD + ? "now" + : "trending", + context: s.context, + icon: s.icon, + title: s.title, + description: s.excerpt, + image: this.normalizeUrl(s.image_src), + referrer: this.stories_referrer, + url: s.url, + score: s.item_score || 1, + spoc_meta: this.show_spocs + ? { campaign_id: s.campaign_id, caps: s.caps } + : {}, + }; + + // Very old cached spocs may not contain an `expiration_timestamp` property + if (s.expiration_timestamp) { + mapped.expiration_timestamp = s.expiration_timestamp; + } + + return mapped; + }) + .sort(this.compareScore); + + return calcResult; + } + + async fetchTopics() { + if (!this.topics_endpoint) { + return null; + } + try { + const response = await fetch(this.topics_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Topics endpoint returned unexpected status: ${response.status}` + ); + } + const body = await response.json(); + const { topics } = body; + if (topics) { + this.topics = topics; + this.topicsLastUpdated = Date.now(); + body._timestamp = this.topicsLastUpdated; + this.cache.set("topics", body); + } + } catch (error) { + console.error(`Failed to fetch topics: ${error.message}`); + } + return this.topics; + } + + dispatchUpdateEvent(shouldBroadcast, data) { + SectionsManager.updateSection(SECTION_ID, data, shouldBroadcast); + } + + compareScore(a, b) { + return b.score - a.score; + } + + updateSettings(settings = {}) { + this.spocsPerNewTabs = settings.spocsPerNewTabs || 1; // Probability of a new tab getting a spoc [0,1] + this.recsExpireTime = settings.recsExpireTime; + } + + // We rotate stories on the client so that + // active stories are at the front of the list, followed by stories that have expired + // impressions i.e. have been displayed for longer than recsExpireTime. + rotate(items) { + if (items.length <= 3) { + return items; + } + + const maxImpressionAge = Math.max( + this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, + DEFAULT_RECS_EXPIRE_TIME + ); + const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + const expired = []; + const active = []; + for (const item of items) { + if ( + impressions[item.guid] && + Date.now() - impressions[item.guid] >= maxImpressionAge + ) { + expired.push(item); + } else { + active.push(item); + } + } + return active.concat(expired); + } + + getApiKeyFromPref(apiKeyPref) { + if (!apiKeyPref) { + return apiKeyPref; + } + + return ( + this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref) + ); + } + + produceFinalEndpointUrl(url, apiKey) { + if (!url) { + return url; + } + if (url.includes("$apiKey") && !apiKey) { + throw new Error(`An API key was specified but none configured: ${url}`); + } + return url.replace("$apiKey", apiKey); + } + + // Need to remove parenthesis from image URLs as React will otherwise + // fail to render them properly as part of the card template. + normalizeUrl(url) { + if (url) { + return url.replace(/\(/g, "%28").replace(/\)/g, "%29"); + } + return url; + } + + shouldShowSpocs() { + return this.show_spocs && this.store.getState().Prefs.values.showSponsored; + } + + dispatchSpocDone(target) { + const action = { type: at.POCKET_WAITING_FOR_SPOC, data: false }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + filterSpocs() { + if (!this.shouldShowSpocs()) { + return []; + } + + if (Math.random() > this.spocsPerNewTabs) { + return []; + } + + if (!this.spocs || !this.spocs.length) { + // We have stories but no spocs so there's nothing to do and this update can be + // removed from the queue. + return []; + } + + // Filter spocs based on frequency caps + const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + let spocs = this.spocs.filter(s => + this.isBelowFrequencyCap(impressions, s) + ); + + // Filter out expired spocs based on `expiration_timestamp` + spocs = spocs.filter(spoc => { + // If cached data is so old it doesn't contain this property, assume the spoc is ok to show + if (!(`expiration_timestamp` in spoc)) { + return true; + } + // `expiration_timestamp` is the number of seconds elapsed since January 1, 1970 00:00:00 UTC + return spoc.expiration_timestamp * 1000 > Date.now(); + }); + + return spocs; + } + + maybeAddSpoc(target) { + const updateContent = () => { + let spocs = this.filterSpocs(); + + if (!spocs.length) { + this.dispatchSpocDone(target); + return false; + } + + // Create a new array with a spoc inserted at index 2 + const section = this.store + .getState() + .Sections.find(s => s.id === SECTION_ID); + let rows = section.rows.slice(0, this.stories.length); + rows.splice(2, 0, Object.assign(spocs[0], { pinned: true })); + + // Send a content update to the target tab + const action = { + type: at.SECTION_UPDATE, + data: Object.assign({ rows }, { id: SECTION_ID }), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + this.dispatchSpocDone(target); + return false; + }; + + if (this.storiesLoaded) { + updateContent(); + } else { + // Delay updating tab content until initial data has been fetched + this.contentUpdateQueue.push(updateContent); + } + } + + // Frequency caps are based on campaigns, which may include multiple spocs. + // We currently support two types of frequency caps: + // - lifetime: Indicates how many times spocs from a campaign can be shown in total + // - period: Indicates how many times spocs from a campaign can be shown within a period + // + // So, for example, the feed configuration below defines that for campaign 1 no more + // than 5 spocs can be show in total, and no more than 2 per hour. + // "campaign_id": 1, + // "caps": { + // "lifetime": 5, + // "campaign": { + // "count": 2, + // "period": 3600 + // } + // } + isBelowFrequencyCap(impressions, spoc) { + const campaignImpressions = impressions[spoc.spoc_meta.campaign_id]; + if (!campaignImpressions) { + return true; + } + + const lifeTimeCap = Math.min( + spoc.spoc_meta.caps && spoc.spoc_meta.caps.lifetime, + MAX_LIFETIME_CAP + ); + const lifeTimeCapExceeded = campaignImpressions.length >= lifeTimeCap; + if (lifeTimeCapExceeded) { + return false; + } + + const campaignCap = + (spoc.spoc_meta.caps && spoc.spoc_meta.caps.campaign) || {}; + const campaignCapExceeded = + campaignImpressions.filter( + i => Date.now() - i < campaignCap.period * 1000 + ).length >= campaignCap.count; + return !campaignCapExceeded; + } + + // Clean up campaign impression pref by removing all campaigns that are no + // longer part of the response, and are therefore considered inactive. + cleanUpCampaignImpressionPref() { + const campaignIds = new Set(this.spocCampaignMap.values()); + this.cleanUpImpressionPref( + id => !campaignIds.has(id), + SPOC_IMPRESSION_TRACKING_PREF + ); + } + + // Clean up rec impression pref by removing all stories that are no + // longer part of the response. + cleanUpTopRecImpressionPref() { + const activeStories = new Set(this.stories.map(s => `${s.guid}`)); + this.cleanUpImpressionPref( + id => !activeStories.has(id), + REC_IMPRESSION_TRACKING_PREF + ); + } + + /** + * Cleans up the provided impression pref (spocs or recs). + * + * @param isExpired predicate (boolean-valued function) that returns whether or not + * the impression for the given key is expired. + * @param pref the impression pref to clean up. + */ + cleanUpImpressionPref(isExpired, pref) { + const impressions = this.readImpressionsPref(pref); + let changed = false; + + Object.keys(impressions).forEach(id => { + if (isExpired(id)) { + changed = true; + delete impressions[id]; + } + }); + + if (changed) { + this.writeImpressionsPref(pref, impressions); + } + } + + // Sets a pref mapping campaign IDs to timestamp arrays. + // The timestamps represent impressions which are used to calculate frequency caps. + recordCampaignImpression(campaignId) { + let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + + const timeStamps = impressions[campaignId] || []; + timeStamps.push(Date.now()); + impressions = Object.assign(impressions, { [campaignId]: timeStamps }); + + this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions); + } + + // Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression). + // We use these timestamps to guarantee a story doesn't stay on top for longer than + // configured in the feed settings (settings.recsExpireTime). + recordTopRecImpressions(topItems) { + let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + let changed = false; + + topItems.forEach(t => { + if (!impressions[t]) { + changed = true; + impressions = Object.assign(impressions, { [t]: Date.now() }); + } + }); + + if (changed) { + this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions); + } + } + + readImpressionsPref(pref) { + const prefVal = this._prefs.get(pref); + return prefVal ? JSON.parse(prefVal) : {}; + } + + writeImpressionsPref(pref, impressions) { + this._prefs.set(pref, JSON.stringify(impressions)); + } + + async removeSpocs() { + // Quick hack so that SPOCS are removed from all open and preloaded tabs when + // they are disabled. The longer term fix should probably be to remove them + // in the Reducer. + await this.clearCache(); + this.uninit(); + this.init(); + } + + lazyLoadTopStories(options = {}) { + let { dsPref, userPref } = options; + if (!dsPref) { + dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF]; + } + if (!userPref) { + userPref = this.store.getState().Prefs.values[PREF_USER_TOPSTORIES]; + } + + try { + this.discoveryStreamEnabled = + JSON.parse(dsPref).enabled && + this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED]; + } catch (e) { + // Load activity stream top stories if fail to determine discovery stream state + this.discoveryStreamEnabled = false; + } + + // Return without invoking initialization if top stories are loaded, or preffed off. + if (this.storiesLoaded || !userPref) { + return; + } + + if (!this.discoveryStreamEnabled && !this.propertiesInitialized) { + this.initializeProperties(); + } + this.init(); + } + + handleDisabled(action) { + switch (action.type) { + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) { + this.lazyLoadTopStories(); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + break; + case at.UNINIT: + this.uninit(); + break; + } + } + + async onAction(action) { + if (this.discoveryStreamEnabled) { + this.handleDisabled(action); + return; + } + switch (action.type) { + // Check discoverystream pref and load activity stream top stories only if needed + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.SYSTEM_TICK: + let stories; + let topics; + if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) { + stories = await this.fetchStories(); + } + if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, false); + break; + case at.UNINIT: + this.uninit(); + break; + case at.NEW_TAB_REHYDRATED: + this.getPocketState(action.meta.fromTarget); + this.maybeAddSpoc(action.meta.fromTarget); + break; + case at.SECTION_OPTIONS_CHANGED: + if (action.data === SECTION_ID) { + await this.clearCache(); + this.uninit(); + this.init(); + } + break; + case at.PLACES_LINK_BLOCKED: + if (this.spocs) { + this.spocs = this.spocs.filter(s => s.url !== action.data.url); + } + break; + case at.TELEMETRY_IMPRESSION_STATS: { + // We want to make sure we only track impressions from Top Stories, + // otherwise unexpected things that are not properly handled can happen. + // Example: Impressions from spocs on Discovery Stream can cause the + // Top Stories impressions pref to continuously grow, see bug #1523408 + if (action.data.source === IMPRESSION_SOURCE) { + const payload = action.data; + const viewImpression = !( + "click" in payload || + "block" in payload || + "pocket" in payload + ); + if (payload.tiles && viewImpression) { + if (this.shouldShowSpocs()) { + payload.tiles.forEach(t => { + if (this.spocCampaignMap.has(t.id)) { + this.recordCampaignImpression(this.spocCampaignMap.get(t.id)); + } + }); + } + const topRecs = payload.tiles + .filter(t => !this.spocCampaignMap.has(t.id)) + .map(t => t.id); + this.recordTopRecImpressions(topRecs); + } + } + break; + } + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + // Check if spocs was disabled. Remove them if they were. + if (action.data.name === "showSponsored" && !action.data.value) { + await this.removeSpocs(); + } + if (action.data.name === "pocketCta") { + this.dispatchPocketCta(action.data.value, true); + } + break; + } + } +} + +const EXPORTED_SYMBOLS = [ + "TopStoriesFeed", + "STORIES_UPDATE_TIME", + "TOPICS_UPDATE_TIME", + "SECTION_ID", + "SPOC_IMPRESSION_TRACKING_PREF", + "REC_IMPRESSION_TRACKING_PREF", + "DEFAULT_RECS_EXPIRE_TIME", +]; diff --git a/browser/components/newtab/lib/UTEventReporting.sys.mjs b/browser/components/newtab/lib/UTEventReporting.sys.mjs new file mode 100644 index 0000000000..8da7824415 --- /dev/null +++ b/browser/components/newtab/lib/UTEventReporting.sys.mjs @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Note: the schema can be found in + * https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml + */ +const EXTRAS_FIELD_NAMES = [ + "addon_version", + "session_id", + "page", + "user_prefs", + "action_position", +]; + +export class UTEventReporting { + constructor() { + Services.telemetry.setEventRecordingEnabled("activity_stream", true); + this.sendUserEvent = this.sendUserEvent.bind(this); + this.sendSessionEndEvent = this.sendSessionEndEvent.bind(this); + } + + _createExtras(data) { + // Make a copy of the given data and delete/modify it as needed. + let utExtras = Object.assign({}, data); + for (let field of Object.keys(utExtras)) { + if (EXTRAS_FIELD_NAMES.includes(field)) { + utExtras[field] = String(utExtras[field]); + continue; + } + delete utExtras[field]; + } + return utExtras; + } + + sendUserEvent(data) { + let mainFields = ["event", "source"]; + let eventFields = mainFields.map(field => String(data[field]) || null); + + Services.telemetry.recordEvent( + "activity_stream", + "event", + ...eventFields, + this._createExtras(data) + ); + } + + sendSessionEndEvent(data) { + Services.telemetry.recordEvent( + "activity_stream", + "end", + "session", + String(data.session_duration), + this._createExtras(data) + ); + } + + uninit() { + Services.telemetry.setEventRecordingEnabled("activity_stream", false); + } +} diff --git a/browser/components/newtab/lib/cache-worker.js b/browser/components/newtab/lib/cache-worker.js new file mode 100644 index 0000000000..afed78cb00 --- /dev/null +++ b/browser/components/newtab/lib/cache-worker.js @@ -0,0 +1,205 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env mozilla/chrome-worker */ + +/* global ReactDOMServer, NewtabRenderUtils */ + +const PAGE_TEMPLATE_RESOURCE_PATH = + "resource://activity-stream/data/content/abouthomecache/page.html.template"; +const SCRIPT_TEMPLATE_RESOURCE_PATH = + "resource://activity-stream/data/content/abouthomecache/script.js.template"; + +// If we don't stub these functions out, React throws warnings in the console +// upon being loaded. +let window = self; +window.requestAnimationFrame = () => {}; +window.cancelAnimationFrame = () => {}; +window.ASRouterMessage = () => { + return Promise.resolve(); +}; +window.ASRouterAddParentListener = () => {}; +window.ASRouterRemoveParentListener = () => {}; + +/* import-globals-from /toolkit/components/workerloader/require.js */ +importScripts("resource://gre/modules/workers/require.js"); + +{ + let oldChromeUtils = ChromeUtils; + + // ChromeUtils is defined inside of a Worker, but we don't want the + // activity-stream.bundle.js to detect it when loading, since that results + // in it attempting to import JSMs on load, which is not allowed in + // a Worker. So we temporarily clear ChromeUtils so that activity-stream.bundle.js + // thinks its being loaded in content scope. + // + // eslint-disable-next-line no-global-assign + ChromeUtils = undefined; + + /* import-globals-from ../vendor/react.js */ + /* import-globals-from ../vendor/react-dom.js */ + /* import-globals-from ../vendor/react-dom-server.js */ + /* import-globals-from ../vendor/redux.js */ + /* import-globals-from ../vendor/react-transition-group.js */ + /* import-globals-from ../vendor/prop-types.js */ + /* import-globals-from ../vendor/react-redux.js */ + /* import-globals-from ../data/content/activity-stream.bundle.js */ + importScripts( + "resource://activity-stream/vendor/react.js", + "resource://activity-stream/vendor/react-dom.js", + "resource://activity-stream/vendor/react-dom-server.js", + "resource://activity-stream/vendor/redux.js", + "resource://activity-stream/vendor/react-transition-group.js", + "resource://activity-stream/vendor/prop-types.js", + "resource://activity-stream/vendor/react-redux.js", + "resource://activity-stream/data/content/activity-stream.bundle.js" + ); + + // eslint-disable-next-line no-global-assign + ChromeUtils = oldChromeUtils; +} + +let PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js"); + +let Agent = { + _templates: null, + + /** + * Synchronously loads the template files off of the file + * system, and returns them as an object. If the Worker has loaded + * these templates before, a cached copy of the templates is returned + * instead. + * + * @return Object + * An object with the following properties: + * + * pageTemplate (String): + * The template for the document markup. + * + * scriptTempate (String): + * The template for the script. + */ + getOrCreateTemplates() { + if (this._templates) { + return this._templates; + } + + const templateResources = new Map([ + ["pageTemplate", PAGE_TEMPLATE_RESOURCE_PATH], + ["scriptTemplate", SCRIPT_TEMPLATE_RESOURCE_PATH], + ]); + + this._templates = {}; + + for (let [name, path] of templateResources) { + const xhr = new XMLHttpRequest(); + // Using a synchronous XHR in a worker is fine. + xhr.open("GET", path, false); + xhr.responseType = "text"; + xhr.send(null); + this._templates[name] = xhr.responseText; + } + + return this._templates; + }, + + /** + * Constructs the cached about:home document using ReactDOMServer. This will + * be called when "construct" messages are sent to this PromiseWorker. + * + * @param state (Object) + * The most recent Activity Stream Redux state. + * @return Object + * An object with the following properties: + * + * page (String): + * The generated markup for the document. + * + * script (String): + * The generated script for the document. + */ + construct(state) { + // If anything in this function throws an exception, PromiseWorker + // runs the risk of leaving the Promise associated with this method + // forever unresolved. This is particularly bad when this method is + // called via AsyncShutdown, since the forever unresolved Promise can + // result in a AsyncShutdown timeout crash. + // + // To help ensure that no matter what, the Promise resolves with something, + // we wrap the whole operation in a try/catch. + try { + return this._construct(state); + } catch (e) { + console.error("about:home startup cache construction failed:", e); + return { page: null, script: null }; + } + }, + + /** + * Internal method that actually does the work of constructing the cached + * about:home document using ReactDOMServer. This should be called from + * `construct` only. + * + * @param state (Object) + * The most recent Activity Stream Redux state. + * @return Object + * An object with the following properties: + * + * page (String): + * The generated markup for the document. + * + * script (String): + * The generated script for the document. + */ + _construct(state) { + state.App.isForStartupCache = true; + + // ReactDOMServer.renderToString expects a Redux store to pull + // the state from, so we mock out a minimal store implementation. + let fakeStore = { + getState() { + return state; + }, + dispatch() {}, + }; + + let markup = ReactDOMServer.renderToString( + NewtabRenderUtils.NewTab({ + store: fakeStore, + isFirstrun: false, + }) + ); + + let { pageTemplate, scriptTemplate } = this.getOrCreateTemplates(); + let cacheTime = new Date().toUTCString(); + let page = pageTemplate + .replace("{{ MARKUP }}", markup) + .replace("{{ CACHE_TIME }}", cacheTime); + let script = scriptTemplate.replace( + "{{ STATE }}", + JSON.stringify(state, null, "\t") + ); + + return { page, script }; + }, +}; + +// This boilerplate connects the PromiseWorker to the Agent so +// that messages from the main thread map to methods on the +// Agent. +let worker = new PromiseWorker.AbstractWorker(); +worker.dispatch = function (method, args = []) { + return Agent[method](...args); +}; +worker.postMessage = function (result, ...transfers) { + self.postMessage(result, ...transfers); +}; +worker.close = function () { + self.close(); +}; + +self.addEventListener("message", msg => worker.handleMessage(msg)); +self.addEventListener("unhandledrejection", function (error) { + throw error.reason; +}); |