summaryrefslogtreecommitdiffstats
path: root/browser/components/newtab/lib
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 09:22:09 +0000
commit43a97878ce14b72f0981164f87f2e35e14151312 (patch)
tree620249daf56c0258faa40cbdcf9cfba06de2a846 /browser/components/newtab/lib
parentInitial commit. (diff)
downloadfirefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz
firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'browser/components/newtab/lib')
-rw-r--r--browser/components/newtab/lib/ASRouter.jsm2096
-rw-r--r--browser/components/newtab/lib/ASRouterDefaultConfig.jsm65
-rw-r--r--browser/components/newtab/lib/ASRouterNewTabHook.jsm120
-rw-r--r--browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm183
-rw-r--r--browser/components/newtab/lib/ASRouterPreferences.jsm259
-rw-r--r--browser/components/newtab/lib/ASRouterTargeting.jsm1055
-rw-r--r--browser/components/newtab/lib/ASRouterTriggerListeners.jsm969
-rw-r--r--browser/components/newtab/lib/AboutPreferences.jsm311
-rw-r--r--browser/components/newtab/lib/ActivityStream.jsm758
-rw-r--r--browser/components/newtab/lib/ActivityStreamMessageChannel.jsm345
-rw-r--r--browser/components/newtab/lib/ActivityStreamPrefs.jsm95
-rw-r--r--browser/components/newtab/lib/ActivityStreamStorage.jsm121
-rw-r--r--browser/components/newtab/lib/CFRMessageProvider.jsm829
-rw-r--r--browser/components/newtab/lib/CFRPageActions.jsm1036
-rw-r--r--browser/components/newtab/lib/DefaultSites.jsm50
-rw-r--r--browser/components/newtab/lib/DiscoveryStreamFeed.jsm2389
-rw-r--r--browser/components/newtab/lib/DownloadsManager.jsm191
-rw-r--r--browser/components/newtab/lib/FaviconFeed.jsm202
-rw-r--r--browser/components/newtab/lib/FeatureCalloutMessages.jsm640
-rw-r--r--browser/components/newtab/lib/FilterAdult.jsm3036
-rw-r--r--browser/components/newtab/lib/HighlightsFeed.jsm357
-rw-r--r--browser/components/newtab/lib/InfoBar.jsm179
-rw-r--r--browser/components/newtab/lib/LinksCache.jsm136
-rw-r--r--browser/components/newtab/lib/MomentsPageHub.jsm174
-rw-r--r--browser/components/newtab/lib/NewTabInit.jsm57
-rw-r--r--browser/components/newtab/lib/OnboardingMessageProvider.jsm1120
-rw-r--r--browser/components/newtab/lib/PageEventManager.jsm97
-rw-r--r--browser/components/newtab/lib/PanelTestProvider.jsm782
-rw-r--r--browser/components/newtab/lib/PersistentCache.jsm95
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm67
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm65
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm292
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js44
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm311
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm1126
-rw-r--r--browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm89
-rw-r--r--browser/components/newtab/lib/PlacesFeed.jsm615
-rw-r--r--browser/components/newtab/lib/PrefsFeed.jsm273
-rw-r--r--browser/components/newtab/lib/RecommendationProvider.jsm133
-rw-r--r--browser/components/newtab/lib/RemoteImages.jsm609
-rw-r--r--browser/components/newtab/lib/RemoteL10n.jsm252
-rw-r--r--browser/components/newtab/lib/Screenshots.jsm144
-rw-r--r--browser/components/newtab/lib/SearchShortcuts.jsm76
-rw-r--r--browser/components/newtab/lib/SectionsManager.jsm720
-rw-r--r--browser/components/newtab/lib/ShortURL.jsm83
-rw-r--r--browser/components/newtab/lib/SiteClassifier.jsm99
-rw-r--r--browser/components/newtab/lib/SnippetsTestMessageProvider.jsm715
-rw-r--r--browser/components/newtab/lib/Spotlight.jsm117
-rw-r--r--browser/components/newtab/lib/Store.jsm190
-rw-r--r--browser/components/newtab/lib/SystemTickFeed.jsm40
-rw-r--r--browser/components/newtab/lib/TelemetryFeed.jsm1313
-rw-r--r--browser/components/newtab/lib/TippyTopProvider.jsm62
-rw-r--r--browser/components/newtab/lib/ToastNotification.jsm118
-rw-r--r--browser/components/newtab/lib/ToolbarBadgeHub.jsm318
-rw-r--r--browser/components/newtab/lib/ToolbarPanelHub.jsm612
-rw-r--r--browser/components/newtab/lib/TopSitesFeed.jsm1409
-rw-r--r--browser/components/newtab/lib/TopStoriesFeed.jsm751
-rw-r--r--browser/components/newtab/lib/UTEventReporting.jsm66
-rw-r--r--browser/components/newtab/lib/cache-worker.js205
59 files changed, 28661 insertions, 0 deletions
diff --git a/browser/components/newtab/lib/ASRouter.jsm b/browser/components/newtab/lib/ASRouter.jsm
new file mode 100644
index 0000000000..964fa1f011
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouter.jsm
@@ -0,0 +1,2096 @@
+/* vim: set ts=2 sw=2 sts=2 et tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+const lazy = {};
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ SnippetsTestMessageProvider:
+ "resource://activity-stream/lib/SnippetsTestMessageProvider.jsm",
+ PanelTestProvider: "resource://activity-stream/lib/PanelTestProvider.jsm",
+ Spotlight: "resource://activity-stream/lib/Spotlight.jsm",
+ ToastNotification: "resource://activity-stream/lib/ToastNotification.jsm",
+ ToolbarBadgeHub: "resource://activity-stream/lib/ToolbarBadgeHub.jsm",
+ ToolbarPanelHub: "resource://activity-stream/lib/ToolbarPanelHub.jsm",
+ MomentsPageHub: "resource://activity-stream/lib/MomentsPageHub.jsm",
+ InfoBar: "resource://activity-stream/lib/InfoBar.jsm",
+ ASRouterTargeting: "resource://activity-stream/lib/ASRouterTargeting.jsm",
+ ASRouterPreferences: "resource://activity-stream/lib/ASRouterPreferences.jsm",
+ TARGETING_PREFERENCES:
+ "resource://activity-stream/lib/ASRouterPreferences.jsm",
+ ASRouterTriggerListeners:
+ "resource://activity-stream/lib/ASRouterTriggerListeners.jsm",
+ KintoHttpClient: "resource://services-common/kinto-http-client.js",
+ Downloader: "resource://services-settings/Attachments.jsm",
+ RemoteImages: "resource://activity-stream/lib/RemoteImages.jsm",
+ RemoteL10n: "resource://activity-stream/lib/RemoteL10n.jsm",
+ ExperimentAPI: "resource://nimbus/ExperimentAPI.jsm",
+ setTimeout: "resource://gre/modules/Timer.jsm",
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+ SpecialMessageActions:
+ "resource://messaging-system/lib/SpecialMessageActions.jsm",
+ TargetingContext: "resource://messaging-system/targeting/Targeting.jsm",
+ Utils: "resource://services-settings/Utils.jsm",
+ MacAttribution: "resource:///modules/MacAttribution.jsm",
+});
+XPCOMUtils.defineLazyServiceGetters(lazy, {
+ BrowserHandler: ["@mozilla.org/browser/clh;1", "nsIBrowserHandler"],
+});
+const { actionCreators: ac } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const { CFRMessageProvider } = ChromeUtils.import(
+ "resource://activity-stream/lib/CFRMessageProvider.jsm"
+);
+const { OnboardingMessageProvider } = ChromeUtils.import(
+ "resource://activity-stream/lib/OnboardingMessageProvider.jsm"
+);
+const { RemoteSettings } = ChromeUtils.import(
+ "resource://services-settings/remote-settings.js"
+);
+const { CFRPageActions } = ChromeUtils.import(
+ "resource://activity-stream/lib/CFRPageActions.jsm"
+);
+const { AttributionCode } = ChromeUtils.import(
+ "resource:///modules/AttributionCode.jsm"
+);
+
+// List of hosts for endpoints that serve router messages.
+// Key is allowed host, value is a name for the endpoint host.
+const DEFAULT_ALLOWLIST_HOSTS = {
+ "activity-stream-icons.services.mozilla.com": "production",
+ "snippets-admin.mozilla.org": "preview",
+};
+const SNIPPETS_ENDPOINT_ALLOWLIST =
+ "browser.newtab.activity-stream.asrouter.allowHosts";
+// Max possible impressions cap for any message
+const MAX_MESSAGE_LIFETIME_CAP = 100;
+
+const LOCAL_MESSAGE_PROVIDERS = {
+ OnboardingMessageProvider,
+ CFRMessageProvider,
+};
+const STARTPAGE_VERSION = "6";
+
+// Remote Settings
+const RS_MAIN_BUCKET = "main";
+const RS_COLLECTION_L10N = "ms-language-packs"; // "ms" stands for Messaging System
+const RS_PROVIDERS_WITH_L10N = ["cfr"];
+const RS_FLUENT_VERSION = "v1";
+const RS_FLUENT_RECORD_PREFIX = `cfr-${RS_FLUENT_VERSION}`;
+const RS_DOWNLOAD_MAX_RETRIES = 2;
+// This is the list of providers for which we want to cache the targeting
+// expression result and reuse between calls. Cache duration is defined in
+// ASRouterTargeting where evaluation takes place.
+const JEXL_PROVIDER_CACHE = new Set(["snippets"]);
+
+// To observe the app locale change notification.
+const TOPIC_INTL_LOCALE_CHANGED = "intl:app-locales-changed";
+const TOPIC_EXPERIMENT_FORCE_ENROLLED = "nimbus:force-enroll";
+// To observe the pref that controls if ASRouter should use the remote Fluent files for l10n.
+const USE_REMOTE_L10N_PREF =
+ "browser.newtabpage.activity-stream.asrouter.useRemoteL10n";
+
+const MESSAGING_EXPERIMENTS_DEFAULT_FEATURES = [
+ "cfr",
+ "fxms-message-1",
+ "fxms-message-2",
+ "fxms-message-3",
+ "infobar",
+ "moments-page",
+ "pbNewtab",
+ "spotlight",
+];
+
+// Experiment groups that need to report the reach event in Messaging-Experiments.
+// If you're adding new groups to it, make sure they're also added in the
+// `messaging_experiments.reach.objects` defined in "toolkit/components/telemetry/Events.yaml"
+const REACH_EVENT_GROUPS = ["cfr", "moments-page", "infobar", "spotlight"];
+const REACH_EVENT_CATEGORY = "messaging_experiments";
+const REACH_EVENT_METHOD = "reach";
+
+const MessageLoaderUtils = {
+ STARTPAGE_VERSION,
+ REMOTE_LOADER_CACHE_KEY: "RemoteLoaderCache",
+ _errors: [],
+
+ reportError(e) {
+ console.error(e);
+ this._errors.push({
+ timestamp: new Date(),
+ error: { message: e.toString(), stack: e.stack },
+ });
+ },
+
+ get errors() {
+ const errors = this._errors;
+ this._errors = [];
+ return errors;
+ },
+
+ /**
+ * _localLoader - Loads messages for a local provider (i.e. one that lives in mozilla central)
+ *
+ * @param {obj} provider An AS router provider
+ * @param {Array} provider.messages An array of messages
+ * @returns {Array} the array of messages
+ */
+ _localLoader(provider) {
+ return provider.messages;
+ },
+
+ async _remoteLoaderCache(storage) {
+ let allCached;
+ try {
+ allCached =
+ (await storage.get(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY)) || {};
+ } catch (e) {
+ // istanbul ignore next
+ MessageLoaderUtils.reportError(e);
+ // istanbul ignore next
+ allCached = {};
+ }
+ return allCached;
+ },
+
+ /**
+ * _remoteLoader - Loads messages for a remote provider
+ *
+ * @param {obj} provider An AS router provider
+ * @param {string} provider.url An endpoint that returns an array of messages as JSON
+ * @param {obj} options.storage A storage object with get() and set() methods for caching.
+ * @returns {Promise} resolves with an array of messages, or an empty array if none could be fetched
+ */
+ async _remoteLoader(provider, options) {
+ let remoteMessages = [];
+ if (provider.url) {
+ const allCached = await MessageLoaderUtils._remoteLoaderCache(
+ options.storage
+ );
+ const cached = allCached[provider.id];
+ let etag;
+
+ if (
+ cached &&
+ cached.url === provider.url &&
+ cached.version === STARTPAGE_VERSION
+ ) {
+ const { lastFetched, messages } = cached;
+ if (
+ !MessageLoaderUtils.shouldProviderUpdate({
+ ...provider,
+ lastUpdated: lastFetched,
+ })
+ ) {
+ // Cached messages haven't expired, return early.
+ return messages;
+ }
+ etag = cached.etag;
+ remoteMessages = messages;
+ }
+
+ let headers = new Headers();
+ if (etag) {
+ headers.set("If-None-Match", etag);
+ }
+
+ let response;
+ try {
+ response = await fetch(provider.url, {
+ headers,
+ credentials: "omit",
+ });
+ } catch (e) {
+ MessageLoaderUtils.reportError(e);
+ }
+ if (
+ response &&
+ response.ok &&
+ response.status >= 200 &&
+ response.status < 400
+ ) {
+ let jsonResponse;
+ try {
+ jsonResponse = await response.json();
+ } catch (e) {
+ MessageLoaderUtils.reportError(e);
+ return remoteMessages;
+ }
+ if (jsonResponse && jsonResponse.messages) {
+ remoteMessages = jsonResponse.messages.map(msg => ({
+ ...msg,
+ provider_url: provider.url,
+ }));
+
+ // Cache the results if this isn't a preview URL.
+ if (provider.updateCycleInMs > 0) {
+ etag = response.headers.get("ETag");
+ const cacheInfo = {
+ messages: remoteMessages,
+ etag,
+ lastFetched: Date.now(),
+ version: STARTPAGE_VERSION,
+ };
+
+ options.storage.set(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY, {
+ ...allCached,
+ [provider.id]: cacheInfo,
+ });
+ }
+ } else {
+ MessageLoaderUtils.reportError(
+ `No messages returned from ${provider.url}.`
+ );
+ }
+ } else if (response) {
+ MessageLoaderUtils.reportError(
+ `Invalid response status ${response.status} from ${provider.url}.`
+ );
+ }
+ }
+ return remoteMessages;
+ },
+
+ /**
+ * _remoteSettingsLoader - Loads messages for a RemoteSettings provider
+ *
+ * Note:
+ * 1). The "cfr" provider requires the Fluent file for l10n, so there is
+ * another file downloading phase for those two providers after their messages
+ * are successfully fetched from Remote Settings. Currently, they share the same
+ * attachment of the record "${RS_FLUENT_RECORD_PREFIX}-${locale}" in the
+ * "ms-language-packs" collection. E.g. for "en-US" with version "v1",
+ * the Fluent file is attched to the record with ID "cfr-v1-en-US".
+ *
+ * 2). The Remote Settings downloader is able to detect the duplicate download
+ * requests for the same attachment and ignore the redundent requests automatically.
+ *
+ * @param {object} provider An AS router provider
+ * @param {string} provider.id The id of the provider
+ * @param {string} provider.collection Remote Settings collection name
+ * @param {object} options
+ * @param {function} options.dispatchCFRAction Action handler function
+ * @returns {Promise<object[]>} Resolves with an array of messages, or an
+ * empty array if none could be fetched
+ */
+ async _remoteSettingsLoader(provider, options) {
+ let messages = [];
+ if (provider.collection) {
+ try {
+ messages = await MessageLoaderUtils._getRemoteSettingsMessages(
+ provider.collection
+ );
+ if (!messages.length) {
+ MessageLoaderUtils._handleRemoteSettingsUndesiredEvent(
+ "ASR_RS_NO_MESSAGES",
+ provider.id,
+ options.dispatchCFRAction
+ );
+ } else if (
+ RS_PROVIDERS_WITH_L10N.includes(provider.id) &&
+ lazy.RemoteL10n.isLocaleSupported(MessageLoaderUtils.locale)
+ ) {
+ const recordId = `${RS_FLUENT_RECORD_PREFIX}-${MessageLoaderUtils.locale}`;
+ const kinto = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL);
+ const record = await kinto
+ .bucket(RS_MAIN_BUCKET)
+ .collection(RS_COLLECTION_L10N)
+ .getRecord(recordId);
+ if (record && record.data) {
+ const downloader = new lazy.Downloader(
+ RS_MAIN_BUCKET,
+ RS_COLLECTION_L10N,
+ "browser",
+ "newtab"
+ );
+ // Await here in order to capture the exceptions for reporting.
+ await downloader.downloadToDisk(record.data, {
+ retries: RS_DOWNLOAD_MAX_RETRIES,
+ });
+ lazy.RemoteL10n.reloadL10n();
+ } else {
+ MessageLoaderUtils._handleRemoteSettingsUndesiredEvent(
+ "ASR_RS_NO_MESSAGES",
+ RS_COLLECTION_L10N,
+ options.dispatchCFRAction
+ );
+ }
+ }
+ } catch (e) {
+ MessageLoaderUtils._handleRemoteSettingsUndesiredEvent(
+ "ASR_RS_ERROR",
+ provider.id,
+ options.dispatchCFRAction
+ );
+ MessageLoaderUtils.reportError(e);
+ }
+ }
+ return messages;
+ },
+
+ /**
+ * Fetch messages from a given collection in Remote Settings.
+ *
+ * @param {string} collection The remote settings collection identifier
+ * @returns {Promise<object[]>} Resolves with an array of messages
+ */
+ _getRemoteSettingsMessages(collection) {
+ return RemoteSettings(collection).get();
+ },
+
+ /**
+ * Return messages from active Nimbus experiments and rollouts.
+ *
+ * @param {object} provider A messaging experiments provider.
+ * @param {string[]?} provider.featureIds
+ * An optional array of Nimbus feature IDs to check for
+ * enrollments. If not provided, we will fall back to the
+ * set of default features. Otherwise, if provided and
+ * empty, we will not ingest messages from any features.
+ *
+ * @return {object[]} The list of messages from active enrollments, as well as
+ * the messages defined in unenrolled branches so that they
+ * reach events can be recorded (if we record reach events
+ * for that feature).
+ */
+ async _experimentsAPILoader(provider) {
+ // Allow tests to override the set of featureIds
+ const featureIds = Array.isArray(provider.featureIds)
+ ? provider.featureIds
+ : MESSAGING_EXPERIMENTS_DEFAULT_FEATURES;
+ let experiments = [];
+ for (const featureId of featureIds) {
+ let featureAPI = lazy.NimbusFeatures[featureId];
+ let experimentData = lazy.ExperimentAPI.getExperimentMetaData({
+ featureId,
+ });
+
+ // We are not enrolled in any experiment or rollout for this feature, so
+ // we can skip the feature.
+ if (
+ !experimentData &&
+ !lazy.ExperimentAPI.getRolloutMetaData({ featureId })
+ ) {
+ continue;
+ }
+
+ let message = featureAPI.getAllVariables();
+
+ if (message?.id) {
+ // Cache the Nimbus feature ID on the message because there is not a 1-1
+ // correspondance between templates and features. This is used when
+ // recording expose events (see |sendTriggerMessage|).
+ message._nimbusFeature = featureId;
+ experiments.push(message);
+ }
+
+ if (!REACH_EVENT_GROUPS.includes(featureId)) {
+ continue;
+ }
+
+ // If we are in a rollout, we do not have sibling branches.
+ if (experimentData) {
+ // Check other sibling branches for triggers, add them to the return
+ // array if found any. The `forReachEvent` label is used to identify
+ // those branches so that they would only used to record the Reach
+ // event.
+ const branches =
+ (await lazy.ExperimentAPI.getAllBranches(experimentData.slug)) || [];
+ for (const branch of branches) {
+ let branchValue = branch[featureId].value;
+ if (
+ branch.slug !== experimentData.branch.slug &&
+ branchValue?.trigger
+ ) {
+ experiments.push({
+ forReachEvent: { sent: false, group: featureId },
+ experimentSlug: experimentData.slug,
+ branchSlug: branch.slug,
+ ...branchValue,
+ });
+ }
+ }
+ }
+ }
+
+ return experiments;
+ },
+
+ _handleRemoteSettingsUndesiredEvent(event, providerId, dispatchCFRAction) {
+ if (dispatchCFRAction) {
+ dispatchCFRAction(
+ ac.ASRouterUserEvent({
+ action: "asrouter_undesired_event",
+ event,
+ message_id: "n/a",
+ event_context: providerId,
+ })
+ );
+ }
+ },
+
+ /**
+ * _getMessageLoader - return the right loading function given the provider's type
+ *
+ * @param {obj} provider An AS Router provider
+ * @returns {func} A loading function
+ */
+ _getMessageLoader(provider) {
+ switch (provider.type) {
+ case "remote":
+ return this._remoteLoader;
+ case "remote-settings":
+ return this._remoteSettingsLoader;
+ case "remote-experiments":
+ return this._experimentsAPILoader;
+ case "local":
+ default:
+ return this._localLoader;
+ }
+ },
+
+ /**
+ * shouldProviderUpdate - Given the current time, should a provider update its messages?
+ *
+ * @param {any} provider An AS Router provider
+ * @param {int} provider.updateCycleInMs The number of milliseconds we should wait between updates
+ * @param {Date} provider.lastUpdated If the provider has been updated, the time the last update occurred
+ * @param {Date} currentTime The time we should check against. (defaults to Date.now())
+ * @returns {bool} Should an update happen?
+ */
+ shouldProviderUpdate(provider, currentTime = Date.now()) {
+ return (
+ !(provider.lastUpdated >= 0) ||
+ currentTime - provider.lastUpdated > provider.updateCycleInMs
+ );
+ },
+
+ async _loadDataForProvider(provider, options) {
+ const loader = this._getMessageLoader(provider);
+ let messages = await loader(provider, options);
+ // istanbul ignore if
+ if (!messages) {
+ messages = [];
+ MessageLoaderUtils.reportError(
+ new Error(
+ `Tried to load messages for ${provider.id} but the result was not an Array.`
+ )
+ );
+ }
+
+ return { messages };
+ },
+
+ /**
+ * loadMessagesForProvider - Load messages for a provider, given the provider's type.
+ *
+ * @param {obj} provider An AS Router provider
+ * @param {string} provider.type An AS Router provider type (defaults to "local")
+ * @param {obj} options.storage A storage object with get() and set() methods for caching.
+ * @param {func} options.dispatchCFRAction dispatch an action the main AS Store
+ * @returns {obj} Returns an object with .messages (an array of messages) and .lastUpdated (the time the messages were updated)
+ */
+ async loadMessagesForProvider(provider, options) {
+ let { messages } = await this._loadDataForProvider(provider, options);
+ // Filter out messages we temporarily want to exclude
+ if (provider.exclude && provider.exclude.length) {
+ messages = messages.filter(
+ message => !provider.exclude.includes(message.id)
+ );
+ }
+ const lastUpdated = Date.now();
+ return {
+ messages: messages
+ .map(messageData => {
+ const message = {
+ weight: 100,
+ ...messageData,
+ groups: messageData.groups || [],
+ provider: provider.id,
+ };
+
+ return message;
+ })
+ .filter(message => message.weight > 0),
+ lastUpdated,
+ errors: MessageLoaderUtils.errors,
+ };
+ },
+
+ /**
+ * cleanupCache - Removes cached data of removed providers.
+ *
+ * @param {Array} providers A list of activer AS Router providers
+ */
+ async cleanupCache(providers, storage) {
+ const ids = providers.filter(p => p.type === "remote").map(p => p.id);
+ const cache = await MessageLoaderUtils._remoteLoaderCache(storage);
+ let dirty = false;
+ for (let id in cache) {
+ if (!ids.includes(id)) {
+ delete cache[id];
+ dirty = true;
+ }
+ }
+ if (dirty) {
+ await storage.set(MessageLoaderUtils.REMOTE_LOADER_CACHE_KEY, cache);
+ }
+ },
+
+ /**
+ * The locale to use for RemoteL10n.
+ *
+ * This may map the app's actual locale into something that RemoteL10n
+ * supports.
+ */
+ get locale() {
+ const localeMap = {
+ "ja-JP-macos": "ja-JP-mac",
+
+ // While it's not a valid locale, "und" is commonly observed on
+ // Linux platforms. Per l10n team, it's reasonable to fallback to
+ // "en-US", therefore, we should allow the fetch for it.
+ und: "en-US",
+ };
+
+ const locale = Services.locale.appLocaleAsBCP47;
+ return localeMap[locale] ?? locale;
+ },
+};
+
+/**
+ * @class _ASRouter - Keeps track of all messages, UI surfaces, and
+ * handles blocking, rotation, etc. Inspecting ASRouter.state will
+ * tell you what the current displayed message is in all UI surfaces.
+ *
+ * Note: This is written as a constructor rather than just a plain object
+ * so that it can be more easily unit tested.
+ */
+class _ASRouter {
+ constructor(localProviders = LOCAL_MESSAGE_PROVIDERS) {
+ this.initialized = false;
+ this.clearChildMessages = null;
+ this.clearChildProviders = null;
+ this.updateAdminState = null;
+ this.sendTelemetry = null;
+ this.dispatchCFRAction = null;
+ this._storage = null;
+ this._resetInitialization();
+ this._state = {
+ providers: [],
+ messageBlockList: [],
+ messageImpressions: {},
+ messages: [],
+ groups: [],
+ errors: [],
+ localeInUse: Services.locale.appLocaleAsBCP47,
+ };
+ this._experimentChangedListeners = new Map();
+ this._triggerHandler = this._triggerHandler.bind(this);
+ this._localProviders = localProviders;
+ this.blockMessageById = this.blockMessageById.bind(this);
+ this.unblockMessageById = this.unblockMessageById.bind(this);
+ this.handleMessageRequest = this.handleMessageRequest.bind(this);
+ this.addImpression = this.addImpression.bind(this);
+ this._handleTargetingError = this._handleTargetingError.bind(this);
+ this.onPrefChange = this.onPrefChange.bind(this);
+ this._onLocaleChanged = this._onLocaleChanged.bind(this);
+ this.isUnblockedMessage = this.isUnblockedMessage.bind(this);
+ this.unblockAll = this.unblockAll.bind(this);
+ this.forceWNPanel = this.forceWNPanel.bind(this);
+ this._onExperimentForceEnrolled = this._onExperimentForceEnrolled.bind(
+ this
+ );
+ this.forcePBWindow = this.forcePBWindow.bind(this);
+ Services.telemetry.setEventRecordingEnabled(REACH_EVENT_CATEGORY, true);
+ }
+
+ async onPrefChange(prefName) {
+ if (lazy.TARGETING_PREFERENCES.includes(prefName)) {
+ let invalidMessages = [];
+ // Notify all tabs of messages that have become invalid after pref change
+ const context = this._getMessagesContext();
+ const targetingContext = new lazy.TargetingContext(context);
+
+ for (const msg of this.state.messages.filter(this.isUnblockedMessage)) {
+ if (!msg.targeting) {
+ continue;
+ }
+ const isMatch = await targetingContext.evalWithDefault(msg.targeting);
+ if (!isMatch) {
+ invalidMessages.push(msg.id);
+ }
+ }
+ this.clearChildMessages(invalidMessages);
+ } else {
+ // Update message providers and fetch new messages on pref change
+ this._loadLocalProviders();
+ let invalidProviders = await this._updateMessageProviders();
+ if (invalidProviders.length) {
+ this.clearChildProviders(invalidProviders);
+ }
+ await this.loadMessagesFromAllProviders();
+ // Any change in user prefs can disable or enable groups
+ await this.setState(state => ({
+ groups: state.groups.map(this._checkGroupEnabled),
+ }));
+ }
+ }
+
+ // Fetch and decode the message provider pref JSON, and update the message providers
+ async _updateMessageProviders() {
+ lazy.ASRouterPreferences.console.debug("entering updateMessageProviders");
+
+ const previousProviders = this.state.providers;
+ const providers = await Promise.all(
+ [
+ // If we have added a `preview` provider, hold onto it
+ ...previousProviders.filter(p => p.id === "preview"),
+ // The provider should be enabled and not have a user preference set to false
+ ...lazy.ASRouterPreferences.providers.filter(
+ p =>
+ p.enabled &&
+ lazy.ASRouterPreferences.getUserPreference(p.id) !== false
+ ),
+ ].map(async _provider => {
+ // make a copy so we don't modify the source of the pref
+ const provider = { ..._provider };
+
+ if (provider.type === "local" && !provider.messages) {
+ // Get the messages from the local message provider
+ const localProvider = this._localProviders[provider.localProvider];
+ provider.messages = [];
+ if (localProvider) {
+ provider.messages = await localProvider.getMessages();
+ }
+ }
+ if (provider.type === "remote" && provider.url) {
+ provider.url = provider.url.replace(
+ /%STARTPAGE_VERSION%/g,
+ STARTPAGE_VERSION
+ );
+ provider.url = Services.urlFormatter.formatURL(provider.url);
+ }
+ if (provider.id === "messaging-experiments") {
+ // By default, the messaging-experiments provider lacks a featureIds
+ // property, so fall back to the list of default features.
+ if (!provider.featureIds) {
+ provider.featureIds = MESSAGING_EXPERIMENTS_DEFAULT_FEATURES;
+ }
+ }
+ // Reset provider update timestamp to force message refresh
+ provider.lastUpdated = undefined;
+ return provider;
+ })
+ );
+
+ const providerIDs = providers.map(p => p.id);
+ let invalidProviders = [];
+
+ // Clear old messages for providers that are no longer enabled
+ for (const prevProvider of previousProviders) {
+ if (!providerIDs.includes(prevProvider.id)) {
+ invalidProviders.push(prevProvider.id);
+ }
+ }
+
+ {
+ // If the feature IDs of the messaging-experiments provider has changed,
+ // then we need to update which features for which we are listening to
+ // changes.
+ const prevExpts = previousProviders.find(
+ p => p.id === "messaging-experiments"
+ );
+ const expts = providers.find(p => p.id === "messaging-experiments");
+
+ this._onFeatureListChanged(
+ prevExpts?.enabled ? prevExpts.featureIds : [],
+ expts?.enabled ? expts.featureIds : []
+ );
+ }
+
+ return this.setState(prevState => ({
+ providers,
+ // Clear any messages from removed providers
+ messages: [
+ ...prevState.messages.filter(message =>
+ providerIDs.includes(message.provider)
+ ),
+ ],
+ })).then(() => invalidProviders);
+ }
+
+ get state() {
+ return this._state;
+ }
+
+ set state(value) {
+ throw new Error(
+ "Do not modify this.state directy. Instead, call this.setState(newState)"
+ );
+ }
+
+ /**
+ * _resetInitialization - adds the following to the instance:
+ * .initialized {bool} Has AS Router been initialized?
+ * .waitForInitialized {Promise} A promise that resolves when initializion is complete
+ * ._finishInitializing {func} A function that, when called, resolves the .waitForInitialized
+ * promise and sets .initialized to true.
+ * @memberof _ASRouter
+ */
+ _resetInitialization() {
+ this.initialized = false;
+ this.initializing = false;
+ this.waitForInitialized = new Promise(resolve => {
+ this._finishInitializing = () => {
+ this.initialized = true;
+ this.initializing = false;
+ resolve();
+ };
+ });
+ }
+
+ /**
+ * Check all provided groups are enabled.
+ * @param groups Set of groups to verify
+ * @returns bool
+ */
+ hasGroupsEnabled(groups = []) {
+ return this.state.groups
+ .filter(({ id }) => groups.includes(id))
+ .every(({ enabled }) => enabled);
+ }
+
+ /**
+ * Verify that the provider block the message through the `exclude` field
+ * @param message Message to verify
+ * @returns bool
+ */
+ isExcludedByProvider(message) {
+ // preview snippets are never excluded
+ if (message.provider === "preview") {
+ return false;
+ }
+ const provider = this.state.providers.find(p => p.id === message.provider);
+ if (!provider) {
+ return true;
+ }
+ if (provider.exclude) {
+ return provider.exclude.includes(message.id);
+ }
+ return false;
+ }
+
+ /**
+ * Takes a group and sets the correct `enabled` state based on message config
+ * and user preferences
+ *
+ * @param {GroupConfig} group
+ * @returns {GroupConfig}
+ */
+ _checkGroupEnabled(group) {
+ return {
+ ...group,
+ enabled:
+ group.enabled &&
+ // And if defined user preferences are true. If multiple prefs are
+ // defined then at least one has to be enabled.
+ (Array.isArray(group.userPreferences)
+ ? group.userPreferences.some(pref =>
+ lazy.ASRouterPreferences.getUserPreference(pref)
+ )
+ : true),
+ };
+ }
+
+ /**
+ * Fetch all message groups and update Router.state.groups.
+ * There are two cases to consider:
+ * 1. The provider needs to update as determined by the update cycle
+ * 2. Some pref change occured which could invalidate one of the existing
+ * groups.
+ */
+ async loadAllMessageGroups() {
+ const provider = this.state.providers.find(
+ p =>
+ p.id === "message-groups" && MessageLoaderUtils.shouldProviderUpdate(p)
+ );
+ let remoteMessages = null;
+ if (provider) {
+ const { messages } = await MessageLoaderUtils._loadDataForProvider(
+ provider,
+ {
+ storage: this._storage,
+ dispatchCFRAction: this.dispatchCFRAction,
+ }
+ );
+ remoteMessages = messages;
+ }
+ await this.setState(state => ({
+ // If fetching remote messages fails we default to existing state.groups.
+ groups: (remoteMessages || state.groups).map(this._checkGroupEnabled),
+ }));
+ }
+
+ /**
+ * loadMessagesFromAllProviders - Loads messages from all providers if they require updates.
+ * Checks the .lastUpdated field on each provider to see if updates are needed
+ * @param toUpdate An optional list of providers to update. This overrides
+ * the checks to determine which providers to update.
+ * @memberof _ASRouter
+ */
+ async loadMessagesFromAllProviders(toUpdate = undefined) {
+ const needsUpdate = Array.isArray(toUpdate)
+ ? toUpdate
+ : this.state.providers.filter(provider =>
+ MessageLoaderUtils.shouldProviderUpdate(provider)
+ );
+ lazy.ASRouterPreferences.console.debug(
+ "entering loadMessagesFromAllProviders"
+ );
+
+ await this.loadAllMessageGroups();
+ // Don't do extra work if we don't need any updates
+ if (needsUpdate.length) {
+ let newState = { messages: [], providers: [] };
+ for (const provider of this.state.providers) {
+ if (needsUpdate.includes(provider)) {
+ const {
+ messages,
+ lastUpdated,
+ errors,
+ } = await MessageLoaderUtils.loadMessagesForProvider(provider, {
+ storage: this._storage,
+ dispatchCFRAction: this.dispatchCFRAction,
+ });
+ newState.providers.push({ ...provider, lastUpdated, errors });
+ newState.messages = [...newState.messages, ...messages];
+ } else {
+ // Skip updating this provider's messages if no update is required
+ let messages = this.state.messages.filter(
+ msg => msg.provider === provider.id
+ );
+ newState.providers.push(provider);
+ newState.messages = [...newState.messages, ...messages];
+ }
+ }
+
+ // Some messages have triggers that require us to initalise trigger listeners
+ const unseenListeners = new Set(lazy.ASRouterTriggerListeners.keys());
+ for (const { trigger } of newState.messages) {
+ if (trigger && lazy.ASRouterTriggerListeners.has(trigger.id)) {
+ lazy.ASRouterTriggerListeners.get(trigger.id).init(
+ this._triggerHandler,
+ trigger.params,
+ trigger.patterns
+ );
+ unseenListeners.delete(trigger.id);
+ }
+ }
+ // We don't need these listeners, but they may have previously been
+ // initialised, so uninitialise them
+ for (const triggerID of unseenListeners) {
+ lazy.ASRouterTriggerListeners.get(triggerID).uninit();
+ }
+
+ // We don't want to cache preview endpoints, remove them after messages are fetched
+ await this.setState(this._removePreviewEndpoint(newState));
+ await this.cleanupImpressions();
+ }
+ return this.state;
+ }
+
+ async _maybeUpdateL10nAttachment() {
+ const { localeInUse } = this.state.localeInUse;
+ const newLocale = Services.locale.appLocaleAsBCP47;
+ if (newLocale !== localeInUse) {
+ const providers = [...this.state.providers];
+ let needsUpdate = false;
+ providers.forEach(provider => {
+ if (RS_PROVIDERS_WITH_L10N.includes(provider.id)) {
+ // Force to refresh the messages as well as the attachment.
+ provider.lastUpdated = undefined;
+ needsUpdate = true;
+ }
+ });
+ if (needsUpdate) {
+ await this.setState({
+ localeInUse: newLocale,
+ providers,
+ });
+ await this.loadMessagesFromAllProviders();
+ }
+ }
+ return this.state;
+ }
+
+ async _onLocaleChanged(subject, topic, data) {
+ await this._maybeUpdateL10nAttachment();
+ }
+
+ observe(aSubject, aTopic, aPrefName) {
+ switch (aPrefName) {
+ case USE_REMOTE_L10N_PREF:
+ CFRPageActions.reloadL10n();
+ break;
+ }
+ }
+
+ toWaitForInitFunc(func) {
+ return (...args) => this.waitForInitialized.then(() => func(...args));
+ }
+
+ /**
+ * init - Initializes the MessageRouter.
+ *
+ * @param {obj} parameters parameters to initialize ASRouter
+ * @memberof _ASRouter
+ */
+ async init({
+ storage,
+ sendTelemetry,
+ clearChildMessages,
+ clearChildProviders,
+ updateAdminState,
+ dispatchCFRAction,
+ }) {
+ if (this.initializing || this.initialized) {
+ return null;
+ }
+ this.initializing = true;
+ this._storage = storage;
+ this.ALLOWLIST_HOSTS = this._loadSnippetsAllowHosts();
+ this.clearChildMessages = this.toWaitForInitFunc(clearChildMessages);
+ this.clearChildProviders = this.toWaitForInitFunc(clearChildProviders);
+ // NOTE: This is only necessary to sync devtools and snippets when devtools is active.
+ this.updateAdminState = this.toWaitForInitFunc(updateAdminState);
+ this.sendTelemetry = sendTelemetry;
+ this.dispatchCFRAction = this.toWaitForInitFunc(dispatchCFRAction);
+
+ lazy.ASRouterPreferences.init();
+ lazy.ASRouterPreferences.addListener(this.onPrefChange);
+ lazy.ToolbarBadgeHub.init(this.waitForInitialized, {
+ handleMessageRequest: this.handleMessageRequest,
+ addImpression: this.addImpression,
+ blockMessageById: this.blockMessageById,
+ unblockMessageById: this.unblockMessageById,
+ sendTelemetry: this.sendTelemetry,
+ });
+ lazy.ToolbarPanelHub.init(this.waitForInitialized, {
+ getMessages: this.handleMessageRequest,
+ sendTelemetry: this.sendTelemetry,
+ });
+ lazy.MomentsPageHub.init(this.waitForInitialized, {
+ handleMessageRequest: this.handleMessageRequest,
+ addImpression: this.addImpression,
+ blockMessageById: this.blockMessageById,
+ sendTelemetry: this.sendTelemetry,
+ });
+
+ this._loadLocalProviders();
+
+ const messageBlockList =
+ (await this._storage.get("messageBlockList")) || [];
+ const messageImpressions =
+ (await this._storage.get("messageImpressions")) || {};
+ const groupImpressions =
+ (await this._storage.get("groupImpressions")) || {};
+ const previousSessionEnd =
+ (await this._storage.get("previousSessionEnd")) || 0;
+
+ await this.setState({
+ messageBlockList,
+ groupImpressions,
+ messageImpressions,
+ previousSessionEnd,
+ ...(lazy.ASRouterPreferences.specialConditions || {}),
+ initialized: false,
+ });
+ await this._updateMessageProviders();
+ await this.loadMessagesFromAllProviders();
+ await MessageLoaderUtils.cleanupCache(this.state.providers, storage);
+
+ lazy.SpecialMessageActions.blockMessageById = this.blockMessageById;
+ Services.obs.addObserver(this._onLocaleChanged, TOPIC_INTL_LOCALE_CHANGED);
+ Services.obs.addObserver(
+ this._onExperimentForceEnrolled,
+ TOPIC_EXPERIMENT_FORCE_ENROLLED
+ );
+ Services.prefs.addObserver(USE_REMOTE_L10N_PREF, this);
+ // sets .initialized to true and resolves .waitForInitialized promise
+ this._finishInitializing();
+ return this.state;
+ }
+
+ uninit() {
+ this._storage.set("previousSessionEnd", Date.now());
+
+ this.clearChildMessages = null;
+ this.clearChildProviders = null;
+ this.updateAdminState = null;
+ this.sendTelemetry = null;
+ this.dispatchCFRAction = null;
+
+ lazy.ASRouterPreferences.removeListener(this.onPrefChange);
+ lazy.ASRouterPreferences.uninit();
+ lazy.ToolbarPanelHub.uninit();
+ lazy.ToolbarBadgeHub.uninit();
+ lazy.MomentsPageHub.uninit();
+
+ // Uninitialise all trigger listeners
+ for (const listener of lazy.ASRouterTriggerListeners.values()) {
+ listener.uninit();
+ }
+ Services.obs.removeObserver(
+ this._onLocaleChanged,
+ TOPIC_INTL_LOCALE_CHANGED
+ );
+ Services.obs.removeObserver(
+ this._onExperimentForceEnrolled,
+ TOPIC_EXPERIMENT_FORCE_ENROLLED
+ );
+ Services.prefs.removeObserver(USE_REMOTE_L10N_PREF, this);
+ // If we added any CFR recommendations, they need to be removed
+ CFRPageActions.clearRecommendations();
+ this._resetInitialization();
+ }
+
+ setState(callbackOrObj) {
+ lazy.ASRouterPreferences.console.debug(
+ "in setState, callbackOrObj = ",
+ callbackOrObj
+ );
+ lazy.ASRouterPreferences.console.trace();
+ const newState =
+ typeof callbackOrObj === "function"
+ ? callbackOrObj(this.state)
+ : callbackOrObj;
+ this._state = {
+ ...this.state,
+ ...newState,
+ };
+ if (lazy.ASRouterPreferences.devtoolsEnabled) {
+ return this.updateTargetingParameters().then(state => {
+ this.updateAdminState(state);
+ return state;
+ });
+ }
+ return Promise.resolve(this.state);
+ }
+
+ updateTargetingParameters() {
+ return this.getTargetingParameters(
+ lazy.ASRouterTargeting.Environment,
+ this._getMessagesContext()
+ ).then(targetingParameters => ({
+ ...this.state,
+ providerPrefs: lazy.ASRouterPreferences.providers,
+ userPrefs: lazy.ASRouterPreferences.getAllUserPreferences(),
+ targetingParameters,
+ errors: this.errors,
+ }));
+ }
+
+ getMessageById(id) {
+ return this.state.messages.find(message => message.id === id);
+ }
+
+ _loadLocalProviders() {
+ // If we're in ASR debug mode add the local test providers
+ if (lazy.ASRouterPreferences.devtoolsEnabled) {
+ this._localProviders = {
+ ...this._localProviders,
+ SnippetsTestMessageProvider: lazy.SnippetsTestMessageProvider,
+ PanelTestProvider: lazy.PanelTestProvider,
+ };
+ }
+ }
+
+ /**
+ * Used by ASRouter Admin returns all ASRouterTargeting.Environment
+ * and ASRouter._getMessagesContext parameters and values
+ */
+ async getTargetingParameters(environment, localContext) {
+ const targetingParameters = {};
+ for (const param of Object.keys(environment)) {
+ targetingParameters[param] = await environment[param];
+ }
+ for (const param of Object.keys(localContext)) {
+ targetingParameters[param] = await localContext[param];
+ }
+
+ return targetingParameters;
+ }
+
+ _handleTargetingError(error, message) {
+ console.error(error);
+ this.dispatchCFRAction(
+ ac.ASRouterUserEvent({
+ message_id: message.id,
+ action: "asrouter_undesired_event",
+ event: "TARGETING_EXPRESSION_ERROR",
+ event_context: {},
+ })
+ );
+ }
+
+ // Return an object containing targeting parameters used to select messages
+ _getMessagesContext() {
+ const { messageImpressions, previousSessionEnd } = this.state;
+
+ return {
+ get messageImpressions() {
+ return messageImpressions;
+ },
+ get previousSessionEnd() {
+ return previousSessionEnd;
+ },
+ };
+ }
+
+ async evaluateExpression({ expression, context }) {
+ const targetingContext = new lazy.TargetingContext(context);
+ let evaluationStatus;
+ try {
+ evaluationStatus = {
+ result: await targetingContext.evalWithDefault(expression),
+ success: true,
+ };
+ } catch (e) {
+ evaluationStatus = { result: e.message, success: false };
+ }
+ return Promise.resolve({ evaluationStatus });
+ }
+
+ unblockAll() {
+ return this.setState({ messageBlockList: [] });
+ }
+
+ isUnblockedMessage(message) {
+ const { state } = this;
+ return (
+ !state.messageBlockList.includes(message.id) &&
+ (!message.campaign ||
+ !state.messageBlockList.includes(message.campaign)) &&
+ this.hasGroupsEnabled(message.groups) &&
+ !this.isExcludedByProvider(message)
+ );
+ }
+
+ // Work out if a message can be shown based on its and its provider's frequency caps.
+ isBelowFrequencyCaps(message) {
+ const { messageImpressions, groupImpressions } = this.state;
+ const impressionsForMessage = messageImpressions[message.id];
+
+ const _belowItemFrequencyCap = this._isBelowItemFrequencyCap(
+ message,
+ impressionsForMessage,
+ MAX_MESSAGE_LIFETIME_CAP
+ );
+ if (!_belowItemFrequencyCap) {
+ lazy.ASRouterPreferences.console.debug(
+ `isBelowFrequencyCaps: capped by item: `,
+ message,
+ "impressions =",
+ impressionsForMessage
+ );
+ }
+
+ const _belowGroupFrequencyCaps = message.groups.every(messageGroup => {
+ const belowThisGroupCap = this._isBelowItemFrequencyCap(
+ this.state.groups.find(({ id }) => id === messageGroup),
+ groupImpressions[messageGroup]
+ );
+
+ if (!belowThisGroupCap) {
+ lazy.ASRouterPreferences.console.debug(
+ `isBelowFrequencyCaps: ${message.id} capped by group ${messageGroup}`
+ );
+ } else {
+ lazy.ASRouterPreferences.console.debug(
+ `isBelowFrequencyCaps: ${message.id} allowed by group ${messageGroup}, groupImpressions = `,
+ groupImpressions
+ );
+ }
+
+ return belowThisGroupCap;
+ });
+
+ return _belowItemFrequencyCap && _belowGroupFrequencyCaps;
+ }
+
+ // Helper for isBelowFrecencyCaps - work out if the frequency cap for the given
+ // item has been exceeded or not
+ _isBelowItemFrequencyCap(item, impressions, maxLifetimeCap = Infinity) {
+ if (item && item.frequency && impressions && impressions.length) {
+ if (
+ item.frequency.lifetime &&
+ impressions.length >= Math.min(item.frequency.lifetime, maxLifetimeCap)
+ ) {
+ lazy.ASRouterPreferences.console.debug(
+ `${item.id} capped by lifetime (${item.frequency.lifetime})`
+ );
+
+ return false;
+ }
+ if (item.frequency.custom) {
+ const now = Date.now();
+ for (const setting of item.frequency.custom) {
+ let { period } = setting;
+ const impressionsInPeriod = impressions.filter(t => now - t < period);
+ if (impressionsInPeriod.length >= setting.cap) {
+ lazy.ASRouterPreferences.console.debug(
+ `${item.id} capped by impressions (${impressionsInPeriod.length}) in period (${period}) >= ${setting.cap}`
+ );
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ async _extraTemplateStrings(originalMessage) {
+ let extraTemplateStrings;
+ let localProvider = this._findProvider(originalMessage.provider);
+ if (localProvider && localProvider.getExtraAttributes) {
+ extraTemplateStrings = await localProvider.getExtraAttributes();
+ }
+
+ return extraTemplateStrings;
+ }
+
+ _findProvider(providerID) {
+ return this._localProviders[
+ this.state.providers.find(i => i.id === providerID).localProvider
+ ];
+ }
+
+ routeCFRMessage(message, browser, trigger, force = false) {
+ if (!message) {
+ return { message: {} };
+ }
+
+ switch (message.template) {
+ case "whatsnew_panel_message":
+ if (force) {
+ lazy.ToolbarPanelHub.forceShowMessage(browser, message);
+ }
+ break;
+ case "cfr_doorhanger":
+ case "milestone_message":
+ if (force) {
+ CFRPageActions.forceRecommendation(
+ browser,
+ message,
+ this.dispatchCFRAction
+ );
+ } else {
+ CFRPageActions.addRecommendation(
+ browser,
+ trigger.param && trigger.param.host,
+ message,
+ this.dispatchCFRAction
+ );
+ }
+ break;
+ case "cfr_urlbar_chiclet":
+ if (force) {
+ CFRPageActions.forceRecommendation(
+ browser,
+ message,
+ this.dispatchCFRAction
+ );
+ } else {
+ CFRPageActions.addRecommendation(
+ browser,
+ null,
+ message,
+ this.dispatchCFRAction
+ );
+ }
+ break;
+ case "toolbar_badge":
+ lazy.ToolbarBadgeHub.registerBadgeNotificationListener(message, {
+ force,
+ });
+ break;
+ case "update_action":
+ lazy.MomentsPageHub.executeAction(message);
+ break;
+ case "infobar":
+ lazy.InfoBar.showInfoBarMessage(
+ browser,
+ message,
+ this.dispatchCFRAction
+ );
+ break;
+ case "spotlight":
+ lazy.Spotlight.showSpotlightDialog(
+ browser,
+ message,
+ this.dispatchCFRAction
+ );
+ break;
+ case "toast_notification":
+ lazy.ToastNotification.showToastNotification(
+ message,
+ this.dispatchCFRAction
+ );
+ break;
+ }
+
+ return { message };
+ }
+
+ addImpression(message) {
+ lazy.ASRouterPreferences.console.debug(
+ `entering addImpression for ${message.id}`
+ );
+
+ const groupsWithFrequency = this.state.groups.filter(
+ ({ frequency, id }) => frequency && message.groups.includes(id)
+ );
+ // We only need to store impressions for messages that have frequency, or
+ // that have providers that have frequency
+ if (message.frequency || groupsWithFrequency.length) {
+ const time = Date.now();
+ return this.setState(state => {
+ const messageImpressions = this._addImpressionForItem(
+ state.messageImpressions,
+ message,
+ "messageImpressions",
+ time
+ );
+ // Initialize this with state.groupImpressions, and then assign the
+ // newly-updated copy to it during each iteration so that
+ // all the changes get captured and either returned or passed into the
+ // _addImpressionsForItem call on the next iteration.
+ let { groupImpressions } = state;
+ for (const group of groupsWithFrequency) {
+ groupImpressions = this._addImpressionForItem(
+ groupImpressions,
+ group,
+ "groupImpressions",
+ time
+ );
+ }
+
+ return { messageImpressions, groupImpressions };
+ });
+ }
+ return Promise.resolve();
+ }
+
+ // Helper for addImpression - calculate the updated impressions object for the given
+ // item, then store it and return it
+ _addImpressionForItem(currentImpressions, item, impressionsString, time) {
+ // The destructuring here is to avoid mutating passed parameters
+ // (see https://redux.js.org/recipes/structuring-reducers/prerequisite-concepts#immutable-data-management)
+ const impressions = { ...currentImpressions };
+ if (item.frequency) {
+ impressions[item.id] = impressions[item.id]
+ ? [...impressions[item.id]]
+ : [];
+ impressions[item.id].push(time);
+ lazy.ASRouterPreferences.console.debug(
+ item.id,
+ "impression added, impressions[item.id]: ",
+ impressions[item.id]
+ );
+
+ this._storage.set(impressionsString, impressions);
+ }
+ return impressions;
+ }
+
+ /**
+ * getLongestPeriod
+ *
+ * @param {obj} item Either an ASRouter message or an ASRouter provider
+ * @returns {int|null} if the item has custom frequency caps, the longest period found in the list of caps.
+ if the item has no custom frequency caps, null
+ * @memberof _ASRouter
+ */
+ getLongestPeriod(item) {
+ if (!item.frequency || !item.frequency.custom) {
+ return null;
+ }
+ return item.frequency.custom.sort((a, b) => b.period - a.period)[0].period;
+ }
+
+ /**
+ * cleanupImpressions - this function cleans up obsolete impressions whenever
+ * messages are refreshed or fetched. It will likely need to be more sophisticated in the future,
+ * but the current behaviour for when both message impressions and provider impressions are
+ * cleared is as follows (where `item` is either `message` or `provider`):
+ *
+ * 1. If the item id for a list of item impressions no longer exists in the ASRouter state, it
+ * will be cleared.
+ * 2. If the item has time-bound frequency caps but no lifetime cap, any item impressions older
+ * than the longest time period will be cleared.
+ */
+ cleanupImpressions() {
+ return this.setState(state => {
+ const messageImpressions = this._cleanupImpressionsForItems(
+ state,
+ state.messages,
+ "messageImpressions"
+ );
+ const groupImpressions = this._cleanupImpressionsForItems(
+ state,
+ state.groups,
+ "groupImpressions"
+ );
+ return { messageImpressions, groupImpressions };
+ });
+ }
+
+ /** _cleanupImpressionsForItems - Helper for cleanupImpressions - calculate the updated
+ /* impressions object for the given items, then store it and return it
+ *
+ * @param {obj} state Reference to ASRouter internal state
+ * @param {array} items Can be messages, providers or groups that we count impressions for
+ * @param {string} impressionsString Key name for entry in state where impressions are stored
+ */
+ _cleanupImpressionsForItems(state, items, impressionsString) {
+ const impressions = { ...state[impressionsString] };
+ let needsUpdate = false;
+ Object.keys(impressions).forEach(id => {
+ const [item] = items.filter(x => x.id === id);
+ // Don't keep impressions for items that no longer exist
+ if (!item || !item.frequency || !Array.isArray(impressions[id])) {
+ lazy.ASRouterPreferences.console.debug(
+ "_cleanupImpressionsForItem: removing impressions for deleted or changed item: ",
+ item
+ );
+ lazy.ASRouterPreferences.console.trace();
+ delete impressions[id];
+ needsUpdate = true;
+ return;
+ }
+ if (!impressions[id].length) {
+ return;
+ }
+ // If we don't want to store impressions older than the longest period
+ if (item.frequency.custom && !item.frequency.lifetime) {
+ lazy.ASRouterPreferences.console.debug(
+ "_cleanupImpressionsForItem: removing impressions older than longest period for item: ",
+ item
+ );
+ const now = Date.now();
+ impressions[id] = impressions[id].filter(
+ t => now - t < this.getLongestPeriod(item)
+ );
+ needsUpdate = true;
+ }
+ });
+ if (needsUpdate) {
+ this._storage.set(impressionsString, impressions);
+ }
+ return impressions;
+ }
+
+ handleMessageRequest({
+ messages: candidates,
+ triggerId,
+ triggerParam,
+ triggerContext,
+ template,
+ provider,
+ ordered = false,
+ returnAll = false,
+ }) {
+ let shouldCache;
+ lazy.ASRouterPreferences.console.debug(
+ "in handleMessageRequest, arguments = ",
+ Array.from(arguments) // eslint-disable-line prefer-rest-params
+ );
+ lazy.ASRouterPreferences.console.trace();
+ const messages =
+ candidates ||
+ this.state.messages.filter(m => {
+ if (provider && m.provider !== provider) {
+ lazy.ASRouterPreferences.console.debug(m.id, " filtered by provider");
+ return false;
+ }
+ if (template && m.template !== template) {
+ lazy.ASRouterPreferences.console.debug(m.id, " filtered by template");
+ return false;
+ }
+ if (triggerId && !m.trigger) {
+ lazy.ASRouterPreferences.console.debug(m.id, " filtered by trigger");
+ return false;
+ }
+ if (triggerId && m.trigger.id !== triggerId) {
+ lazy.ASRouterPreferences.console.debug(
+ m.id,
+ " filtered by triggerId"
+ );
+ return false;
+ }
+ if (!this.isUnblockedMessage(m)) {
+ lazy.ASRouterPreferences.console.debug(
+ m.id,
+ " filtered because blocked"
+ );
+ return false;
+ }
+ if (!this.isBelowFrequencyCaps(m)) {
+ lazy.ASRouterPreferences.console.debug(
+ m.id,
+ " filtered because capped"
+ );
+ return false;
+ }
+
+ if (shouldCache !== false) {
+ shouldCache = JEXL_PROVIDER_CACHE.has(m.provider);
+ }
+
+ return true;
+ });
+
+ if (!messages.length) {
+ return returnAll ? messages : null;
+ }
+
+ const context = this._getMessagesContext();
+
+ // Find a message that matches the targeting context as well as the trigger context (if one is provided)
+ // If no trigger is provided, we should find a message WITHOUT a trigger property defined.
+ return lazy.ASRouterTargeting.findMatchingMessage({
+ messages,
+ trigger: triggerId && {
+ id: triggerId,
+ param: triggerParam,
+ context: triggerContext,
+ },
+ context,
+ onError: this._handleTargetingError,
+ ordered,
+ shouldCache,
+ returnAll,
+ });
+ }
+
+ setMessageById({ id, ...data }, force, browser) {
+ return this.routeCFRMessage(this.getMessageById(id), browser, data, force);
+ }
+
+ blockMessageById(idOrIds) {
+ lazy.ASRouterPreferences.console.debug(
+ "blockMessageById called, idOrIds = ",
+ idOrIds
+ );
+ lazy.ASRouterPreferences.console.trace();
+
+ const idsToBlock = Array.isArray(idOrIds) ? idOrIds : [idOrIds];
+
+ return this.setState(state => {
+ const messageBlockList = [...state.messageBlockList];
+ const messageImpressions = { ...state.messageImpressions };
+
+ idsToBlock.forEach(id => {
+ const message = state.messages.find(m => m.id === id);
+ const idToBlock = message && message.campaign ? message.campaign : id;
+ if (!messageBlockList.includes(idToBlock)) {
+ messageBlockList.push(idToBlock);
+ }
+
+ // When a message is blocked, its impressions should be cleared as well
+ delete messageImpressions[id];
+ });
+
+ this._storage.set("messageBlockList", messageBlockList);
+ this._storage.set("messageImpressions", messageImpressions);
+ return { messageBlockList, messageImpressions };
+ });
+ }
+
+ unblockMessageById(idOrIds) {
+ const idsToUnblock = Array.isArray(idOrIds) ? idOrIds : [idOrIds];
+
+ return this.setState(state => {
+ const messageBlockList = [...state.messageBlockList];
+ idsToUnblock
+ .map(id => state.messages.find(m => m.id === id))
+ // Remove all `id`s (or `campaign`s for snippets) from the message
+ // block list
+ .forEach(message => {
+ const idToUnblock =
+ message && message.campaign ? message.campaign : message.id;
+ messageBlockList.splice(messageBlockList.indexOf(idToUnblock), 1);
+ });
+
+ this._storage.set("messageBlockList", messageBlockList);
+ return { messageBlockList };
+ });
+ }
+
+ resetGroupsState() {
+ const newGroupImpressions = {};
+ for (let { id } of this.state.groups) {
+ newGroupImpressions[id] = [];
+ }
+ // Update storage
+ this._storage.set("groupImpressions", newGroupImpressions);
+ return this.setState(({ groups }) => ({
+ groupImpressions: newGroupImpressions,
+ }));
+ }
+
+ resetMessageState() {
+ const newMessageImpressions = {};
+ for (let { id } of this.state.messages) {
+ newMessageImpressions[id] = [];
+ }
+ // Update storage
+ this._storage.set("messageImpressions", newMessageImpressions);
+ return this.setState(() => ({
+ messageImpressions: newMessageImpressions,
+ }));
+ }
+
+ _validPreviewEndpoint(url) {
+ try {
+ const endpoint = new URL(url);
+ if (!this.ALLOWLIST_HOSTS[endpoint.host]) {
+ console.error(
+ `The preview URL host ${endpoint.host} is not in the list of allowed hosts.`
+ );
+ }
+ if (endpoint.protocol !== "https:") {
+ console.error("The URL protocol is not https.");
+ }
+ return (
+ endpoint.protocol === "https:" && this.ALLOWLIST_HOSTS[endpoint.host]
+ );
+ } catch (e) {
+ return false;
+ }
+ }
+
+ // Ensure we switch to the Onboarding message after RTAMO addon was installed
+ _updateOnboardingState() {
+ let addonInstallObs = (subject, topic) => {
+ Services.obs.removeObserver(
+ addonInstallObs,
+ "webextension-install-notify"
+ );
+ };
+ Services.obs.addObserver(addonInstallObs, "webextension-install-notify");
+ }
+
+ _loadSnippetsAllowHosts() {
+ let additionalHosts = [];
+ const allowPrefValue = Services.prefs.getStringPref(
+ SNIPPETS_ENDPOINT_ALLOWLIST,
+ ""
+ );
+ try {
+ additionalHosts = JSON.parse(allowPrefValue);
+ } catch (e) {
+ if (allowPrefValue) {
+ console.error(
+ `Pref ${SNIPPETS_ENDPOINT_ALLOWLIST} value is not valid JSON`
+ );
+ }
+ }
+
+ if (!additionalHosts.length) {
+ return DEFAULT_ALLOWLIST_HOSTS;
+ }
+
+ // If there are additional hosts we want to allow, add them as
+ // `preview` so that the updateCycle is 0
+ return additionalHosts.reduce(
+ (allow_hosts, host) => {
+ allow_hosts[host] = "preview";
+ Services.console.logStringMessage(
+ `Adding ${host} to list of allowed hosts.`
+ );
+ return allow_hosts;
+ },
+ { ...DEFAULT_ALLOWLIST_HOSTS }
+ );
+ }
+
+ // To be passed to ASRouterTriggerListeners
+ _triggerHandler(browser, trigger) {
+ // Disable ASRouterTriggerListeners in kiosk mode.
+ if (lazy.BrowserHandler.kiosk) {
+ return Promise.resolve();
+ }
+ return this.sendTriggerMessage({ ...trigger, browser });
+ }
+
+ _removePreviewEndpoint(state) {
+ state.providers = state.providers.filter(p => p.id !== "preview");
+ return state;
+ }
+
+ addPreviewEndpoint(url, browser) {
+ const providers = [...this.state.providers];
+ if (
+ this._validPreviewEndpoint(url) &&
+ !providers.find(p => p.url === url)
+ ) {
+ // When you view a preview snippet we want to hide all real content -
+ // sending EnterSnippetsPreviewMode puts this browser tab in that state.
+ browser.sendMessageToActor("EnterSnippetsPreviewMode", {}, "ASRouter");
+ providers.push({
+ id: "preview",
+ type: "remote",
+ enabled: true,
+ url,
+ updateCycleInMs: 0,
+ });
+ return this.setState({ providers });
+ }
+ return Promise.resolve();
+ }
+
+ /**
+ * forceAttribution - this function should only be called from within about:newtab#asrouter.
+ * It forces the browser attribution to be set to something specified in asrouter admin
+ * tools, and reloads the providers in order to get messages that are dependant on this
+ * attribution data (see Return to AMO flow in bug 1475354 for example). Note - OSX and Windows only
+ * @param {data} Object an object containing the attribtion data that came from asrouter admin page
+ */
+ async forceAttribution(data) {
+ // Extract the parameters from data that will make up the referrer url
+ const attributionData = AttributionCode.allowedCodeKeys
+ .map(key => `${key}=${encodeURIComponent(data[key] || "")}`)
+ .join("&");
+ if (AppConstants.platform === "win") {
+ // The whole attribution data is encoded (again) for windows
+ await AttributionCode.writeAttributionFile(
+ encodeURIComponent(attributionData)
+ );
+ } else if (AppConstants.platform === "macosx") {
+ let appPath = lazy.MacAttribution.applicationPath;
+ let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService(
+ Ci.nsIMacAttributionService
+ );
+
+ // The attribution data is treated as a url query for mac
+ let referrer = `https://www.mozilla.org/anything/?${attributionData}`;
+
+ // This sets the Attribution to be the referrer
+ attributionSvc.setReferrerUrl(appPath, referrer, true);
+
+ // Delete attribution data file
+ await AttributionCode.deleteFileAsync();
+ }
+
+ // Clear cache call is only possible in a testing environment
+ Services.env.set("XPCSHELL_TEST_PROFILE_DIR", "testing");
+
+ // Clear and refresh Attribution, and then fetch the messages again to update
+ AttributionCode._clearCache();
+ await AttributionCode.getAttrDataAsync();
+ await this._updateMessageProviders();
+ return this.loadMessagesFromAllProviders();
+ }
+
+ async sendPBNewTabMessage({ tabId, hideDefault }) {
+ let message = null;
+ const PromoInfo = {
+ FOCUS: { enabledPref: "browser.promo.focus.enabled" },
+ VPN: { enabledPref: "browser.vpn_promo.enabled" },
+ PIN: { enabledPref: "browser.promo.pin.enabled" },
+ };
+ await this.loadMessagesFromAllProviders();
+
+ // If message has hideDefault property set to true
+ // remove from state all pb_newtab messages with type default
+ if (hideDefault) {
+ await this.setState(state => ({
+ messages: state.messages.filter(
+ m => !(m.template === "pb_newtab" && m.type === "default")
+ ),
+ }));
+ }
+
+ // Remove from state pb_newtab messages with PromoType disabled
+ await this.setState(state => ({
+ messages: state.messages.filter(
+ m =>
+ !(
+ m.template === "pb_newtab" &&
+ !Services.prefs.getBoolPref(
+ PromoInfo[m.content?.promoType]?.enabledPref,
+ true
+ )
+ )
+ ),
+ }));
+
+ const telemetryObject = { tabId };
+ TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ message = await this.handleMessageRequest({
+ template: "pb_newtab",
+ });
+ TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+
+ // Format urls if any are defined
+ ["infoLinkUrl"].forEach(key => {
+ if (message?.content?.[key]) {
+ message.content[key] = Services.urlFormatter.formatURL(
+ message.content[key]
+ );
+ }
+ });
+
+ return { message };
+ }
+
+ async sendNewTabMessage({ endpoint, tabId, browser }) {
+ let message;
+
+ // Load preview endpoint for snippets if one is sent
+ if (endpoint) {
+ await this.addPreviewEndpoint(endpoint.url, browser);
+ }
+
+ // Load all messages
+ await this.loadMessagesFromAllProviders();
+
+ if (endpoint) {
+ message = await this.handleMessageRequest({ provider: "preview" });
+
+ // We don't want to cache preview messages, remove them after we selected the message to show
+ if (message) {
+ await this.setState(state => ({
+ messages: state.messages.filter(m => m.id !== message.id),
+ }));
+ }
+ } else {
+ const telemetryObject = { tabId };
+ TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ message = await this.handleMessageRequest({ provider: "snippets" });
+ TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ }
+
+ return this.routeCFRMessage(message, browser, undefined, false);
+ }
+
+ _recordReachEvent(message) {
+ const messageGroup = message.forReachEvent.group;
+ // Events telemetry only accepts understores for the event `object`
+ const underscored = messageGroup.split("-").join("_");
+ const extra = { branches: message.branchSlug };
+ Services.telemetry.recordEvent(
+ REACH_EVENT_CATEGORY,
+ REACH_EVENT_METHOD,
+ underscored,
+ message.experimentSlug,
+ extra
+ );
+ }
+
+ async sendTriggerMessage({ tabId, browser, ...trigger }) {
+ await this.loadMessagesFromAllProviders();
+
+ const telemetryObject = { tabId };
+ TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ // Return all the messages so that it can record the Reach event
+ const messages =
+ (await this.handleMessageRequest({
+ triggerId: trigger.id,
+ triggerParam: trigger.param,
+ triggerContext: trigger.context,
+ returnAll: true,
+ })) || [];
+ TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+
+ // Record the Reach event for all the messages with `forReachEvent`,
+ // only send the first message without forReachEvent to the target
+ const nonReachMessages = [];
+ for (const message of messages) {
+ if (message.forReachEvent) {
+ if (!message.forReachEvent.sent) {
+ this._recordReachEvent(message);
+ message.forReachEvent.sent = true;
+ }
+ } else {
+ nonReachMessages.push(message);
+ }
+ }
+
+ if (nonReachMessages.length) {
+ let featureId = nonReachMessages[0]._nimbusFeature;
+ if (featureId) {
+ lazy.NimbusFeatures[featureId].recordExposureEvent({ once: true });
+ }
+ }
+
+ return this.routeCFRMessage(
+ nonReachMessages[0] || null,
+ browser,
+ trigger,
+ false
+ );
+ }
+
+ async forceWNPanel(browser) {
+ let win = browser.ownerGlobal;
+ await lazy.ToolbarPanelHub.enableToolbarButton();
+
+ win.PanelUI.showSubView(
+ "PanelUI-whatsNew",
+ win.document.getElementById("whats-new-menu-button")
+ );
+
+ let panel = win.document.getElementById("customizationui-widget-panel");
+ // Set the attribute to keep the panel open
+ panel.setAttribute("noautohide", true);
+ }
+
+ async closeWNPanel(browser) {
+ let win = browser.ownerGlobal;
+ let panel = win.document.getElementById("customizationui-widget-panel");
+ // Set the attribute to allow the panel to close
+ panel.setAttribute("noautohide", false);
+ // Removing the button is enough to close the panel.
+ await lazy.ToolbarPanelHub._hideToolbarButton(win);
+ }
+
+ async _onExperimentForceEnrolled(subject, topic, slug) {
+ const experimentProvider = this.state.providers.find(
+ p => p.id === "messaging-experiments"
+ );
+ if (!experimentProvider.enabled) {
+ return;
+ }
+
+ const branch = lazy.ExperimentAPI.getActiveBranch({ slug });
+ const features = branch.features ?? [branch.feature];
+ const featureIds = features.map(feature => feature.featureId);
+
+ this._onFeaturesUpdated(...featureIds);
+
+ await this.loadMessagesFromAllProviders([experimentProvider]);
+ }
+
+ /**
+ * Handle a change to the list of featureIds that the messaging-experiments
+ * provider is watching.
+ *
+ * This normally occurs when ASRouter update message providers, which happens
+ * every startup and when the messaging-experiment provider pref changes.
+ *
+ * On startup, |oldFeatures| will be an empty array and we will subscribe to
+ * everything in |newFeatures|.
+ *
+ * When the pref changes, we unsubscribe from |oldFeatures - newFeatures| and
+ * subscribe to |newFeatures - oldFeatures|. Features that are listed in both
+ * sets do not have their subscription status changed. Pref changes are mostly
+ * during unit tests.
+ *
+ * @param {string[]} oldFeatures The list of feature IDs we were previously
+ * listening to for new experiments.
+ * @param {string[]} newFeatures The list of feature IDs we are now listening
+ * to for new experiments.
+ */
+ _onFeatureListChanged(oldFeatures, newFeatures) {
+ for (const featureId of oldFeatures) {
+ if (!newFeatures.includes(featureId)) {
+ const listener = this._experimentChangedListeners.get(featureId);
+ this._experimentChangedListeners.delete(featureId);
+ lazy.NimbusFeatures[featureId].off(listener);
+ }
+ }
+
+ const newlySubscribed = [];
+
+ for (const featureId of newFeatures) {
+ if (!oldFeatures.includes(featureId)) {
+ const listener = () => this._onFeaturesUpdated(featureId);
+ this._experimentChangedListeners.set(featureId, listener);
+ lazy.NimbusFeatures[featureId].onUpdate(listener);
+
+ newlySubscribed.push(featureId);
+ }
+ }
+
+ // Check for any messages present in the newly subscribed to Nimbus features
+ // so we can prefetch their remote images (if any).
+ this._onFeaturesUpdated(...newlySubscribed);
+ }
+
+ /**
+ * Handle updated experiment features.
+ *
+ * If there are messages for the feature, RemoteImages will prefetch any
+ * images.
+ *
+ * @param {string[]} featureIds The feature IDs that have been updated.
+ */
+ _onFeaturesUpdated(...featureIds) {
+ const messages = [];
+
+ for (const featureId of featureIds) {
+ const featureAPI = lazy.NimbusFeatures[featureId];
+ // If there is no active experiment for the feature, this will return
+ // null.
+ if (lazy.ExperimentAPI.getExperimentMetaData({ featureId })) {
+ // Otherwise, getAllVariables() will return the JSON blob for the
+ // message.
+ messages.push(featureAPI.getAllVariables());
+ }
+ }
+
+ // We are not awaiting this because we want these images to load in the
+ // background.
+ if (messages.length) {
+ lazy.RemoteImages.prefetchImagesFor(messages);
+ }
+ }
+
+ async forcePBWindow(browser, msg) {
+ const privateBrowserOpener = await new Promise((
+ resolveOnContentBrowserCreated // wrap this in a promise to give back the right browser
+ ) =>
+ browser.ownerGlobal.openTrustedLinkIn(
+ "about:privatebrowsing?debug",
+ "window",
+ {
+ private: true,
+ triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal(
+ {}
+ ),
+ csp: null,
+ resolveOnContentBrowserCreated,
+ opener: "devtools",
+ }
+ )
+ );
+
+ lazy.setTimeout(() => {
+ // setTimeout is necessary to make sure the private browsing window has a chance to open before the message is sent
+ privateBrowserOpener.browsingContext.currentWindowGlobal
+ .getActor("AboutPrivateBrowsing")
+ .sendAsyncMessage("ShowDevToolsMessage", msg);
+ }, 100);
+
+ return privateBrowserOpener;
+ }
+}
+
+/**
+ * ASRouter - singleton instance of _ASRouter that controls all messages
+ * in the new tab page.
+ */
+const ASRouter = new _ASRouter();
+
+const EXPORTED_SYMBOLS = ["_ASRouter", "ASRouter", "MessageLoaderUtils"];
diff --git a/browser/components/newtab/lib/ASRouterDefaultConfig.jsm b/browser/components/newtab/lib/ASRouterDefaultConfig.jsm
new file mode 100644
index 0000000000..6caff6f4a2
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterDefaultConfig.jsm
@@ -0,0 +1,65 @@
+/* vim: set ts=2 sw=2 sts=2 et tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const EXPORTED_SYMBOLS = ["ASRouterDefaultConfig"];
+
+const { ASRouter } = ChromeUtils.import(
+ "resource://activity-stream/lib/ASRouter.jsm"
+);
+const { TelemetryFeed } = ChromeUtils.import(
+ "resource://activity-stream/lib/TelemetryFeed.jsm"
+);
+const { ASRouterParentProcessMessageHandler } = ChromeUtils.import(
+ "resource://activity-stream/lib/ASRouterParentProcessMessageHandler.jsm"
+);
+const { SpecialMessageActions } = ChromeUtils.import(
+ "resource://messaging-system/lib/SpecialMessageActions.jsm"
+);
+const { ASRouterPreferences } = ChromeUtils.import(
+ "resource://activity-stream/lib/ASRouterPreferences.jsm"
+);
+const { QueryCache } = ChromeUtils.import(
+ "resource://activity-stream/lib/ASRouterTargeting.jsm"
+);
+const { ActivityStreamStorage } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamStorage.jsm"
+);
+
+const createStorage = async telemetryFeed => {
+ const dbStore = new ActivityStreamStorage({
+ storeNames: ["sectionPrefs", "snippets"],
+ telemetry: {
+ handleUndesiredEvent: e => telemetryFeed.SendASRouterUndesiredEvent(e),
+ },
+ });
+ // Accessing the db causes the object stores to be created / migrated.
+ // This needs to happen before other instances try to access the db, which
+ // would update only a subset of the stores to the latest version.
+ try {
+ await dbStore.db; // eslint-disable-line no-unused-expressions
+ } catch (e) {
+ return Promise.reject(e);
+ }
+ return dbStore.getDbTable("snippets");
+};
+
+const ASRouterDefaultConfig = () => {
+ const router = ASRouter;
+ const telemetry = new TelemetryFeed();
+ const messageHandler = new ASRouterParentProcessMessageHandler({
+ router,
+ preferences: ASRouterPreferences,
+ specialMessageActions: SpecialMessageActions,
+ queryCache: QueryCache,
+ sendTelemetry: telemetry.onAction.bind(telemetry),
+ });
+ return {
+ router,
+ messageHandler,
+ createStorage: createStorage.bind(null, telemetry),
+ };
+};
diff --git a/browser/components/newtab/lib/ASRouterNewTabHook.jsm b/browser/components/newtab/lib/ASRouterNewTabHook.jsm
new file mode 100644
index 0000000000..78eb465725
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterNewTabHook.jsm
@@ -0,0 +1,120 @@
+/* vim: set ts=2 sw=2 sts=2 et tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const EXPORTED_SYMBOLS = ["ASRouterNewTabHook"];
+
+class ASRouterNewTabHookInstance {
+ constructor() {
+ this._newTabMessageHandler = null;
+ this._parentProcessMessageHandler = null;
+ this._router = null;
+ this._clearChildMessages = (...params) =>
+ this._newTabMessageHandler === null
+ ? Promise.resolve()
+ : this._newTabMessageHandler.clearChildMessages(...params);
+ this._clearChildProviders = (...params) =>
+ this._newTabMessageHandler === null
+ ? Promise.resolve()
+ : this._newTabMessageHandler.clearChildProviders(...params);
+ this._updateAdminState = (...params) =>
+ this._newTabMessageHandler === null
+ ? Promise.resolve()
+ : this._newTabMessageHandler.updateAdminState(...params);
+ }
+
+ /**
+ * Params:
+ * object - {
+ * messageHandler: message handler for parent process messages
+ * {
+ * handleCFRAction: Responds to CFR action and returns a Promise
+ * handleTelemetry: Logs telemetry events and returns nothing
+ * },
+ * router: ASRouter instance
+ * createStorage: function to create DB storage for ASRouter
+ * }
+ */
+ async initialize({ messageHandler, router, createStorage }) {
+ this._parentProcessMessageHandler = messageHandler;
+ this._router = router;
+ if (!this._router.initialized) {
+ const storage = await createStorage();
+ await this._router.init({
+ storage,
+ sendTelemetry: this._parentProcessMessageHandler.handleTelemetry,
+ dispatchCFRAction: this._parentProcessMessageHandler.handleCFRAction,
+ clearChildMessages: this._clearChildMessages,
+ clearChildProviders: this._clearChildProviders,
+ updateAdminState: this._updateAdminState,
+ });
+ }
+ }
+
+ destroy() {
+ if (this._router?.initialized) {
+ this.disconnect();
+ this._router.uninit();
+ }
+ }
+
+ /**
+ * Connects new tab message handler to hook.
+ * Note: Should only ever be called on an initialized instance
+ * Params:
+ * newTabMessageHandler - {
+ * clearChildMessages: clears child messages and returns Promise
+ * clearChildProviders: clears child providers and returns Promise.
+ * updateAdminState: updates admin state and returns Promise
+ * }
+ * Returns: parentProcessMessageHandler
+ */
+ connect(newTabMessageHandler) {
+ this._newTabMessageHandler = newTabMessageHandler;
+ return this._parentProcessMessageHandler;
+ }
+
+ /**
+ * Disconnects new tab message handler from hook.
+ */
+ disconnect() {
+ this._newTabMessageHandler = null;
+ }
+}
+
+class AwaitSingleton {
+ constructor() {
+ this.instance = null;
+ const initialized = new Promise(resolve => {
+ this.setInstance = instance => {
+ this.setInstance = () => {};
+ this.instance = instance;
+ resolve(instance);
+ };
+ });
+ this.getInstance = () => initialized;
+ }
+}
+
+const ASRouterNewTabHook = (() => {
+ const singleton = new AwaitSingleton();
+ const instance = new ASRouterNewTabHookInstance();
+ return {
+ getInstance: singleton.getInstance,
+
+ /**
+ * Param:
+ * params - see ASRouterNewTabHookInstance.init
+ */
+ createInstance: async params => {
+ await instance.initialize(params);
+ singleton.setInstance(instance);
+ },
+
+ destroy: () => {
+ instance.destroy();
+ },
+ };
+})();
diff --git a/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm b/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm
new file mode 100644
index 0000000000..77068db008
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm
@@ -0,0 +1,183 @@
+/* vim: set ts=2 sw=2 sts=2 et tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const EXPORTED_SYMBOLS = ["ASRouterParentProcessMessageHandler"];
+
+const { ASRouterPreferences } = ChromeUtils.import(
+ "resource://activity-stream/lib/ASRouterPreferences.jsm"
+);
+
+const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/ActorConstants.sys.mjs"
+);
+
+class ASRouterParentProcessMessageHandler {
+ constructor({
+ router,
+ preferences,
+ specialMessageActions,
+ queryCache,
+ sendTelemetry,
+ }) {
+ this._router = router;
+ this._preferences = preferences;
+ this._specialMessageActions = specialMessageActions;
+ this._queryCache = queryCache;
+ this.handleTelemetry = sendTelemetry;
+ this.handleMessage = this.handleMessage.bind(this);
+ this.handleCFRAction = this.handleCFRAction.bind(this);
+ }
+
+ handleCFRAction({ type, data }, browser) {
+ switch (type) {
+ case msg.INFOBAR_TELEMETRY:
+ case msg.TOOLBAR_BADGE_TELEMETRY:
+ case msg.TOOLBAR_PANEL_TELEMETRY:
+ case msg.MOMENTS_PAGE_TELEMETRY:
+ case msg.DOORHANGER_TELEMETRY:
+ case msg.SPOTLIGHT_TELEMETRY:
+ case msg.TOAST_NOTIFICATION_TELEMETRY: {
+ return this.handleTelemetry({ type, data });
+ }
+ default: {
+ return this.handleMessage(type, data, { browser });
+ }
+ }
+ }
+
+ handleMessage(name, data, { id: tabId, browser } = { browser: null }) {
+ switch (name) {
+ case msg.AS_ROUTER_TELEMETRY_USER_EVENT:
+ return this.handleTelemetry({
+ type: msg.AS_ROUTER_TELEMETRY_USER_EVENT,
+ data,
+ });
+ case msg.BLOCK_MESSAGE_BY_ID: {
+ ASRouterPreferences.console.debug(
+ "handleMesssage(): about to block, data = ",
+ data
+ );
+ ASRouterPreferences.console.trace();
+
+ // Block the message but don't dismiss it in case the action taken has
+ // another state that needs to be visible
+ return this._router
+ .blockMessageById(data.id)
+ .then(() => !data.preventDismiss);
+ }
+ case msg.USER_ACTION: {
+ // This is to support ReturnToAMO
+ if (data.type === "INSTALL_ADDON_FROM_URL") {
+ this._router._updateOnboardingState();
+ }
+ return this._specialMessageActions.handleAction(data, browser);
+ }
+ case msg.IMPRESSION: {
+ return this._router.addImpression(data);
+ }
+ case msg.TRIGGER: {
+ return this._router.sendTriggerMessage({
+ ...(data && data.trigger),
+ tabId,
+ browser,
+ });
+ }
+ case msg.PBNEWTAB_MESSAGE_REQUEST: {
+ return this._router.sendPBNewTabMessage({
+ ...data,
+ tabId,
+ browser,
+ });
+ }
+ case msg.NEWTAB_MESSAGE_REQUEST: {
+ return this._router.sendNewTabMessage({
+ ...data,
+ tabId,
+ browser,
+ });
+ }
+
+ // ADMIN Messages
+ case msg.ADMIN_CONNECT_STATE: {
+ if (data && data.endpoint) {
+ return this._router
+ .addPreviewEndpoint(data.endpoint.url)
+ .then(() => this._router.loadMessagesFromAllProviders());
+ }
+ return this._router.updateTargetingParameters();
+ }
+ case msg.UNBLOCK_MESSAGE_BY_ID: {
+ return this._router.unblockMessageById(data.id);
+ }
+ case msg.UNBLOCK_ALL: {
+ return this._router.unblockAll();
+ }
+ case msg.BLOCK_BUNDLE: {
+ return this._router.blockMessageById(data.bundle.map(b => b.id));
+ }
+ case msg.UNBLOCK_BUNDLE: {
+ return this._router.setState(state => {
+ const messageBlockList = [...state.messageBlockList];
+ for (let message of data.bundle) {
+ messageBlockList.splice(messageBlockList.indexOf(message.id), 1);
+ }
+ this._router._storage.set("messageBlockList", messageBlockList);
+ return { messageBlockList };
+ });
+ }
+ case msg.DISABLE_PROVIDER: {
+ this._preferences.enableOrDisableProvider(data, false);
+ return Promise.resolve();
+ }
+ case msg.ENABLE_PROVIDER: {
+ this._preferences.enableOrDisableProvider(data, true);
+ return Promise.resolve();
+ }
+ case msg.EVALUATE_JEXL_EXPRESSION: {
+ return this._router.evaluateExpression(data);
+ }
+ case msg.EXPIRE_QUERY_CACHE: {
+ this._queryCache.expireAll();
+ return Promise.resolve();
+ }
+ case msg.FORCE_ATTRIBUTION: {
+ return this._router.forceAttribution(data);
+ }
+ case msg.FORCE_PRIVATE_BROWSING_WINDOW: {
+ return this._router.forcePBWindow(browser, data.message);
+ }
+ case msg.FORCE_WHATSNEW_PANEL: {
+ return this._router.forceWNPanel(browser);
+ }
+ case msg.CLOSE_WHATSNEW_PANEL: {
+ return this._router.closeWNPanel(browser);
+ }
+ case msg.MODIFY_MESSAGE_JSON: {
+ return this._router.routeCFRMessage(data.content, browser, data, true);
+ }
+ case msg.OVERRIDE_MESSAGE: {
+ return this._router.setMessageById(data, true, browser);
+ }
+ case msg.RESET_PROVIDER_PREF: {
+ this._preferences.resetProviderPref();
+ return Promise.resolve();
+ }
+ case msg.SET_PROVIDER_USER_PREF: {
+ this._preferences.setUserPreference(data.id, data.value);
+ return Promise.resolve();
+ }
+ case msg.RESET_GROUPS_STATE: {
+ return this._router
+ .resetGroupsState(data)
+ .then(() => this._router.loadMessagesFromAllProviders());
+ }
+ default: {
+ return Promise.reject(new Error(`Unknown message received: ${name}`));
+ }
+ }
+ }
+}
diff --git a/browser/components/newtab/lib/ASRouterPreferences.jsm b/browser/components/newtab/lib/ASRouterPreferences.jsm
new file mode 100644
index 0000000000..c2fc071fbd
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterPreferences.jsm
@@ -0,0 +1,259 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const PROVIDER_PREF_BRANCH =
+ "browser.newtabpage.activity-stream.asrouter.providers.";
+const DEVTOOLS_PREF =
+ "browser.newtabpage.activity-stream.asrouter.devtoolsEnabled";
+
+/**
+ * Use `ASRouterPreferences.console.debug()` and friends from ASRouter files to
+ * log messages during development. See LOG_LEVELS in ConsoleAPI.jsm for the
+ * available methods as well as the available values for this pref.
+ */
+const DEBUG_PREF = "browser.newtabpage.activity-stream.asrouter.debugLogLevel";
+
+const FXA_USERNAME_PREF = "services.sync.username";
+
+const DEFAULT_STATE = {
+ _initialized: false,
+ _providers: null,
+ _providerPrefBranch: PROVIDER_PREF_BRANCH,
+ _devtoolsEnabled: null,
+ _devtoolsPref: DEVTOOLS_PREF,
+};
+
+const USER_PREFERENCES = {
+ snippets: "browser.newtabpage.activity-stream.feeds.snippets",
+ cfrAddons: "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons",
+ cfrFeatures:
+ "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features",
+};
+
+// Preferences that influence targeting attributes. When these change we need
+// to re-evaluate if the message targeting still matches
+const TARGETING_PREFERENCES = [FXA_USERNAME_PREF];
+
+const TEST_PROVIDERS = [
+ {
+ id: "snippets_local_testing",
+ type: "local",
+ localProvider: "SnippetsTestMessageProvider",
+ enabled: true,
+ },
+ {
+ id: "panel_local_testing",
+ type: "local",
+ localProvider: "PanelTestProvider",
+ enabled: true,
+ },
+];
+
+class _ASRouterPreferences {
+ constructor() {
+ Object.assign(this, DEFAULT_STATE);
+ this._callbacks = new Set();
+
+ XPCOMUtils.defineLazyGetter(this, "console", () => {
+ let { ConsoleAPI } = ChromeUtils.importESModule(
+ "resource://gre/modules/Console.sys.mjs"
+ );
+ let consoleOptions = {
+ maxLogLevel: "error",
+ maxLogLevelPref: DEBUG_PREF,
+ prefix: "ASRouter",
+ };
+ return new ConsoleAPI(consoleOptions);
+ });
+ }
+
+ _transformPersonalizedCfrScores(value) {
+ let result = {};
+ try {
+ result = JSON.parse(value);
+ } catch (e) {
+ console.error(e);
+ }
+ return result;
+ }
+
+ _getProviderConfig() {
+ const prefList = Services.prefs.getChildList(this._providerPrefBranch);
+ return prefList.reduce((filtered, pref) => {
+ let value;
+ try {
+ value = JSON.parse(Services.prefs.getStringPref(pref, ""));
+ } catch (e) {
+ console.error(
+ `Could not parse ASRouter preference. Try resetting ${pref} in about:config.`
+ );
+ }
+ if (value) {
+ filtered.push(value);
+ }
+ return filtered;
+ }, []);
+ }
+
+ get providers() {
+ if (!this._initialized || this._providers === null) {
+ const config = this._getProviderConfig();
+ const providers = config.map(provider => Object.freeze(provider));
+ if (this.devtoolsEnabled) {
+ providers.unshift(...TEST_PROVIDERS);
+ }
+ this._providers = Object.freeze(providers);
+ }
+
+ return this._providers;
+ }
+
+ enableOrDisableProvider(id, value) {
+ const providers = this._getProviderConfig();
+ const config = providers.find(p => p.id === id);
+ if (!config) {
+ console.error(
+ `Cannot set enabled state for '${id}' because the pref ${this._providerPrefBranch}${id} does not exist or is not correctly formatted.`
+ );
+ return;
+ }
+
+ Services.prefs.setStringPref(
+ this._providerPrefBranch + id,
+ JSON.stringify({ ...config, enabled: value })
+ );
+ }
+
+ resetProviderPref() {
+ for (const pref of Services.prefs.getChildList(this._providerPrefBranch)) {
+ Services.prefs.clearUserPref(pref);
+ }
+ for (const id of Object.keys(USER_PREFERENCES)) {
+ Services.prefs.clearUserPref(USER_PREFERENCES[id]);
+ }
+ }
+
+ /**
+ * Bug 1800087 - Migrate the ASRouter message provider prefs' values to the
+ * current format (provider.bucket -> provider.collection).
+ *
+ * TODO (Bug 1800937): Remove migration code after the next watershed release.
+ */
+ _migrateProviderPrefs() {
+ const prefList = Services.prefs.getChildList(this._providerPrefBranch);
+ for (const pref of prefList) {
+ if (!Services.prefs.prefHasUserValue(pref)) {
+ continue;
+ }
+ try {
+ let value = JSON.parse(Services.prefs.getStringPref(pref, ""));
+ if (value && "bucket" in value && !("collection" in value)) {
+ const { bucket, ...rest } = value;
+ Services.prefs.setStringPref(
+ pref,
+ JSON.stringify({
+ ...rest,
+ collection: bucket,
+ })
+ );
+ }
+ } catch (e) {
+ Services.prefs.clearUserPref(pref);
+ }
+ }
+ }
+
+ get devtoolsEnabled() {
+ if (!this._initialized || this._devtoolsEnabled === null) {
+ this._devtoolsEnabled = Services.prefs.getBoolPref(
+ this._devtoolsPref,
+ false
+ );
+ }
+ return this._devtoolsEnabled;
+ }
+
+ observe(aSubject, aTopic, aPrefName) {
+ if (aPrefName && aPrefName.startsWith(this._providerPrefBranch)) {
+ this._providers = null;
+ } else if (aPrefName === this._devtoolsPref) {
+ this._providers = null;
+ this._devtoolsEnabled = null;
+ }
+ this._callbacks.forEach(cb => cb(aPrefName));
+ }
+
+ getUserPreference(name) {
+ const prefName = USER_PREFERENCES[name] || name;
+ return Services.prefs.getBoolPref(prefName, true);
+ }
+
+ getAllUserPreferences() {
+ const values = {};
+ for (const id of Object.keys(USER_PREFERENCES)) {
+ values[id] = this.getUserPreference(id);
+ }
+ return values;
+ }
+
+ setUserPreference(providerId, value) {
+ if (!USER_PREFERENCES[providerId]) {
+ return;
+ }
+ Services.prefs.setBoolPref(USER_PREFERENCES[providerId], value);
+ }
+
+ addListener(callback) {
+ this._callbacks.add(callback);
+ }
+
+ removeListener(callback) {
+ this._callbacks.delete(callback);
+ }
+
+ init() {
+ if (this._initialized) {
+ return;
+ }
+ this._migrateProviderPrefs();
+ Services.prefs.addObserver(this._providerPrefBranch, this);
+ Services.prefs.addObserver(this._devtoolsPref, this);
+ for (const id of Object.keys(USER_PREFERENCES)) {
+ Services.prefs.addObserver(USER_PREFERENCES[id], this);
+ }
+ for (const targetingPref of TARGETING_PREFERENCES) {
+ Services.prefs.addObserver(targetingPref, this);
+ }
+ this._initialized = true;
+ }
+
+ uninit() {
+ if (this._initialized) {
+ Services.prefs.removeObserver(this._providerPrefBranch, this);
+ Services.prefs.removeObserver(this._devtoolsPref, this);
+ for (const id of Object.keys(USER_PREFERENCES)) {
+ Services.prefs.removeObserver(USER_PREFERENCES[id], this);
+ }
+ for (const targetingPref of TARGETING_PREFERENCES) {
+ Services.prefs.removeObserver(targetingPref, this);
+ }
+ }
+ Object.assign(this, DEFAULT_STATE);
+ this._callbacks.clear();
+ }
+}
+
+const ASRouterPreferences = new _ASRouterPreferences();
+
+const EXPORTED_SYMBOLS = [
+ "_ASRouterPreferences",
+ "ASRouterPreferences",
+ "TEST_PROVIDERS",
+ "TARGETING_PREFERENCES",
+];
diff --git a/browser/components/newtab/lib/ASRouterTargeting.jsm b/browser/components/newtab/lib/ASRouterTargeting.jsm
new file mode 100644
index 0000000000..6682aaf534
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterTargeting.jsm
@@ -0,0 +1,1055 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const FXA_ENABLED_PREF = "identity.fxaccounts.enabled";
+const DISTRIBUTION_ID_PREF = "distribution.id";
+const DISTRIBUTION_ID_CHINA_REPACK = "MozillaOnline";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+const { NewTabUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/NewTabUtils.sys.mjs"
+);
+const { ShellService } = ChromeUtils.import(
+ "resource:///modules/ShellService.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ BuiltInThemes: "resource:///modules/BuiltInThemes.sys.mjs",
+ ProfileAge: "resource://gre/modules/ProfileAge.sys.mjs",
+ Region: "resource://gre/modules/Region.sys.mjs",
+ TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs",
+ TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ ASRouterPreferences: "resource://activity-stream/lib/ASRouterPreferences.jsm",
+ AddonManager: "resource://gre/modules/AddonManager.jsm",
+ ClientEnvironment: "resource://normandy/lib/ClientEnvironment.jsm",
+ AttributionCode: "resource:///modules/AttributionCode.jsm",
+ TargetingContext: "resource://messaging-system/targeting/Targeting.jsm",
+ HomePage: "resource:///modules/HomePage.jsm",
+ AboutNewTab: "resource:///modules/AboutNewTab.jsm",
+ BrowserWindowTracker: "resource:///modules/BrowserWindowTracker.jsm",
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "CustomizableUI",
+ "resource:///modules/CustomizableUI.jsm"
+);
+
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.import(
+ "resource://gre/modules/FxAccounts.jsm"
+ ).getFxAccountsSingleton();
+});
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "cfrFeaturesUserPref",
+ "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features",
+ true
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "cfrAddonsUserPref",
+ "browser.newtabpage.activity-stream.asrouter.userprefs.cfr.addons",
+ true
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "isWhatsNewPanelEnabled",
+ "browser.messaging-system.whatsNewPanel.enabled",
+ false
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "hasAccessedFxAPanel",
+ "identity.fxaccounts.toolbar.accessed",
+ false
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "clientsDevicesDesktop",
+ "services.sync.clients.devices.desktop",
+ 0
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "clientsDevicesMobile",
+ "services.sync.clients.devices.mobile",
+ 0
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "syncNumClients",
+ "services.sync.numClients",
+ 0
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "devtoolsSelfXSSCount",
+ "devtools.selfxss.count",
+ 0
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "isFxAEnabled",
+ FXA_ENABLED_PREF,
+ true
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "isXPIInstallEnabled",
+ "xpinstall.enabled",
+ true
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "snippetsUserPref",
+ "browser.newtabpage.activity-stream.feeds.snippets",
+ false
+);
+
+XPCOMUtils.defineLazyServiceGetters(lazy, {
+ AUS: ["@mozilla.org/updates/update-service;1", "nsIApplicationUpdateService"],
+ BrowserHandler: ["@mozilla.org/browser/clh;1", "nsIBrowserHandler"],
+ TrackingDBService: [
+ "@mozilla.org/tracking-db-service;1",
+ "nsITrackingDBService",
+ ],
+ UpdateCheckSvc: ["@mozilla.org/updates/update-checker;1", "nsIUpdateChecker"],
+});
+
+const FXA_USERNAME_PREF = "services.sync.username";
+
+const { activityStreamProvider: asProvider } = NewTabUtils;
+
+const FXA_ATTACHED_CLIENTS_UPDATE_INTERVAL = 4 * 60 * 60 * 1000; // Four hours
+const FRECENT_SITES_UPDATE_INTERVAL = 6 * 60 * 60 * 1000; // Six hours
+const FRECENT_SITES_IGNORE_BLOCKED = false;
+const FRECENT_SITES_NUM_ITEMS = 25;
+const FRECENT_SITES_MIN_FRECENCY = 100;
+
+const CACHE_EXPIRATION = 5 * 60 * 1000;
+const jexlEvaluationCache = new Map();
+
+/**
+ * CachedTargetingGetter
+ * @param property {string} Name of the method
+ * @param options {any=} Options passed to the method
+ * @param updateInterval {number?} Update interval for query. Defaults to FRECENT_SITES_UPDATE_INTERVAL
+ */
+function CachedTargetingGetter(
+ property,
+ options = null,
+ updateInterval = FRECENT_SITES_UPDATE_INTERVAL,
+ getter = asProvider
+) {
+ return {
+ _lastUpdated: 0,
+ _value: null,
+ // For testing
+ expire() {
+ this._lastUpdated = 0;
+ this._value = null;
+ },
+ async get() {
+ const now = Date.now();
+ if (now - this._lastUpdated >= updateInterval) {
+ this._value = await getter[property](options);
+ this._lastUpdated = now;
+ }
+ return this._value;
+ },
+ };
+}
+
+function CacheListAttachedOAuthClients() {
+ return {
+ _lastUpdated: 0,
+ _value: null,
+ expire() {
+ this._lastUpdated = 0;
+ this._value = null;
+ },
+ get() {
+ const now = Date.now();
+ if (now - this._lastUpdated >= FXA_ATTACHED_CLIENTS_UPDATE_INTERVAL) {
+ this._value = new Promise(resolve => {
+ lazy.fxAccounts
+ .listAttachedOAuthClients()
+ .then(clients => {
+ resolve(clients);
+ })
+ .catch(() => resolve([]));
+ });
+ this._lastUpdated = now;
+ }
+ return this._value;
+ },
+ };
+}
+
+function CheckBrowserNeedsUpdate(
+ updateInterval = FRECENT_SITES_UPDATE_INTERVAL
+) {
+ const checker = {
+ _lastUpdated: 0,
+ _value: null,
+ // For testing. Avoid update check network call.
+ setUp(value) {
+ this._lastUpdated = Date.now();
+ this._value = value;
+ },
+ expire() {
+ this._lastUpdated = 0;
+ this._value = null;
+ },
+ async get() {
+ const now = Date.now();
+ if (
+ !AppConstants.MOZ_UPDATER ||
+ now - this._lastUpdated < updateInterval
+ ) {
+ return this._value;
+ }
+ if (!lazy.AUS.canCheckForUpdates) {
+ return false;
+ }
+ this._lastUpdated = now;
+ let check = lazy.UpdateCheckSvc.checkForUpdates(
+ lazy.UpdateCheckSvc.FOREGROUND_CHECK
+ );
+ let result = await check.result;
+ if (!result.succeeded) {
+ throw result.request;
+ }
+ checker._value = !!result.updates.length;
+ return checker._value;
+ },
+ };
+
+ return checker;
+}
+
+const QueryCache = {
+ expireAll() {
+ Object.keys(this.queries).forEach(query => {
+ this.queries[query].expire();
+ });
+ Object.keys(this.getters).forEach(key => {
+ this.getters[key].expire();
+ });
+ },
+ queries: {
+ TopFrecentSites: new CachedTargetingGetter("getTopFrecentSites", {
+ ignoreBlocked: FRECENT_SITES_IGNORE_BLOCKED,
+ numItems: FRECENT_SITES_NUM_ITEMS,
+ topsiteFrecency: FRECENT_SITES_MIN_FRECENCY,
+ onePerDomain: true,
+ includeFavicon: false,
+ }),
+ TotalBookmarksCount: new CachedTargetingGetter("getTotalBookmarksCount"),
+ CheckBrowserNeedsUpdate: new CheckBrowserNeedsUpdate(),
+ RecentBookmarks: new CachedTargetingGetter("getRecentBookmarks"),
+ ListAttachedOAuthClients: new CacheListAttachedOAuthClients(),
+ UserMonthlyActivity: new CachedTargetingGetter("getUserMonthlyActivity"),
+ },
+ getters: {
+ doesAppNeedPin: new CachedTargetingGetter(
+ "doesAppNeedPin",
+ null,
+ FRECENT_SITES_UPDATE_INTERVAL,
+ ShellService
+ ),
+ doesAppNeedPrivatePin: new CachedTargetingGetter(
+ "doesAppNeedPin",
+ true,
+ FRECENT_SITES_UPDATE_INTERVAL,
+ ShellService
+ ),
+ isDefaultBrowser: new CachedTargetingGetter(
+ "isDefaultBrowser",
+ null,
+ FRECENT_SITES_UPDATE_INTERVAL,
+ ShellService
+ ),
+ currentThemes: new CachedTargetingGetter(
+ "getAddonsByTypes",
+ ["theme"],
+ FRECENT_SITES_UPDATE_INTERVAL,
+ lazy.AddonManager // eslint-disable-line mozilla/valid-lazy
+ ),
+ },
+};
+
+/**
+ * sortMessagesByWeightedRank
+ *
+ * Each message has an associated weight, which is guaranteed to be strictly
+ * positive. Sort the messages so that higher weighted messages are more likely
+ * to come first.
+ *
+ * Specifically, sort them so that the probability of message x_1 with weight
+ * w_1 appearing before message x_2 with weight w_2 is (w_1 / (w_1 + w_2)).
+ *
+ * This is equivalent to requiring that x_1 appearing before x_2 is (w_1 / w_2)
+ * "times" as likely as x_2 appearing before x_1.
+ *
+ * See Bug 1484996, Comment 2 for a justification of the method.
+ *
+ * @param {Array} messages - A non-empty array of messages to sort, all with
+ * strictly positive weights
+ * @returns the sorted array
+ */
+function sortMessagesByWeightedRank(messages) {
+ return messages
+ .map(message => ({
+ message,
+ rank: Math.pow(Math.random(), 1 / message.weight),
+ }))
+ .sort((a, b) => b.rank - a.rank)
+ .map(({ message }) => message);
+}
+
+/**
+ * getSortedMessages - Given an array of Messages, applies sorting and filtering rules
+ * in expected order.
+ *
+ * @param {Array<Message>} messages
+ * @param {{}} options
+ * @param {boolean} options.ordered - Should .order be used instead of random weighted sorting?
+ * @returns {Array<Message>}
+ */
+function getSortedMessages(messages, options = {}) {
+ let { ordered } = { ordered: false, ...options };
+ let result = messages;
+
+ if (!ordered) {
+ result = sortMessagesByWeightedRank(result);
+ }
+
+ result.sort((a, b) => {
+ // Next, sort by priority
+ if (a.priority > b.priority || (!isNaN(a.priority) && isNaN(b.priority))) {
+ return -1;
+ }
+ if (a.priority < b.priority || (isNaN(a.priority) && !isNaN(b.priority))) {
+ return 1;
+ }
+
+ // Sort messages with targeting expressions higher than those with none
+ if (a.targeting && !b.targeting) {
+ return -1;
+ }
+ if (!a.targeting && b.targeting) {
+ return 1;
+ }
+
+ // Next, sort by order *ascending* if ordered = true
+ if (ordered) {
+ if (a.order > b.order || (!isNaN(a.order) && isNaN(b.order))) {
+ return 1;
+ }
+ if (a.order < b.order || (isNaN(a.order) && !isNaN(b.order))) {
+ return -1;
+ }
+ }
+
+ return 0;
+ });
+
+ return result;
+}
+
+/**
+ * parseAboutPageURL - Parse a URL string retrieved from about:home and about:new, returns
+ * its type (web extenstion or custom url) and the parsed url(s)
+ *
+ * @param {string} url - A URL string for home page or newtab page
+ * @returns {Object} {
+ * isWebExt: boolean,
+ * isCustomUrl: boolean,
+ * urls: Array<{url: string, host: string}>
+ * }
+ */
+function parseAboutPageURL(url) {
+ let ret = {
+ isWebExt: false,
+ isCustomUrl: false,
+ urls: [],
+ };
+ if (url.startsWith("moz-extension://")) {
+ ret.isWebExt = true;
+ ret.urls.push({ url, host: "" });
+ } else {
+ // The home page URL could be either a single URL or a list of "|" separated URLs.
+ // Note that it should work with "about:home" and "about:blank", in which case the
+ // "host" is set as an empty string.
+ for (const _url of url.split("|")) {
+ if (!["about:home", "about:newtab", "about:blank"].includes(_url)) {
+ ret.isCustomUrl = true;
+ }
+ try {
+ const parsedURL = new URL(_url);
+ const host = parsedURL.hostname.replace(/^www\./i, "");
+ ret.urls.push({ url: _url, host });
+ } catch (e) {}
+ }
+ // If URL parsing failed, just return the given url with an empty host
+ if (!ret.urls.length) {
+ ret.urls.push({ url, host: "" });
+ }
+ }
+
+ return ret;
+}
+
+const TargetingGetters = {
+ get locale() {
+ return Services.locale.appLocaleAsBCP47;
+ },
+ get localeLanguageCode() {
+ return (
+ Services.locale.appLocaleAsBCP47 &&
+ Services.locale.appLocaleAsBCP47.substr(0, 2)
+ );
+ },
+ get browserSettings() {
+ const { settings } = lazy.TelemetryEnvironment.currentEnvironment;
+ return {
+ update: settings.update,
+ };
+ },
+ get attributionData() {
+ // Attribution is determined at startup - so we can use the cached attribution at this point
+ return lazy.AttributionCode.getCachedAttributionData();
+ },
+ get currentDate() {
+ return new Date();
+ },
+ get profileAgeCreated() {
+ return lazy.ProfileAge().then(times => times.created);
+ },
+ get profileAgeReset() {
+ return lazy.ProfileAge().then(times => times.reset);
+ },
+ get usesFirefoxSync() {
+ return Services.prefs.prefHasUserValue(FXA_USERNAME_PREF);
+ },
+ get isFxAEnabled() {
+ return lazy.isFxAEnabled;
+ },
+ get isFxASignedIn() {
+ return new Promise(resolve => {
+ if (!lazy.isFxAEnabled) {
+ resolve(false);
+ }
+ if (Services.prefs.getStringPref(FXA_USERNAME_PREF, "")) {
+ resolve(true);
+ }
+ lazy.fxAccounts
+ .getSignedInUser()
+ .then(data => resolve(!!data))
+ .catch(e => resolve(false));
+ });
+ },
+ get sync() {
+ return {
+ desktopDevices: lazy.clientsDevicesDesktop,
+ mobileDevices: lazy.clientsDevicesMobile,
+ totalDevices: lazy.syncNumClients,
+ };
+ },
+ get xpinstallEnabled() {
+ // This is needed for all add-on recommendations, to know if we allow xpi installs in the first place
+ return lazy.isXPIInstallEnabled;
+ },
+ get addonsInfo() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ if (bts?.isBackgroundTaskMode) {
+ return { addons: {}, isFullData: true };
+ }
+
+ return lazy.AddonManager.getActiveAddons(["extension", "service"]).then(
+ ({ addons, fullData }) => {
+ const info = {};
+ for (const addon of addons) {
+ info[addon.id] = {
+ version: addon.version,
+ type: addon.type,
+ isSystem: addon.isSystem,
+ isWebExtension: addon.isWebExtension,
+ };
+ if (fullData) {
+ Object.assign(info[addon.id], {
+ name: addon.name,
+ userDisabled: addon.userDisabled,
+ installDate: addon.installDate,
+ });
+ }
+ }
+ return { addons: info, isFullData: fullData };
+ }
+ );
+ },
+ get searchEngines() {
+ const NONE = { installed: [], current: "" };
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ if (bts?.isBackgroundTaskMode) {
+ return Promise.resolve(NONE);
+ }
+ return new Promise(resolve => {
+ // Note: calling init ensures this code is only executed after Search has been initialized
+ Services.search
+ .getAppProvidedEngines()
+ .then(engines => {
+ resolve({
+ current: Services.search.defaultEngine.identifier,
+ installed: engines.map(engine => engine.identifier),
+ });
+ })
+ .catch(() => resolve(NONE));
+ });
+ },
+ get isDefaultBrowser() {
+ return QueryCache.getters.isDefaultBrowser.get().catch(() => null);
+ },
+ get devToolsOpenedCount() {
+ return lazy.devtoolsSelfXSSCount;
+ },
+ get topFrecentSites() {
+ return QueryCache.queries.TopFrecentSites.get().then(sites =>
+ sites.map(site => ({
+ url: site.url,
+ host: new URL(site.url).hostname,
+ frecency: site.frecency,
+ lastVisitDate: site.lastVisitDate,
+ }))
+ );
+ },
+ get recentBookmarks() {
+ return QueryCache.queries.RecentBookmarks.get();
+ },
+ get pinnedSites() {
+ return NewTabUtils.pinnedLinks.links.map(site =>
+ site
+ ? {
+ url: site.url,
+ host: new URL(site.url).hostname,
+ searchTopSite: site.searchTopSite,
+ }
+ : {}
+ );
+ },
+ get providerCohorts() {
+ return lazy.ASRouterPreferences.providers.reduce((prev, current) => {
+ prev[current.id] = current.cohort || "";
+ return prev;
+ }, {});
+ },
+ get totalBookmarksCount() {
+ return QueryCache.queries.TotalBookmarksCount.get();
+ },
+ get firefoxVersion() {
+ return parseInt(AppConstants.MOZ_APP_VERSION.match(/\d+/), 10);
+ },
+ get region() {
+ return lazy.Region.home || "";
+ },
+ get needsUpdate() {
+ return QueryCache.queries.CheckBrowserNeedsUpdate.get();
+ },
+ get hasPinnedTabs() {
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ if (win.closed || !win.ownerGlobal.gBrowser) {
+ continue;
+ }
+ if (win.ownerGlobal.gBrowser.visibleTabs.filter(t => t.pinned).length) {
+ return true;
+ }
+ }
+
+ return false;
+ },
+ get hasAccessedFxAPanel() {
+ return lazy.hasAccessedFxAPanel;
+ },
+ get isWhatsNewPanelEnabled() {
+ return lazy.isWhatsNewPanelEnabled;
+ },
+ get userPrefs() {
+ return {
+ cfrFeatures: lazy.cfrFeaturesUserPref,
+ cfrAddons: lazy.cfrAddonsUserPref,
+ snippets: lazy.snippetsUserPref,
+ };
+ },
+ get totalBlockedCount() {
+ return lazy.TrackingDBService.sumAllEvents();
+ },
+ get blockedCountByType() {
+ const idToTextMap = new Map([
+ [Ci.nsITrackingDBService.TRACKERS_ID, "trackerCount"],
+ [Ci.nsITrackingDBService.TRACKING_COOKIES_ID, "cookieCount"],
+ [Ci.nsITrackingDBService.CRYPTOMINERS_ID, "cryptominerCount"],
+ [Ci.nsITrackingDBService.FINGERPRINTERS_ID, "fingerprinterCount"],
+ [Ci.nsITrackingDBService.SOCIAL_ID, "socialCount"],
+ ]);
+
+ const dateTo = new Date();
+ const dateFrom = new Date(dateTo.getTime() - 42 * 24 * 60 * 60 * 1000);
+ return lazy.TrackingDBService.getEventsByDateRange(dateFrom, dateTo).then(
+ eventsByDate => {
+ let totalEvents = {};
+ for (let blockedType of idToTextMap.values()) {
+ totalEvents[blockedType] = 0;
+ }
+
+ return eventsByDate.reduce((acc, day) => {
+ const type = day.getResultByName("type");
+ const count = day.getResultByName("count");
+ acc[idToTextMap.get(type)] = acc[idToTextMap.get(type)] + count;
+ return acc;
+ }, totalEvents);
+ }
+ );
+ },
+ get attachedFxAOAuthClients() {
+ return this.usesFirefoxSync
+ ? QueryCache.queries.ListAttachedOAuthClients.get()
+ : [];
+ },
+ get platformName() {
+ return AppConstants.platform;
+ },
+ get isChinaRepack() {
+ return (
+ Services.prefs
+ .getDefaultBranch(null)
+ .getCharPref(DISTRIBUTION_ID_PREF, "default") ===
+ DISTRIBUTION_ID_CHINA_REPACK
+ );
+ },
+ get userId() {
+ return lazy.ClientEnvironment.userId;
+ },
+ get profileRestartCount() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ if (bts?.isBackgroundTaskMode) {
+ return 0;
+ }
+ // Counter starts at 1 when a profile is created, substract 1 so the value
+ // returned matches expectations
+ return (
+ lazy.TelemetrySession.getMetadata("targeting").profileSubsessionCounter -
+ 1
+ );
+ },
+ get homePageSettings() {
+ const url = lazy.HomePage.get();
+ const { isWebExt, isCustomUrl, urls } = parseAboutPageURL(url);
+
+ return {
+ isWebExt,
+ isCustomUrl,
+ urls,
+ isDefault: lazy.HomePage.isDefault,
+ isLocked: lazy.HomePage.locked,
+ };
+ },
+ get newtabSettings() {
+ const url = lazy.AboutNewTab.newTabURL;
+ const { isWebExt, isCustomUrl, urls } = parseAboutPageURL(url);
+
+ return {
+ isWebExt,
+ isCustomUrl,
+ isDefault: lazy.AboutNewTab.activityStreamEnabled,
+ url: urls[0].url,
+ host: urls[0].host,
+ };
+ },
+ get isFissionExperimentEnabled() {
+ return (
+ Services.appinfo.fissionExperimentStatus ===
+ Ci.nsIXULRuntime.eExperimentStatusTreatment
+ );
+ },
+ get activeNotifications() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ if (bts?.isBackgroundTaskMode) {
+ // This might need to hook into the alert service to enumerate relevant
+ // persistent native notifications.
+ return false;
+ }
+
+ let window = lazy.BrowserWindowTracker.getTopWindow();
+
+ // Technically this doesn't mean we have active notifications,
+ // but because we use !activeNotifications to check for conflicts, this should return true
+ if (!window) {
+ return true;
+ }
+
+ if (
+ window.gURLBar?.view.isOpen ||
+ window.gNotificationBox?.currentNotification ||
+ window.gBrowser.getNotificationBox()?.currentNotification
+ ) {
+ return true;
+ }
+
+ return false;
+ },
+
+ get isMajorUpgrade() {
+ return lazy.BrowserHandler.majorUpgrade;
+ },
+
+ get hasActiveEnterprisePolicies() {
+ return Services.policies.status === Services.policies.ACTIVE;
+ },
+
+ get userMonthlyActivity() {
+ return QueryCache.queries.UserMonthlyActivity.get();
+ },
+
+ get doesAppNeedPin() {
+ return QueryCache.getters.doesAppNeedPin.get();
+ },
+
+ get doesAppNeedPrivatePin() {
+ return QueryCache.getters.doesAppNeedPrivatePin.get();
+ },
+
+ /**
+ * Is this invocation running in background task mode?
+ *
+ * @return {boolean} `true` if running in background task mode.
+ */
+ get isBackgroundTaskMode() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ return !!bts?.isBackgroundTaskMode;
+ },
+
+ /**
+ * A non-empty task name if this invocation is running in background
+ * task mode, or `null` if this invocation is not running in
+ * background task mode.
+ *
+ * @return {string|null} background task name or `null`.
+ */
+ get backgroundTaskName() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ return bts?.backgroundTaskName();
+ },
+
+ get userPrefersReducedMotion() {
+ let window = Services.appShell.hiddenDOMWindow;
+ return window?.matchMedia("(prefers-reduced-motion: reduce)")?.matches;
+ },
+ /**
+ * Is there an active Colorway collection?
+ * @return {boolean} `true` if an active collection exists.
+ */
+ get colorwaysActive() {
+ return !!lazy.BuiltInThemes.findActiveColorwayCollection();
+ },
+ /**
+ * Has the user enabled an active Colorway as their theme?
+ * @return {boolean} `true` if an active theme from the current
+ * collection is enabled.
+ */
+ get userEnabledActiveColorway() {
+ let bts = Cc["@mozilla.org/backgroundtasks;1"]?.getService(
+ Ci.nsIBackgroundTasks
+ );
+ if (bts?.isBackgroundTaskMode) {
+ return Promise.resolve(false);
+ }
+ return QueryCache.getters.currentThemes.get().then(themes => {
+ let themeId = themes.find(theme => theme.isActive)?.id;
+ return !!(
+ themeId && lazy.BuiltInThemes.isColorwayFromCurrentCollection(themeId)
+ );
+ });
+ },
+ /**
+ * Whether or not the user is in the Major Release 2022 holdback study.
+ */
+ get inMr2022Holdback() {
+ return (
+ lazy.NimbusFeatures.majorRelease2022.getVariable("onboarding") === false
+ );
+ },
+ /**
+ * The distribution id, if any.
+ * @return {string}
+ */
+ get distributionId() {
+ return Services.prefs
+ .getDefaultBranch(null)
+ .getCharPref("distribution.id", "");
+ },
+
+ /** Where the Firefox View button is shown, if at all.
+ * @return {string} container of the button if it is shown in the toolbar/overflow menu
+ * @return {string} `null` if the button has been removed
+ */
+ get fxViewButtonAreaType() {
+ let button = lazy.CustomizableUI.getWidget("firefox-view-button");
+ return button.areaType;
+ },
+};
+
+const ASRouterTargeting = {
+ Environment: TargetingGetters,
+
+ /**
+ * Snapshot the current targeting environment.
+ *
+ * Asynchronous getters are handled. Getters that throw or reject
+ * are ignored.
+ *
+ * @param {object} target - the environment to snapshot.
+ * @return {object} snapshot of target with `environment` object and `version`
+ * integer.
+ */
+ async getEnvironmentSnapshot(target = ASRouterTargeting.Environment) {
+ // One promise for each named property. Label promises with property name.
+ let promises = Object.keys(target).map(async name => {
+ // Each promise needs to check if we're shutting down when it is evaluated.
+ if (Services.startup.shuttingDown) {
+ throw new Error("shutting down, so not querying targeting environment");
+ }
+ return [name, await target[name]];
+ });
+
+ // Ignore properties that are rejected.
+ let results = await Promise.allSettled(promises);
+
+ let environment = {};
+ for (let result of results) {
+ if (result.status === "fulfilled") {
+ let [name, value] = result.value;
+ environment[name] = value;
+ }
+ }
+
+ // Should we need to migrate in the future.
+ const snapshot = { environment, version: 1 };
+
+ return snapshot;
+ },
+
+ isTriggerMatch(trigger = {}, candidateMessageTrigger = {}) {
+ if (trigger.id !== candidateMessageTrigger.id) {
+ return false;
+ } else if (
+ !candidateMessageTrigger.params &&
+ !candidateMessageTrigger.patterns
+ ) {
+ return true;
+ }
+
+ if (!trigger.param) {
+ return false;
+ }
+
+ return (
+ (candidateMessageTrigger.params &&
+ trigger.param.host &&
+ candidateMessageTrigger.params.includes(trigger.param.host)) ||
+ (candidateMessageTrigger.params &&
+ trigger.param.type &&
+ candidateMessageTrigger.params.filter(t => t === trigger.param.type)
+ .length) ||
+ (candidateMessageTrigger.params &&
+ trigger.param.type &&
+ candidateMessageTrigger.params.filter(
+ t => (t & trigger.param.type) === t
+ ).length) ||
+ (candidateMessageTrigger.patterns &&
+ trigger.param.url &&
+ new MatchPatternSet(candidateMessageTrigger.patterns).matches(
+ trigger.param.url
+ ))
+ );
+ },
+
+ /**
+ * getCachedEvaluation - Return a cached jexl evaluation if available
+ *
+ * @param {string} targeting JEXL expression to lookup
+ * @returns {obj|null} Object with value result or null if not available
+ */
+ getCachedEvaluation(targeting) {
+ if (jexlEvaluationCache.has(targeting)) {
+ const { timestamp, value } = jexlEvaluationCache.get(targeting);
+ if (Date.now() - timestamp <= CACHE_EXPIRATION) {
+ return { value };
+ }
+ jexlEvaluationCache.delete(targeting);
+ }
+
+ return null;
+ },
+
+ /**
+ * checkMessageTargeting - Checks is a message's targeting parameters are satisfied
+ *
+ * @param {*} message An AS router message
+ * @param {obj} targetingContext a TargetingContext instance complete with eval environment
+ * @param {func} onError A function to handle errors (takes two params; error, message)
+ * @param {boolean} shouldCache Should the JEXL evaluations be cached and reused.
+ * @returns
+ */
+ async checkMessageTargeting(message, targetingContext, onError, shouldCache) {
+ lazy.ASRouterPreferences.console.debug(
+ "in checkMessageTargeting, arguments = ",
+ Array.from(arguments) // eslint-disable-line prefer-rest-params
+ );
+
+ // If no targeting is specified,
+ if (!message.targeting) {
+ return true;
+ }
+ let result;
+ try {
+ if (shouldCache) {
+ result = this.getCachedEvaluation(message.targeting);
+ if (result) {
+ return result.value;
+ }
+ }
+ // Used to report the source of the targeting error in the case of
+ // undesired events
+ targetingContext.setTelemetrySource(message.id);
+ result = await targetingContext.evalWithDefault(message.targeting);
+ if (shouldCache) {
+ jexlEvaluationCache.set(message.targeting, {
+ timestamp: Date.now(),
+ value: result,
+ });
+ }
+ } catch (error) {
+ if (onError) {
+ onError(error, message);
+ }
+ console.error(error);
+ result = false;
+ }
+ return result;
+ },
+
+ _isMessageMatch(
+ message,
+ trigger,
+ targetingContext,
+ onError,
+ shouldCache = false
+ ) {
+ return (
+ message &&
+ (trigger
+ ? this.isTriggerMatch(trigger, message.trigger)
+ : !message.trigger) &&
+ // If a trigger expression was passed to this function, the message should match it.
+ // Otherwise, we should choose a message with no trigger property (i.e. a message that can show up at any time)
+ this.checkMessageTargeting(
+ message,
+ targetingContext,
+ onError,
+ shouldCache
+ )
+ );
+ },
+
+ /**
+ * findMatchingMessage - Given an array of messages, returns one message
+ * whos targeting expression evaluates to true
+ *
+ * @param {Array<Message>} messages An array of AS router messages
+ * @param {trigger} string A trigger expression if a message for that trigger is desired
+ * @param {obj|null} context A FilterExpression context. Defaults to TargetingGetters above.
+ * @param {func} onError A function to handle errors (takes two params; error, message)
+ * @param {func} ordered An optional param when true sort message by order specified in message
+ * @param {boolean} shouldCache Should the JEXL evaluations be cached and reused.
+ * @param {boolean} returnAll Should we return all matching messages, not just the first one found.
+ * @returns {obj|Array<Message>} If returnAll is false, a single message. If returnAll is true, an array of messages.
+ */
+ async findMatchingMessage({
+ messages,
+ trigger = {},
+ context = {},
+ onError,
+ ordered = false,
+ shouldCache = false,
+ returnAll = false,
+ }) {
+ const sortedMessages = getSortedMessages(messages, { ordered });
+ lazy.ASRouterPreferences.console.debug(
+ "in findMatchingMessage, sortedMessages = ",
+ sortedMessages
+ );
+ const matching = returnAll ? [] : null;
+ const targetingContext = new lazy.TargetingContext(
+ lazy.TargetingContext.combineContexts(
+ context,
+ this.Environment,
+ trigger.context || {}
+ )
+ );
+
+ const isMatch = candidate =>
+ this._isMessageMatch(
+ candidate,
+ trigger,
+ targetingContext,
+ onError,
+ shouldCache
+ );
+
+ for (const candidate of sortedMessages) {
+ if (await isMatch(candidate)) {
+ // If not returnAll, we should return the first message we find that matches.
+ if (!returnAll) {
+ return candidate;
+ }
+
+ matching.push(candidate);
+ }
+ }
+ return matching;
+ },
+};
+
+const EXPORTED_SYMBOLS = [
+ "ASRouterTargeting",
+ "QueryCache",
+ "CachedTargetingGetter",
+ "getSortedMessages",
+];
diff --git a/browser/components/newtab/lib/ASRouterTriggerListeners.jsm b/browser/components/newtab/lib/ASRouterTriggerListeners.jsm
new file mode 100644
index 0000000000..a366a64659
--- /dev/null
+++ b/browser/components/newtab/lib/ASRouterTriggerListeners.jsm
@@ -0,0 +1,969 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AboutReaderParent: "resource:///actors/AboutReaderParent.sys.mjs",
+ BrowserUtils: "resource://gre/modules/BrowserUtils.sys.mjs",
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+ clearTimeout: "resource://gre/modules/Timer.sys.mjs",
+ setTimeout: "resource://gre/modules/Timer.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ EveryWindow: "resource:///modules/EveryWindow.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "log", () => {
+ const { Logger } = ChromeUtils.import(
+ "resource://messaging-system/lib/Logger.jsm"
+ );
+ return new Logger("ASRouterTriggerListeners");
+});
+
+const FEW_MINUTES = 15 * 60 * 1000; // 15 mins
+
+function isPrivateWindow(win) {
+ return (
+ !(win instanceof Ci.nsIDOMWindow) ||
+ win.closed ||
+ lazy.PrivateBrowsingUtils.isWindowPrivate(win)
+ );
+}
+
+/**
+ * Check current location against the list of allowed hosts
+ * Additionally verify for redirects and check original request URL against
+ * the list.
+ *
+ * @returns {object} - {host, url} pair that matched the list of allowed hosts
+ */
+function checkURLMatch(aLocationURI, { hosts, matchPatternSet }, aRequest) {
+ // If checks pass we return a match
+ let match;
+ try {
+ match = { host: aLocationURI.host, url: aLocationURI.spec };
+ } catch (e) {
+ // nsIURI.host can throw for non-nsStandardURL nsIURIs
+ return false;
+ }
+
+ // Check current location against allowed hosts
+ if (hosts.has(match.host)) {
+ return match;
+ }
+
+ if (matchPatternSet) {
+ if (matchPatternSet.matches(match.url)) {
+ return match;
+ }
+ }
+
+ // Nothing else to check, return early
+ if (!aRequest) {
+ return false;
+ }
+
+ // The original URL at the start of the request
+ const originalLocation = aRequest.QueryInterface(Ci.nsIChannel).originalURI;
+ // We have been redirected
+ if (originalLocation.spec !== aLocationURI.spec) {
+ return (
+ hosts.has(originalLocation.host) && {
+ host: originalLocation.host,
+ url: originalLocation.spec,
+ }
+ );
+ }
+
+ return false;
+}
+
+function createMatchPatternSet(patterns, flags) {
+ try {
+ return new MatchPatternSet(new Set(patterns), flags);
+ } catch (e) {
+ console.error(e);
+ }
+ return new MatchPatternSet([]);
+}
+
+/**
+ * A Map from trigger IDs to singleton trigger listeners. Each listener must
+ * have idempotent `init` and `uninit` methods.
+ */
+const ASRouterTriggerListeners = new Map([
+ [
+ "openArticleURL",
+ {
+ id: "openArticleURL",
+ _initialized: false,
+ _triggerHandler: null,
+ _hosts: new Set(),
+ _matchPatternSet: null,
+ readerModeEvent: "Reader:UpdateReaderButton",
+
+ init(triggerHandler, hosts, patterns) {
+ if (!this._initialized) {
+ this.receiveMessage = this.receiveMessage.bind(this);
+ lazy.AboutReaderParent.addMessageListener(this.readerModeEvent, this);
+ this._triggerHandler = triggerHandler;
+ this._initialized = true;
+ }
+ if (patterns) {
+ this._matchPatternSet = createMatchPatternSet([
+ ...(this._matchPatternSet ? this._matchPatternSet.patterns : []),
+ ...patterns,
+ ]);
+ }
+ if (hosts) {
+ hosts.forEach(h => this._hosts.add(h));
+ }
+ },
+
+ receiveMessage({ data, target }) {
+ if (data && data.isArticle) {
+ const match = checkURLMatch(target.currentURI, {
+ hosts: this._hosts,
+ matchPatternSet: this._matchPatternSet,
+ });
+ if (match) {
+ this._triggerHandler(target, { id: this.id, param: match });
+ }
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ lazy.AboutReaderParent.removeMessageListener(
+ this.readerModeEvent,
+ this
+ );
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._hosts = new Set();
+ this._matchPatternSet = null;
+ }
+ },
+ },
+ ],
+ [
+ "openBookmarkedURL",
+ {
+ id: "openBookmarkedURL",
+ _initialized: false,
+ _triggerHandler: null,
+ _hosts: new Set(),
+ bookmarkEvent: "bookmark-icon-updated",
+
+ init(triggerHandler) {
+ if (!this._initialized) {
+ Services.obs.addObserver(this, this.bookmarkEvent);
+ this._triggerHandler = triggerHandler;
+ this._initialized = true;
+ }
+ },
+
+ observe(subject, topic, data) {
+ if (topic === this.bookmarkEvent && data === "starred") {
+ const browser = Services.wm.getMostRecentBrowserWindow();
+ if (browser) {
+ this._triggerHandler(browser.gBrowser.selectedBrowser, {
+ id: this.id,
+ });
+ }
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ Services.obs.removeObserver(this, this.bookmarkEvent);
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._hosts = new Set();
+ }
+ },
+ },
+ ],
+ [
+ "frequentVisits",
+ {
+ id: "frequentVisits",
+ _initialized: false,
+ _triggerHandler: null,
+ _hosts: null,
+ _matchPatternSet: null,
+ _visits: null,
+
+ init(triggerHandler, hosts = [], patterns) {
+ if (!this._initialized) {
+ this.onTabSwitch = this.onTabSwitch.bind(this);
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.addEventListener("TabSelect", this.onTabSwitch);
+ win.gBrowser.addTabsProgressListener(this);
+ }
+ },
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.removeEventListener("TabSelect", this.onTabSwitch);
+ win.gBrowser.removeTabsProgressListener(this);
+ }
+ }
+ );
+ this._visits = new Map();
+ this._initialized = true;
+ }
+ this._triggerHandler = triggerHandler;
+ if (patterns) {
+ this._matchPatternSet = createMatchPatternSet([
+ ...(this._matchPatternSet ? this._matchPatternSet.patterns : []),
+ ...patterns,
+ ]);
+ }
+ if (this._hosts) {
+ hosts.forEach(h => this._hosts.add(h));
+ } else {
+ this._hosts = new Set(hosts); // Clone the hosts to avoid unexpected behaviour
+ }
+ },
+
+ /* _updateVisits - Record visit timestamps for websites that match `this._hosts` and only
+ * if it's been more than FEW_MINUTES since the last visit.
+ * @param {string} host - Location host of current selected tab
+ * @returns {boolean} - If the new visit has been recorded
+ */
+ _updateVisits(host) {
+ const visits = this._visits.get(host);
+
+ if (visits && Date.now() - visits[0] > FEW_MINUTES) {
+ this._visits.set(host, [Date.now(), ...visits]);
+ return true;
+ }
+ if (!visits) {
+ this._visits.set(host, [Date.now()]);
+ return true;
+ }
+
+ return false;
+ },
+
+ onTabSwitch(event) {
+ if (!event.target.ownerGlobal.gBrowser) {
+ return;
+ }
+
+ const { gBrowser } = event.target.ownerGlobal;
+ const match = checkURLMatch(gBrowser.currentURI, {
+ hosts: this._hosts,
+ matchPatternSet: this._matchPatternSet,
+ });
+ if (match) {
+ this.triggerHandler(gBrowser.selectedBrowser, match);
+ }
+ },
+
+ triggerHandler(aBrowser, match) {
+ const updated = this._updateVisits(match.host);
+
+ // If the previous visit happend less than FEW_MINUTES ago
+ // no updates were made, no need to trigger the handler
+ if (!updated) {
+ return;
+ }
+
+ this._triggerHandler(aBrowser, {
+ id: this.id,
+ param: match,
+ context: {
+ // Remapped to {host, timestamp} because JEXL operators can only
+ // filter over collections (arrays of objects)
+ recentVisits: this._visits
+ .get(match.host)
+ .map(timestamp => ({ host: match.host, timestamp })),
+ },
+ });
+ },
+
+ onLocationChange(aBrowser, aWebProgress, aRequest, aLocationURI, aFlags) {
+ // Some websites trigger redirect events after they finish loading even
+ // though the location remains the same. This results in onLocationChange
+ // events to be fired twice.
+ const isSameDocument = !!(
+ aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT
+ );
+ if (aWebProgress.isTopLevel && !isSameDocument) {
+ const match = checkURLMatch(
+ aLocationURI,
+ { hosts: this._hosts, matchPatternSet: this._matchPatternSet },
+ aRequest
+ );
+ if (match) {
+ this.triggerHandler(aBrowser, match);
+ }
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ lazy.EveryWindow.unregisterCallback(this.id);
+
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._hosts = null;
+ this._matchPatternSet = null;
+ this._visits = null;
+ }
+ },
+ },
+ ],
+
+ /**
+ * Attach listeners to every browser window to detect location changes, and
+ * notify the trigger handler whenever we navigate to a URL with a hostname
+ * we're looking for.
+ */
+ [
+ "openURL",
+ {
+ id: "openURL",
+ _initialized: false,
+ _triggerHandler: null,
+ _hosts: null,
+ _matchPatternSet: null,
+ _visits: null,
+
+ /*
+ * If the listener is already initialised, `init` will replace the trigger
+ * handler and add any new hosts to `this._hosts`.
+ */
+ init(triggerHandler, hosts = [], patterns) {
+ if (!this._initialized) {
+ this.onLocationChange = this.onLocationChange.bind(this);
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.addEventListener("TabSelect", this.onTabSwitch);
+ win.gBrowser.addTabsProgressListener(this);
+ }
+ },
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.removeEventListener("TabSelect", this.onTabSwitch);
+ win.gBrowser.removeTabsProgressListener(this);
+ }
+ }
+ );
+
+ this._visits = new Map();
+ this._initialized = true;
+ }
+ this._triggerHandler = triggerHandler;
+ if (patterns) {
+ this._matchPatternSet = createMatchPatternSet([
+ ...(this._matchPatternSet ? this._matchPatternSet.patterns : []),
+ ...patterns,
+ ]);
+ }
+ if (this._hosts) {
+ hosts.forEach(h => this._hosts.add(h));
+ } else {
+ this._hosts = new Set(hosts); // Clone the hosts to avoid unexpected behaviour
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ lazy.EveryWindow.unregisterCallback(this.id);
+
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._hosts = null;
+ this._matchPatternSet = null;
+ this._visits = null;
+ }
+ },
+
+ onLocationChange(aBrowser, aWebProgress, aRequest, aLocationURI, aFlags) {
+ // Some websites trigger redirect events after they finish loading even
+ // though the location remains the same. This results in onLocationChange
+ // events to be fired twice.
+ const isSameDocument = !!(
+ aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT
+ );
+ if (aWebProgress.isTopLevel && !isSameDocument) {
+ const match = checkURLMatch(
+ aLocationURI,
+ { hosts: this._hosts, matchPatternSet: this._matchPatternSet },
+ aRequest
+ );
+ if (match) {
+ let visitsCount = (this._visits.get(match.url) || 0) + 1;
+ this._visits.set(match.url, visitsCount);
+ this._triggerHandler(aBrowser, {
+ id: this.id,
+ param: match,
+ context: { visitsCount },
+ });
+ }
+ }
+ },
+ },
+ ],
+
+ /**
+ * Add an observer notification to notify the trigger handler whenever the user
+ * saves or updates a login via the login capture doorhanger.
+ */
+ [
+ "newSavedLogin",
+ {
+ _initialized: false,
+ _triggerHandler: null,
+
+ /**
+ * If the listener is already initialised, `init` will replace the trigger
+ * handler.
+ */
+ init(triggerHandler) {
+ if (!this._initialized) {
+ Services.obs.addObserver(this, "LoginStats:NewSavedPassword");
+ Services.obs.addObserver(this, "LoginStats:LoginUpdateSaved");
+ this._initialized = true;
+ }
+ this._triggerHandler = triggerHandler;
+ },
+
+ uninit() {
+ if (this._initialized) {
+ Services.obs.removeObserver(this, "LoginStats:NewSavedPassword");
+ Services.obs.removeObserver(this, "LoginStats:LoginUpdateSaved");
+
+ this._initialized = false;
+ this._triggerHandler = null;
+ }
+ },
+
+ observe(aSubject, aTopic, aData) {
+ if (aSubject.currentURI.asciiHost === "accounts.firefox.com") {
+ // Don't notify about saved logins on the FxA login origin since this
+ // trigger is used to promote login Sync and getting a recommendation
+ // to enable Sync during the sign up process is a bad UX.
+ return;
+ }
+
+ switch (aTopic) {
+ case "LoginStats:NewSavedPassword": {
+ this._triggerHandler(aSubject, {
+ id: "newSavedLogin",
+ context: { type: "save" },
+ });
+ break;
+ }
+ case "LoginStats:LoginUpdateSaved": {
+ this._triggerHandler(aSubject, {
+ id: "newSavedLogin",
+ context: { type: "update" },
+ });
+ break;
+ }
+ default: {
+ throw new Error(`Unexpected observer notification: ${aTopic}`);
+ }
+ }
+ },
+ },
+ ],
+
+ [
+ "contentBlocking",
+ {
+ _initialized: false,
+ _triggerHandler: null,
+ _events: [],
+ _sessionPageLoad: 0,
+ onLocationChange: null,
+
+ init(triggerHandler, params, patterns) {
+ params.forEach(p => this._events.push(p));
+
+ if (!this._initialized) {
+ Services.obs.addObserver(this, "SiteProtection:ContentBlockingEvent");
+ Services.obs.addObserver(
+ this,
+ "SiteProtection:ContentBlockingMilestone"
+ );
+ this.onLocationChange = this._onLocationChange.bind(this);
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.gBrowser.addTabsProgressListener(this);
+ }
+ },
+ win => {
+ if (!isPrivateWindow(win)) {
+ win.gBrowser.removeTabsProgressListener(this);
+ }
+ }
+ );
+
+ this._initialized = true;
+ }
+ this._triggerHandler = triggerHandler;
+ },
+
+ uninit() {
+ if (this._initialized) {
+ Services.obs.removeObserver(
+ this,
+ "SiteProtection:ContentBlockingEvent"
+ );
+ Services.obs.removeObserver(
+ this,
+ "SiteProtection:ContentBlockingMilestone"
+ );
+ lazy.EveryWindow.unregisterCallback(this.id);
+ this.onLocationChange = null;
+ this._initialized = false;
+ }
+ this._triggerHandler = null;
+ this._events = [];
+ this._sessionPageLoad = 0;
+ },
+
+ observe(aSubject, aTopic, aData) {
+ switch (aTopic) {
+ case "SiteProtection:ContentBlockingEvent":
+ const { browser, host, event } = aSubject.wrappedJSObject;
+ if (this._events.filter(e => (e & event) === e).length) {
+ this._triggerHandler(browser, {
+ id: "contentBlocking",
+ param: {
+ host,
+ type: event,
+ },
+ context: {
+ pageLoad: this._sessionPageLoad,
+ },
+ });
+ }
+ break;
+ case "SiteProtection:ContentBlockingMilestone":
+ if (this._events.includes(aSubject.wrappedJSObject.event)) {
+ this._triggerHandler(
+ Services.wm.getMostRecentBrowserWindow().gBrowser
+ .selectedBrowser,
+ {
+ id: "contentBlocking",
+ context: {
+ pageLoad: this._sessionPageLoad,
+ },
+ param: {
+ type: aSubject.wrappedJSObject.event,
+ },
+ }
+ );
+ }
+ break;
+ }
+ },
+
+ _onLocationChange(
+ aBrowser,
+ aWebProgress,
+ aRequest,
+ aLocationURI,
+ aFlags
+ ) {
+ // Some websites trigger redirect events after they finish loading even
+ // though the location remains the same. This results in onLocationChange
+ // events to be fired twice.
+ const isSameDocument = !!(
+ aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT
+ );
+ if (
+ ["http", "https"].includes(aLocationURI.scheme) &&
+ aWebProgress.isTopLevel &&
+ !isSameDocument
+ ) {
+ this._sessionPageLoad += 1;
+ }
+ },
+ },
+ ],
+
+ [
+ "captivePortalLogin",
+ {
+ id: "captivePortalLogin",
+ _initialized: false,
+ _triggerHandler: null,
+
+ _shouldShowCaptivePortalVPNPromo() {
+ return lazy.BrowserUtils.shouldShowVPNPromo();
+ },
+
+ init(triggerHandler) {
+ if (!this._initialized) {
+ Services.obs.addObserver(this, "captive-portal-login-success");
+ this._initialized = true;
+ }
+ this._triggerHandler = triggerHandler;
+ },
+
+ observe(aSubject, aTopic, aData) {
+ switch (aTopic) {
+ case "captive-portal-login-success":
+ const browser = Services.wm.getMostRecentBrowserWindow();
+ // The check is here rather than in init because some
+ // folks leave their browsers running for a long time,
+ // eg from before leaving on a plane trip to after landing
+ // in the new destination, and the current region may have
+ // changed since init time.
+ if (browser && this._shouldShowCaptivePortalVPNPromo()) {
+ this._triggerHandler(browser.gBrowser.selectedBrowser, {
+ id: this.id,
+ });
+ }
+ break;
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ this._triggerHandler = null;
+ this._initialized = false;
+ Services.obs.removeObserver(this, "captive-portal-login-success");
+ }
+ },
+ },
+ ],
+
+ [
+ "preferenceObserver",
+ {
+ id: "preferenceObserver",
+ _initialized: false,
+ _triggerHandler: null,
+ _observedPrefs: [],
+
+ init(triggerHandler, prefs) {
+ if (!this._initialized) {
+ this._triggerHandler = triggerHandler;
+ this._initialized = true;
+ }
+ prefs.forEach(pref => {
+ this._observedPrefs.push(pref);
+ Services.prefs.addObserver(pref, this);
+ });
+ },
+
+ observe(aSubject, aTopic, aData) {
+ switch (aTopic) {
+ case "nsPref:changed":
+ const browser = Services.wm.getMostRecentBrowserWindow();
+ if (browser && this._observedPrefs.includes(aData)) {
+ this._triggerHandler(browser.gBrowser.selectedBrowser, {
+ id: this.id,
+ param: {
+ type: aData,
+ },
+ });
+ }
+ break;
+ }
+ },
+
+ uninit() {
+ if (this._initialized) {
+ this._observedPrefs.forEach(pref =>
+ Services.prefs.removeObserver(pref, this)
+ );
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._observedPrefs = [];
+ }
+ },
+ },
+ ],
+ [
+ "nthTabClosed",
+ {
+ id: "nthTabClosed",
+ _initialized: false,
+ _triggerHandler: null,
+ // Number of tabs the user closed this session
+ _closedTabs: 0,
+
+ init(triggerHandler) {
+ this._triggerHandler = triggerHandler;
+ if (!this._initialized) {
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ win.addEventListener("TabClose", this);
+ },
+ win => {
+ win.removeEventListener("TabClose", this);
+ }
+ );
+ this._initialized = true;
+ }
+ },
+ handleEvent(event) {
+ if (this._initialized) {
+ if (!event.target.ownerGlobal.gBrowser) {
+ return;
+ }
+ const { gBrowser } = event.target.ownerGlobal;
+ this._closedTabs++;
+ this._triggerHandler(gBrowser.selectedBrowser, {
+ id: this.id,
+ context: { tabsClosedCount: this._closedTabs },
+ });
+ }
+ },
+ uninit() {
+ if (this._initialized) {
+ lazy.EveryWindow.unregisterCallback(this.id);
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._closedTabs = 0;
+ }
+ },
+ },
+ ],
+ [
+ "activityAfterIdle",
+ {
+ id: "activityAfterIdle",
+ _initialized: false,
+ _triggerHandler: null,
+ _idleService: null,
+ // Optimization - only report idle state after one minute of idle time.
+ // This represents a minimum idleForMilliseconds of 60000.
+ _idleThreshold: 60,
+ _idleSince: null,
+ _quietSince: null,
+ _awaitingVisibilityChange: false,
+ // Fire the trigger 2 seconds after activity resumes to ensure user is
+ // actively using the browser when it fires.
+ _triggerDelay: 2000,
+ _triggerTimeout: null,
+ // We may get an idle notification immediately after waking from sleep.
+ // The idle time in such a case will be the amount of time since the last
+ // user interaction, which was before the computer went to sleep. We want
+ // to ignore them in that case, so we ignore idle notifications that
+ // happen within 1 second of the last wake notification.
+ _wakeDelay: 1000,
+ _lastWakeTime: null,
+ _listenedEvents: ["visibilitychange", "TabClose", "TabAttrModified"],
+ // When the OS goes to sleep or the process is suspended, we want to drop
+ // the idle time, since the time between sleep and wake is expected to be
+ // very long (e.g. overnight). Otherwise, this would trigger on the first
+ // activity after waking/resuming, counting sleep as idle time. This
+ // basically means each session starts with a fresh idle time.
+ _observedTopics: [
+ "sleep_notification",
+ "suspend_process_notification",
+ "wake_notification",
+ "resume_process_notification",
+ "mac_app_activate",
+ ],
+
+ get _isVisible() {
+ return [...Services.wm.getEnumerator("navigator:browser")].some(
+ win => !win.closed && !win.document?.hidden
+ );
+ },
+ get _soundPlaying() {
+ return [...Services.wm.getEnumerator("navigator:browser")].some(win =>
+ win.gBrowser?.tabs.some(tab => tab.soundPlaying)
+ );
+ },
+ init(triggerHandler) {
+ this._triggerHandler = triggerHandler;
+ // Instantiate this here instead of with a lazy service getter so we can
+ // stub it in tests (otherwise we'd have to wait up to 6 minutes for an
+ // idle notification in certain test environments).
+ if (!this._idleService) {
+ this._idleService = Cc[
+ "@mozilla.org/widget/useridleservice;1"
+ ].getService(Ci.nsIUserIdleService);
+ }
+ if (
+ !this._initialized &&
+ !lazy.PrivateBrowsingUtils.permanentPrivateBrowsing
+ ) {
+ this._idleService.addIdleObserver(this, this._idleThreshold);
+ for (let topic of this._observedTopics) {
+ Services.obs.addObserver(this, topic);
+ }
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ for (let ev of this._listenedEvents) {
+ win.addEventListener(ev, this);
+ }
+ },
+ win => {
+ for (let ev of this._listenedEvents) {
+ win.removeEventListener(ev, this);
+ }
+ }
+ );
+ if (!this._soundPlaying) {
+ this._quietSince = Date.now();
+ }
+ this._initialized = true;
+ this.log("Initialized: ", {
+ idleTime: this._idleService.idleTime,
+ quietSince: this._quietSince,
+ });
+ }
+ },
+ observe(subject, topic, data) {
+ if (this._initialized) {
+ this.log("Heard observer notification: ", {
+ subject,
+ topic,
+ data,
+ idleTime: this._idleService.idleTime,
+ idleSince: this._idleSince,
+ quietSince: this._quietSince,
+ lastWakeTime: this._lastWakeTime,
+ });
+ switch (topic) {
+ case "idle":
+ const now = Date.now();
+ // If the idle notification is within 1 second of the last wake
+ // notification, ignore it. We do this to avoid counting time the
+ // computer spent asleep as "idle time"
+ const isImmediatelyAfterWake =
+ this._lastWakeTime &&
+ now - this._lastWakeTime < this._wakeDelay;
+ if (!isImmediatelyAfterWake) {
+ this._idleSince = now - subject.idleTime;
+ }
+ break;
+ case "active":
+ // Trigger when user returns from being idle.
+ if (this._isVisible) {
+ this._onActive();
+ this._idleSince = null;
+ this._lastWakeTime = null;
+ } else if (this._idleSince) {
+ // If the window is not visible, we want to wait until it is
+ // visible before triggering.
+ this._awaitingVisibilityChange = true;
+ }
+ break;
+ // OS/process notifications
+ case "wake_notification":
+ case "resume_process_notification":
+ case "mac_app_activate":
+ this._lastWakeTime = Date.now();
+ // Fall through to reset idle time.
+ default:
+ this._idleSince = null;
+ }
+ }
+ },
+ handleEvent(event) {
+ if (this._initialized) {
+ switch (event.type) {
+ case "visibilitychange":
+ if (this._awaitingVisibilityChange && this._isVisible) {
+ this._onActive();
+ this._idleSince = null;
+ this._lastWakeTime = null;
+ this._awaitingVisibilityChange = false;
+ }
+ break;
+ case "TabAttrModified":
+ // Listen for DOMAudioPlayback* events.
+ if (!event.detail?.changed?.includes("soundplaying")) {
+ break;
+ }
+ // fall through
+ case "TabClose":
+ this.log("Tab sound changed: ", {
+ event,
+ idleTime: this._idleService.idleTime,
+ idleSince: this._idleSince,
+ quietSince: this._quietSince,
+ });
+ // Maybe update time if a tab closes with sound playing.
+ if (this._soundPlaying) {
+ this._quietSince = null;
+ } else if (!this._quietSince) {
+ this._quietSince = Date.now();
+ }
+ }
+ }
+ },
+ _onActive() {
+ this.log("User is active: ", {
+ idleTime: this._idleService.idleTime,
+ idleSince: this._idleSince,
+ quietSince: this._quietSince,
+ lastWakeTime: this._lastWakeTime,
+ });
+ if (this._idleSince && this._quietSince) {
+ const win = Services.wm.getMostRecentBrowserWindow();
+ if (win && !isPrivateWindow(win) && !this._triggerTimeout) {
+ // Number of ms since the last user interaction/audio playback
+ const idleForMilliseconds =
+ Date.now() - Math.min(this._idleSince, this._quietSince);
+ this._triggerTimeout = lazy.setTimeout(() => {
+ this._triggerHandler(win.gBrowser.selectedBrowser, {
+ id: this.id,
+ context: { idleForMilliseconds },
+ });
+ this._triggerTimeout = null;
+ }, this._triggerDelay);
+ }
+ }
+ },
+ uninit() {
+ if (this._initialized) {
+ this._idleService.removeIdleObserver(this, this._idleThreshold);
+ for (let topic of this._observedTopics) {
+ Services.obs.removeObserver(this, topic);
+ }
+ lazy.EveryWindow.unregisterCallback(this.id);
+ lazy.clearTimeout(this._triggerTimeout);
+ this._triggerTimeout = null;
+ this._initialized = false;
+ this._triggerHandler = null;
+ this._idleSince = null;
+ this._quietSince = null;
+ this._lastWakeTime = null;
+ this._awaitingVisibilityChange = false;
+ this.log("Uninitialized");
+ }
+ },
+ log(...args) {
+ lazy.log.debug("Idle trigger :>>", ...args);
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIObserver",
+ "nsISupportsWeakReference",
+ ]),
+ },
+ ],
+]);
+
+const EXPORTED_SYMBOLS = ["ASRouterTriggerListeners"];
diff --git a/browser/components/newtab/lib/AboutPreferences.jsm b/browser/components/newtab/lib/AboutPreferences.jsm
new file mode 100644
index 0000000000..4906728eb2
--- /dev/null
+++ b/browser/components/newtab/lib/AboutPreferences.jsm
@@ -0,0 +1,311 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const HTML_NS = "http://www.w3.org/1999/xhtml";
+const PREFERENCES_LOADED_EVENT = "home-pane-loaded";
+
+const lazy = {};
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+// These "section" objects are formatted in a way to be similar to the ones from
+// SectionsManager to construct the preferences view.
+const PREFS_BEFORE_SECTIONS = () => [
+ {
+ id: "search",
+ pref: {
+ feed: "showSearch",
+ titleString: "home-prefs-search-header",
+ },
+ icon: "chrome://global/skin/icons/search-glass.svg",
+ },
+ {
+ id: "topsites",
+ pref: {
+ feed: "feeds.topsites",
+ titleString: "home-prefs-shortcuts-header",
+ descString: "home-prefs-shortcuts-description",
+ get nestedPrefs() {
+ return Services.prefs.getBoolPref("browser.topsites.useRemoteSetting")
+ ? [
+ {
+ name: "showSponsoredTopSites",
+ titleString: "home-prefs-shortcuts-by-option-sponsored",
+ eventSource: "SPONSORED_TOP_SITES",
+ },
+ ]
+ : [];
+ },
+ },
+ icon: "chrome://browser/skin/topsites.svg",
+ maxRows: 4,
+ rowsPref: "topSitesRows",
+ eventSource: "TOP_SITES",
+ },
+];
+
+const PREFS_AFTER_SECTIONS = () => [
+ {
+ id: "snippets",
+ pref: {
+ feed: "feeds.snippets",
+ titleString: "home-prefs-snippets-header",
+ descString: "home-prefs-snippets-description-new",
+ },
+ icon: "chrome://global/skin/icons/info.svg",
+ eventSource: "SNIPPETS",
+ },
+];
+
+class AboutPreferences {
+ init() {
+ Services.obs.addObserver(this, PREFERENCES_LOADED_EVENT);
+ }
+
+ uninit() {
+ Services.obs.removeObserver(this, PREFERENCES_LOADED_EVENT);
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ case at.SETTINGS_OPEN:
+ action._target.browser.ownerGlobal.openPreferences("paneHome");
+ break;
+ // This is used to open the web extension settings page for an extension
+ case at.OPEN_WEBEXT_SETTINGS:
+ action._target.browser.ownerGlobal.BrowserOpenAddonsMgr(
+ `addons://detail/${encodeURIComponent(action.data)}`
+ );
+ break;
+ }
+ }
+
+ handleDiscoverySettings(sections) {
+ // Deep copy object to not modify original Sections state in store
+ let sectionsCopy = JSON.parse(JSON.stringify(sections));
+ sectionsCopy.forEach(obj => {
+ if (obj.id === "topstories") {
+ obj.rowsPref = "";
+ }
+ });
+ return sectionsCopy;
+ }
+
+ setupUserEvent(element, eventSource) {
+ element.addEventListener("command", e => {
+ const { checked } = e.target;
+ if (typeof checked === "boolean") {
+ this.store.dispatch(
+ ac.UserEvent({
+ event: "PREF_CHANGED",
+ source: eventSource,
+ value: { status: checked, menu_source: "ABOUT_PREFERENCES" },
+ })
+ );
+ }
+ });
+ }
+
+ observe(window) {
+ const discoveryStreamConfig = this.store.getState().DiscoveryStream.config;
+ let sections = this.store.getState().Sections;
+
+ if (discoveryStreamConfig.enabled) {
+ sections = this.handleDiscoverySettings(sections);
+ }
+
+ const featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {};
+
+ this.renderPreferences(window, [
+ ...PREFS_BEFORE_SECTIONS(featureConfig),
+ ...sections,
+ ...PREFS_AFTER_SECTIONS(featureConfig),
+ ]);
+ }
+
+ /**
+ * Render preferences to an about:preferences content window with the provided
+ * preferences structure.
+ */
+ renderPreferences({ document, Preferences, gHomePane }, prefStructure) {
+ // Helper to create a new element and append it
+ const createAppend = (tag, parent, options) =>
+ parent.appendChild(document.createXULElement(tag, options));
+
+ // Helper to get fluentIDs sometimes encase in an object
+ const getString = message =>
+ typeof message !== "object" ? message : message.id;
+
+ // Helper to link a UI element to a preference for updating
+ const linkPref = (element, name, type) => {
+ const fullPref = `browser.newtabpage.activity-stream.${name}`;
+ element.setAttribute("preference", fullPref);
+ Preferences.add({ id: fullPref, type });
+
+ // Prevent changing the UI if the preference can't be changed
+ element.disabled = Preferences.get(fullPref).locked;
+ };
+
+ // Insert a new group immediately after the homepage one
+ const homeGroup = document.getElementById("homepageGroup");
+ const contentsGroup = homeGroup.insertAdjacentElement(
+ "afterend",
+ homeGroup.cloneNode()
+ );
+ contentsGroup.id = "homeContentsGroup";
+ contentsGroup.setAttribute("data-subcategory", "contents");
+ const homeHeader = createAppend("label", contentsGroup).appendChild(
+ document.createElementNS(HTML_NS, "h2")
+ );
+ document.l10n.setAttributes(homeHeader, "home-prefs-content-header2");
+
+ const homeDescription = createAppend("description", contentsGroup);
+ document.l10n.setAttributes(
+ homeDescription,
+ "home-prefs-content-description2"
+ );
+
+ // Add preferences for each section
+ prefStructure.forEach(sectionData => {
+ const {
+ id,
+ pref: prefData,
+ icon = "webextension",
+ maxRows,
+ rowsPref,
+ shouldHidePref,
+ eventSource,
+ } = sectionData;
+ const { feed: name, titleString = {}, descString, nestedPrefs = [] } =
+ prefData || {};
+
+ // Don't show any sections that we don't want to expose in preferences UI
+ if (shouldHidePref) {
+ return;
+ }
+
+ // Use full icon spec for certain protocols or fall back to packaged icon
+ const iconUrl = !icon.search(/^(chrome|moz-extension|resource):/)
+ ? icon
+ : `chrome://activity-stream/content/data/content/assets/glyph-${icon}-16.svg`;
+
+ // Add the main preference for turning on/off a section
+ const sectionVbox = createAppend("vbox", contentsGroup);
+ sectionVbox.setAttribute("data-subcategory", id);
+ const checkbox = createAppend("checkbox", sectionVbox);
+ checkbox.classList.add("section-checkbox");
+ checkbox.setAttribute("src", iconUrl);
+ // Setup a user event if we have an event source for this pref.
+ if (eventSource) {
+ this.setupUserEvent(checkbox, eventSource);
+ }
+ document.l10n.setAttributes(
+ checkbox,
+ getString(titleString),
+ titleString.values
+ );
+
+ linkPref(checkbox, name, "bool");
+
+ // Specially add a link for stories
+ if (id === "topstories") {
+ const sponsoredHbox = createAppend("hbox", sectionVbox);
+ sponsoredHbox.setAttribute("align", "center");
+ sponsoredHbox.appendChild(checkbox);
+ checkbox.classList.add("tail-with-learn-more");
+
+ const link = createAppend("label", sponsoredHbox, { is: "text-link" });
+ link.classList.add("learn-sponsored");
+ link.setAttribute("href", sectionData.pref.learnMore.link.href);
+ document.l10n.setAttributes(link, sectionData.pref.learnMore.link.id);
+ }
+
+ // Add more details for the section (e.g., description, more prefs)
+ const detailVbox = createAppend("vbox", sectionVbox);
+ detailVbox.classList.add("indent");
+ if (descString) {
+ const label = createAppend("label", detailVbox);
+ label.classList.add("indent");
+ document.l10n.setAttributes(
+ label,
+ getString(descString),
+ descString.values
+ );
+
+ // Add a rows dropdown if we have a pref to control and a maximum
+ if (rowsPref && maxRows) {
+ const detailHbox = createAppend("hbox", detailVbox);
+ detailHbox.setAttribute("align", "center");
+ label.setAttribute("flex", 1);
+ detailHbox.appendChild(label);
+
+ // Add box so the search tooltip is positioned correctly
+ const tooltipBox = createAppend("hbox", detailHbox);
+
+ // Add appropriate number of localized entries to the dropdown
+ const menulist = createAppend("menulist", tooltipBox);
+ menulist.setAttribute("crop", "none");
+ const menupopup = createAppend("menupopup", menulist);
+ for (let num = 1; num <= maxRows; num++) {
+ const item = createAppend("menuitem", menupopup);
+ document.l10n.setAttributes(
+ item,
+ "home-prefs-sections-rows-option",
+ { num }
+ );
+ item.setAttribute("value", num);
+ }
+ linkPref(menulist, rowsPref, "int");
+ }
+ }
+
+ const subChecks = [];
+ const fullName = `browser.newtabpage.activity-stream.${sectionData.pref.feed}`;
+ const pref = Preferences.get(fullName);
+
+ // Add a checkbox pref for any nested preferences
+ nestedPrefs.forEach(nested => {
+ const subcheck = createAppend("checkbox", detailVbox);
+ // Setup a user event if we have an event source for this pref.
+ if (nested.eventSource) {
+ this.setupUserEvent(subcheck, nested.eventSource);
+ }
+ subcheck.classList.add("indent");
+ document.l10n.setAttributes(subcheck, nested.titleString);
+ linkPref(subcheck, nested.name, "bool");
+ subChecks.push(subcheck);
+ subcheck.disabled = !pref._value;
+ subcheck.hidden = nested.hidden;
+ });
+
+ // Disable any nested checkboxes if the parent pref is not enabled.
+ pref.on("change", () => {
+ subChecks.forEach(subcheck => {
+ subcheck.disabled = !pref._value;
+ });
+ });
+ });
+
+ // Update the visibility of the Restore Defaults btn based on checked prefs
+ gHomePane.toggleRestoreDefaultsBtn();
+ }
+}
+
+const EXPORTED_SYMBOLS = ["AboutPreferences", "PREFERENCES_LOADED_EVENT"];
diff --git a/browser/components/newtab/lib/ActivityStream.jsm b/browser/components/newtab/lib/ActivityStream.jsm
new file mode 100644
index 0000000000..bb9960a639
--- /dev/null
+++ b/browser/components/newtab/lib/ActivityStream.jsm
@@ -0,0 +1,758 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "DEFAULT_SITES",
+ "resource://activity-stream/lib/DefaultSites.jsm"
+);
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Region: "resource://gre/modules/Region.sys.mjs",
+});
+
+// NB: Eagerly load modules that will be loaded/constructed/initialized in the
+// common case to avoid the overhead of wrapping and detecting lazy loading.
+const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "AboutPreferences",
+ "resource://activity-stream/lib/AboutPreferences.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "DefaultPrefs",
+ "resource://activity-stream/lib/ActivityStreamPrefs.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "NewTabInit",
+ "resource://activity-stream/lib/NewTabInit.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "SectionsFeed",
+ "resource://activity-stream/lib/SectionsManager.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "RecommendationProvider",
+ "resource://activity-stream/lib/RecommendationProvider.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PlacesFeed",
+ "resource://activity-stream/lib/PlacesFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PrefsFeed",
+ "resource://activity-stream/lib/PrefsFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Store",
+ "resource://activity-stream/lib/Store.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "SystemTickFeed",
+ "resource://activity-stream/lib/SystemTickFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "TelemetryFeed",
+ "resource://activity-stream/lib/TelemetryFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "FaviconFeed",
+ "resource://activity-stream/lib/FaviconFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "TopSitesFeed",
+ "resource://activity-stream/lib/TopSitesFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "TopStoriesFeed",
+ "resource://activity-stream/lib/TopStoriesFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "HighlightsFeed",
+ "resource://activity-stream/lib/HighlightsFeed.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "DiscoveryStreamFeed",
+ "resource://activity-stream/lib/DiscoveryStreamFeed.jsm"
+);
+
+const REGION_STORIES_CONFIG =
+ "browser.newtabpage.activity-stream.discoverystream.region-stories-config";
+const REGION_STORIES_BLOCK =
+ "browser.newtabpage.activity-stream.discoverystream.region-stories-block";
+const REGION_SPOCS_CONFIG =
+ "browser.newtabpage.activity-stream.discoverystream.region-spocs-config";
+const REGION_BASIC_CONFIG =
+ "browser.newtabpage.activity-stream.discoverystream.region-basic-config";
+const LOCALE_LIST_CONFIG =
+ "browser.newtabpage.activity-stream.discoverystream.locale-list-config";
+
+// Determine if spocs should be shown for a geo/locale
+function showSpocs({ geo }) {
+ const spocsGeoString =
+ Services.prefs.getStringPref(REGION_SPOCS_CONFIG) || "";
+ const spocsGeo = spocsGeoString.split(",").map(s => s.trim());
+ return spocsGeo.includes(geo);
+}
+
+// Configure default Activity Stream prefs with a plain `value` or a `getValue`
+// that computes a value. A `value_local_dev` is used for development defaults.
+const PREFS_CONFIG = new Map([
+ [
+ "default.sites",
+ {
+ title:
+ "Comma-separated list of default top sites to fill in behind visited sites",
+ getValue: ({ geo }) =>
+ lazy.DEFAULT_SITES.get(lazy.DEFAULT_SITES.has(geo) ? geo : ""),
+ },
+ ],
+ [
+ "feeds.section.topstories.options",
+ {
+ title: "Configuration options for top stories feed",
+ // This is a dynamic pref as it depends on the feed being shown or not
+ getValue: args =>
+ JSON.stringify({
+ api_key_pref: "extensions.pocket.oAuthConsumerKey",
+ // Use the opposite value as what default value the feed would have used
+ hidden: !PREFS_CONFIG.get("feeds.system.topstories").getValue(args),
+ provider_icon: "chrome://global/skin/icons/pocket.svg",
+ provider_name: "Pocket",
+ read_more_endpoint:
+ "https://getpocket.com/explore/trending?src=fx_new_tab",
+ stories_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=${
+ args.locale
+ }&feed_variant=${
+ showSpocs(args) ? "default_spocs_on" : "default_spocs_off"
+ }`,
+ stories_referrer: "https://getpocket.com/recommendations",
+ topics_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/trending-topics?version=2&consumer_key=$apiKey&locale_lang=${args.locale}`,
+ show_spocs: showSpocs(args),
+ }),
+ },
+ ],
+ [
+ "feeds.topsites",
+ {
+ title: "Displays Top Sites on the New Tab Page",
+ value: true,
+ },
+ ],
+ [
+ "hideTopSitesTitle",
+ {
+ title:
+ "Hide the top sites section's title, including the section and collapse icons",
+ value: false,
+ },
+ ],
+ [
+ "showSponsored",
+ {
+ title:
+ "Show sponsored cards in spoc experiment (show_spocs in topstories.options has to be set to true as well)",
+ value: true,
+ },
+ ],
+ [
+ "showSponsoredTopSites",
+ {
+ title: "Show sponsored top sites",
+ value: true,
+ },
+ ],
+ [
+ "pocketCta",
+ {
+ title: "Pocket cta and button for logged out users.",
+ value: JSON.stringify({
+ cta_button: "",
+ cta_text: "",
+ cta_url: "",
+ use_cta: false,
+ }),
+ },
+ ],
+ [
+ "showSearch",
+ {
+ title: "Show the Search bar",
+ value: true,
+ },
+ ],
+ [
+ "feeds.snippets",
+ {
+ title: "Show snippets on activity stream",
+ value: false,
+ },
+ ],
+ [
+ "topSitesRows",
+ {
+ title: "Number of rows of Top Sites to display",
+ value: 1,
+ },
+ ],
+ [
+ "telemetry",
+ {
+ title: "Enable system error and usage data collection",
+ value: true,
+ value_local_dev: false,
+ },
+ ],
+ [
+ "telemetry.ut.events",
+ {
+ title: "Enable Unified Telemetry event data collection",
+ value: AppConstants.EARLY_BETA_OR_EARLIER,
+ value_local_dev: false,
+ },
+ ],
+ [
+ "telemetry.structuredIngestion.endpoint",
+ {
+ title: "Structured Ingestion telemetry server endpoint",
+ value: "https://incoming.telemetry.mozilla.org/submit",
+ },
+ ],
+ [
+ "section.highlights.includeVisited",
+ {
+ title:
+ "Boolean flag that decides whether or not to show visited pages in highlights.",
+ value: true,
+ },
+ ],
+ [
+ "section.highlights.includeBookmarks",
+ {
+ title:
+ "Boolean flag that decides whether or not to show bookmarks in highlights.",
+ value: true,
+ },
+ ],
+ [
+ "section.highlights.includePocket",
+ {
+ title:
+ "Boolean flag that decides whether or not to show saved Pocket stories in highlights.",
+ value: true,
+ },
+ ],
+ [
+ "section.highlights.includeDownloads",
+ {
+ title:
+ "Boolean flag that decides whether or not to show saved recent Downloads in highlights.",
+ value: true,
+ },
+ ],
+ [
+ "section.highlights.rows",
+ {
+ title: "Number of rows of Highlights to display",
+ value: 1,
+ },
+ ],
+ [
+ "section.topstories.rows",
+ {
+ title: "Number of rows of Top Stories to display",
+ value: 1,
+ },
+ ],
+ [
+ "sectionOrder",
+ {
+ title: "The rendering order for the sections",
+ value: "topsites,topstories,highlights",
+ },
+ ],
+ [
+ "improvesearch.noDefaultSearchTile",
+ {
+ title: "Remove tiles that are the same as the default search",
+ value: true,
+ },
+ ],
+ [
+ "improvesearch.topSiteSearchShortcuts.searchEngines",
+ {
+ title:
+ "An ordered, comma-delimited list of search shortcuts that we should try and pin",
+ // This pref is dynamic as the shortcuts vary depending on the region
+ getValue: ({ geo }) => {
+ if (!geo) {
+ return "";
+ }
+ const searchShortcuts = [];
+ if (geo === "CN") {
+ searchShortcuts.push("baidu");
+ } else if (["BY", "KZ", "RU", "TR"].includes(geo)) {
+ searchShortcuts.push("yandex");
+ } else {
+ searchShortcuts.push("google");
+ }
+ if (["DE", "FR", "GB", "IT", "JP", "US"].includes(geo)) {
+ searchShortcuts.push("amazon");
+ }
+ return searchShortcuts.join(",");
+ },
+ },
+ ],
+ [
+ "improvesearch.topSiteSearchShortcuts.havePinned",
+ {
+ title:
+ "A comma-delimited list of search shortcuts that have previously been pinned",
+ value: "",
+ },
+ ],
+ [
+ "asrouter.devtoolsEnabled",
+ {
+ title: "Are the asrouter devtools enabled?",
+ value: false,
+ },
+ ],
+ [
+ "asrouter.providers.onboarding",
+ {
+ title: "Configuration for onboarding provider",
+ value: JSON.stringify({
+ id: "onboarding",
+ type: "local",
+ localProvider: "OnboardingMessageProvider",
+ enabled: true,
+ // Block specific messages from this local provider
+ exclude: [],
+ }),
+ },
+ ],
+ // See browser/app/profile/firefox.js for other ASR preferences. They must be defined there to enable roll-outs.
+ [
+ "discoverystream.flight.blocks",
+ {
+ title: "Track flight blocks",
+ skipBroadcast: true,
+ value: "{}",
+ },
+ ],
+ [
+ "discoverystream.config",
+ {
+ title: "Configuration for the new pocket new tab",
+ getValue: ({ geo, locale }) => {
+ return JSON.stringify({
+ api_key_pref: "extensions.pocket.oAuthConsumerKey",
+ collapsible: true,
+ enabled: true,
+ show_spocs: showSpocs({ geo }),
+ hardcoded_layout: true,
+ // This is currently an exmple layout used for dev purposes.
+ layout_endpoint:
+ "https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic",
+ });
+ },
+ },
+ ],
+ [
+ "discoverystream.endpoints",
+ {
+ title:
+ "Endpoint prefixes (comma-separated) that are allowed to be requested",
+ value: "https://getpocket.cdn.mozilla.net/,https://spocs.getpocket.com/",
+ },
+ ],
+ [
+ "discoverystream.isCollectionDismissible",
+ {
+ title: "Allows Pocket story collections to be dismissed",
+ value: false,
+ },
+ ],
+ [
+ "discoverystream.region-basic-layout",
+ {
+ title: "Decision to use basic layout based on region.",
+ getValue: ({ geo }) => {
+ const preffedRegionsString =
+ Services.prefs.getStringPref(REGION_BASIC_CONFIG) || "";
+ // If no regions are set to basic,
+ // we don't need to bother checking against the region.
+ // We are also not concerned if geo is not set,
+ // because stories are going to be empty until we have geo.
+ if (!preffedRegionsString) {
+ return false;
+ }
+ const preffedRegions = preffedRegionsString
+ .split(",")
+ .map(s => s.trim());
+
+ return preffedRegions.includes(geo);
+ },
+ },
+ ],
+ [
+ "discoverystream.spoc.impressions",
+ {
+ title: "Track spoc impressions",
+ skipBroadcast: true,
+ value: "{}",
+ },
+ ],
+ [
+ "discoverystream.endpointSpocsClear",
+ {
+ title:
+ "Endpoint for when a user opts-out of sponsored content to delete the user's data from the ad server.",
+ value: "https://spocs.getpocket.com/user",
+ },
+ ],
+ [
+ "discoverystream.rec.impressions",
+ {
+ title: "Track rec impressions",
+ skipBroadcast: true,
+ value: "{}",
+ },
+ ],
+ [
+ "showRecentSaves",
+ {
+ title: "Control whether a user wants recent saves visible on Newtab",
+ value: true,
+ },
+ ],
+]);
+
+// Array of each feed's FEEDS_CONFIG factory and values to add to PREFS_CONFIG
+const FEEDS_DATA = [
+ {
+ name: "aboutpreferences",
+ factory: () => new lazy.AboutPreferences(),
+ title: "about:preferences rendering",
+ value: true,
+ },
+ {
+ name: "newtabinit",
+ factory: () => new lazy.NewTabInit(),
+ title: "Sends a copy of the state to each new tab that is opened",
+ value: true,
+ },
+ {
+ name: "places",
+ factory: () => new lazy.PlacesFeed(),
+ title: "Listens for and relays various Places-related events",
+ value: true,
+ },
+ {
+ name: "prefs",
+ factory: () => new lazy.PrefsFeed(PREFS_CONFIG),
+ title: "Preferences",
+ value: true,
+ },
+ {
+ name: "sections",
+ factory: () => new lazy.SectionsFeed(),
+ title: "Manages sections",
+ value: true,
+ },
+ {
+ name: "section.highlights",
+ factory: () => new lazy.HighlightsFeed(),
+ title: "Fetches content recommendations from places db",
+ value: false,
+ },
+ {
+ name: "system.topstories",
+ factory: () =>
+ new lazy.TopStoriesFeed(PREFS_CONFIG.get("discoverystream.config")),
+ title:
+ "System pref that fetches content recommendations from a configurable content provider",
+ // Dynamically determine if Pocket should be shown for a geo / locale
+ getValue: ({ geo, locale }) => {
+ // If we don't have geo, we don't want to flash the screen with stories while geo loads.
+ // Best to display nothing until geo is ready.
+ if (!geo) {
+ return false;
+ }
+ const preffedRegionsBlockString =
+ Services.prefs.getStringPref(REGION_STORIES_BLOCK) || "";
+ const preffedRegionsString =
+ Services.prefs.getStringPref(REGION_STORIES_CONFIG) || "";
+ const preffedLocaleListString =
+ Services.prefs.getStringPref(LOCALE_LIST_CONFIG) || "";
+ const preffedBlockRegions = preffedRegionsBlockString
+ .split(",")
+ .map(s => s.trim());
+ const preffedRegions = preffedRegionsString.split(",").map(s => s.trim());
+ const preffedLocales = preffedLocaleListString
+ .split(",")
+ .map(s => s.trim());
+ const locales = {
+ US: ["en-CA", "en-GB", "en-US"],
+ CA: ["en-CA", "en-GB", "en-US"],
+ GB: ["en-CA", "en-GB", "en-US"],
+ AU: ["en-CA", "en-GB", "en-US"],
+ NZ: ["en-CA", "en-GB", "en-US"],
+ IN: ["en-CA", "en-GB", "en-US"],
+ IE: ["en-CA", "en-GB", "en-US"],
+ ZA: ["en-CA", "en-GB", "en-US"],
+ CH: ["de"],
+ BE: ["de"],
+ DE: ["de"],
+ AT: ["de"],
+ IT: ["it"],
+ FR: ["fr"],
+ ES: ["es"],
+ PL: ["pl"],
+ JP: ["ja", "ja-JP-mac"],
+ }[geo];
+
+ const regionBlocked = preffedBlockRegions.includes(geo);
+ const localeEnabled = locale && preffedLocales.includes(locale);
+ const regionEnabled =
+ preffedRegions.includes(geo) && !!locales && locales.includes(locale);
+ return !regionBlocked && (localeEnabled || regionEnabled);
+ },
+ },
+ {
+ name: "systemtick",
+ factory: () => new lazy.SystemTickFeed(),
+ title: "Produces system tick events to periodically check for data expiry",
+ value: true,
+ },
+ {
+ name: "telemetry",
+ factory: () => new lazy.TelemetryFeed(),
+ title: "Relays telemetry-related actions to PingCentre",
+ value: true,
+ },
+ {
+ name: "favicon",
+ factory: () => new lazy.FaviconFeed(),
+ title: "Fetches tippy top manifests from remote service",
+ value: true,
+ },
+ {
+ name: "system.topsites",
+ factory: () => new lazy.TopSitesFeed(),
+ title: "Queries places and gets metadata for Top Sites section",
+ value: true,
+ },
+ {
+ name: "recommendationprovider",
+ factory: () => new lazy.RecommendationProvider(),
+ title: "Handles setup and interaction for the personality provider",
+ value: true,
+ },
+ {
+ name: "discoverystreamfeed",
+ factory: () => new lazy.DiscoveryStreamFeed(),
+ title: "Handles new pocket ui for the new tab page",
+ value: true,
+ },
+];
+
+const FEEDS_CONFIG = new Map();
+for (const config of FEEDS_DATA) {
+ const pref = `feeds.${config.name}`;
+ FEEDS_CONFIG.set(pref, config.factory);
+ PREFS_CONFIG.set(pref, config);
+}
+
+class ActivityStream {
+ /**
+ * constructor - Initializes an instance of ActivityStream
+ */
+ constructor() {
+ this.initialized = false;
+ this.store = new lazy.Store();
+ this.feeds = FEEDS_CONFIG;
+ this._defaultPrefs = new lazy.DefaultPrefs(PREFS_CONFIG);
+ }
+
+ init() {
+ try {
+ this._updateDynamicPrefs();
+ this._defaultPrefs.init();
+ Services.obs.addObserver(this, "intl:app-locales-changed");
+
+ // Look for outdated user pref values that might have been accidentally
+ // persisted when restoring the original pref value at the end of an
+ // experiment across versions with a different default value.
+ const DS_CONFIG =
+ "browser.newtabpage.activity-stream.discoverystream.config";
+ if (
+ Services.prefs.prefHasUserValue(DS_CONFIG) &&
+ [
+ // Firefox 66
+ `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.com/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`,
+ // Firefox 67
+ `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`,
+ // Firefox 68
+ `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","collapsible":true,"enabled":false,"show_spocs":true,"hardcoded_layout":true,"personalized":false,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`,
+ ].includes(Services.prefs.getStringPref(DS_CONFIG))
+ ) {
+ Services.prefs.clearUserPref(DS_CONFIG);
+ }
+
+ // Hook up the store and let all feeds and pages initialize
+ this.store.init(
+ this.feeds,
+ ac.BroadcastToContent({
+ type: at.INIT,
+ data: {
+ locale: this.locale,
+ },
+ meta: {
+ isStartup: true,
+ },
+ }),
+ { type: at.UNINIT }
+ );
+
+ this.initialized = true;
+ } catch (e) {
+ // TelemetryFeed could be unavailable if the telemetry is disabled, or
+ // the telemetry feed is not yet initialized.
+ const telemetryFeed = this.store.feeds.get("feeds.telemetry");
+ if (telemetryFeed) {
+ telemetryFeed.handleUndesiredEvent({
+ data: { event: "ADDON_INIT_FAILED" },
+ });
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Check if an old pref has a custom value to migrate. Clears the pref so that
+ * it's the default after migrating (to avoid future need to migrate).
+ *
+ * @param oldPrefName {string} Pref to check and migrate
+ * @param cbIfNotDefault {function} Callback that gets the current pref value
+ */
+ _migratePref(oldPrefName, cbIfNotDefault) {
+ // Nothing to do if the user doesn't have a custom value
+ if (!Services.prefs.prefHasUserValue(oldPrefName)) {
+ return;
+ }
+
+ // Figure out what kind of pref getter to use
+ let prefGetter;
+ switch (Services.prefs.getPrefType(oldPrefName)) {
+ case Services.prefs.PREF_BOOL:
+ prefGetter = "getBoolPref";
+ break;
+ case Services.prefs.PREF_INT:
+ prefGetter = "getIntPref";
+ break;
+ case Services.prefs.PREF_STRING:
+ prefGetter = "getStringPref";
+ break;
+ }
+
+ // Give the callback the current value then clear the pref
+ cbIfNotDefault(Services.prefs[prefGetter](oldPrefName));
+ Services.prefs.clearUserPref(oldPrefName);
+ }
+
+ uninit() {
+ if (this.geo === "") {
+ Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC);
+ }
+
+ Services.obs.removeObserver(this, "intl:app-locales-changed");
+
+ this.store.uninit();
+ this.initialized = false;
+ }
+
+ _updateDynamicPrefs() {
+ // Save the geo pref if we have it
+ if (lazy.Region.home) {
+ this.geo = lazy.Region.home;
+ } else if (this.geo !== "") {
+ // Watch for geo changes and use a dummy value for now
+ Services.obs.addObserver(this, lazy.Region.REGION_TOPIC);
+ this.geo = "";
+ }
+
+ this.locale = Services.locale.appLocaleAsBCP47;
+
+ // Update the pref config of those with dynamic values
+ for (const pref of PREFS_CONFIG.keys()) {
+ // Only need to process dynamic prefs
+ const prefConfig = PREFS_CONFIG.get(pref);
+ if (!prefConfig.getValue) {
+ continue;
+ }
+
+ // Have the dynamic pref just reuse using existing default, e.g., those
+ // set via Autoconfig or policy
+ try {
+ const existingDefault = this._defaultPrefs.get(pref);
+ if (existingDefault !== undefined && prefConfig.value === undefined) {
+ prefConfig.getValue = () => existingDefault;
+ }
+ } catch (ex) {
+ // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing
+ // default branch to believe there's a type) but no actual default value
+ }
+
+ // Compute the dynamic value (potentially generic based on dummy geo)
+ const newValue = prefConfig.getValue({
+ geo: this.geo,
+ locale: this.locale,
+ });
+
+ // If there's an existing value and it has changed, that means we need to
+ // overwrite the default with the new value.
+ if (prefConfig.value !== undefined && prefConfig.value !== newValue) {
+ this._defaultPrefs.set(pref, newValue);
+ }
+
+ prefConfig.value = newValue;
+ }
+ }
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "intl:app-locales-changed":
+ case lazy.Region.REGION_TOPIC:
+ this._updateDynamicPrefs();
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["ActivityStream", "PREFS_CONFIG"];
diff --git a/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm b/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm
new file mode 100644
index 0000000000..9f6ec301b4
--- /dev/null
+++ b/browser/components/newtab/lib/ActivityStreamMessageChannel.jsm
@@ -0,0 +1,345 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const lazy = {};
+
+// TODO delete this?
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "AboutNewTab",
+ "resource:///modules/AboutNewTab.jsm"
+);
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AboutHomeStartupCache: "resource:///modules/BrowserGlue.sys.mjs",
+});
+
+const { RemotePages } = ChromeUtils.import(
+ "resource://gre/modules/remotepagemanager/RemotePageManagerParent.jsm"
+);
+
+const {
+ actionCreators: ac,
+ actionTypes: at,
+ actionUtils: au,
+} = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const ABOUT_NEW_TAB_URL = "about:newtab";
+const ABOUT_HOME_URL = "about:home";
+
+const DEFAULT_OPTIONS = {
+ dispatch(action) {
+ throw new Error(
+ `\nMessageChannel: Received action ${action.type}, but no dispatcher was defined.\n`
+ );
+ },
+ pageURL: ABOUT_NEW_TAB_URL,
+ outgoingMessageName: "ActivityStream:MainToContent",
+ incomingMessageName: "ActivityStream:ContentToMain",
+};
+
+class ActivityStreamMessageChannel {
+ /**
+ * ActivityStreamMessageChannel - This module connects a Redux store to a RemotePageManager in Firefox.
+ * Call .createChannel to start the connection, and .destroyChannel to destroy it.
+ * You should use the BroadcastToContent, AlsoToOneContent, and AlsoToMain action creators
+ * in common/Actions.sys.mjs to help you create actions that will be automatically routed
+ * to the correct location.
+ *
+ * @param {object} options
+ * @param {function} options.dispatch The dispatch method from a Redux store
+ * @param {string} options.pageURL The URL to which a RemotePageManager should be attached.
+ * Note that if it is about:newtab, the existing RemotePageManager
+ * for about:newtab will also be disabled
+ * @param {string} options.outgoingMessageName The name of the message sent to child processes
+ * @param {string} options.incomingMessageName The name of the message received from child processes
+ * @return {ActivityStreamMessageChannel}
+ */
+ constructor(options = {}) {
+ Object.assign(this, DEFAULT_OPTIONS, options);
+ this.channel = null;
+
+ this.middleware = this.middleware.bind(this);
+ this.onMessage = this.onMessage.bind(this);
+ this.onNewTabLoad = this.onNewTabLoad.bind(this);
+ this.onNewTabUnload = this.onNewTabUnload.bind(this);
+ this.onNewTabInit = this.onNewTabInit.bind(this);
+ }
+
+ /**
+ * middleware - Redux middleware that looks for AlsoToOneContent and BroadcastToContent type
+ * actions, and sends them out.
+ *
+ * @param {object} store A redux store
+ * @return {function} Redux middleware
+ */
+ middleware(store) {
+ return next => action => {
+ const skipMain = action.meta && action.meta.skipMain;
+ if (!this.channel && !skipMain) {
+ next(action);
+ return;
+ }
+ if (au.isSendToOneContent(action)) {
+ this.send(action);
+ } else if (au.isBroadcastToContent(action)) {
+ this.broadcast(action);
+ } else if (au.isSendToPreloaded(action)) {
+ this.sendToPreloaded(action);
+ }
+
+ if (!skipMain) {
+ next(action);
+ }
+ };
+ }
+
+ /**
+ * onActionFromContent - Handler for actions from a content processes
+ *
+ * @param {object} action A Redux action
+ * @param {string} targetId The portID of the port that sent the message
+ */
+ onActionFromContent(action, targetId) {
+ this.dispatch(ac.AlsoToMain(action, this.validatePortID(targetId)));
+ }
+
+ /**
+ * broadcast - Sends an action to all ports
+ *
+ * @param {object} action A Redux action
+ */
+ broadcast(action) {
+ // We're trying to update all tabs, so signal the AboutHomeStartupCache
+ // that its likely time to refresh the cache.
+ lazy.AboutHomeStartupCache.onPreloadedNewTabMessage();
+
+ this.channel.sendAsyncMessage(this.outgoingMessageName, action);
+ }
+
+ /**
+ * send - Sends an action to a specific port
+ *
+ * @param {obj} action A redux action; it should contain a portID in the meta.toTarget property
+ */
+ send(action) {
+ const targetId = action.meta && action.meta.toTarget;
+ const target = this.getTargetById(targetId);
+ try {
+ target.sendAsyncMessage(this.outgoingMessageName, action);
+ } catch (e) {
+ // The target page is closed/closing by the user or test, so just ignore.
+ }
+ }
+
+ /**
+ * A valid portID is a combination of process id and port
+ * https://searchfox.org/mozilla-central/rev/196560b95f191b48ff7cba7c2ba9237bba6b5b6a/toolkit/components/remotepagemanager/RemotePageManagerChild.jsm#14
+ */
+ validatePortID(id) {
+ if (typeof id !== "string" || !id.includes(":")) {
+ console.error("Invalid portID");
+ }
+
+ return id;
+ }
+
+ /**
+ * getIdByTarget - Retrieve the id of a message target, if it exists in this.targets
+ *
+ * @param {obj} targetObj A message target
+ * @return {string|null} The unique id of the target, if it exists.
+ */
+ getTargetById(id) {
+ this.validatePortID(id);
+ for (let port of this.channel.messagePorts) {
+ if (port.portID === id) {
+ return port;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * sendToPreloaded - Sends an action to each preloaded browser, if any
+ *
+ * @param {obj} action A redux action
+ */
+ sendToPreloaded(action) {
+ // We're trying to update the preloaded about:newtab, so signal
+ // the AboutHomeStartupCache that its likely time to refresh
+ // the cache.
+ lazy.AboutHomeStartupCache.onPreloadedNewTabMessage();
+
+ const preloadedBrowsers = this.getPreloadedBrowser();
+ if (preloadedBrowsers && action.data) {
+ for (let preloadedBrowser of preloadedBrowsers) {
+ try {
+ preloadedBrowser.sendAsyncMessage(this.outgoingMessageName, action);
+ } catch (e) {
+ // The preloaded page is no longer available, so just ignore.
+ }
+ }
+ }
+ }
+
+ /**
+ * getPreloadedBrowser - Retrieve the port of any preloaded browsers
+ *
+ * @return {Array|null} An array of ports belonging to the preloaded browsers, or null
+ * if there aren't any preloaded browsers
+ */
+ getPreloadedBrowser() {
+ let preloadedPorts = [];
+ for (let port of this.channel.messagePorts) {
+ if (this.isPreloadedBrowser(port.browser)) {
+ preloadedPorts.push(port);
+ }
+ }
+ return preloadedPorts.length ? preloadedPorts : null;
+ }
+
+ /**
+ * isPreloadedBrowser - Returns true if the passed browser has been preloaded
+ * for faster rendering of new tabs.
+ *
+ * @param {<browser>} A <browser> to check.
+ * @return {bool} True if the browser is preloaded.
+ * if there aren't any preloaded browsers
+ */
+ isPreloadedBrowser(browser) {
+ return browser.getAttribute("preloadedState") === "preloaded";
+ }
+
+ /**
+ * createChannel - Create RemotePages channel to establishing message passing
+ * between the main process and child pages
+ */
+ createChannel() {
+ // Receive AboutNewTab's Remote Pages instance, if it exists, on override
+ const channel =
+ this.pageURL === ABOUT_NEW_TAB_URL &&
+ lazy.AboutNewTab.overridePageListener(true);
+ this.channel =
+ channel || new RemotePages([ABOUT_HOME_URL, ABOUT_NEW_TAB_URL]);
+ this.channel.addMessageListener("RemotePage:Init", this.onNewTabInit);
+ this.channel.addMessageListener("RemotePage:Load", this.onNewTabLoad);
+ this.channel.addMessageListener("RemotePage:Unload", this.onNewTabUnload);
+ this.channel.addMessageListener(this.incomingMessageName, this.onMessage);
+ }
+
+ simulateMessagesForExistingTabs() {
+ // Some pages might have already loaded, so we won't get the usual message
+ for (const target of this.channel.messagePorts) {
+ const simulatedMsg = {
+ target: Object.assign({ simulated: true }, target),
+ };
+ this.onNewTabInit(simulatedMsg);
+ if (target.loaded) {
+ this.onNewTabLoad(simulatedMsg);
+ }
+ }
+ }
+
+ /**
+ * destroyChannel - Destroys the RemotePages channel
+ */
+ destroyChannel() {
+ this.channel.removeMessageListener("RemotePage:Init", this.onNewTabInit);
+ this.channel.removeMessageListener("RemotePage:Load", this.onNewTabLoad);
+ this.channel.removeMessageListener(
+ "RemotePage:Unload",
+ this.onNewTabUnload
+ );
+ this.channel.removeMessageListener(
+ this.incomingMessageName,
+ this.onMessage
+ );
+ if (this.pageURL === ABOUT_NEW_TAB_URL) {
+ lazy.AboutNewTab.reset(this.channel);
+ } else {
+ this.channel.destroy();
+ }
+ this.channel = null;
+ }
+
+ /**
+ * onNewTabInit - Handler for special RemotePage:Init message fired
+ * by RemotePages
+ *
+ * @param {obj} msg The messsage from a page that was just initialized
+ */
+ onNewTabInit(msg) {
+ this.onActionFromContent(
+ {
+ type: at.NEW_TAB_INIT,
+ data: msg.target,
+ },
+ msg.target.portID
+ );
+ }
+
+ /**
+ * onNewTabLoad - Handler for special RemotePage:Load message fired by RemotePages
+ *
+ * @param {obj} msg The messsage from a page that was just loaded
+ */
+ onNewTabLoad(msg) {
+ let { browser } = msg.target;
+ if (
+ this.isPreloadedBrowser(browser) &&
+ browser.ownerGlobal.windowState !== browser.ownerGlobal.STATE_MINIMIZED &&
+ !browser.ownerGlobal.isFullyOccluded
+ ) {
+ // As a perceived performance optimization, if this loaded Activity Stream
+ // happens to be a preloaded browser in a window that is not minimized or
+ // occluded, have it render its layers to the compositor now to increase
+ // the odds that by the time we switch to the tab, the layers are already
+ // ready to present to the user.
+ browser.renderLayers = true;
+ }
+
+ this.onActionFromContent({ type: at.NEW_TAB_LOAD }, msg.target.portID);
+ }
+
+ /**
+ * onNewTabUnloadLoad - Handler for special RemotePage:Unload message fired by RemotePages
+ *
+ * @param {obj} msg The messsage from a page that was just unloaded
+ */
+ onNewTabUnload(msg) {
+ this.onActionFromContent({ type: at.NEW_TAB_UNLOAD }, msg.target.portID);
+ }
+
+ /**
+ * onMessage - Handles custom messages from content. It expects all messages to
+ * be formatted as Redux actions, and dispatches them to this.store
+ *
+ * @param {obj} msg A custom message from content
+ * @param {obj} msg.action A Redux action (e.g. {type: "HELLO_WORLD"})
+ * @param {obj} msg.target A message target
+ */
+ onMessage(msg) {
+ const { portID } = msg.target;
+ if (!msg.data || !msg.data.type) {
+ console.error(
+ new Error(`Received an improperly formatted message from ${portID}`)
+ );
+ return;
+ }
+ let action = {};
+ Object.assign(action, msg.data);
+ // target is used to access a browser reference that came from the content
+ // and should only be used in feeds (not reducers)
+ action._target = msg.target;
+ this.onActionFromContent(action, portID);
+ }
+}
+
+const EXPORTED_SYMBOLS = ["ActivityStreamMessageChannel", "DEFAULT_OPTIONS"];
diff --git a/browser/components/newtab/lib/ActivityStreamPrefs.jsm b/browser/components/newtab/lib/ActivityStreamPrefs.jsm
new file mode 100644
index 0000000000..8614671903
--- /dev/null
+++ b/browser/components/newtab/lib/ActivityStreamPrefs.jsm
@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+const { Preferences } = ChromeUtils.importESModule(
+ "resource://gre/modules/Preferences.sys.mjs"
+);
+
+const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream.";
+
+class Prefs extends Preferences {
+ /**
+ * Prefs - A wrapper around Preferences that always sets the branch to
+ * ACTIVITY_STREAM_PREF_BRANCH
+ */
+ constructor(branch = ACTIVITY_STREAM_PREF_BRANCH) {
+ super({ branch });
+ this._branchObservers = new Map();
+ }
+
+ ignoreBranch(listener) {
+ const observer = this._branchObservers.get(listener);
+ this._prefBranch.removeObserver("", observer);
+ this._branchObservers.delete(listener);
+ }
+
+ observeBranch(listener) {
+ const observer = (subject, topic, pref) => {
+ listener.onPrefChanged(pref, this.get(pref));
+ };
+ this._prefBranch.addObserver("", observer);
+ this._branchObservers.set(listener, observer);
+ }
+}
+
+class DefaultPrefs extends Preferences {
+ /**
+ * DefaultPrefs - A helper for setting and resetting default prefs for the add-on
+ *
+ * @param {Map} config A Map with {string} key of the pref name and {object}
+ * value with the following pref properties:
+ * {string} .title (optional) A description of the pref
+ * {bool|string|number} .value The default value for the pref
+ * @param {string} branch (optional) The pref branch (defaults to ACTIVITY_STREAM_PREF_BRANCH)
+ */
+ constructor(config, branch = ACTIVITY_STREAM_PREF_BRANCH) {
+ super({
+ branch,
+ defaultBranch: true,
+ });
+ this._config = config;
+ }
+
+ /**
+ * init - Set default prefs for all prefs in the config
+ */
+ init() {
+ // Local developer builds (with the default mozconfig) aren't OFFICIAL
+ const IS_UNOFFICIAL_BUILD = !AppConstants.MOZILLA_OFFICIAL;
+
+ for (const pref of this._config.keys()) {
+ try {
+ // Avoid replacing existing valid default pref values, e.g., those set
+ // via Autoconfig or policy
+ if (this.get(pref) !== undefined) {
+ continue;
+ }
+ } catch (ex) {
+ // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing
+ // default branch to believe there's a type) but no actual default value
+ }
+
+ const prefConfig = this._config.get(pref);
+ let value;
+ if (IS_UNOFFICIAL_BUILD && "value_local_dev" in prefConfig) {
+ value = prefConfig.value_local_dev;
+ } else {
+ value = prefConfig.value;
+ }
+
+ try {
+ this.set(pref, value);
+ } catch (ex) {
+ // Potentially the user somehow set an unexpected value type, so we fail
+ // to set a default of our expected type
+ }
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["DefaultPrefs", "Prefs"];
diff --git a/browser/components/newtab/lib/ActivityStreamStorage.jsm b/browser/components/newtab/lib/ActivityStreamStorage.jsm
new file mode 100644
index 0000000000..f34e289f8a
--- /dev/null
+++ b/browser/components/newtab/lib/ActivityStreamStorage.jsm
@@ -0,0 +1,121 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ IndexedDB: "resource://gre/modules/IndexedDB.sys.mjs",
+});
+
+class ActivityStreamStorage {
+ /**
+ * @param storeNames Array of strings used to create all the required stores
+ */
+ constructor({ storeNames, telemetry }) {
+ if (!storeNames) {
+ throw new Error("storeNames required");
+ }
+
+ this.dbName = "ActivityStream";
+ this.dbVersion = 3;
+ this.storeNames = storeNames;
+ this.telemetry = telemetry;
+ }
+
+ get db() {
+ return this._db || (this._db = this.createOrOpenDb());
+ }
+
+ /**
+ * Public method that binds the store required by the consumer and exposes
+ * the private db getters and setters.
+ *
+ * @param storeName String name of desired store
+ */
+ getDbTable(storeName) {
+ if (this.storeNames.includes(storeName)) {
+ return {
+ get: this._get.bind(this, storeName),
+ getAll: this._getAll.bind(this, storeName),
+ set: this._set.bind(this, storeName),
+ };
+ }
+
+ throw new Error(`Store name ${storeName} does not exist.`);
+ }
+
+ async _getStore(storeName) {
+ return (await this.db).objectStore(storeName, "readwrite");
+ }
+
+ _get(storeName, key) {
+ return this._requestWrapper(async () =>
+ (await this._getStore(storeName)).get(key)
+ );
+ }
+
+ _getAll(storeName) {
+ return this._requestWrapper(async () =>
+ (await this._getStore(storeName)).getAll()
+ );
+ }
+
+ _set(storeName, key, value) {
+ return this._requestWrapper(async () =>
+ (await this._getStore(storeName)).put(value, key)
+ );
+ }
+
+ _openDatabase() {
+ return lazy.IndexedDB.open(this.dbName, { version: this.dbVersion }, db => {
+ // If provided with array of objectStore names we need to create all the
+ // individual stores
+ this.storeNames.forEach(store => {
+ if (!db.objectStoreNames.contains(store)) {
+ this._requestWrapper(() => db.createObjectStore(store));
+ }
+ });
+ });
+ }
+
+ /**
+ * createOrOpenDb - Open a db (with this.dbName) if it exists.
+ * If it does not exist, create it.
+ * If an error occurs, deleted the db and attempt to
+ * re-create it.
+ * @returns Promise that resolves with a db instance
+ */
+ async createOrOpenDb() {
+ try {
+ const db = await this._openDatabase();
+ return db;
+ } catch (e) {
+ if (this.telemetry) {
+ this.telemetry.handleUndesiredEvent({ event: "INDEXEDDB_OPEN_FAILED" });
+ }
+ await lazy.IndexedDB.deleteDatabase(this.dbName);
+ return this._openDatabase();
+ }
+ }
+
+ async _requestWrapper(request) {
+ let result = null;
+ try {
+ result = await request();
+ } catch (e) {
+ if (this.telemetry) {
+ this.telemetry.handleUndesiredEvent({ event: "TRANSACTION_FAILED" });
+ }
+ throw e;
+ }
+
+ return result;
+ }
+}
+
+function getDefaultOptions(options) {
+ return { collapsed: !!options.collapsed };
+}
+
+const EXPORTED_SYMBOLS = ["ActivityStreamStorage", "getDefaultOptions"];
diff --git a/browser/components/newtab/lib/CFRMessageProvider.jsm b/browser/components/newtab/lib/CFRMessageProvider.jsm
new file mode 100644
index 0000000000..dde7c3e194
--- /dev/null
+++ b/browser/components/newtab/lib/CFRMessageProvider.jsm
@@ -0,0 +1,829 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+const FACEBOOK_CONTAINER_PARAMS = {
+ existing_addons: [
+ "@contain-facebook",
+ "{bb1b80be-e6b3-40a1-9b6e-9d4073343f0b}",
+ "{a50d61ca-d27b-437a-8b52-5fd801a0a88b}",
+ ],
+ open_urls: ["www.facebook.com", "facebook.com"],
+ sumo_path: "extensionrecommendations",
+ min_frecency: 10000,
+};
+const GOOGLE_TRANSLATE_PARAMS = {
+ existing_addons: [
+ "jid1-93WyvpgvxzGATw@jetpack",
+ "{087ef4e1-4286-4be6-9aa3-8d6c420ee1db}",
+ "{4170faaa-ee87-4a0e-b57a-1aec49282887}",
+ "jid1-TMndP6cdKgxLcQ@jetpack",
+ "s3google@translator",
+ "{9c63d15c-b4d9-43bd-b223-37f0a1f22e2a}",
+ "translator@zoli.bod",
+ "{8cda9ce6-7893-4f47-ac70-a65215cec288}",
+ "simple-translate@sienori",
+ "@translatenow",
+ "{a79fafce-8da6-4685-923f-7ba1015b8748})",
+ "{8a802b5a-eeab-11e2-a41d-b0096288709b}",
+ "jid0-fbHwsGfb6kJyq2hj65KnbGte3yT@jetpack",
+ "storetranslate.plugin@gmail.com",
+ "jid1-r2tWDbSkq8AZK1@jetpack",
+ "{b384b75c-c978-4c4d-b3cf-62a82d8f8f12}",
+ "jid1-f7dnBeTj8ElpWQ@jetpack",
+ "{dac8a935-4775-4918-9205-5c0600087dc4}",
+ "gtranslation2@slam.com",
+ "{e20e0de5-1667-4df4-bd69-705720e37391}",
+ "{09e26ae9-e9c1-477c-80a6-99934212f2fe}",
+ "mgxtranslator@magemagix.com",
+ "gtranslatewins@mozilla.org",
+ ],
+ open_urls: ["translate.google.com"],
+ sumo_path: "extensionrecommendations",
+ min_frecency: 10000,
+};
+const YOUTUBE_ENHANCE_PARAMS = {
+ existing_addons: [
+ "enhancerforyoutube@maximerf.addons.mozilla.org",
+ "{dc8f61ab-5e98-4027-98ef-bb2ff6060d71}",
+ "{7b1bf0b6-a1b9-42b0-b75d-252036438bdc}",
+ "jid0-UVAeBCfd34Kk5usS8A1CBiobvM8@jetpack",
+ "iridium@particlecore.github.io",
+ "jid1-ss6kLNCbNz6u0g@jetpack",
+ "{1cf918d2-f4ea-4b4f-b34e-455283fef19f}",
+ ],
+ open_urls: ["www.youtube.com", "youtube.com"],
+ sumo_path: "extensionrecommendations",
+ min_frecency: 10000,
+};
+const WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS = {
+ existing_addons: [
+ "@wikipediacontextmenusearch",
+ "{ebf47fc8-01d8-4dba-aa04-2118402f4b20}",
+ "{5737a280-b359-4e26-95b0-adec5915a854}",
+ "olivier.debroqueville@gmail.com",
+ "{3923146e-98cb-472b-9c13-f6849d34d6b8}",
+ ],
+ open_urls: ["www.wikipedia.org", "wikipedia.org"],
+ sumo_path: "extensionrecommendations",
+ min_frecency: 10000,
+};
+const REDDIT_ENHANCEMENT_PARAMS = {
+ existing_addons: ["jid1-xUfzOsOFlzSOXg@jetpack"],
+ open_urls: ["www.reddit.com", "reddit.com"],
+ sumo_path: "extensionrecommendations",
+ min_frecency: 10000,
+};
+
+const CFR_MESSAGES = [
+ {
+ id: "FACEBOOK_CONTAINER_3",
+ template: "cfr_doorhanger",
+ groups: ["cfr-message-provider"],
+ content: {
+ layout: "addon_recommendation",
+ category: "cfrAddons",
+ bucket_id: "CFR_M1",
+ notification_text: {
+ string_id: "cfr-doorhanger-extension-notification2",
+ },
+ heading_text: { string_id: "cfr-doorhanger-extension-heading" },
+ info_icon: {
+ label: { string_id: "cfr-doorhanger-extension-sumo-link" },
+ sumo_path: FACEBOOK_CONTAINER_PARAMS.sumo_path,
+ },
+ addon: {
+ id: "954390",
+ title: "Facebook Container",
+ icon: "chrome://browser/skin/addons/addon-install-downloading.svg",
+ rating: 4.6,
+ users: 299019,
+ author: "Mozilla",
+ amo_url: "https://addons.mozilla.org/firefox/addon/facebook-container/",
+ },
+ text:
+ "Stop Facebook from tracking your activity across the web. Use Facebook the way you normally do without annoying ads following you around.",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-extension-ok-button" },
+ action: {
+ type: "INSTALL_ADDON_FROM_URL",
+ data: { url: "https://example.com", telemetrySource: "amo" },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfraddons" },
+ },
+ },
+ ],
+ },
+ },
+ frequency: { lifetime: 3 },
+ targeting: `
+ localeLanguageCode == "en" &&
+ (xpinstallEnabled == true) &&
+ (${JSON.stringify(
+ FACEBOOK_CONTAINER_PARAMS.existing_addons
+ )} intersect addonsInfo.addons|keys)|length == 0 &&
+ (${JSON.stringify(
+ FACEBOOK_CONTAINER_PARAMS.open_urls
+ )} intersect topFrecentSites[.frecency >= ${
+ FACEBOOK_CONTAINER_PARAMS.min_frecency
+ }]|mapToProperty('host'))|length > 0`,
+ trigger: { id: "openURL", params: FACEBOOK_CONTAINER_PARAMS.open_urls },
+ },
+ {
+ id: "GOOGLE_TRANSLATE_3",
+ groups: ["cfr-message-provider"],
+ template: "cfr_doorhanger",
+ content: {
+ layout: "addon_recommendation",
+ category: "cfrAddons",
+ bucket_id: "CFR_M1",
+ notification_text: {
+ string_id: "cfr-doorhanger-extension-notification2",
+ },
+ heading_text: { string_id: "cfr-doorhanger-extension-heading" },
+ info_icon: {
+ label: { string_id: "cfr-doorhanger-extension-sumo-link" },
+ sumo_path: GOOGLE_TRANSLATE_PARAMS.sumo_path,
+ },
+ addon: {
+ id: "445852",
+ title: "To Google Translate",
+ icon: "chrome://browser/skin/addons/addon-install-downloading.svg",
+ rating: 4.1,
+ users: 313474,
+ author: "Juan Escobar",
+ amo_url:
+ "https://addons.mozilla.org/firefox/addon/to-google-translate/",
+ },
+ text:
+ "Instantly translate any webpage text. Simply highlight the text, right-click to open the context menu, and choose a text or aural translation.",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-extension-ok-button" },
+ action: {
+ type: "INSTALL_ADDON_FROM_URL",
+ data: { url: "https://example.com", telemetrySource: "amo" },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfraddons" },
+ },
+ },
+ ],
+ },
+ },
+ frequency: { lifetime: 3 },
+ targeting: `
+ localeLanguageCode == "en" &&
+ (xpinstallEnabled == true) &&
+ (${JSON.stringify(
+ GOOGLE_TRANSLATE_PARAMS.existing_addons
+ )} intersect addonsInfo.addons|keys)|length == 0 &&
+ (${JSON.stringify(
+ GOOGLE_TRANSLATE_PARAMS.open_urls
+ )} intersect topFrecentSites[.frecency >= ${
+ GOOGLE_TRANSLATE_PARAMS.min_frecency
+ }]|mapToProperty('host'))|length > 0`,
+ trigger: { id: "openURL", params: GOOGLE_TRANSLATE_PARAMS.open_urls },
+ },
+ {
+ id: "YOUTUBE_ENHANCE_3",
+ groups: ["cfr-message-provider"],
+ template: "cfr_doorhanger",
+ content: {
+ layout: "addon_recommendation",
+ category: "cfrAddons",
+ bucket_id: "CFR_M1",
+ notification_text: {
+ string_id: "cfr-doorhanger-extension-notification2",
+ },
+ heading_text: { string_id: "cfr-doorhanger-extension-heading" },
+ info_icon: {
+ label: { string_id: "cfr-doorhanger-extension-sumo-link" },
+ sumo_path: YOUTUBE_ENHANCE_PARAMS.sumo_path,
+ },
+ addon: {
+ id: "700308",
+ title: "Enhancer for YouTube\u2122",
+ icon: "chrome://browser/skin/addons/addon-install-downloading.svg",
+ rating: 4.8,
+ users: 357328,
+ author: "Maxime RF",
+ amo_url:
+ "https://addons.mozilla.org/firefox/addon/enhancer-for-youtube/",
+ },
+ text:
+ "Take control of your YouTube experience. Automatically block annoying ads, set playback speed and volume, remove annotations, and more.",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-extension-ok-button" },
+ action: {
+ type: "INSTALL_ADDON_FROM_URL",
+ data: { url: "https://example.com", telemetrySource: "amo" },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfraddons" },
+ },
+ },
+ ],
+ },
+ },
+ frequency: { lifetime: 3 },
+ targeting: `
+ localeLanguageCode == "en" &&
+ (xpinstallEnabled == true) &&
+ (${JSON.stringify(
+ YOUTUBE_ENHANCE_PARAMS.existing_addons
+ )} intersect addonsInfo.addons|keys)|length == 0 &&
+ (${JSON.stringify(
+ YOUTUBE_ENHANCE_PARAMS.open_urls
+ )} intersect topFrecentSites[.frecency >= ${
+ YOUTUBE_ENHANCE_PARAMS.min_frecency
+ }]|mapToProperty('host'))|length > 0`,
+ trigger: { id: "openURL", params: YOUTUBE_ENHANCE_PARAMS.open_urls },
+ },
+ {
+ id: "WIKIPEDIA_CONTEXT_MENU_SEARCH_3",
+ groups: ["cfr-message-provider"],
+ template: "cfr_doorhanger",
+ exclude: true,
+ content: {
+ layout: "addon_recommendation",
+ category: "cfrAddons",
+ bucket_id: "CFR_M1",
+ notification_text: {
+ string_id: "cfr-doorhanger-extension-notification2",
+ },
+ heading_text: { string_id: "cfr-doorhanger-extension-heading" },
+ info_icon: {
+ label: { string_id: "cfr-doorhanger-extension-sumo-link" },
+ sumo_path: WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.sumo_path,
+ },
+ addon: {
+ id: "659026",
+ title: "Wikipedia Context Menu Search",
+ icon: "chrome://browser/skin/addons/addon-install-downloading.svg",
+ rating: 4.9,
+ users: 3095,
+ author: "Nick Diedrich",
+ amo_url:
+ "https://addons.mozilla.org/firefox/addon/wikipedia-context-menu-search/",
+ },
+ text:
+ "Get to a Wikipedia page fast, from anywhere on the web. Just highlight any webpage text and right-click to open the context menu to start a Wikipedia search.",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-extension-ok-button" },
+ action: {
+ type: "INSTALL_ADDON_FROM_URL",
+ data: { url: "https://example.com", telemetrySource: "amo" },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfraddons" },
+ },
+ },
+ ],
+ },
+ },
+ frequency: { lifetime: 3 },
+ targeting: `
+ localeLanguageCode == "en" &&
+ (xpinstallEnabled == true) &&
+ (${JSON.stringify(
+ WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.existing_addons
+ )} intersect addonsInfo.addons|keys)|length == 0 &&
+ (${JSON.stringify(
+ WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.open_urls
+ )} intersect topFrecentSites[.frecency >= ${
+ WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.min_frecency
+ }]|mapToProperty('host'))|length > 0`,
+ trigger: {
+ id: "openURL",
+ params: WIKIPEDIA_CONTEXT_MENU_SEARCH_PARAMS.open_urls,
+ },
+ },
+ {
+ id: "REDDIT_ENHANCEMENT_3",
+ groups: ["cfr-message-provider"],
+ template: "cfr_doorhanger",
+ exclude: true,
+ content: {
+ layout: "addon_recommendation",
+ category: "cfrAddons",
+ bucket_id: "CFR_M1",
+ notification_text: {
+ string_id: "cfr-doorhanger-extension-notification2",
+ },
+ heading_text: { string_id: "cfr-doorhanger-extension-heading" },
+ info_icon: {
+ label: { string_id: "cfr-doorhanger-extension-sumo-link" },
+ sumo_path: REDDIT_ENHANCEMENT_PARAMS.sumo_path,
+ },
+ addon: {
+ id: "387429",
+ title: "Reddit Enhancement Suite",
+ icon: "chrome://browser/skin/addons/addon-install-downloading.svg",
+ rating: 4.6,
+ users: 258129,
+ author: "honestbleeps",
+ amo_url:
+ "https://addons.mozilla.org/firefox/addon/reddit-enhancement-suite/",
+ },
+ text:
+ "New features include Inline Image Viewer, Never Ending Reddit (never click 'next page' again), Keyboard Navigation, Account Switcher, and User Tagger.",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-extension-ok-button" },
+ action: {
+ type: "INSTALL_ADDON_FROM_URL",
+ data: { url: "https://example.com", telemetrySource: "amo" },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfraddons" },
+ },
+ },
+ ],
+ },
+ },
+ frequency: { lifetime: 3 },
+ targeting: `
+ localeLanguageCode == "en" &&
+ (xpinstallEnabled == true) &&
+ (${JSON.stringify(
+ REDDIT_ENHANCEMENT_PARAMS.existing_addons
+ )} intersect addonsInfo.addons|keys)|length == 0 &&
+ (${JSON.stringify(
+ REDDIT_ENHANCEMENT_PARAMS.open_urls
+ )} intersect topFrecentSites[.frecency >= ${
+ REDDIT_ENHANCEMENT_PARAMS.min_frecency
+ }]|mapToProperty('host'))|length > 0`,
+ trigger: { id: "openURL", params: REDDIT_ENHANCEMENT_PARAMS.open_urls },
+ },
+ {
+ id: "DOH_ROLLOUT_CONFIRMATION",
+ groups: ["cfr-message-provider"],
+ targeting: `
+ "doh-rollout.enabled"|preferenceValue &&
+ !"doh-rollout.disable-heuristics"|preferenceValue &&
+ !"doh-rollout.skipHeuristicsCheck"|preferenceValue &&
+ !"doh-rollout.doorhanger-decision"|preferenceValue
+ `,
+ template: "cfr_doorhanger",
+ content: {
+ skip_address_bar_notifier: true,
+ persistent_doorhanger: true,
+ anchor_id: "PanelUI-menu-button",
+ layout: "icon_and_message",
+ text: { string_id: "cfr-doorhanger-doh-body" },
+ icon: "chrome://global/skin/icons/security.svg",
+ buttons: {
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-doh-secondary-button" },
+ action: {
+ type: "DISABLE_DOH",
+ },
+ },
+ ],
+ primary: {
+ label: { string_id: "cfr-doorhanger-doh-primary-button-2" },
+ action: {
+ type: "ACCEPT_DOH",
+ },
+ },
+ },
+ bucket_id: "DOH_ROLLOUT_CONFIRMATION",
+ heading_text: { string_id: "cfr-doorhanger-doh-header" },
+ info_icon: {
+ label: {
+ string_id: "cfr-doorhanger-extension-sumo-link",
+ },
+ sumo_path: "extensionrecommendations",
+ },
+ notification_text: "Message from Firefox",
+ category: "cfrFeatures",
+ },
+ trigger: {
+ id: "openURL",
+ patterns: ["*://*/*"],
+ },
+ },
+ {
+ id: "SAVE_LOGIN",
+ groups: ["cfr-message-provider"],
+ frequency: {
+ lifetime: 3,
+ },
+ targeting:
+ "(!type || type == 'save') && isFxAEnabled == true && usesFirefoxSync == false",
+ template: "cfr_doorhanger",
+ content: {
+ layout: "icon_and_message",
+ text: "Securely store and sync your passwords to all your devices.",
+ icon: "chrome://browser/content/aboutlogins/icons/intro-illustration.svg",
+ icon_class: "cfr-doorhanger-large-icon",
+ buttons: {
+ secondary: [
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-cancel-button",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: {
+ category: "general-cfrfeatures",
+ },
+ },
+ },
+ ],
+ primary: {
+ label: {
+ value: "Turn on Sync",
+ attributes: { accesskey: "T" },
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: {
+ category: "sync",
+ entrypoint: "cfr-save-login",
+ },
+ },
+ },
+ },
+ bucket_id: "CFR_SAVE_LOGIN",
+ heading_text: "Never Lose a Password Again",
+ info_icon: {
+ label: {
+ string_id: "cfr-doorhanger-extension-sumo-link",
+ },
+ sumo_path: "extensionrecommendations",
+ },
+ notification_text: {
+ string_id: "cfr-doorhanger-feature-notification",
+ },
+ category: "cfrFeatures",
+ },
+ trigger: {
+ id: "newSavedLogin",
+ },
+ },
+ {
+ id: "UPDATE_LOGIN",
+ groups: ["cfr-message-provider"],
+ frequency: {
+ lifetime: 3,
+ },
+ targeting:
+ "type == 'update' && isFxAEnabled == true && usesFirefoxSync == false",
+ template: "cfr_doorhanger",
+ content: {
+ layout: "icon_and_message",
+ text: "Securely store and sync your passwords to all your devices.",
+ icon: "chrome://browser/content/aboutlogins/icons/intro-illustration.svg",
+ icon_class: "cfr-doorhanger-large-icon",
+ buttons: {
+ secondary: [
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-cancel-button",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: {
+ category: "general-cfrfeatures",
+ },
+ },
+ },
+ ],
+ primary: {
+ label: {
+ value: "Turn on Sync",
+ attributes: { accesskey: "T" },
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: {
+ category: "sync",
+ entrypoint: "cfr-update-login",
+ },
+ },
+ },
+ },
+ bucket_id: "CFR_UPDATE_LOGIN",
+ heading_text: "Never Lose a Password Again",
+ info_icon: {
+ label: {
+ string_id: "cfr-doorhanger-extension-sumo-link",
+ },
+ sumo_path: "extensionrecommendations",
+ },
+ notification_text: {
+ string_id: "cfr-doorhanger-feature-notification",
+ },
+ category: "cfrFeatures",
+ },
+ trigger: {
+ id: "newSavedLogin",
+ },
+ },
+ {
+ id: "MILESTONE_MESSAGE",
+ groups: ["cfr-message-provider"],
+ template: "milestone_message",
+ content: {
+ layout: "short_message",
+ category: "cfrFeatures",
+ anchor_id: "tracking-protection-icon-box",
+ skip_address_bar_notifier: true,
+ bucket_id: "CFR_MILESTONE_MESSAGE",
+ heading_text: { string_id: "cfr-doorhanger-milestone-heading2" },
+ notification_text: "",
+ text: "",
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-milestone-ok-button" },
+ action: { type: "OPEN_PROTECTION_REPORT" },
+ event: "PROTECTION",
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-milestone-close-button" },
+ action: { type: "CANCEL" },
+ event: "DISMISS",
+ },
+ ],
+ },
+ },
+ targeting: "pageLoad >= 1",
+ frequency: {
+ lifetime: 7, // Length of privacy.contentBlocking.cfr-milestone.milestones pref
+ },
+ trigger: {
+ id: "contentBlocking",
+ params: ["ContentBlockingMilestone"],
+ },
+ },
+ {
+ id: "HEARTBEAT_TACTIC_2",
+ groups: ["cfr-message-provider"],
+ template: "cfr_urlbar_chiclet",
+ content: {
+ layout: "chiclet_open_url",
+ category: "cfrHeartbeat",
+ bucket_id: "HEARTBEAT_TACTIC_2",
+ notification_text: "Improve Firefox",
+ active_color: "#595e91",
+ action: {
+ url: "http://example.com/%VERSION%/",
+ where: "tabshifted",
+ },
+ },
+ targeting: "false",
+ frequency: {
+ lifetime: 3,
+ },
+ trigger: {
+ id: "openURL",
+ patterns: ["*://*/*"],
+ },
+ },
+ {
+ id: "HOMEPAGE_REMEDIATION_82",
+ groups: ["cfr-message-provider"],
+ frequency: {
+ lifetime: 3,
+ },
+ targeting:
+ "!homePageSettings.isDefault && homePageSettings.isCustomUrl && homePageSettings.urls[.host == 'google.com']|length > 0 && visitsCount >= 3 && userPrefs.cfrFeatures",
+ template: "cfr_doorhanger",
+ content: {
+ layout: "icon_and_message",
+ text:
+ "Update your homepage to search Google while also being able to search your Firefox history and bookmarks.",
+ icon: "chrome://global/skin/icons/search-glass.svg",
+ buttons: {
+ secondary: [
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-cancel-button",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: {
+ category: "general-cfrfeatures",
+ },
+ },
+ },
+ ],
+ primary: {
+ label: {
+ value: "Activate now",
+ attributes: {
+ accesskey: "A",
+ },
+ },
+ action: {
+ type: "CONFIGURE_HOMEPAGE",
+ data: {
+ homePage: "default",
+ newtab: "default",
+ layout: {
+ search: true,
+ topsites: false,
+ highlights: false,
+ topstories: false,
+ snippets: false,
+ },
+ },
+ },
+ },
+ },
+ bucket_id: "HOMEPAGE_REMEDIATION_82",
+ heading_text: "A better search experience",
+ info_icon: {
+ label: {
+ string_id: "cfr-doorhanger-extension-sumo-link",
+ },
+ sumo_path: "extensionrecommendations",
+ },
+ notification_text: {
+ string_id: "cfr-doorhanger-feature-notification",
+ },
+ category: "cfrFeatures",
+ },
+ trigger: {
+ id: "openURL",
+ params: ["google.com", "www.google.com"],
+ },
+ },
+ {
+ id: "INFOBAR_ACTION_86",
+ groups: ["cfr-message-provider"],
+ targeting: "false",
+ template: "infobar",
+ content: {
+ type: "global",
+ text: { string_id: "default-browser-notification-message" },
+ buttons: [
+ {
+ label: { string_id: "default-browser-notification-button" },
+ primary: true,
+ accessKey: "O",
+ action: {
+ type: "SET_DEFAULT_BROWSER",
+ },
+ },
+ ],
+ },
+ trigger: { id: "defaultBrowserCheck" },
+ },
+ {
+ id: "PREF_OBSERVER_MESSAGE_94",
+ groups: ["cfr-message-provider"],
+ targeting: "true",
+ template: "infobar",
+ content: {
+ type: "global",
+ text: "This is a message triggered when a pref value changes",
+ buttons: [
+ {
+ label: "OK",
+ primary: true,
+ accessKey: "O",
+ action: {
+ type: "CANCEL",
+ },
+ },
+ ],
+ },
+ trigger: { id: "preferenceObserver", params: ["foo.bar"] },
+ },
+];
+
+const CFRMessageProvider = {
+ getMessages() {
+ return Promise.resolve(CFR_MESSAGES.filter(msg => !msg.exclude));
+ },
+};
+
+const EXPORTED_SYMBOLS = ["CFRMessageProvider"];
diff --git a/browser/components/newtab/lib/CFRPageActions.jsm b/browser/components/newtab/lib/CFRPageActions.jsm
new file mode 100644
index 0000000000..74e53c899a
--- /dev/null
+++ b/browser/components/newtab/lib/CFRPageActions.jsm
@@ -0,0 +1,1036 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ RemoteL10n: "resource://activity-stream/lib/RemoteL10n.jsm",
+ CustomizableUI: "resource:///modules/CustomizableUI.jsm",
+});
+
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "TrackingDBService",
+ "@mozilla.org/tracking-db-service;1",
+ "nsITrackingDBService"
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "milestones",
+ "browser.contentblocking.cfr-milestone.milestones",
+ "[]",
+ null,
+ JSON.parse
+);
+
+const POPUP_NOTIFICATION_ID = "contextual-feature-recommendation";
+const SUMO_BASE_URL = Services.urlFormatter.formatURLPref(
+ "app.support.baseURL"
+);
+const ADDONS_API_URL =
+ "https://services.addons.mozilla.org/api/v4/addons/addon";
+
+const DELAY_BEFORE_EXPAND_MS = 1000;
+const CATEGORY_ICONS = {
+ cfrAddons: "webextensions-icon",
+ cfrFeatures: "recommendations-icon",
+ cfrHeartbeat: "highlights-icon",
+};
+
+/**
+ * A WeakMap from browsers to {host, recommendation} pairs. Recommendations are
+ * defined in the ExtensionDoorhanger.schema.json.
+ *
+ * A recommendation is specific to a browser and host and is active until the
+ * given browser is closed or the user navigates (within that browser) away from
+ * the host.
+ */
+let RecommendationMap = new WeakMap();
+
+/**
+ * A WeakMap from windows to their CFR PageAction.
+ */
+let PageActionMap = new WeakMap();
+
+/**
+ * We need one PageAction for each window
+ */
+class PageAction {
+ constructor(win, dispatchCFRAction) {
+ this.window = win;
+
+ this.urlbar = win.gURLBar; // The global URLBar object
+ this.urlbarinput = win.gURLBar.textbox; // The URLBar DOM node
+
+ this.container = win.document.getElementById(
+ "contextual-feature-recommendation"
+ );
+ this.button = win.document.getElementById("cfr-button");
+ this.label = win.document.getElementById("cfr-label");
+
+ // This should NOT be use directly to dispatch message-defined actions attached to buttons.
+ // Please use dispatchUserAction instead.
+ this._dispatchCFRAction = dispatchCFRAction;
+
+ this._popupStateChange = this._popupStateChange.bind(this);
+ this._collapse = this._collapse.bind(this);
+ this._cfrUrlbarButtonClick = this._cfrUrlbarButtonClick.bind(this);
+ this._executeNotifierAction = this._executeNotifierAction.bind(this);
+ this.dispatchUserAction = this.dispatchUserAction.bind(this);
+
+ // Saved timeout IDs for scheduled state changes, so they can be cancelled
+ this.stateTransitionTimeoutIDs = [];
+
+ XPCOMUtils.defineLazyGetter(this, "isDarkTheme", () => {
+ try {
+ return this.window.document.documentElement.hasAttribute(
+ "lwt-toolbar-field-brighttext"
+ );
+ } catch (e) {
+ return false;
+ }
+ });
+ }
+
+ addImpression(recommendation) {
+ this._dispatchImpression(recommendation);
+ // Only send an impression ping upon the first expansion.
+ // Note that when the user clicks on the "show" button on the asrouter admin
+ // page (both `bucket_id` and `id` will be set as null), we don't want to send
+ // the impression ping in that case.
+ if (!!recommendation.id && !!recommendation.content.bucket_id) {
+ this._sendTelemetry({
+ message_id: recommendation.id,
+ bucket_id: recommendation.content.bucket_id,
+ event: "IMPRESSION",
+ });
+ }
+ }
+
+ reloadL10n() {
+ lazy.RemoteL10n.reloadL10n();
+ }
+
+ async showAddressBarNotifier(recommendation, shouldExpand = false) {
+ this.container.hidden = false;
+
+ let notificationText = await this.getStrings(
+ recommendation.content.notification_text
+ );
+ this.label.value = notificationText;
+ if (notificationText.attributes) {
+ this.button.setAttribute(
+ "tooltiptext",
+ notificationText.attributes.tooltiptext
+ );
+ // For a11y, we want the more descriptive text.
+ this.container.setAttribute(
+ "aria-label",
+ notificationText.attributes.tooltiptext
+ );
+ }
+ this.container.setAttribute(
+ "data-cfr-icon",
+ CATEGORY_ICONS[recommendation.content.category]
+ );
+ if (recommendation.content.active_color) {
+ this.container.style.setProperty(
+ "--cfr-active-color",
+ recommendation.content.active_color
+ );
+ }
+
+ // Wait for layout to flush to avoid a synchronous reflow then calculate the
+ // label width. We can safely get the width even though the recommendation is
+ // collapsed; the label itself remains full width (with its overflow hidden)
+ let [{ width }] = await this.window.promiseDocumentFlushed(() =>
+ this.label.getClientRects()
+ );
+ this.urlbarinput.style.setProperty("--cfr-label-width", `${width}px`);
+
+ this.container.addEventListener("click", this._cfrUrlbarButtonClick);
+ // Collapse the recommendation on url bar focus in order to free up more
+ // space to display and edit the url
+ this.urlbar.addEventListener("focus", this._collapse);
+
+ if (shouldExpand) {
+ this._clearScheduledStateChanges();
+
+ // After one second, expand
+ this._expand(DELAY_BEFORE_EXPAND_MS);
+
+ this.addImpression(recommendation);
+ }
+
+ if (notificationText.attributes) {
+ this.window.A11yUtils.announce({
+ raw: notificationText.attributes["a11y-announcement"],
+ source: this.container,
+ });
+ }
+ }
+
+ hideAddressBarNotifier() {
+ this.container.hidden = true;
+ this._clearScheduledStateChanges();
+ this.urlbarinput.removeAttribute("cfr-recommendation-state");
+ this.container.removeEventListener("click", this._cfrUrlbarButtonClick);
+ this.urlbar.removeEventListener("focus", this._collapse);
+ if (this.currentNotification) {
+ this.window.PopupNotifications.remove(this.currentNotification);
+ this.currentNotification = null;
+ }
+ }
+
+ _expand(delay) {
+ if (delay > 0) {
+ this.stateTransitionTimeoutIDs.push(
+ this.window.setTimeout(() => {
+ this.urlbarinput.setAttribute("cfr-recommendation-state", "expanded");
+ }, delay)
+ );
+ } else {
+ // Non-delayed state change overrides any scheduled state changes
+ this._clearScheduledStateChanges();
+ this.urlbarinput.setAttribute("cfr-recommendation-state", "expanded");
+ }
+ }
+
+ _collapse(delay) {
+ if (delay > 0) {
+ this.stateTransitionTimeoutIDs.push(
+ this.window.setTimeout(() => {
+ if (
+ this.urlbarinput.getAttribute("cfr-recommendation-state") ===
+ "expanded"
+ ) {
+ this.urlbarinput.setAttribute(
+ "cfr-recommendation-state",
+ "collapsed"
+ );
+ }
+ }, delay)
+ );
+ } else {
+ // Non-delayed state change overrides any scheduled state changes
+ this._clearScheduledStateChanges();
+ if (
+ this.urlbarinput.getAttribute("cfr-recommendation-state") === "expanded"
+ ) {
+ this.urlbarinput.setAttribute("cfr-recommendation-state", "collapsed");
+ }
+ }
+ }
+
+ _clearScheduledStateChanges() {
+ while (this.stateTransitionTimeoutIDs.length) {
+ // clearTimeout is safe even with invalid/expired IDs
+ this.window.clearTimeout(this.stateTransitionTimeoutIDs.pop());
+ }
+ }
+
+ // This is called when the popup closes as a result of interaction _outside_
+ // the popup, e.g. by hitting <esc>
+ _popupStateChange(state) {
+ if (state === "shown") {
+ if (this._autoFocus) {
+ this.window.document.commandDispatcher.advanceFocusIntoSubtree(
+ this.currentNotification.owner.panel
+ );
+ this._autoFocus = false;
+ }
+ } else if (state === "removed") {
+ if (this.currentNotification) {
+ this.window.PopupNotifications.remove(this.currentNotification);
+ this.currentNotification = null;
+ }
+ } else if (state === "dismissed") {
+ const message = RecommendationMap.get(this.currentNotification?.browser);
+ this._sendTelemetry({
+ message_id: message?.id,
+ bucket_id: message?.content.bucket_id,
+ event: "DISMISS",
+ });
+ this._collapse();
+ }
+ }
+
+ shouldShowDoorhanger(recommendation) {
+ if (recommendation.content.layout === "chiclet_open_url") {
+ return false;
+ }
+
+ return true;
+ }
+
+ dispatchUserAction(action) {
+ this._dispatchCFRAction(
+ { type: "USER_ACTION", data: action },
+ this.window.gBrowser.selectedBrowser
+ );
+ }
+
+ _dispatchImpression(message) {
+ this._dispatchCFRAction({ type: "IMPRESSION", data: message });
+ }
+
+ _sendTelemetry(ping) {
+ this._dispatchCFRAction({
+ type: "DOORHANGER_TELEMETRY",
+ data: { action: "cfr_user_event", source: "CFR", ...ping },
+ });
+ }
+
+ _blockMessage(messageID) {
+ this._dispatchCFRAction({
+ type: "BLOCK_MESSAGE_BY_ID",
+ data: { id: messageID },
+ });
+ }
+
+ maybeLoadCustomElement(win) {
+ if (!win.customElements.get("remote-text")) {
+ Services.scriptloader.loadSubScript(
+ "resource://activity-stream/data/custom-elements/paragraph.js",
+ win
+ );
+ }
+ }
+
+ /**
+ * getStrings - Handles getting the localized strings vs message overrides.
+ * If string_id is not defined it assumes you passed in an override
+ * message and it just returns it.
+ * If subAttribute is provided, the string for it is returned.
+ * @return A string. One of 1) passed in string 2) a String object with
+ * attributes property if there are attributes 3) the sub attribute.
+ */
+ async getStrings(string, subAttribute = "") {
+ if (!string.string_id) {
+ if (subAttribute) {
+ if (string.attributes) {
+ return string.attributes[subAttribute];
+ }
+
+ console.error(`String ${string.value} does not contain any attributes`);
+ return subAttribute;
+ }
+
+ if (typeof string.value === "string") {
+ const stringWithAttributes = new String(string.value); // eslint-disable-line no-new-wrappers
+ stringWithAttributes.attributes = string.attributes;
+ return stringWithAttributes;
+ }
+
+ return string;
+ }
+
+ const [localeStrings] = await lazy.RemoteL10n.l10n.formatMessages([
+ {
+ id: string.string_id,
+ args: string.args,
+ },
+ ]);
+
+ const mainString = new String(localeStrings.value); // eslint-disable-line no-new-wrappers
+ if (localeStrings.attributes) {
+ const attributes = localeStrings.attributes.reduce((acc, attribute) => {
+ acc[attribute.name] = attribute.value;
+ return acc;
+ }, {});
+ mainString.attributes = attributes;
+ }
+
+ return subAttribute ? mainString.attributes[subAttribute] : mainString;
+ }
+
+ async _setAddonAuthorAndRating(document, content) {
+ const author = this.window.document.getElementById(
+ "cfr-notification-author"
+ );
+ const footerFilledStars = this.window.document.getElementById(
+ "cfr-notification-footer-filled-stars"
+ );
+ const footerEmptyStars = this.window.document.getElementById(
+ "cfr-notification-footer-empty-stars"
+ );
+ const footerUsers = this.window.document.getElementById(
+ "cfr-notification-footer-users"
+ );
+ const footerSpacer = this.window.document.getElementById(
+ "cfr-notification-footer-spacer"
+ );
+
+ author.textContent = await this.getStrings({
+ string_id: "cfr-doorhanger-extension-author",
+ args: { name: content.addon.author },
+ });
+
+ const { rating } = content.addon;
+ if (rating) {
+ const MAX_RATING = 5;
+ const STARS_WIDTH = 17 * MAX_RATING;
+ const calcWidth = stars => `${(stars / MAX_RATING) * STARS_WIDTH}px`;
+ footerFilledStars.style.width = calcWidth(rating);
+ footerEmptyStars.style.width = calcWidth(MAX_RATING - rating);
+
+ const ratingString = await this.getStrings(
+ {
+ string_id: "cfr-doorhanger-extension-rating",
+ args: { total: rating },
+ },
+ "tooltiptext"
+ );
+ footerFilledStars.setAttribute("tooltiptext", ratingString);
+ footerEmptyStars.setAttribute("tooltiptext", ratingString);
+ } else {
+ footerFilledStars.style.width = "";
+ footerEmptyStars.style.width = "";
+ footerFilledStars.removeAttribute("tooltiptext");
+ footerEmptyStars.removeAttribute("tooltiptext");
+ }
+
+ const { users } = content.addon;
+ if (users) {
+ footerUsers.setAttribute(
+ "value",
+ await this.getStrings({
+ string_id: "cfr-doorhanger-extension-total-users",
+ args: { total: users },
+ })
+ );
+ footerUsers.hidden = false;
+ } else {
+ // Prevent whitespace around empty label from affecting other spacing
+ footerUsers.hidden = true;
+ footerUsers.removeAttribute("value");
+ }
+
+ // Spacer pushes the link to the opposite end when there's other content
+
+ footerSpacer.hidden = !rating && !users;
+ }
+
+ _createElementAndAppend({ type, id }, parent) {
+ let element = this.window.document.createXULElement(type);
+ if (id) {
+ element.setAttribute("id", id);
+ }
+ parent.appendChild(element);
+ return element;
+ }
+
+ async _renderMilestonePopup(message, browser) {
+ this.maybeLoadCustomElement(this.window);
+
+ let { content, id } = message;
+ let { primary, secondary } = content.buttons;
+ let earliestDate = await lazy.TrackingDBService.getEarliestRecordedDate();
+ let timestamp = new Date().getTime(earliestDate);
+ let panelTitle = "";
+ let headerLabel = this.window.document.getElementById(
+ "cfr-notification-header-label"
+ );
+ let reachedMilestone = 0;
+ let totalSaved = await lazy.TrackingDBService.sumAllEvents();
+ for (let milestone of lazy.milestones) {
+ if (totalSaved >= milestone) {
+ reachedMilestone = milestone;
+ }
+ }
+ if (headerLabel.firstChild) {
+ headerLabel.firstChild.remove();
+ }
+ headerLabel.appendChild(
+ lazy.RemoteL10n.createElement(this.window.document, "span", {
+ content: message.content.heading_text,
+ attributes: {
+ blockedCount: reachedMilestone,
+ date: timestamp,
+ },
+ })
+ );
+
+ // Use the message layout as a CSS selector to hide different parts of the
+ // notification template markup
+ this.window.document
+ .getElementById("contextual-feature-recommendation-notification")
+ .setAttribute("data-notification-category", content.layout);
+ this.window.document
+ .getElementById("contextual-feature-recommendation-notification")
+ .setAttribute("data-notification-bucket", content.bucket_id);
+
+ let primaryBtnString = await this.getStrings(primary.label);
+ let primaryActionCallback = () => {
+ this.dispatchUserAction(primary.action);
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "CLICK_BUTTON",
+ });
+
+ RecommendationMap.delete(browser);
+ // Invalidate the pref after the user interacts with the button.
+ // We don't need to show the illustration in the privacy panel.
+ Services.prefs.clearUserPref(
+ "browser.contentblocking.cfr-milestone.milestone-shown-time"
+ );
+ };
+
+ let secondaryBtnString = await this.getStrings(secondary[0].label);
+ let secondaryActionsCallback = () => {
+ this.dispatchUserAction(secondary[0].action);
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "DISMISS",
+ });
+ RecommendationMap.delete(browser);
+ };
+
+ let mainAction = {
+ label: primaryBtnString,
+ accessKey: primaryBtnString.attributes.accesskey,
+ callback: primaryActionCallback,
+ };
+
+ let secondaryActions = [
+ {
+ label: secondaryBtnString,
+ accessKey: secondaryBtnString.attributes.accesskey,
+ callback: secondaryActionsCallback,
+ },
+ ];
+
+ // Actually show the notification
+ this.currentNotification = this.window.PopupNotifications.show(
+ browser,
+ POPUP_NOTIFICATION_ID,
+ panelTitle,
+ "cfr",
+ mainAction,
+ secondaryActions,
+ {
+ hideClose: true,
+ persistWhileVisible: true,
+ }
+ );
+ Services.prefs.setIntPref(
+ "browser.contentblocking.cfr-milestone.milestone-achieved",
+ reachedMilestone
+ );
+ Services.prefs.setStringPref(
+ "browser.contentblocking.cfr-milestone.milestone-shown-time",
+ Date.now().toString()
+ );
+ }
+
+ // eslint-disable-next-line max-statements
+ async _renderPopup(message, browser) {
+ this.maybeLoadCustomElement(this.window);
+
+ const { id, content } = message;
+
+ const headerLabel = this.window.document.getElementById(
+ "cfr-notification-header-label"
+ );
+ const headerLink = this.window.document.getElementById(
+ "cfr-notification-header-link"
+ );
+ const headerImage = this.window.document.getElementById(
+ "cfr-notification-header-image"
+ );
+ const footerText = this.window.document.getElementById(
+ "cfr-notification-footer-text"
+ );
+ const footerLink = this.window.document.getElementById(
+ "cfr-notification-footer-learn-more-link"
+ );
+ const { primary, secondary } = content.buttons;
+ let primaryActionCallback;
+ let persistent = !!content.persistent_doorhanger;
+ let options = { persistent, persistWhileVisible: persistent };
+ let panelTitle;
+
+ headerLabel.value = await this.getStrings(content.heading_text);
+ if (content.info_icon) {
+ headerLink.setAttribute(
+ "href",
+ SUMO_BASE_URL + content.info_icon.sumo_path
+ );
+ headerImage.setAttribute(
+ "tooltiptext",
+ await this.getStrings(content.info_icon.label, "tooltiptext")
+ );
+ }
+ headerLink.onclick = () =>
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "RATIONALE",
+ });
+ // Use the message layout as a CSS selector to hide different parts of the
+ // notification template markup
+ this.window.document
+ .getElementById("contextual-feature-recommendation-notification")
+ .setAttribute("data-notification-category", content.layout);
+ this.window.document
+ .getElementById("contextual-feature-recommendation-notification")
+ .setAttribute("data-notification-bucket", content.bucket_id);
+
+ switch (content.layout) {
+ case "icon_and_message":
+ const author = this.window.document.getElementById(
+ "cfr-notification-author"
+ );
+ if (author.firstChild) {
+ author.firstChild.remove();
+ }
+ author.appendChild(
+ lazy.RemoteL10n.createElement(this.window.document, "span", {
+ content: content.text,
+ })
+ );
+ primaryActionCallback = () => {
+ this._blockMessage(id);
+ this.dispatchUserAction(primary.action);
+ this.hideAddressBarNotifier();
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "ENABLE",
+ });
+ RecommendationMap.delete(browser);
+ };
+
+ let getIcon = () => {
+ if (content.icon_dark_theme && this.isDarkTheme) {
+ return content.icon_dark_theme;
+ }
+ return content.icon;
+ };
+
+ let learnMoreURL = content.learn_more
+ ? SUMO_BASE_URL + content.learn_more
+ : null;
+
+ panelTitle = await this.getStrings(content.heading_text);
+ options = {
+ popupIconURL: getIcon(),
+ popupIconClass: content.icon_class,
+ learnMoreURL,
+ ...options,
+ };
+ break;
+ default:
+ panelTitle = await this.getStrings(content.addon.title);
+ await this._setAddonAuthorAndRating(this.window.document, content);
+ if (footerText.firstChild) {
+ footerText.firstChild.remove();
+ }
+ // Main body content of the dropdown
+ footerText.appendChild(
+ lazy.RemoteL10n.createElement(this.window.document, "span", {
+ content: content.text,
+ })
+ );
+ options = { popupIconURL: content.addon.icon, ...options };
+
+ footerLink.value = await this.getStrings({
+ string_id: "cfr-doorhanger-extension-learn-more-link",
+ });
+ footerLink.setAttribute("href", content.addon.amo_url);
+ footerLink.onclick = () =>
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "LEARN_MORE",
+ });
+
+ primaryActionCallback = async () => {
+ // eslint-disable-next-line no-use-before-define
+ primary.action.data.url = await CFRPageActions._fetchLatestAddonVersion(
+ content.addon.id
+ );
+ this._blockMessage(id);
+ this.dispatchUserAction(primary.action);
+ this.hideAddressBarNotifier();
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "INSTALL",
+ });
+ RecommendationMap.delete(browser);
+ };
+ }
+
+ const primaryBtnStrings = await this.getStrings(primary.label);
+ const mainAction = {
+ label: primaryBtnStrings,
+ accessKey: primaryBtnStrings.attributes.accesskey,
+ callback: primaryActionCallback,
+ };
+
+ let _renderSecondaryButtonAction = async (event, button) => {
+ let label = await this.getStrings(button.label);
+ let { attributes } = label;
+
+ return {
+ label,
+ accessKey: attributes.accesskey,
+ callback: () => {
+ if (button.action) {
+ this.dispatchUserAction(button.action);
+ } else {
+ this._blockMessage(id);
+ this.hideAddressBarNotifier();
+ RecommendationMap.delete(browser);
+ }
+
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event,
+ });
+ // We want to collapse if needed when we dismiss
+ this._collapse();
+ },
+ };
+ };
+
+ // For each secondary action, define default telemetry event
+ const defaultSecondaryEvent = ["DISMISS", "BLOCK", "MANAGE"];
+ const secondaryActions = await Promise.all(
+ secondary.map((button, i) => {
+ return _renderSecondaryButtonAction(
+ button.event || defaultSecondaryEvent[i],
+ button
+ );
+ })
+ );
+
+ // If the recommendation button is focused, it was probably activated via
+ // the keyboard. Therefore, focus the first element in the notification when
+ // it appears.
+ // We don't use the autofocus option provided by PopupNotifications.show
+ // because it doesn't focus the first element; i.e. the user still has to
+ // press tab once. That's not good enough, especially for screen reader
+ // users. Instead, we handle this ourselves in _popupStateChange.
+ this._autoFocus = this.window.document.activeElement === this.container;
+
+ // Actually show the notification
+ this.currentNotification = this.window.PopupNotifications.show(
+ browser,
+ POPUP_NOTIFICATION_ID,
+ panelTitle,
+ "cfr",
+ mainAction,
+ secondaryActions,
+ {
+ ...options,
+ hideClose: true,
+ eventCallback: this._popupStateChange,
+ }
+ );
+ }
+
+ _executeNotifierAction(browser, message) {
+ switch (message.content.layout) {
+ case "chiclet_open_url":
+ this._dispatchCFRAction(
+ {
+ type: "USER_ACTION",
+ data: {
+ type: "OPEN_URL",
+ data: {
+ args: message.content.action.url,
+ where: message.content.action.where,
+ },
+ },
+ },
+ this.window
+ );
+ break;
+ }
+
+ this._blockMessage(message.id);
+ this.hideAddressBarNotifier();
+ RecommendationMap.delete(browser);
+ }
+
+ /**
+ * Respond to a user click on the recommendation by showing a doorhanger/
+ * popup notification or running the action defined in the message
+ */
+ async _cfrUrlbarButtonClick(event) {
+ const browser = this.window.gBrowser.selectedBrowser;
+ if (!RecommendationMap.has(browser)) {
+ // There's no recommendation for this browser, so the user shouldn't have
+ // been able to click
+ this.hideAddressBarNotifier();
+ return;
+ }
+ const message = RecommendationMap.get(browser);
+ const { id, content } = message;
+
+ this._sendTelemetry({
+ message_id: id,
+ bucket_id: content.bucket_id,
+ event: "CLICK_DOORHANGER",
+ });
+
+ if (this.shouldShowDoorhanger(message)) {
+ // The recommendation should remain either collapsed or expanded while the
+ // doorhanger is showing
+ this._clearScheduledStateChanges(browser, message);
+ await this.showPopup();
+ } else {
+ await this._executeNotifierAction(browser, message);
+ }
+ }
+
+ async showPopup() {
+ const browser = this.window.gBrowser.selectedBrowser;
+ const message = RecommendationMap.get(browser);
+ const { content } = message;
+ let anchor;
+
+ // A hacky way of setting the popup anchor outside the usual url bar icon box
+ // See https://searchfox.org/mozilla-central/rev/847b64cc28b74b44c379f9bff4f415b97da1c6d7/toolkit/modules/PopupNotifications.jsm#42
+ //If the anchor has been moved to the overflow menu ('menu-panel') and an alt_anchor_id has been provided, we want to use the alt_anchor_id
+
+ if (
+ content.alt_anchor_id &&
+ lazy.CustomizableUI.getWidget(content.anchor_id).areaType.includes(
+ "panel"
+ )
+ ) {
+ anchor = this.window.document.getElementById(content.alt_anchor_id);
+ } else {
+ anchor =
+ this.window.document.getElementById(content.anchor_id) ||
+ this.container;
+ }
+ browser.cfrpopupnotificationanchor = anchor;
+
+ await this._renderPopup(message, browser);
+ }
+
+ async showMilestonePopup() {
+ const browser = this.window.gBrowser.selectedBrowser;
+ const message = RecommendationMap.get(browser);
+ const { content } = message;
+
+ // A hacky way of setting the popup anchor outside the usual url bar icon box
+ // See https://searchfox.org/mozilla-central/rev/c5c002f81f08a73e04868e0c2bf0eb113f200b03/toolkit/modules/PopupNotifications.sys.mjs#40
+ browser.cfrpopupnotificationanchor =
+ this.window.document.getElementById(content.anchor_id) || this.container;
+
+ await this._renderMilestonePopup(message, browser);
+ return true;
+ }
+}
+
+function isHostMatch(browser, host) {
+ return (
+ browser.documentURI.scheme.startsWith("http") &&
+ browser.documentURI.host === host
+ );
+}
+
+const CFRPageActions = {
+ // For testing purposes
+ RecommendationMap,
+ PageActionMap,
+
+ /**
+ * To be called from browser.js on a location change, passing in the browser
+ * that's been updated
+ */
+ updatePageActions(browser) {
+ const win = browser.ownerGlobal;
+ const pageAction = PageActionMap.get(win);
+ if (!pageAction || browser !== win.gBrowser.selectedBrowser) {
+ return;
+ }
+ if (RecommendationMap.has(browser)) {
+ const recommendation = RecommendationMap.get(browser);
+ if (
+ !recommendation.content.skip_address_bar_notifier &&
+ (isHostMatch(browser, recommendation.host) ||
+ // If there is no host associated we assume we're back on a tab
+ // that had a CFR message so we should show it again
+ !recommendation.host)
+ ) {
+ // The browser has a recommendation specified with this host, so show
+ // the page action
+ pageAction.showAddressBarNotifier(recommendation);
+ } else if (!recommendation.content.persistent_doorhanger) {
+ if (recommendation.retain) {
+ // Keep the recommendation first time the user navigates away just in
+ // case they will go back to the previous page
+ pageAction.hideAddressBarNotifier();
+ recommendation.retain = false;
+ } else {
+ // The user has navigated away from the specified host in the given
+ // browser, so the recommendation is no longer valid and should be removed
+ RecommendationMap.delete(browser);
+ pageAction.hideAddressBarNotifier();
+ }
+ }
+ } else {
+ // There's no recommendation specified for this browser, so hide the page action
+ pageAction.hideAddressBarNotifier();
+ }
+ },
+
+ /**
+ * Fetch the URL to the latest add-on xpi so the recommendation can download it.
+ * @param id The add-on ID
+ * @return A string for the URL that was fetched
+ */
+ async _fetchLatestAddonVersion(id) {
+ let url = null;
+ try {
+ const response = await fetch(`${ADDONS_API_URL}/${id}/`, {
+ credentials: "omit",
+ });
+ if (response.status !== 204 && response.ok) {
+ const json = await response.json();
+ url = json.current_version.files[0].url;
+ }
+ } catch (e) {
+ console.error(
+ "Failed to get the latest add-on version for this recommendation"
+ );
+ }
+ return url;
+ },
+
+ /**
+ * Force a recommendation to be shown. Should only happen via the Admin page.
+ * @param browser The browser for the recommendation
+ * @param recommendation The recommendation to show
+ * @param dispatchCFRAction A function to dispatch resulting actions to
+ * @return Did adding the recommendation succeed?
+ */
+ async forceRecommendation(browser, recommendation, dispatchCFRAction) {
+ // If we are forcing via the Admin page, the browser comes in a different format
+ const win = browser.ownerGlobal;
+ const { id, content } = recommendation;
+ RecommendationMap.set(browser, {
+ id,
+ content,
+ retain: true,
+ });
+ if (!PageActionMap.has(win)) {
+ PageActionMap.set(win, new PageAction(win, dispatchCFRAction));
+ }
+
+ if (content.skip_address_bar_notifier) {
+ if (recommendation.template === "milestone_message") {
+ await PageActionMap.get(win).showMilestonePopup();
+ PageActionMap.get(win).addImpression(recommendation);
+ } else {
+ await PageActionMap.get(win).showPopup();
+ PageActionMap.get(win).addImpression(recommendation);
+ }
+ } else {
+ await PageActionMap.get(win).showAddressBarNotifier(recommendation, true);
+ }
+ return true;
+ },
+
+ /**
+ * Add a recommendation specific to the given browser and host.
+ * @param browser The browser for the recommendation
+ * @param host The host for the recommendation
+ * @param recommendation The recommendation to show
+ * @param dispatchCFRAction A function to dispatch resulting actions to
+ * @return Did adding the recommendation succeed?
+ */
+ async addRecommendation(browser, host, recommendation, dispatchCFRAction) {
+ const win = browser.ownerGlobal;
+ if (lazy.PrivateBrowsingUtils.isWindowPrivate(win)) {
+ return false;
+ }
+ if (
+ browser !== win.gBrowser.selectedBrowser ||
+ // We can have recommendations without URL restrictions
+ (host && !isHostMatch(browser, host))
+ ) {
+ return false;
+ }
+ if (RecommendationMap.has(browser)) {
+ // Don't replace an existing message
+ return false;
+ }
+ const { id, content } = recommendation;
+ RecommendationMap.set(browser, {
+ id,
+ host,
+ content,
+ retain: true,
+ });
+ if (!PageActionMap.has(win)) {
+ PageActionMap.set(win, new PageAction(win, dispatchCFRAction));
+ }
+
+ if (content.skip_address_bar_notifier) {
+ if (recommendation.template === "milestone_message") {
+ await PageActionMap.get(win).showMilestonePopup();
+ PageActionMap.get(win).addImpression(recommendation);
+ } else {
+ // Tracking protection messages
+ await PageActionMap.get(win).showPopup();
+ PageActionMap.get(win).addImpression(recommendation);
+ }
+ } else {
+ // Doorhanger messages
+ await PageActionMap.get(win).showAddressBarNotifier(recommendation, true);
+ }
+ return true;
+ },
+
+ /**
+ * Clear all recommendations and hide all PageActions
+ */
+ clearRecommendations() {
+ // WeakMaps aren't iterable so we have to test all existing windows
+ for (const win of Services.wm.getEnumerator("navigator:browser")) {
+ if (win.closed || !PageActionMap.has(win)) {
+ continue;
+ }
+ PageActionMap.get(win).hideAddressBarNotifier();
+ }
+ // WeakMaps don't have a `clear` method
+ PageActionMap = new WeakMap();
+ RecommendationMap = new WeakMap();
+ this.PageActionMap = PageActionMap;
+ this.RecommendationMap = RecommendationMap;
+ },
+
+ /**
+ * Reload the l10n Fluent files for all PageActions
+ */
+ reloadL10n() {
+ for (const win of Services.wm.getEnumerator("navigator:browser")) {
+ if (win.closed || !PageActionMap.has(win)) {
+ continue;
+ }
+ PageActionMap.get(win).reloadL10n();
+ }
+ },
+};
+
+const EXPORTED_SYMBOLS = ["CFRPageActions", "PageAction"];
diff --git a/browser/components/newtab/lib/DefaultSites.jsm b/browser/components/newtab/lib/DefaultSites.jsm
new file mode 100644
index 0000000000..f52cab278d
--- /dev/null
+++ b/browser/components/newtab/lib/DefaultSites.jsm
@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const DEFAULT_SITES_MAP = new Map([
+ // This first item is the global list fallback for any unexpected geos
+ [
+ "",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.reddit.com/,https://www.amazon.com/,https://twitter.com/",
+ ],
+ [
+ "US",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://twitter.com/",
+ ],
+ [
+ "CA",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://www.amazon.ca/,https://twitter.com/",
+ ],
+ [
+ "DE",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.de/,https://www.ebay.de/,https://www.wikipedia.org/,https://www.reddit.com/",
+ ],
+ [
+ "PL",
+ "https://www.youtube.com/,https://www.facebook.com/,https://allegro.pl/,https://www.wikipedia.org/,https://www.olx.pl/,https://www.wykop.pl/",
+ ],
+ [
+ "RU",
+ "https://vk.com/,https://www.youtube.com/,https://ok.ru/,https://www.avito.ru/,https://www.aliexpress.com/,https://www.wikipedia.org/",
+ ],
+ [
+ "GB",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.amazon.co.uk/,https://www.bbc.co.uk/,https://www.ebay.co.uk/",
+ ],
+ [
+ "FR",
+ "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.amazon.fr/,https://www.leboncoin.fr/,https://twitter.com/",
+ ],
+ [
+ "CN",
+ "https://www.baidu.com/,https://www.zhihu.com/,https://www.ifeng.com/,https://weibo.com/,https://www.ctrip.com/,https://www.iqiyi.com/",
+ ],
+]);
+
+const EXPORTED_SYMBOLS = ["DEFAULT_SITES"];
+
+// Immutable for export.
+const DEFAULT_SITES = Object.freeze(DEFAULT_SITES_MAP);
diff --git a/browser/components/newtab/lib/DiscoveryStreamFeed.jsm b/browser/components/newtab/lib/DiscoveryStreamFeed.jsm
new file mode 100644
index 0000000000..1f1aec6bdc
--- /dev/null
+++ b/browser/components/newtab/lib/DiscoveryStreamFeed.jsm
@@ -0,0 +1,2389 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const lazy = {};
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+ Region: "resource://gre/modules/Region.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "RemoteSettings",
+ "resource://services-settings/remote-settings.js"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "pktApi",
+ "chrome://pocket/content/pktApi.jsm"
+);
+const { setTimeout, clearTimeout } = ChromeUtils.importESModule(
+ "resource://gre/modules/Timer.sys.mjs"
+);
+const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PersistentCache",
+ "resource://activity-stream/lib/PersistentCache.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "ExperimentAPI",
+ "resource://nimbus/ExperimentAPI.jsm"
+);
+
+const CACHE_KEY = "discovery_stream";
+const LAYOUT_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
+const STARTUP_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week
+const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
+const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
+const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
+const MIN_PERSONALIZATION_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours
+const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
+const FETCH_TIMEOUT = 45 * 1000;
+const SPOCS_URL = "https://spocs.getpocket.com/spocs";
+const PREF_CONFIG = "discoverystream.config";
+const PREF_ENDPOINTS = "discoverystream.endpoints";
+const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId";
+const PREF_ENABLED = "discoverystream.enabled";
+const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout";
+const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint";
+const PREF_SPOCS_ENDPOINT_QUERY = "discoverystream.spocs-endpoint-query";
+const PREF_REGION_BASIC_LAYOUT = "discoverystream.region-basic-layout";
+const PREF_USER_TOPSTORIES = "feeds.section.topstories";
+const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories";
+const PREF_USER_TOPSITES = "feeds.topsites";
+const PREF_SYSTEM_TOPSITES = "feeds.system.topsites";
+const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear";
+const PREF_SHOW_SPONSORED = "showSponsored";
+const PREF_SHOW_SPONSORED_TOPSITES = "showSponsoredTopSites";
+const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions";
+const PREF_FLIGHT_BLOCKS = "discoverystream.flight.blocks";
+const PREF_REC_IMPRESSIONS = "discoverystream.rec.impressions";
+const PREF_COLLECTIONS_ENABLED =
+ "discoverystream.sponsored-collections.enabled";
+const PREF_POCKET_BUTTON = "extensions.pocket.enabled";
+const PREF_COLLECTION_DISMISSIBLE = "discoverystream.isCollectionDismissible";
+const PREF_PERSONALIZATION = "discoverystream.personalization.enabled";
+const PREF_PERSONALIZATION_OVERRIDE =
+ "discoverystream.personalization.override";
+
+let getHardcodedLayout;
+
+class DiscoveryStreamFeed {
+ constructor() {
+ // Internal state for checking if we've intialized all our data
+ this.loaded = false;
+
+ // Persistent cache for remote endpoint data.
+ this.cache = new lazy.PersistentCache(CACHE_KEY, true);
+ this.locale = Services.locale.appLocaleAsBCP47;
+ this._impressionId = this.getOrCreateImpressionId();
+ // Internal in-memory cache for parsing json prefs.
+ this._prefCache = {};
+ }
+
+ getOrCreateImpressionId() {
+ let impressionId = Services.prefs.getCharPref(PREF_IMPRESSION_ID, "");
+ if (!impressionId) {
+ impressionId = String(Services.uuid.generateUUID());
+ Services.prefs.setCharPref(PREF_IMPRESSION_ID, impressionId);
+ }
+ return impressionId;
+ }
+
+ finalLayoutEndpoint(url, apiKey) {
+ if (url.includes("$apiKey") && !apiKey) {
+ throw new Error(
+ `Layout Endpoint - An API key was specified but none configured: ${url}`
+ );
+ }
+ return url.replace("$apiKey", apiKey);
+ }
+
+ get config() {
+ if (this._prefCache.config) {
+ return this._prefCache.config;
+ }
+ try {
+ this._prefCache.config = JSON.parse(
+ this.store.getState().Prefs.values[PREF_CONFIG]
+ );
+ const layoutUrl = this._prefCache.config.layout_endpoint;
+
+ const apiKeyPref = this._prefCache.config.api_key_pref;
+ if (layoutUrl && apiKeyPref) {
+ const apiKey = Services.prefs.getCharPref(apiKeyPref, "");
+ this._prefCache.config.layout_endpoint = this.finalLayoutEndpoint(
+ layoutUrl,
+ apiKey
+ );
+ }
+ } catch (e) {
+ // istanbul ignore next
+ this._prefCache.config = {};
+ // istanbul ignore next
+ console.error(
+ `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config. ${e}`
+ );
+ }
+ this._prefCache.config.enabled =
+ this._prefCache.config.enabled &&
+ this.store.getState().Prefs.values[PREF_ENABLED];
+
+ return this._prefCache.config;
+ }
+
+ resetConfigDefauts() {
+ this.store.dispatch({
+ type: at.CLEAR_PREF,
+ data: {
+ name: PREF_CONFIG,
+ },
+ });
+ }
+
+ get region() {
+ return lazy.Region.home;
+ }
+
+ get showSpocs() {
+ // High level overall sponsored check, if one of these is true,
+ // we know we need some sort of spoc control setup.
+ return this.showSponsoredStories || this.showSponsoredTopsites;
+ }
+
+ get showSponsoredStories() {
+ // Combine user-set sponsored opt-out with Mozilla-set config
+ return (
+ this.store.getState().Prefs.values[PREF_SHOW_SPONSORED] &&
+ this.config.show_spocs
+ );
+ }
+
+ get showSponsoredTopsites() {
+ const placements = this.getPlacements();
+ // Combine user-set sponsored opt-out with placement data
+ return !!(
+ this.store.getState().Prefs.values[PREF_SHOW_SPONSORED_TOPSITES] &&
+ placements.find(placement => placement.name === "sponsored-topsites")
+ );
+ }
+
+ get showStories() {
+ // Combine user-set stories opt-out with Mozilla-set config
+ return (
+ this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] &&
+ this.store.getState().Prefs.values[PREF_USER_TOPSTORIES]
+ );
+ }
+
+ get showTopsites() {
+ // Combine user-set topsites opt-out with Mozilla-set config
+ return (
+ this.store.getState().Prefs.values[PREF_SYSTEM_TOPSITES] &&
+ this.store.getState().Prefs.values[PREF_USER_TOPSITES]
+ );
+ }
+
+ get personalized() {
+ // If stories are not displayed, no point in trying to personalize them.
+ if (!this.showStories) {
+ return false;
+ }
+ const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.spocsPersonalized;
+ const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.recsPersonalized;
+ const personalization = this.store.getState().Prefs.values[
+ PREF_PERSONALIZATION
+ ];
+
+ // There is a server sent flag to keep personalization on.
+ // If the server stops sending this, we turn personalization off,
+ // until the server starts returning the signal.
+ const overrideState = this.store.getState().Prefs.values[
+ PREF_PERSONALIZATION_OVERRIDE
+ ];
+
+ return (
+ personalization &&
+ !overrideState &&
+ !!this.recommendationProvider &&
+ (spocsPersonalized || recsPersonalized)
+ );
+ }
+
+ get recommendationProvider() {
+ if (this._recommendationProvider) {
+ return this._recommendationProvider;
+ }
+ this._recommendationProvider = this.store.feeds.get(
+ "feeds.recommendationprovider"
+ );
+ return this._recommendationProvider;
+ }
+
+ setupConfig(isStartup = false) {
+ // Send the initial state of the pref on our reducer
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_CONFIG_SETUP,
+ data: this.config,
+ meta: {
+ isStartup,
+ },
+ })
+ );
+ }
+
+ setupPrefs(isStartup = false) {
+ const pocketNewtabExperiment = lazy.ExperimentAPI.getExperimentMetaData({
+ featureId: "pocketNewtab",
+ });
+
+ const pocketNewtabRollout = lazy.ExperimentAPI.getRolloutMetaData({
+ featureId: "pocketNewtab",
+ });
+
+ // We want to know if the user is in an experiment or rollout,
+ // but we prioritize experiments over rollouts.
+ const experimentMetaData = pocketNewtabExperiment || pocketNewtabRollout;
+
+ let utmSource = "pocket-newtab";
+ let utmCampaign = experimentMetaData?.slug;
+ let utmContent = experimentMetaData?.branch?.slug;
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_EXPERIMENT_DATA,
+ data: {
+ utmSource,
+ utmCampaign,
+ utmContent,
+ },
+ meta: {
+ isStartup,
+ },
+ })
+ );
+
+ const pocketButtonEnabled = Services.prefs.getBoolPref(PREF_POCKET_BUTTON);
+
+ const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {};
+ const { region } = this.store.getState().Prefs.values;
+
+ this.setupSpocsCacheUpdateTime();
+ const saveToPocketCardRegions = nimbusConfig.saveToPocketCardRegions
+ ?.split(",")
+ .map(s => s.trim());
+ const saveToPocketCard =
+ pocketButtonEnabled &&
+ (nimbusConfig.saveToPocketCard ||
+ saveToPocketCardRegions?.includes(region));
+
+ const hideDescriptionsRegions = nimbusConfig.hideDescriptionsRegions
+ ?.split(",")
+ .map(s => s.trim());
+ const hideDescriptions =
+ nimbusConfig.hideDescriptions ||
+ hideDescriptionsRegions?.includes(region);
+
+ // We don't BroadcastToContent for this, as the changes may
+ // shift around elements on an open newtab the user is currently reading.
+ // So instead we AlsoToPreloaded so the next tab is updated.
+ // This is because setupPrefs is called by the system and not a user interaction.
+ this.store.dispatch(
+ ac.AlsoToPreloaded({
+ type: at.DISCOVERY_STREAM_PREFS_SETUP,
+ data: {
+ recentSavesEnabled: nimbusConfig.recentSavesEnabled,
+ pocketButtonEnabled,
+ saveToPocketCard,
+ hideDescriptions,
+ compactImages: nimbusConfig.compactImages,
+ imageGradient: nimbusConfig.imageGradient,
+ newSponsoredLabel: nimbusConfig.newSponsoredLabel,
+ titleLines: nimbusConfig.titleLines,
+ descLines: nimbusConfig.descLines,
+ readTime: nimbusConfig.readTime,
+ },
+ meta: {
+ isStartup,
+ },
+ })
+ );
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE,
+ data: {
+ value: this.store.getState().Prefs.values[
+ PREF_COLLECTION_DISMISSIBLE
+ ],
+ },
+ meta: {
+ isStartup,
+ },
+ })
+ );
+ }
+
+ async setupPocketState(target) {
+ let dispatch = action =>
+ this.store.dispatch(ac.OnlyToOneContent(action, target));
+ const isUserLoggedIn = lazy.pktApi.isUserLoggedIn();
+ dispatch({
+ type: at.DISCOVERY_STREAM_POCKET_STATE_SET,
+ data: {
+ isUserLoggedIn,
+ },
+ });
+
+ // If we're not logged in, don't bother fetching recent saves, we're done.
+ if (isUserLoggedIn) {
+ let recentSaves = await lazy.pktApi.getRecentSavesCache();
+ if (recentSaves) {
+ // We have cache, so we can use those.
+ dispatch({
+ type: at.DISCOVERY_STREAM_RECENT_SAVES,
+ data: {
+ recentSaves,
+ },
+ });
+ } else {
+ // We don't have cache, so fetch fresh stories.
+ lazy.pktApi.getRecentSaves({
+ success(data) {
+ dispatch({
+ type: at.DISCOVERY_STREAM_RECENT_SAVES,
+ data: {
+ recentSaves: data,
+ },
+ });
+ },
+ error(error) {},
+ });
+ }
+ }
+ }
+
+ uninitPrefs() {
+ // Reset in-memory cache
+ this._prefCache = {};
+ }
+
+ async fetchFromEndpoint(rawEndpoint, options = {}) {
+ if (!rawEndpoint) {
+ console.error("Tried to fetch endpoint but none was configured.");
+ return null;
+ }
+
+ const apiKeyPref = this._prefCache.config.api_key_pref;
+ const apiKey = Services.prefs.getCharPref(apiKeyPref, "");
+
+ // The server somtimes returns this value already replaced, but we try this for two reasons:
+ // 1. Layout endpoints are not from the server.
+ // 2. Hardcoded layouts don't have this already done for us.
+ const endpoint = rawEndpoint
+ .replace("$apiKey", apiKey)
+ .replace("$locale", this.locale)
+ .replace("$region", this.region);
+
+ try {
+ // Make sure the requested endpoint is allowed
+ const allowed = this.store
+ .getState()
+ .Prefs.values[PREF_ENDPOINTS].split(",");
+ if (!allowed.some(prefix => endpoint.startsWith(prefix))) {
+ throw new Error(`Not one of allowed prefixes (${allowed})`);
+ }
+
+ const controller = new AbortController();
+ const { signal } = controller;
+
+ const fetchPromise = fetch(endpoint, {
+ ...options,
+ credentials: "omit",
+ signal,
+ });
+ // istanbul ignore next
+ const timeoutId = setTimeout(() => {
+ controller.abort();
+ }, FETCH_TIMEOUT);
+
+ const response = await fetchPromise;
+ if (!response.ok) {
+ throw new Error(`Unexpected status (${response.status})`);
+ }
+ clearTimeout(timeoutId);
+ return response.json();
+ } catch (error) {
+ console.error(`Failed to fetch ${endpoint}: ${error.message}`);
+ }
+ return null;
+ }
+
+ get spocsCacheUpdateTime() {
+ if (this._spocsCacheUpdateTime) {
+ return this._spocsCacheUpdateTime;
+ }
+ this.setupSpocsCacheUpdateTime();
+ return this._spocsCacheUpdateTime;
+ }
+
+ setupSpocsCacheUpdateTime() {
+ const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {};
+ const { spocsCacheTimeout } = nimbusConfig;
+ const MAX_TIMEOUT = 30;
+ const MIN_TIMEOUT = 5;
+ // We do a bit of min max checking the the configured value is between
+ // 5 and 30 minutes, to protect against unreasonable values.
+ if (
+ spocsCacheTimeout &&
+ spocsCacheTimeout <= MAX_TIMEOUT &&
+ spocsCacheTimeout >= MIN_TIMEOUT
+ ) {
+ // This value is in minutes, but we want ms.
+ this._spocsCacheUpdateTime = spocsCacheTimeout * 60 * 1000;
+ } else {
+ // The const is already in ms.
+ this._spocsCacheUpdateTime = SPOCS_FEEDS_UPDATE_TIME;
+ }
+ }
+
+ /**
+ * Returns true if data in the cache for a particular key has expired or is missing.
+ * @param {object} cachedData data returned from cache.get()
+ * @param {string} key a cache key
+ * @param {string?} url for "feed" only, the URL of the feed.
+ * @param {boolean} is this check done at initial browser load
+ */
+ isExpired({ cachedData, key, url, isStartup }) {
+ const { layout, spocs, feeds } = cachedData;
+ const updateTimePerComponent = {
+ layout: LAYOUT_UPDATE_TIME,
+ spocs: this.spocsCacheUpdateTime,
+ feed: COMPONENT_FEEDS_UPDATE_TIME,
+ };
+ const EXPIRATION_TIME = isStartup
+ ? STARTUP_CACHE_EXPIRE_TIME
+ : updateTimePerComponent[key];
+ switch (key) {
+ case "layout":
+ // This never needs to expire, as it's not expected to change.
+ if (this.config.hardcoded_layout) {
+ return false;
+ }
+ return !layout || !(Date.now() - layout.lastUpdated < EXPIRATION_TIME);
+ case "spocs":
+ return !spocs || !(Date.now() - spocs.lastUpdated < EXPIRATION_TIME);
+ case "feed":
+ return (
+ !feeds ||
+ !feeds[url] ||
+ !(Date.now() - feeds[url].lastUpdated < EXPIRATION_TIME)
+ );
+ default:
+ // istanbul ignore next
+ throw new Error(`${key} is not a valid key`);
+ }
+ }
+
+ async _checkExpirationPerComponent() {
+ const cachedData = (await this.cache.get()) || {};
+ const { feeds } = cachedData;
+ return {
+ layout: this.isExpired({ cachedData, key: "layout" }),
+ spocs: this.showSpocs && this.isExpired({ cachedData, key: "spocs" }),
+ feeds:
+ this.showStories &&
+ (!feeds ||
+ Object.keys(feeds).some(url =>
+ this.isExpired({ cachedData, key: "feed", url })
+ )),
+ };
+ }
+
+ /**
+ * Returns true if any data for the cached endpoints has expired or is missing.
+ */
+ async checkIfAnyCacheExpired() {
+ const expirationPerComponent = await this._checkExpirationPerComponent();
+ return (
+ expirationPerComponent.layout ||
+ expirationPerComponent.spocs ||
+ expirationPerComponent.feeds
+ );
+ }
+
+ async fetchLayout(isStartup) {
+ const cachedData = (await this.cache.get()) || {};
+ let { layout } = cachedData;
+ if (this.isExpired({ cachedData, key: "layout", isStartup })) {
+ const layoutResponse = await this.fetchFromEndpoint(
+ this.config.layout_endpoint
+ );
+ if (layoutResponse && layoutResponse.layout) {
+ layout = {
+ lastUpdated: Date.now(),
+ spocs: layoutResponse.spocs,
+ layout: layoutResponse.layout,
+ status: "success",
+ };
+
+ await this.cache.set("layout", layout);
+ } else {
+ console.error("No response for response.layout prop");
+ }
+ }
+ return layout;
+ }
+
+ updatePlacements(sendUpdate, layout, isStartup = false) {
+ const placements = [];
+ const placementsMap = {};
+ for (const row of layout.filter(r => r.components && r.components.length)) {
+ for (const component of row.components.filter(
+ c => c.placement && c.spocs
+ )) {
+ // If we find a valid placement, we set it to this value.
+ let placement;
+
+ // We need to check to see if this placement is on or not.
+ // If this placement has a prefs array, check against that.
+ if (component.spocs.prefs) {
+ // Check every pref in the array to see if this placement is turned on.
+ if (
+ component.spocs.prefs.length &&
+ component.spocs.prefs.every(
+ p => this.store.getState().Prefs.values[p]
+ )
+ ) {
+ // This placement is on.
+ placement = component.placement;
+ }
+ } else if (this.showSponsoredStories) {
+ // If we do not have a prefs array, use old check.
+ // This is because Pocket spocs uses an old non pref method.
+ placement = component.placement;
+ }
+
+ // Validate this placement and check for dupes.
+ if (placement?.name && !placementsMap[placement.name]) {
+ placementsMap[placement.name] = placement;
+ placements.push(placement);
+ }
+ }
+ }
+
+ // Update placements data.
+ // Even if we have no placements, we still want to update it to clear it.
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_SPOCS_PLACEMENTS,
+ data: { placements },
+ meta: {
+ isStartup,
+ },
+ });
+ }
+
+ /**
+ * Adds a query string to a URL.
+ * A query can be any string literal accepted by https://developer.mozilla.org/docs/Web/API/URLSearchParams
+ * Examples: "?foo=1&bar=2", "&foo=1&bar=2", "foo=1&bar=2", "?bar=2" or "bar=2"
+ */
+ addEndpointQuery(url, query) {
+ if (!query) {
+ return url;
+ }
+
+ const urlObject = new URL(url);
+ const params = new URLSearchParams(query);
+
+ for (let [key, val] of params.entries()) {
+ urlObject.searchParams.append(key, val);
+ }
+
+ return urlObject.toString();
+ }
+
+ parseGridPositions(csvPositions) {
+ let gridPositions;
+
+ // Only accept parseable non-negative integers
+ try {
+ gridPositions = csvPositions.map(index => {
+ let parsedInt = parseInt(index, 10);
+
+ if (!isNaN(parsedInt) && parsedInt >= 0) {
+ return parsedInt;
+ }
+
+ throw new Error("Bad input");
+ });
+ } catch (e) {
+ // Catch spoc positions that are not numbers or negative, and do nothing.
+ // We have hard coded backup positions.
+ gridPositions = undefined;
+ }
+
+ return gridPositions;
+ }
+
+ async loadLayout(sendUpdate, isStartup) {
+ let layoutResp = {};
+ let url = "";
+
+ if (!this.config.hardcoded_layout) {
+ layoutResp = await this.fetchLayout(isStartup);
+ }
+
+ if (!layoutResp || !layoutResp.layout) {
+ const isBasicLayout =
+ this.config.hardcoded_basic_layout ||
+ this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT] ||
+ this.store.getState().Prefs.values[PREF_REGION_BASIC_LAYOUT];
+
+ const sponsoredCollectionsEnabled = this.store.getState().Prefs.values[
+ PREF_COLLECTIONS_ENABLED
+ ];
+
+ const pocketConfig =
+ this.store.getState().Prefs.values?.pocketConfig || {};
+
+ let items = isBasicLayout ? 3 : 21;
+ if (pocketConfig.fourCardLayout || pocketConfig.hybridLayout) {
+ items = isBasicLayout ? 4 : 24;
+ }
+
+ const prepConfArr = arr => {
+ return arr
+ ?.split(",")
+ .filter(item => item)
+ .map(item => parseInt(item, 10));
+ };
+
+ const spocAdTypes = prepConfArr(pocketConfig.spocAdTypes);
+ const spocZoneIds = prepConfArr(pocketConfig.spocZoneIds);
+ const spocTopsitesAdTypes = prepConfArr(pocketConfig.spocTopsitesAdTypes);
+ const spocTopsitesZoneIds = prepConfArr(pocketConfig.spocTopsitesZoneIds);
+ const { spocSiteId } = pocketConfig;
+ let spocPlacementData;
+ let spocTopsitesPlacementData;
+ let spocsUrl;
+
+ if (spocAdTypes?.length && spocZoneIds?.length) {
+ spocPlacementData = {
+ ad_types: spocAdTypes,
+ zone_ids: spocZoneIds,
+ };
+ }
+
+ if (spocTopsitesAdTypes?.length && spocTopsitesZoneIds?.length) {
+ spocTopsitesPlacementData = {
+ ad_types: spocTopsitesAdTypes,
+ zone_ids: spocTopsitesZoneIds,
+ };
+ }
+
+ if (spocSiteId) {
+ const newUrl = new URL(SPOCS_URL);
+ newUrl.searchParams.set("site", spocSiteId);
+ spocsUrl = newUrl.href;
+ }
+
+ // Set a hardcoded layout if one is needed.
+ // Changing values in this layout in memory object is unnecessary.
+ layoutResp = getHardcodedLayout({
+ spocsUrl,
+ items,
+ sponsoredCollectionsEnabled,
+ spocPlacementData,
+ spocTopsitesPlacementData,
+ spocPositions: this.parseGridPositions(
+ pocketConfig.spocPositions?.split(`,`)
+ ),
+ widgetPositions: this.parseGridPositions(
+ pocketConfig.widgetPositions?.split(`,`)
+ ),
+ widgetData: [
+ ...(this.locale.startsWith("en-") ? [{ type: "TopicsWidget" }] : []),
+ ],
+ hybridLayout: pocketConfig.hybridLayout,
+ hideCardBackground: pocketConfig.hideCardBackground,
+ fourCardLayout: pocketConfig.fourCardLayout,
+ newFooterSection: pocketConfig.newFooterSection,
+ compactGrid: pocketConfig.compactGrid,
+ // For now essentialReadsHeader and editorsPicksHeader are English only.
+ essentialReadsHeader:
+ this.locale.startsWith("en-") && pocketConfig.essentialReadsHeader,
+ editorsPicksHeader:
+ this.locale.startsWith("en-") && pocketConfig.editorsPicksHeader,
+ });
+ }
+
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_LAYOUT_UPDATE,
+ data: layoutResp,
+ meta: {
+ isStartup,
+ },
+ });
+
+ if (layoutResp.spocs) {
+ url =
+ this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] ||
+ this.config.spocs_endpoint ||
+ layoutResp.spocs.url;
+
+ const spocsEndpointQuery = this.store.getState().Prefs.values[
+ PREF_SPOCS_ENDPOINT_QUERY
+ ];
+
+ // For QA, testing, or debugging purposes, there may be a query string to add.
+ url = this.addEndpointQuery(url, spocsEndpointQuery);
+
+ if (
+ url &&
+ url !== this.store.getState().DiscoveryStream.spocs.spocs_endpoint
+ ) {
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT,
+ data: {
+ url,
+ },
+ meta: {
+ isStartup,
+ },
+ });
+ this.updatePlacements(sendUpdate, layoutResp.layout, isStartup);
+ }
+ }
+ }
+
+ /**
+ * buildFeedPromise - Adds the promise result to newFeeds and
+ * pushes a promise to newsFeedsPromises.
+ * @param {Object} Has both newFeedsPromises (Array) and newFeeds (Object)
+ * @param {Boolean} isStartup We have different cache handling for startup.
+ * @returns {Function} We return a function so we can contain
+ * the scope for isStartup and the promises object.
+ * Combines feed results and promises for each component with a feed.
+ */
+ buildFeedPromise(
+ { newFeedsPromises, newFeeds },
+ isStartup = false,
+ sendUpdate
+ ) {
+ return component => {
+ const { url } = component.feed;
+
+ if (!newFeeds[url]) {
+ // We initially stub this out so we don't fetch dupes,
+ // we then fill in with the proper object inside the promise.
+ newFeeds[url] = {};
+ const feedPromise = this.getComponentFeed(url, isStartup);
+
+ feedPromise
+ .then(feed => {
+ // If we stored the result of filter in feed cache as it happened,
+ // I think we could reduce doing this for cache fetches.
+ // Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606277
+ newFeeds[url] = this.filterRecommendations(feed);
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_FEED_UPDATE,
+ data: {
+ feed: newFeeds[url],
+ url,
+ },
+ meta: {
+ isStartup,
+ },
+ });
+ })
+ .catch(
+ /* istanbul ignore next */ error => {
+ console.error(
+ `Error trying to load component feed ${url}: ${error}`
+ );
+ }
+ );
+ newFeedsPromises.push(feedPromise);
+ }
+ };
+ }
+
+ filterRecommendations(feed) {
+ if (
+ feed &&
+ feed.data &&
+ feed.data.recommendations &&
+ feed.data.recommendations.length
+ ) {
+ const { data: recommendations } = this.filterBlocked(
+ feed.data.recommendations
+ );
+ return {
+ ...feed,
+ data: {
+ ...feed.data,
+ recommendations,
+ },
+ };
+ }
+ return feed;
+ }
+
+ /**
+ * reduceFeedComponents - Filters out components with no feeds, and combines
+ * all feeds on this component with the feeds from other components.
+ * @param {Boolean} isStartup We have different cache handling for startup.
+ * @returns {Function} We return a function so we can contain the scope for isStartup.
+ * Reduces feeds into promises and feed data.
+ */
+ reduceFeedComponents(isStartup, sendUpdate) {
+ return (accumulator, row) => {
+ row.components
+ .filter(component => component && component.feed)
+ .forEach(this.buildFeedPromise(accumulator, isStartup, sendUpdate));
+ return accumulator;
+ };
+ }
+
+ /**
+ * buildFeedPromises - Filters out rows with no components,
+ * and gets us a promise for each unique feed.
+ * @param {Object} layout This is the Discovery Stream layout object.
+ * @param {Boolean} isStartup We have different cache handling for startup.
+ * @returns {Object} An object with newFeedsPromises (Array) and newFeeds (Object),
+ * we can Promise.all newFeedsPromises to get completed data in newFeeds.
+ */
+ buildFeedPromises(layout, isStartup, sendUpdate) {
+ const initialData = {
+ newFeedsPromises: [],
+ newFeeds: {},
+ };
+ return layout
+ .filter(row => row && row.components)
+ .reduce(this.reduceFeedComponents(isStartup, sendUpdate), initialData);
+ }
+
+ async loadComponentFeeds(sendUpdate, isStartup = false) {
+ const { DiscoveryStream } = this.store.getState();
+
+ if (!DiscoveryStream || !DiscoveryStream.layout) {
+ return;
+ }
+
+ // Reset the flag that indicates whether or not at least one API request
+ // was issued to fetch the component feed in `getComponentFeed()`.
+ this.componentFeedFetched = false;
+ const { newFeedsPromises, newFeeds } = this.buildFeedPromises(
+ DiscoveryStream.layout,
+ isStartup,
+ sendUpdate
+ );
+
+ // Each promise has a catch already built in, so no need to catch here.
+ await Promise.all(newFeedsPromises);
+
+ if (this.componentFeedFetched) {
+ this.cleanUpTopRecImpressionPref(newFeeds);
+ }
+ await this.cache.set("feeds", newFeeds);
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_FEEDS_UPDATE,
+ meta: {
+ isStartup,
+ },
+ });
+ }
+
+ getPlacements() {
+ const { placements } = this.store.getState().DiscoveryStream.spocs;
+ return placements;
+ }
+
+ // I wonder, can this be better as a reducer?
+ // See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717
+ placementsForEach(callback) {
+ this.getPlacements().forEach(callback);
+ }
+
+ // Bug 1567271 introduced meta data on a list of spocs.
+ // This involved moving the spocs array into an items prop.
+ // However, old data could still be returned, and cached data might also be old.
+ // For ths reason, we want to ensure if we don't find an items array,
+ // we use the previous array placement, and then stub out title and context to empty strings.
+ // We need to do this *after* both fresh fetches and cached data to reduce repetition.
+ normalizeSpocsItems(spocs) {
+ const items = spocs.items || spocs;
+ const title = spocs.title || "";
+ const context = spocs.context || "";
+ const sponsor = spocs.sponsor || "";
+ // We do not stub sponsored_by_override with an empty string. It is an override, and an empty string
+ // explicitly means to override the client to display an empty string.
+ // An empty string is not an no op in this case. Undefined is the proper no op here.
+ const { sponsored_by_override } = spocs;
+ // Undefined is fine here. It's optional and only used by collections.
+ // If we leave it out, you get a collection that cannot be dismissed.
+ const { flight_id } = spocs;
+ return {
+ items,
+ title,
+ context,
+ sponsor,
+ sponsored_by_override,
+ ...(flight_id ? { flight_id } : {}),
+ };
+ }
+
+ // This turns personalization on/off if the server sends the override command.
+ // The server sends a true signal to keep personalization on. So a malfunctioning
+ // server would more likely mistakenly turn off personalization, and not turn it on.
+ // This is safer, because the override is for cases where personalization is causing issues.
+ // So having it mistakenly go off is safe, but it mistakenly going on could be bad.
+ personalizationOverride(overrideCommand) {
+ // Are we currently in an override state.
+ // This is useful to know if we want to do a cleanup.
+ const overrideState = this.store.getState().Prefs.values[
+ PREF_PERSONALIZATION_OVERRIDE
+ ];
+
+ // Is this profile currently set to be personalized.
+ const personalization = this.store.getState().Prefs.values[
+ PREF_PERSONALIZATION
+ ];
+
+ // If we have an override command, profile is currently personalized,
+ // and is not currently being overridden, we can set the override pref.
+ if (overrideCommand && personalization && !overrideState) {
+ this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION_OVERRIDE, true));
+ }
+
+ // This is if we need to revert an override and do cleanup.
+ // We do this if we are in an override state,
+ // but not currently receiving the override signal.
+ if (!overrideCommand && overrideState) {
+ this.store.dispatch({
+ type: at.CLEAR_PREF,
+ data: { name: PREF_PERSONALIZATION_OVERRIDE },
+ });
+ }
+ }
+
+ updateSponsoredCollectionsPref(collectionEnabled = false) {
+ const currentState = this.store.getState().Prefs.values[
+ PREF_COLLECTIONS_ENABLED
+ ];
+
+ // If the current state does not match the new state, update the pref.
+ if (currentState !== collectionEnabled) {
+ this.store.dispatch(
+ ac.SetPref(PREF_COLLECTIONS_ENABLED, collectionEnabled)
+ );
+ }
+ }
+
+ async loadSpocs(sendUpdate, isStartup) {
+ const cachedData = (await this.cache.get()) || {};
+ let spocsState;
+
+ const placements = this.getPlacements();
+
+ if (this.showSpocs && placements?.length) {
+ spocsState = cachedData.spocs;
+ if (this.isExpired({ cachedData, key: "spocs", isStartup })) {
+ const endpoint = this.store.getState().DiscoveryStream.spocs
+ .spocs_endpoint;
+
+ const headers = new Headers();
+ headers.append("content-type", "application/json");
+
+ const apiKeyPref = this._prefCache.config.api_key_pref;
+ const apiKey = Services.prefs.getCharPref(apiKeyPref, "");
+
+ const spocsResponse = await this.fetchFromEndpoint(endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({
+ pocket_id: this._impressionId,
+ version: 2,
+ consumer_key: apiKey,
+ ...(placements.length ? { placements } : {}),
+ }),
+ });
+
+ if (spocsResponse) {
+ spocsState = {
+ lastUpdated: Date.now(),
+ spocs: {
+ ...spocsResponse,
+ },
+ };
+
+ if (spocsResponse.settings && spocsResponse.settings.feature_flags) {
+ this.personalizationOverride(
+ // The server's old signal was for a version override.
+ // When we removed version 1, version 2 was now the defacto only version.
+ // Without a version 1, the override is now a command to turn off personalization.
+ !spocsResponse.settings.feature_flags.spoc_v2
+ );
+ this.updateSponsoredCollectionsPref(
+ spocsResponse.settings.feature_flags.collections
+ );
+ }
+
+ const spocsResultPromises = this.getPlacements().map(
+ async placement => {
+ const freshSpocs = spocsState.spocs[placement.name];
+
+ if (!freshSpocs) {
+ return;
+ }
+
+ // spocs can be returns as an array, or an object with an items array.
+ // We want to normalize this so all our spocs have an items array.
+ // There can also be some meta data for title and context.
+ // This is mostly because of backwards compat.
+ const {
+ items: normalizedSpocsItems,
+ title,
+ context,
+ sponsor,
+ sponsored_by_override,
+ } = this.normalizeSpocsItems(freshSpocs);
+
+ if (!normalizedSpocsItems || !normalizedSpocsItems.length) {
+ // In the case of old data, we still want to ensure we normalize the data structure,
+ // even if it's empty. We expect the empty data to be an object with items array,
+ // and not just an empty array.
+ spocsState.spocs = {
+ ...spocsState.spocs,
+ [placement.name]: {
+ title,
+ context,
+ items: [],
+ },
+ };
+ return;
+ }
+
+ // Migrate flight_id
+ const { data: migratedSpocs } = this.migrateFlightId(
+ normalizedSpocsItems
+ );
+
+ const { data: capResult } = this.frequencyCapSpocs(migratedSpocs);
+
+ const { data: blockedResults } = this.filterBlocked(capResult);
+
+ const { data: scoredResults } = await this.scoreItems(
+ blockedResults,
+ "spocs"
+ );
+
+ spocsState.spocs = {
+ ...spocsState.spocs,
+ [placement.name]: {
+ title,
+ context,
+ sponsor,
+ sponsored_by_override,
+ items: scoredResults,
+ },
+ };
+ }
+ );
+ await Promise.all(spocsResultPromises);
+
+ this.cleanUpFlightImpressionPref(spocsState.spocs);
+ await this.cache.set("spocs", {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.spocs,
+ });
+ } else {
+ console.error("No response for spocs_endpoint prop");
+ }
+ }
+ }
+
+ // Use good data if we have it, otherwise nothing.
+ // We can have no data if spocs set to off.
+ // We can have no data if request fails and there is no good cache.
+ // We want to send an update spocs or not, so client can render something.
+ spocsState =
+ spocsState && spocsState.spocs
+ ? spocsState
+ : {
+ lastUpdated: Date.now(),
+ spocs: {},
+ };
+
+ sendUpdate({
+ type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
+ data: {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.spocs,
+ },
+ meta: {
+ isStartup,
+ },
+ });
+ }
+
+ async clearSpocs() {
+ const endpoint = this.store.getState().Prefs.values[
+ PREF_SPOCS_CLEAR_ENDPOINT
+ ];
+ if (!endpoint) {
+ return;
+ }
+ const headers = new Headers();
+ headers.append("content-type", "application/json");
+
+ await this.fetchFromEndpoint(endpoint, {
+ method: "DELETE",
+ headers,
+ body: JSON.stringify({
+ pocket_id: this._impressionId,
+ }),
+ });
+ }
+
+ /*
+ * This just re hydrates the provider from cache.
+ * We can call this on startup because it's generally fast.
+ * It reports to devtools the last time the data in the cache was updated.
+ */
+ async loadPersonalizationScoresCache(isStartup = false) {
+ const cachedData = (await this.cache.get()) || {};
+ const { personalization } = cachedData;
+
+ if (this.personalized && personalization && personalization.scores) {
+ this.recommendationProvider.setProvider(personalization.scores);
+
+ this.personalizationLastUpdated = personalization._timestamp;
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED,
+ data: {
+ lastUpdated: this.personalizationLastUpdated,
+ },
+ meta: {
+ isStartup,
+ },
+ })
+ );
+ }
+ }
+
+ /*
+ * This creates a new recommendationProvider using fresh data,
+ * It's run on a last updated timer. This is the opposite of loadPersonalizationScoresCache.
+ * This is also much slower so we only trigger this in the background on idle-daily.
+ * It causes new profiles to pick up personalization slowly because the first time
+ * a new profile is run you don't have any old cache to use, so it needs to wait for the first
+ * idle-daily. Older profiles can rely on cache during the idle-daily gap. Idle-daily is
+ * usually run once every 24 hours.
+ */
+ async updatePersonalizationScores() {
+ if (
+ !this.personalized ||
+ Date.now() - this.personalizationLastUpdated <
+ MIN_PERSONALIZATION_UPDATE_TIME
+ ) {
+ return;
+ }
+
+ this.recommendationProvider.setProvider();
+
+ await this.recommendationProvider.init();
+
+ const personalization = { scores: this.recommendationProvider.getScores() };
+ this.personalizationLastUpdated = Date.now();
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED,
+ data: {
+ lastUpdated: this.personalizationLastUpdated,
+ },
+ })
+ );
+ personalization._timestamp = this.personalizationLastUpdated;
+ this.cache.set("personalization", personalization);
+ }
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "idle-daily":
+ this.updatePersonalizationScores();
+ break;
+ case "nsPref:changed":
+ // If the Pocket button was turned on or off, we need to update the cards
+ // because cards show menu options for the Pocket button that need to be removed.
+ if (data === PREF_POCKET_BUTTON) {
+ this.configReset();
+ }
+ break;
+ }
+ }
+
+ /*
+ * This function is used to sort any type of story, both spocs and recs.
+ * This uses hierarchical sorting, first sorting by priority, then by score within a priority.
+ * This function could be sorting an array of spocs or an array of recs.
+ * A rec would have priority undefined, and a spoc would probably have a priority set.
+ * Priority is sorted ascending, so low numbers are the highest priority.
+ * Score is sorted descending, so high numbers are the highest score.
+ * Undefined priority values are considered the lowest priority.
+ * A negative priority is considered the same as undefined, lowest priority.
+ * A negative priority is unlikely and not currently supported or expected.
+ * A negative score is a possible use case.
+ */
+ sortItem(a, b) {
+ // If the priorities are the same, sort based on score.
+ // If both item priorities are undefined,
+ // we can safely sort via score.
+ if (a.priority === b.priority) {
+ return b.score - a.score;
+ } else if (!a.priority || a.priority <= 0) {
+ // If priority is undefined or an unexpected value,
+ // consider it lowest priority.
+ return 1;
+ } else if (!b.priority || b.priority <= 0) {
+ // Also consider this case lowest priority.
+ return -1;
+ }
+ // Our primary sort for items with priority.
+ return a.priority - b.priority;
+ }
+
+ async scoreItems(items, type) {
+ const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.spocsPersonalized;
+ const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.recsPersonalized;
+ const personalizedByType =
+ type === "feed" ? recsPersonalized : spocsPersonalized;
+
+ const data = (
+ await Promise.all(
+ items.map(item => this.scoreItem(item, personalizedByType))
+ )
+ )
+ // Sort by highest scores.
+ .sort(this.sortItem);
+
+ return { data };
+ }
+
+ async scoreItem(item, personalizedByType) {
+ item.score = item.item_score;
+ if (item.score !== 0 && !item.score) {
+ item.score = 1;
+ }
+ if (this.personalized && personalizedByType) {
+ await this.recommendationProvider.calculateItemRelevanceScore(item);
+ }
+ return item;
+ }
+
+ filterBlocked(data) {
+ if (data && data.length) {
+ let flights = this.readDataPref(PREF_FLIGHT_BLOCKS);
+ const filteredItems = data.filter(item => {
+ const blocked =
+ lazy.NewTabUtils.blockedLinks.isBlocked({ url: item.url }) ||
+ flights[item.flight_id];
+ return !blocked;
+ });
+ return { data: filteredItems };
+ }
+ return { data };
+ }
+
+ // For backwards compatibility, older spoc endpoint don't have flight_id,
+ // but instead had campaign_id we can use
+ //
+ // @param {Object} data An object that might have a SPOCS array.
+ // @returns {Object} An object with a property `data` as the result.
+ migrateFlightId(spocs) {
+ if (spocs && spocs.length) {
+ return {
+ data: spocs.map(s => {
+ return {
+ ...s,
+ ...(s.flight_id || s.campaign_id
+ ? {
+ flight_id: s.flight_id || s.campaign_id,
+ }
+ : {}),
+ ...(s.caps
+ ? {
+ caps: {
+ ...s.caps,
+ flight: s.caps.flight || s.caps.campaign,
+ },
+ }
+ : {}),
+ };
+ }),
+ };
+ }
+ return { data: spocs };
+ }
+
+ // Filter spocs based on frequency caps
+ //
+ // @param {Object} data An object that might have a SPOCS array.
+ // @returns {Object} An object with a property `data` as the result, and a property
+ // `filterItems` as the frequency capped items.
+ frequencyCapSpocs(spocs) {
+ if (spocs && spocs.length) {
+ const impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS);
+ const caps = [];
+ const result = spocs.filter(s => {
+ const isBelow = this.isBelowFrequencyCap(impressions, s);
+ if (!isBelow) {
+ caps.push(s);
+ }
+ return isBelow;
+ });
+ // send caps to redux if any.
+ if (caps.length) {
+ this.store.dispatch({
+ type: at.DISCOVERY_STREAM_SPOCS_CAPS,
+ data: caps,
+ });
+ }
+ return { data: result, filtered: caps };
+ }
+ return { data: spocs, filtered: [] };
+ }
+
+ // Frequency caps are based on flight, which may include multiple spocs.
+ // We currently support two types of frequency caps:
+ // - lifetime: Indicates how many times spocs from a flight can be shown in total
+ // - period: Indicates how many times spocs from a flight can be shown within a period
+ //
+ // So, for example, the feed configuration below defines that for flight 1 no more
+ // than 5 spocs can be shown in total, and no more than 2 per hour.
+ // "flight_id": 1,
+ // "caps": {
+ // "lifetime": 5,
+ // "flight": {
+ // "count": 2,
+ // "period": 3600
+ // }
+ // }
+ isBelowFrequencyCap(impressions, spoc) {
+ const flightImpressions = impressions[spoc.flight_id];
+ if (!flightImpressions) {
+ return true;
+ }
+
+ const lifetime = spoc.caps && spoc.caps.lifetime;
+
+ const lifeTimeCap = Math.min(
+ lifetime || MAX_LIFETIME_CAP,
+ MAX_LIFETIME_CAP
+ );
+ const lifeTimeCapExceeded = flightImpressions.length >= lifeTimeCap;
+ if (lifeTimeCapExceeded) {
+ return false;
+ }
+
+ const flightCap = spoc.caps && spoc.caps.flight;
+ if (flightCap) {
+ const flightCapExceeded =
+ flightImpressions.filter(i => Date.now() - i < flightCap.period * 1000)
+ .length >= flightCap.count;
+ return !flightCapExceeded;
+ }
+ return true;
+ }
+
+ async retryFeed(feed) {
+ const { url } = feed;
+ const result = await this.getComponentFeed(url);
+ const newFeed = this.filterRecommendations(result);
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_FEED_UPDATE,
+ data: {
+ feed: newFeed,
+ url,
+ },
+ })
+ );
+ }
+
+ async getComponentFeed(feedUrl, isStartup) {
+ const cachedData = (await this.cache.get()) || {};
+ const { feeds } = cachedData;
+
+ let feed = feeds ? feeds[feedUrl] : null;
+ if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) {
+ const feedResponse = await this.fetchFromEndpoint(feedUrl);
+ if (feedResponse) {
+ const { data: scoredItems } = await this.scoreItems(
+ feedResponse.recommendations,
+ "feed"
+ );
+ const { recsExpireTime } = feedResponse.settings;
+ const recommendations = this.rotate(scoredItems, recsExpireTime);
+ this.componentFeedFetched = true;
+ feed = {
+ lastUpdated: Date.now(),
+ data: {
+ settings: feedResponse.settings,
+ recommendations,
+ status: "success",
+ },
+ };
+ } else {
+ console.error("No response for feed");
+ }
+ }
+
+ // If we have no feed at this point, both fetch and cache failed for some reason.
+ return (
+ feed || {
+ data: {
+ status: "failed",
+ },
+ }
+ );
+ }
+
+ /**
+ * Called at startup to update cached data in the background.
+ */
+ async _maybeUpdateCachedData() {
+ const expirationPerComponent = await this._checkExpirationPerComponent();
+ // Pass in `store.dispatch` to send the updates only to main
+ if (expirationPerComponent.layout) {
+ await this.loadLayout(this.store.dispatch);
+ }
+ if (expirationPerComponent.spocs) {
+ await this.loadSpocs(this.store.dispatch);
+ }
+ if (expirationPerComponent.feeds) {
+ await this.loadComponentFeeds(this.store.dispatch);
+ }
+ }
+
+ /**
+ * @typedef {Object} RefreshAll
+ * @property {boolean} updateOpenTabs - Sends updates to open tabs immediately if true,
+ * updates in background if false
+ * @property {boolean} isStartup - When the function is called at browser startup
+ *
+ * Refreshes layout, component feeds, and spocs in order if caches have expired.
+ * @param {RefreshAll} options
+ */
+ async refreshAll(options = {}) {
+ const personalizationCacheLoadPromise = this.loadPersonalizationScoresCache(
+ options.isStartup
+ );
+
+ const spocsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.spocsPersonalized;
+ const recsPersonalized = this.store.getState().Prefs.values?.pocketConfig
+ ?.recsPersonalized;
+
+ let expirationPerComponent = {};
+ if (this.personalized) {
+ // We store this before we refresh content.
+ // This way, we can know what and if something got updated,
+ // so we can know to score the results.
+ expirationPerComponent = await this._checkExpirationPerComponent();
+ }
+ await this.refreshContent(options);
+
+ if (this.personalized) {
+ // personalizationCacheLoadPromise is probably done, because of the refreshContent await above,
+ // but to be sure, we should check that it's done, without making the parent function wait.
+ personalizationCacheLoadPromise.then(() => {
+ // If we don't have expired stories or feeds, we don't need to score after init.
+ // If we do have expired stories, we want to score after init.
+ // In both cases, we don't want these to block the parent function.
+ // This is why we store the promise, and call then to do our scoring work.
+ const initPromise = this.recommendationProvider.init();
+ initPromise.then(() => {
+ // Both scoreFeeds and scoreSpocs are promises,
+ // but they don't need to wait for each other.
+ // We can just fire them and forget at this point.
+ const { feeds, spocs } = this.store.getState().DiscoveryStream;
+ if (
+ recsPersonalized &&
+ feeds.loaded &&
+ expirationPerComponent.feeds
+ ) {
+ this.scoreFeeds(feeds);
+ }
+ if (
+ spocsPersonalized &&
+ spocs.loaded &&
+ expirationPerComponent.spocs
+ ) {
+ this.scoreSpocs(spocs);
+ }
+ });
+ });
+ }
+ }
+
+ async scoreFeeds(feedsState) {
+ if (feedsState.data) {
+ const feeds = {};
+ const feedsPromises = Object.keys(feedsState.data).map(url => {
+ let feed = feedsState.data[url];
+ const feedPromise = this.scoreItems(feed.data.recommendations, "feed");
+ feedPromise.then(({ data: scoredItems }) => {
+ const { recsExpireTime } = feed.data.settings;
+ const recommendations = this.rotate(scoredItems, recsExpireTime);
+ feed = {
+ ...feed,
+ data: {
+ ...feed.data,
+ recommendations,
+ },
+ };
+
+ feeds[url] = feed;
+
+ this.store.dispatch(
+ ac.AlsoToPreloaded({
+ type: at.DISCOVERY_STREAM_FEED_UPDATE,
+ data: {
+ feed,
+ url,
+ },
+ })
+ );
+ });
+ return feedPromise;
+ });
+ await Promise.all(feedsPromises);
+ await this.cache.set("feeds", feeds);
+ }
+ }
+
+ async scoreSpocs(spocsState) {
+ const spocsResultPromises = this.getPlacements().map(async placement => {
+ const nextSpocs = spocsState.data[placement.name] || {};
+ const { items } = nextSpocs;
+
+ if (!items || !items.length) {
+ return;
+ }
+
+ const { data: scoreResult } = await this.scoreItems(items, "spocs");
+
+ spocsState.data = {
+ ...spocsState.data,
+ [placement.name]: {
+ ...nextSpocs,
+ items: scoreResult,
+ },
+ };
+ });
+ await Promise.all(spocsResultPromises);
+
+ // Update cache here so we don't need to re calculate scores on loads from cache.
+ // Related Bug 1606276
+ await this.cache.set("spocs", {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.data,
+ });
+ this.store.dispatch(
+ ac.AlsoToPreloaded({
+ type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
+ data: {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.data,
+ },
+ })
+ );
+ }
+
+ async refreshContent(options = {}) {
+ const { updateOpenTabs, isStartup } = options;
+
+ const dispatch = updateOpenTabs
+ ? action => this.store.dispatch(ac.BroadcastToContent(action))
+ : this.store.dispatch;
+
+ await this.loadLayout(dispatch, isStartup);
+ if (this.showStories || this.showTopsites) {
+ const promises = [];
+ // We could potentially have either or both sponsored topsites or stories.
+ // We only make one fetch, and control which to request when we fetch.
+ // So for now we only care if we need to make this request at all.
+ const spocsPromise = this.loadSpocs(dispatch, isStartup).catch(error =>
+ console.error(`Error trying to load spocs feeds: ${error}`)
+ );
+ promises.push(spocsPromise);
+ if (this.showStories) {
+ const storiesPromise = this.loadComponentFeeds(
+ dispatch,
+ isStartup
+ ).catch(error =>
+ console.error(`Error trying to load component feeds: ${error}`)
+ );
+ promises.push(storiesPromise);
+ }
+ await Promise.all(promises);
+ if (isStartup) {
+ await this._maybeUpdateCachedData();
+ }
+ }
+ }
+
+ // We have to rotate stories on the client so that
+ // active stories are at the front of the list, followed by stories that have expired
+ // impressions i.e. have been displayed for longer than recsExpireTime.
+ rotate(recommendations, recsExpireTime) {
+ const maxImpressionAge = Math.max(
+ recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME,
+ DEFAULT_RECS_EXPIRE_TIME
+ );
+ const impressions = this.readDataPref(PREF_REC_IMPRESSIONS);
+ const expired = [];
+ const active = [];
+ for (const item of recommendations) {
+ if (
+ impressions[item.id] &&
+ Date.now() - impressions[item.id] >= maxImpressionAge
+ ) {
+ expired.push(item);
+ } else {
+ active.push(item);
+ }
+ }
+ return active.concat(expired);
+ }
+
+ enableStories() {
+ if (this.config.enabled && this.loaded) {
+ // If stories are being re enabled, ensure we have stories.
+ this.refreshAll({ updateOpenTabs: true });
+ }
+ }
+
+ async enable() {
+ await this.refreshAll({ updateOpenTabs: true, isStartup: true });
+ Services.obs.addObserver(this, "idle-daily");
+ this.loaded = true;
+ }
+
+ async reset() {
+ this.resetDataPrefs();
+ await this.resetCache();
+ if (this.loaded) {
+ Services.obs.removeObserver(this, "idle-daily");
+ }
+ this.resetState();
+ }
+
+ async resetCache() {
+ await this.resetAllCache();
+ }
+
+ async resetContentCache() {
+ await this.cache.set("layout", {});
+ await this.cache.set("feeds", {});
+ await this.cache.set("spocs", {});
+ }
+
+ async resetAllCache() {
+ await this.resetContentCache();
+ await this.cache.set("personalization", {});
+ }
+
+ resetDataPrefs() {
+ this.writeDataPref(PREF_SPOC_IMPRESSIONS, {});
+ this.writeDataPref(PREF_REC_IMPRESSIONS, {});
+ this.writeDataPref(PREF_FLIGHT_BLOCKS, {});
+ }
+
+ resetState() {
+ // Reset reducer
+ this.store.dispatch(
+ ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET })
+ );
+ this.setupPrefs(false /* isStartup */);
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE,
+ data: {
+ value: this.store.getState().Prefs.values[
+ PREF_COLLECTION_DISMISSIBLE
+ ],
+ },
+ })
+ );
+ this.personalizationLastUpdated = null;
+ this.loaded = false;
+ }
+
+ async onPrefChange() {
+ // We always want to clear the cache/state if the pref has changed
+ await this.reset();
+ if (this.config.enabled) {
+ // Load data from all endpoints
+ await this.enable();
+ }
+ }
+
+ // This is a request to change the config from somewhere.
+ // Can be from a spefic pref related to Discovery Stream,
+ // or can be a generic request from an external feed that
+ // something changed.
+ configReset() {
+ this._prefCache.config = null;
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_CONFIG_CHANGE,
+ data: this.config,
+ })
+ );
+ }
+
+ recordFlightImpression(flightId) {
+ let impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS);
+
+ const timeStamps = impressions[flightId] || [];
+ timeStamps.push(Date.now());
+ impressions = { ...impressions, [flightId]: timeStamps };
+
+ this.writeDataPref(PREF_SPOC_IMPRESSIONS, impressions);
+ }
+
+ recordTopRecImpressions(recId) {
+ let impressions = this.readDataPref(PREF_REC_IMPRESSIONS);
+ if (!impressions[recId]) {
+ impressions = { ...impressions, [recId]: Date.now() };
+ this.writeDataPref(PREF_REC_IMPRESSIONS, impressions);
+ }
+ }
+
+ recordBlockFlightId(flightId) {
+ const flights = this.readDataPref(PREF_FLIGHT_BLOCKS);
+ if (!flights[flightId]) {
+ flights[flightId] = 1;
+ this.writeDataPref(PREF_FLIGHT_BLOCKS, flights);
+ }
+ }
+
+ cleanUpFlightImpressionPref(data) {
+ let flightIds = [];
+ this.placementsForEach(placement => {
+ const newSpocs = data[placement.name];
+ if (!newSpocs) {
+ return;
+ }
+
+ const items = newSpocs.items || [];
+ flightIds = [...flightIds, ...items.map(s => `${s.flight_id}`)];
+ });
+ if (flightIds && flightIds.length) {
+ this.cleanUpImpressionPref(
+ id => !flightIds.includes(id),
+ PREF_SPOC_IMPRESSIONS
+ );
+ }
+ }
+
+ // Clean up rec impression pref by removing all stories that are no
+ // longer part of the response.
+ cleanUpTopRecImpressionPref(newFeeds) {
+ // Need to build a single list of stories.
+ const activeStories = Object.keys(newFeeds)
+ .filter(currentValue => newFeeds[currentValue].data)
+ .reduce((accumulator, currentValue) => {
+ const { recommendations } = newFeeds[currentValue].data;
+ return accumulator.concat(recommendations.map(i => `${i.id}`));
+ }, []);
+ this.cleanUpImpressionPref(
+ id => !activeStories.includes(id),
+ PREF_REC_IMPRESSIONS
+ );
+ }
+
+ writeDataPref(pref, impressions) {
+ this.store.dispatch(ac.SetPref(pref, JSON.stringify(impressions)));
+ }
+
+ readDataPref(pref) {
+ const prefVal = this.store.getState().Prefs.values[pref];
+ return prefVal ? JSON.parse(prefVal) : {};
+ }
+
+ cleanUpImpressionPref(isExpired, pref) {
+ const impressions = this.readDataPref(pref);
+ let changed = false;
+
+ Object.keys(impressions).forEach(id => {
+ if (isExpired(id)) {
+ changed = true;
+ delete impressions[id];
+ }
+ });
+
+ if (changed) {
+ this.writeDataPref(pref, impressions);
+ }
+ }
+
+ onCollectionsChanged() {
+ // Update layout, and reload any off screen tabs.
+ // This does not change any existing open tabs.
+ // It also doesn't update any spoc or rec data, just the layout.
+ const dispatch = action => this.store.dispatch(ac.AlsoToPreloaded(action));
+ this.loadLayout(dispatch, false);
+ }
+
+ async onPrefChangedAction(action) {
+ switch (action.data.name) {
+ case PREF_CONFIG:
+ case PREF_ENABLED:
+ case PREF_HARDCODED_BASIC_LAYOUT:
+ case PREF_SPOCS_ENDPOINT:
+ case PREF_SPOCS_ENDPOINT_QUERY:
+ case PREF_PERSONALIZATION:
+ // This is a config reset directly related to Discovery Stream pref.
+ this.configReset();
+ break;
+ case PREF_COLLECTIONS_ENABLED:
+ this.onCollectionsChanged();
+ break;
+ case PREF_USER_TOPSITES:
+ case PREF_SYSTEM_TOPSITES:
+ if (
+ !(
+ this.showTopsites ||
+ (this.showStories && this.showSponsoredStories)
+ )
+ ) {
+ // Ensure we delete any remote data potentially related to spocs.
+ this.clearSpocs();
+ }
+ break;
+ case PREF_USER_TOPSTORIES:
+ case PREF_SYSTEM_TOPSTORIES:
+ if (
+ !(
+ this.showStories ||
+ (this.showTopsites && this.showSponsoredTopsites)
+ )
+ ) {
+ // Ensure we delete any remote data potentially related to spocs.
+ this.clearSpocs();
+ }
+ if (action.data.value) {
+ this.enableStories();
+ }
+ break;
+ // Check if spocs was disabled. Remove them if they were.
+ case PREF_SHOW_SPONSORED:
+ case PREF_SHOW_SPONSORED_TOPSITES:
+ const dispatch = update =>
+ this.store.dispatch(ac.BroadcastToContent(update));
+ // We refresh placements data because one of the spocs were turned off.
+ this.updatePlacements(
+ dispatch,
+ this.store.getState().DiscoveryStream.layout
+ );
+ // Currently the order of this is important.
+ // We need to check this after updatePlacements is called,
+ // because some of the spoc logic depends on the result of placement updates.
+ if (
+ !(
+ (this.showSponsoredStories ||
+ (this.showTopSites && this.showSponsoredTopSites)) &&
+ (this.showSponsoredTopsites ||
+ (this.showStories && this.showSponsoredStories))
+ )
+ ) {
+ // Ensure we delete any remote data potentially related to spocs.
+ this.clearSpocs();
+ }
+ // Placements have changed so consider spocs expired, and reload them.
+ await this.cache.set("spocs", {});
+ await this.loadSpocs(dispatch);
+ break;
+ }
+ }
+
+ async onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ // During the initialization of Firefox:
+ // 1. Set-up listeners and initialize the redux state for config;
+ this.setupConfig(true /* isStartup */);
+ this.setupPrefs(true /* isStartup */);
+ // 2. If config.enabled is true, start loading data.
+ if (this.config.enabled) {
+ await this.enable();
+ }
+ Services.prefs.addObserver(PREF_POCKET_BUTTON, this);
+ break;
+ case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK:
+ case at.SYSTEM_TICK:
+ // Only refresh if we loaded once in .enable()
+ if (
+ this.config.enabled &&
+ this.loaded &&
+ (await this.checkIfAnyCacheExpired())
+ ) {
+ await this.refreshAll({ updateOpenTabs: false });
+ }
+ break;
+ case at.DISCOVERY_STREAM_DEV_IDLE_DAILY:
+ Services.obs.notifyObservers(null, "idle-daily");
+ break;
+ case at.DISCOVERY_STREAM_DEV_SYNC_RS:
+ lazy.RemoteSettings.pollChanges();
+ break;
+ case at.DISCOVERY_STREAM_DEV_EXPIRE_CACHE:
+ // Personalization scores update at a slower interval than content, so in order to debug,
+ // we want to be able to expire just content to trigger the earlier expire times.
+ await this.resetContentCache();
+ break;
+ case at.DISCOVERY_STREAM_CONFIG_SET_VALUE:
+ // Use the original string pref to then set a value instead of
+ // this.config which has some modifications
+ this.store.dispatch(
+ ac.SetPref(
+ PREF_CONFIG,
+ JSON.stringify({
+ ...JSON.parse(this.store.getState().Prefs.values[PREF_CONFIG]),
+ [action.data.name]: action.data.value,
+ })
+ )
+ );
+ break;
+ case at.DISCOVERY_STREAM_POCKET_STATE_INIT:
+ this.setupPocketState(action.meta.fromTarget);
+ break;
+ case at.DISCOVERY_STREAM_CONFIG_RESET:
+ // This is a generic config reset likely related to an external feed pref.
+ this.configReset();
+ break;
+ case at.DISCOVERY_STREAM_CONFIG_RESET_DEFAULTS:
+ this.resetConfigDefauts();
+ break;
+ case at.DISCOVERY_STREAM_RETRY_FEED:
+ this.retryFeed(action.data.feed);
+ break;
+ case at.DISCOVERY_STREAM_CONFIG_CHANGE:
+ // When the config pref changes, load or unload data as needed.
+ await this.onPrefChange();
+ break;
+ case at.DISCOVERY_STREAM_IMPRESSION_STATS:
+ if (
+ action.data.tiles &&
+ action.data.tiles[0] &&
+ action.data.tiles[0].id
+ ) {
+ this.recordTopRecImpressions(action.data.tiles[0].id);
+ }
+ break;
+ case at.DISCOVERY_STREAM_SPOC_IMPRESSION:
+ if (this.showSpocs) {
+ this.recordFlightImpression(action.data.flightId);
+
+ // Apply frequency capping to SPOCs in the redux store, only update the
+ // store if the SPOCs are changed.
+ const spocsState = this.store.getState().DiscoveryStream.spocs;
+
+ let frequencyCapped = [];
+ this.placementsForEach(placement => {
+ const spocs = spocsState.data[placement.name];
+ if (!spocs || !spocs.items) {
+ return;
+ }
+
+ const { data: capResult, filtered } = this.frequencyCapSpocs(
+ spocs.items
+ );
+ frequencyCapped = [...frequencyCapped, ...filtered];
+
+ spocsState.data = {
+ ...spocsState.data,
+ [placement.name]: {
+ ...spocs,
+ items: capResult,
+ },
+ };
+ });
+
+ if (frequencyCapped.length) {
+ // Update cache here so we don't need to re calculate frequency caps on loads from cache.
+ await this.cache.set("spocs", {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.data,
+ });
+
+ this.store.dispatch(
+ ac.AlsoToPreloaded({
+ type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
+ data: {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.data,
+ },
+ })
+ );
+ }
+ }
+ break;
+ // This is fired from the browser, it has no concept of spocs, flight or pocket.
+ // We match the blocked url with our available spoc urls to see if there is a match.
+ // I suspect we *could* instead do this in BLOCK_URL but I'm not sure.
+ case at.PLACES_LINK_BLOCKED:
+ if (this.showSpocs) {
+ let blockedItems = [];
+ const spocsState = this.store.getState().DiscoveryStream.spocs;
+
+ this.placementsForEach(placement => {
+ const spocs = spocsState.data[placement.name];
+ if (spocs && spocs.items && spocs.items.length) {
+ const blockedResults = [];
+ const blocks = spocs.items.filter(s => {
+ const blocked = s.url === action.data.url;
+ if (!blocked) {
+ blockedResults.push(s);
+ }
+ return blocked;
+ });
+
+ blockedItems = [...blockedItems, ...blocks];
+
+ spocsState.data = {
+ ...spocsState.data,
+ [placement.name]: {
+ ...spocs,
+ items: blockedResults,
+ },
+ };
+ }
+ });
+
+ if (blockedItems.length) {
+ // Update cache here so we don't need to re calculate blocks on loads from cache.
+ await this.cache.set("spocs", {
+ lastUpdated: spocsState.lastUpdated,
+ spocs: spocsState.data,
+ });
+
+ // If we're blocking a spoc, we want open tabs to have
+ // a slightly different treatment from future tabs.
+ // AlsoToPreloaded updates the source data and preloaded tabs with a new spoc.
+ // BroadcastToContent updates open tabs with a non spoc instead of a new spoc.
+ this.store.dispatch(
+ ac.AlsoToPreloaded({
+ type: at.DISCOVERY_STREAM_LINK_BLOCKED,
+ data: action.data,
+ })
+ );
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_SPOC_BLOCKED,
+ data: action.data,
+ })
+ );
+ break;
+ }
+ }
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_LINK_BLOCKED,
+ data: action.data,
+ })
+ );
+ break;
+ case at.UNINIT:
+ // When this feed is shutting down:
+ this.uninitPrefs();
+ this._recommendationProvider = null;
+ Services.prefs.removeObserver(PREF_POCKET_BUTTON, this);
+ break;
+ case at.BLOCK_URL: {
+ // If we block a story that also has a flight_id
+ // we want to record that as blocked too.
+ // This is because a single flight might have slightly different urls.
+ action.data.forEach(site => {
+ const { flight_id } = site;
+ if (flight_id) {
+ this.recordBlockFlightId(flight_id);
+ }
+ });
+ break;
+ }
+ case at.PREF_CHANGED:
+ await this.onPrefChangedAction(action);
+ if (action.data.name === "pocketConfig") {
+ await this.onPrefChange();
+ this.setupPrefs(false /* isStartup */);
+ }
+ break;
+ }
+ }
+}
+
+/* This function generates a hardcoded layout each call.
+ This is because modifying the original object would
+ persist across pref changes and system_tick updates.
+
+ NOTE: There is some branching logic in the template.
+ `spocsUrl` Changing the url for spocs is used for adding a siteId query param.
+ `items` How many items to include in the primary card grid.
+ `spocPositions` Changes the position of spoc cards.
+ `spocPlacementData` Used to set the spoc content.
+ `spocTopsitesPlacementData` Used to set spoc content for topsites.
+ `sponsoredCollectionsEnabled` Tuns on and off the sponsored collection section.
+ `hybridLayout` Changes cards to smaller more compact cards only for specific breakpoints.
+ `hideCardBackground` Removes Pocket card background and borders.
+ `fourCardLayout` Enable four Pocket cards per row.
+ `newFooterSection` Changes the layout of the topics section.
+ `compactGrid` Reduce the number of pixels between the Pocket cards.
+ `essentialReadsHeader` Updates the Pocket section header and title to say "Today’s Essential Reads", moves the "Recommended by Pocket" header to the right side.
+ `editorsPicksHeader` Updates the Pocket section header and title to say "Editor’s Picks", if used with essentialReadsHeader, creates a second section 2 rows down for editorsPicks.
+*/
+getHardcodedLayout = ({
+ spocsUrl = SPOCS_URL,
+ items = 21,
+ spocPositions = [1, 5, 7, 11, 18, 20],
+ spocPlacementData = { ad_types: [3617], zone_ids: [217758, 217995] },
+ spocTopsitesPlacementData,
+ widgetPositions = [],
+ widgetData = [],
+ sponsoredCollectionsEnabled = false,
+ hybridLayout = false,
+ hideCardBackground = false,
+ fourCardLayout = false,
+ newFooterSection = false,
+ compactGrid = false,
+ essentialReadsHeader = false,
+ editorsPicksHeader = false,
+}) => ({
+ lastUpdate: Date.now(),
+ spocs: {
+ url: spocsUrl,
+ },
+ layout: [
+ {
+ width: 12,
+ components: [
+ {
+ type: "TopSites",
+ header: {
+ title: {
+ id: "newtab-section-header-topsites",
+ },
+ },
+ ...(spocTopsitesPlacementData
+ ? {
+ placement: {
+ name: "sponsored-topsites",
+ ad_types: spocTopsitesPlacementData.ad_types,
+ zone_ids: spocTopsitesPlacementData.zone_ids,
+ },
+ spocs: {
+ probability: 1,
+ prefs: [PREF_SHOW_SPONSORED_TOPSITES],
+ positions: [
+ {
+ index: 1,
+ },
+ ],
+ },
+ }
+ : {}),
+ properties: {},
+ },
+ ...(sponsoredCollectionsEnabled
+ ? [
+ {
+ type: "CollectionCardGrid",
+ properties: {
+ items: 3,
+ },
+ header: {
+ title: "",
+ },
+ placement: {
+ name: "sponsored-collection",
+ ad_types: [3617],
+ zone_ids: [217759, 218031],
+ },
+ spocs: {
+ probability: 1,
+ positions: [
+ {
+ index: 0,
+ },
+ {
+ index: 1,
+ },
+ {
+ index: 2,
+ },
+ ],
+ },
+ },
+ ]
+ : []),
+ {
+ type: "Message",
+ essentialReadsHeader,
+ editorsPicksHeader,
+ header: {
+ title: {
+ id: "newtab-section-header-pocket",
+ values: { provider: "Pocket" },
+ },
+ subtitle: "",
+ link_text: {
+ id: "newtab-pocket-learn-more",
+ },
+ link_url: "https://getpocket.com/firefox/new_tab_learn_more",
+ icon: "chrome://global/skin/icons/pocket.svg",
+ },
+ properties: {},
+ styles: {
+ ".ds-message": "margin-bottom: -20px",
+ },
+ },
+ {
+ type: "CardGrid",
+ properties: {
+ items,
+ hybridLayout,
+ hideCardBackground,
+ fourCardLayout,
+ compactGrid,
+ essentialReadsHeader,
+ editorsPicksHeader,
+ },
+ widgets: {
+ positions: widgetPositions.map(position => {
+ return { index: position };
+ }),
+ data: widgetData,
+ },
+ cta_variant: "link",
+ header: {
+ title: "",
+ },
+ placement: {
+ name: "spocs",
+ ad_types: spocPlacementData.ad_types,
+ zone_ids: spocPlacementData.zone_ids,
+ },
+ feed: {
+ embed_reference: null,
+ url:
+ "https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=$locale&region=$region&count=30",
+ },
+ spocs: {
+ probability: 1,
+ positions: spocPositions.map(position => {
+ return { index: position };
+ }),
+ },
+ },
+ {
+ type: "Navigation",
+ newFooterSection,
+ properties: {
+ alignment: "left-align",
+ links: [
+ {
+ name: "Self improvement",
+ url:
+ "https://getpocket.com/explore/self-improvement?utm_source=pocket-newtab",
+ },
+ {
+ name: "Food",
+ url:
+ "https://getpocket.com/explore/food?utm_source=pocket-newtab",
+ },
+ {
+ name: "Entertainment",
+ url:
+ "https://getpocket.com/explore/entertainment?utm_source=pocket-newtab",
+ },
+ {
+ name: "Health & fitness",
+ url:
+ "https://getpocket.com/explore/health?utm_source=pocket-newtab",
+ },
+ {
+ name: "Science",
+ url:
+ "https://getpocket.com/explore/science?utm_source=pocket-newtab",
+ },
+ {
+ name: "More recommendations ›",
+ url: "https://getpocket.com/explore?utm_source=pocket-newtab",
+ },
+ ],
+ extraLinks: [
+ {
+ name: "Career",
+ url:
+ "https://getpocket.com/explore/career?utm_source=pocket-newtab",
+ },
+ {
+ name: "Technology",
+ url:
+ "https://getpocket.com/explore/technology?utm_source=pocket-newtab",
+ },
+ ],
+ privacyNoticeURL: {
+ url:
+ "https://www.mozilla.org/privacy/firefox/#suggest-relevant-content",
+ title: {
+ id: "newtab-section-menu-privacy-notice",
+ },
+ },
+ },
+ header: {
+ title: {
+ id: "newtab-pocket-read-more",
+ },
+ },
+ styles: {
+ ".ds-navigation": "margin-top: -10px;",
+ },
+ },
+ ...(newFooterSection
+ ? [
+ {
+ type: "PrivacyLink",
+ properties: {
+ url: "https://www.mozilla.org/privacy/firefox/",
+ title: {
+ id: "newtab-section-menu-privacy-notice",
+ },
+ },
+ },
+ ]
+ : []),
+ ],
+ },
+ ],
+});
+
+const EXPORTED_SYMBOLS = ["DiscoveryStreamFeed"];
diff --git a/browser/components/newtab/lib/DownloadsManager.jsm b/browser/components/newtab/lib/DownloadsManager.jsm
new file mode 100644
index 0000000000..6c7b756e0d
--- /dev/null
+++ b/browser/components/newtab/lib/DownloadsManager.jsm
@@ -0,0 +1,191 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ DownloadsCommon: "resource:///modules/DownloadsCommon.sys.mjs",
+ DownloadsViewUI: "resource:///modules/DownloadsViewUI.sys.mjs",
+ FileUtils: "resource://gre/modules/FileUtils.sys.mjs",
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+});
+
+const DOWNLOAD_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for downloads changed events
+
+class DownloadsManager {
+ constructor(store) {
+ this._downloadData = null;
+ this._store = null;
+ this._downloadItems = new Map();
+ this._downloadTimer = null;
+ }
+
+ setTimeout(callback, delay) {
+ let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+ timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT);
+ return timer;
+ }
+
+ formatDownload(download) {
+ let referrer = download.source.referrerInfo?.originalReferrer?.spec || null;
+ return {
+ hostname: new URL(download.source.url).hostname,
+ url: download.source.url,
+ path: download.target.path,
+ title: lazy.DownloadsViewUI.getDisplayName(download),
+ description:
+ lazy.DownloadsViewUI.getSizeWithUnits(download) ||
+ lazy.DownloadsCommon.strings.sizeUnknown,
+ referrer,
+ date_added: download.endTime,
+ };
+ }
+
+ init(store) {
+ this._store = store;
+ this._downloadData = lazy.DownloadsCommon.getData(
+ null /* null for non-private downloads */,
+ true,
+ false,
+ true
+ );
+ this._downloadData.addView(this);
+ }
+
+ onDownloadAdded(download) {
+ if (!this._downloadItems.has(download.source.url)) {
+ this._downloadItems.set(download.source.url, download);
+
+ // On startup, all existing downloads fire this notification, so debounce them
+ if (this._downloadTimer) {
+ this._downloadTimer.delay = DOWNLOAD_CHANGED_DELAY_TIME;
+ } else {
+ this._downloadTimer = this.setTimeout(() => {
+ this._downloadTimer = null;
+ this._store.dispatch({ type: at.DOWNLOAD_CHANGED });
+ }, DOWNLOAD_CHANGED_DELAY_TIME);
+ }
+ }
+ }
+
+ onDownloadRemoved(download) {
+ if (this._downloadItems.has(download.source.url)) {
+ this._downloadItems.delete(download.source.url);
+ this._store.dispatch({ type: at.DOWNLOAD_CHANGED });
+ }
+ }
+
+ async getDownloads(
+ threshold,
+ {
+ numItems = this._downloadItems.size,
+ onlySucceeded = false,
+ onlyExists = false,
+ }
+ ) {
+ if (!threshold) {
+ return [];
+ }
+ let results = [];
+
+ // Only get downloads within the time threshold specified and sort by recency
+ const downloadThreshold = Date.now() - threshold;
+ let downloads = [...this._downloadItems.values()]
+ .filter(download => download.endTime > downloadThreshold)
+ .sort((download1, download2) => download1.endTime < download2.endTime);
+
+ for (const download of downloads) {
+ // Ignore blocked links, but allow long (data:) uris to avoid high CPU
+ if (
+ download.source.url.length < 10000 &&
+ lazy.NewTabUtils.blockedLinks.isBlocked(download.source)
+ ) {
+ continue;
+ }
+
+ // Only include downloads where the file still exists
+ if (onlyExists) {
+ // Refresh download to ensure the 'exists' attribute is up to date
+ await download.refresh();
+ if (!download.target.exists) {
+ continue;
+ }
+ }
+ // Only include downloads that were completed successfully
+ if (onlySucceeded) {
+ if (!download.succeeded) {
+ continue;
+ }
+ }
+ const formattedDownloadForHighlights = this.formatDownload(download);
+ results.push(formattedDownloadForHighlights);
+ if (results.length === numItems) {
+ break;
+ }
+ }
+ return results;
+ }
+
+ uninit() {
+ if (this._downloadData) {
+ this._downloadData.removeView(this);
+ this._downloadData = null;
+ }
+ if (this._downloadTimer) {
+ this._downloadTimer.cancel();
+ this._downloadTimer = null;
+ }
+ }
+
+ onAction(action) {
+ let doDownloadAction = callback => {
+ let download = this._downloadItems.get(action.data.url);
+ if (download) {
+ callback(download);
+ }
+ };
+
+ switch (action.type) {
+ case at.COPY_DOWNLOAD_LINK:
+ doDownloadAction(download => {
+ lazy.DownloadsCommon.copyDownloadLink(download);
+ });
+ break;
+ case at.REMOVE_DOWNLOAD_FILE:
+ doDownloadAction(download => {
+ lazy.DownloadsCommon.deleteDownload(download).catch(console.error);
+ });
+ break;
+ case at.SHOW_DOWNLOAD_FILE:
+ doDownloadAction(download => {
+ lazy.DownloadsCommon.showDownloadedFile(
+ new lazy.FileUtils.File(download.target.path)
+ );
+ });
+ break;
+ case at.OPEN_DOWNLOAD_FILE:
+ const win = action._target.browser.ownerGlobal;
+ const openWhere =
+ action.data.event && win.whereToOpenLink(action.data.event);
+ doDownloadAction(download => {
+ lazy.DownloadsCommon.openDownload(download, {
+ // Replace "current" or unknown value with "tab" as the default behavior
+ // for opening downloads when handled internally
+ openWhere: ["window", "tab", "tabshifted"].includes(openWhere)
+ ? openWhere
+ : "tab",
+ });
+ });
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ }
+}
+const EXPORTED_SYMBOLS = ["DownloadsManager"];
diff --git a/browser/components/newtab/lib/FaviconFeed.jsm b/browser/components/newtab/lib/FaviconFeed.jsm
new file mode 100644
index 0000000000..ad82762846
--- /dev/null
+++ b/browser/components/newtab/lib/FaviconFeed.jsm
@@ -0,0 +1,202 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { getDomain } = ChromeUtils.import(
+ "resource://activity-stream/lib/TippyTopProvider.jsm"
+);
+const { RemoteSettings } = ChromeUtils.import(
+ "resource://services-settings/remote-settings.js"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+const MIN_FAVICON_SIZE = 96;
+
+/**
+ * Get favicon info (uri and size) for a uri from Places.
+ *
+ * @param uri {nsIURI} Page to check for favicon data
+ * @returns A promise of an object (possibly null) containing the data
+ */
+function getFaviconInfo(uri) {
+ return new Promise(resolve =>
+ lazy.PlacesUtils.favicons.getFaviconDataForPage(
+ uri,
+ // Package up the icon data in an object if we have it; otherwise null
+ (iconUri, faviconLength, favicon, mimeType, faviconSize) =>
+ resolve(iconUri ? { iconUri, faviconSize } : null),
+ lazy.NewTabUtils.activityStreamProvider.THUMB_FAVICON_SIZE
+ )
+ );
+}
+
+/**
+ * Fetches visit paths for a given URL from its most recent visit in Places.
+ *
+ * Note that this includes the URL itself as well as all the following
+ * permenent&temporary redirected URLs if any.
+ *
+ * @param {String} a URL string
+ *
+ * @returns {Array} Returns an array containing objects as
+ * {int} visit_id: ID of the visit in moz_historyvisits.
+ * {String} url: URL of the redirected URL.
+ */
+async function fetchVisitPaths(url) {
+ const query = `
+ WITH RECURSIVE path(visit_id)
+ AS (
+ SELECT v.id
+ FROM moz_places h
+ JOIN moz_historyvisits v
+ ON v.place_id = h.id
+ WHERE h.url_hash = hash(:url) AND h.url = :url
+ AND v.visit_date = h.last_visit_date
+
+ UNION
+
+ SELECT id
+ FROM moz_historyvisits
+ JOIN path
+ ON visit_id = from_visit
+ WHERE visit_type IN
+ (${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_PERMANENT},
+ ${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_TEMPORARY})
+ )
+ SELECT visit_id, (
+ SELECT (
+ SELECT url
+ FROM moz_places
+ WHERE id = place_id)
+ FROM moz_historyvisits
+ WHERE id = visit_id) AS url
+ FROM path
+ `;
+
+ const visits = await lazy.NewTabUtils.activityStreamProvider.executePlacesQuery(
+ query,
+ {
+ columns: ["visit_id", "url"],
+ params: { url },
+ }
+ );
+ return visits;
+}
+
+/**
+ * Fetch favicon for a url by following its redirects in Places.
+ *
+ * This can improve the rich icon coverage for Top Sites since Places only
+ * associates the favicon to the final url if the original one gets redirected.
+ * Note this is not an urgent request, hence it is dispatched to the main
+ * thread idle handler to avoid any possible performance impact.
+ */
+async function fetchIconFromRedirects(url) {
+ const visitPaths = await fetchVisitPaths(url);
+ if (visitPaths.length > 1) {
+ const lastVisit = visitPaths.pop();
+ const redirectedUri = Services.io.newURI(lastVisit.url);
+ const iconInfo = await getFaviconInfo(redirectedUri);
+ if (iconInfo && iconInfo.faviconSize >= MIN_FAVICON_SIZE) {
+ lazy.PlacesUtils.favicons.setAndFetchFaviconForPage(
+ Services.io.newURI(url),
+ iconInfo.iconUri,
+ false,
+ lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+ null,
+ Services.scriptSecurityManager.getSystemPrincipal()
+ );
+ }
+ }
+}
+
+class FaviconFeed {
+ constructor() {
+ this._queryForRedirects = new Set();
+ }
+
+ /**
+ * fetchIcon attempts to fetch a rich icon for the given url from two sources.
+ * First, it looks up the tippy top feed, if it's still missing, then it queries
+ * the places for rich icon with its most recent visit in order to deal with
+ * the redirected visit. See Bug 1421428 for more details.
+ */
+ async fetchIcon(url) {
+ // Avoid initializing and fetching icons if prefs are turned off
+ if (!this.shouldFetchIcons) {
+ return;
+ }
+
+ const site = await this.getSite(getDomain(url));
+ if (!site) {
+ if (!this._queryForRedirects.has(url)) {
+ this._queryForRedirects.add(url);
+ Services.tm.idleDispatchToMainThread(() => fetchIconFromRedirects(url));
+ }
+ return;
+ }
+
+ let iconUri = Services.io.newURI(site.image_url);
+ // The #tippytop is to be able to identify them for telemetry.
+ iconUri = iconUri
+ .mutate()
+ .setRef("tippytop")
+ .finalize();
+ lazy.PlacesUtils.favicons.setAndFetchFaviconForPage(
+ Services.io.newURI(url),
+ iconUri,
+ false,
+ lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE,
+ null,
+ Services.scriptSecurityManager.getSystemPrincipal()
+ );
+ }
+
+ /**
+ * Get the site tippy top data from Remote Settings.
+ */
+ async getSite(domain) {
+ const sites = await this.tippyTop.get({
+ filters: { domain },
+ syncIfEmpty: false,
+ });
+ return sites.length ? sites[0] : null;
+ }
+
+ /**
+ * Get the tippy top collection from Remote Settings.
+ */
+ get tippyTop() {
+ if (!this._tippyTop) {
+ this._tippyTop = RemoteSettings("tippytop");
+ }
+ return this._tippyTop;
+ }
+
+ /**
+ * Determine if we should be fetching and saving icons.
+ */
+ get shouldFetchIcons() {
+ return Services.prefs.getBoolPref("browser.chrome.site_icons");
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.RICH_ICON_MISSING:
+ this.fetchIcon(action.data.url);
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["FaviconFeed", "fetchIconFromRedirects"];
diff --git a/browser/components/newtab/lib/FeatureCalloutMessages.jsm b/browser/components/newtab/lib/FeatureCalloutMessages.jsm
new file mode 100644
index 0000000000..b1b6a22ac9
--- /dev/null
+++ b/browser/components/newtab/lib/FeatureCalloutMessages.jsm
@@ -0,0 +1,640 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// Eventually, make this a messaging system
+// provider instead of adding these message
+// into OnboardingMessageProvider.jsm
+const FIREFOX_VIEW_PREF = "browser.firefox-view.feature-tour";
+const PDFJS_PREF = "browser.pdfjs.feature-tour";
+// Empty screens are included as placeholders to ensure step
+// indicator shows the correct number of total steps in the tour
+const PDF_SOURCE = `(source || "") | regExpMatch('(?<!q\=.+)\.pdf') | length > 0`;
+const EMPTY_SCREEN = { content: {} };
+const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000;
+
+// Generate a JEXL targeting string based on the current screen
+// id found in a given Feature Callout tour progress preference
+// and the `complete` property being true
+const matchCurrentScreenTargeting = (prefName, screenId) => {
+ return `'${prefName}' | preferenceValue | regExpMatch('(?<=screen\"\:)"(.*)(?=",)')[1] == '${screenId}' && '${prefName}' | preferenceValue | regExpMatch('(?<=complete\"\:)(.*)(?=})')[1] != "true"`;
+};
+
+/**
+ * add24HourImpressionJEXLTargeting -
+ * Creates a "hasn't been viewed in > 24 hours"
+ * JEXL string and adds it to each message specified
+ *
+ * @param {array} messageIds - IDs of messages that the targeting string will be added to
+ * @param {string} prefix - The prefix of messageIDs that will used to create the JEXL string
+ * @param {array} messages - The array of messages that will be edited
+ * @returns {array} - The array of messages with the appropriate targeting strings edited
+ */
+function add24HourImpressionJEXLTargeting(
+ messageIds,
+ prefix,
+ uneditedMessages
+) {
+ let noImpressionsIn24HoursString = uneditedMessages
+ .filter(message => message.id.startsWith(prefix))
+ .map(
+ message =>
+ // If the last impression is null or if epoch time
+ // of the impression is < current time - 24hours worth of MS
+ `(messageImpressions.${message.id}[messageImpressions.${
+ message.id
+ } | length - 1] == null || messageImpressions.${
+ message.id
+ }[messageImpressions.${message.id} | length - 1] < ${Date.now() -
+ ONE_DAY_IN_MS})`
+ )
+ .join(" && ");
+
+ // We're appending the string here instead of using
+ // template strings to avoid a recursion error from
+ // using the 'messages' variable within itself
+ return uneditedMessages.map(message => {
+ if (messageIds.includes(message.id)) {
+ message.targeting += `&& ${noImpressionsIn24HoursString}`;
+ }
+
+ return message;
+ });
+}
+
+// Exporting the about:firefoxview messages as a method here
+// acts as a safety guard against mutations of the original objects
+const MESSAGES = () => {
+ let messages = [
+ {
+ id: "FIREFOX_VIEW_SPOTLIGHT",
+ template: "spotlight",
+ content: {
+ id: "FIREFOX_VIEW_PROMO",
+ template: "multistage",
+ modal: "tab",
+ screens: [
+ {
+ id: "DEFAULT_MODAL_UI",
+ content: {
+ title: {
+ fontSize: "32px",
+ fontWeight: 400,
+ string_id: "firefoxview-spotlight-promo-title",
+ },
+ subtitle: {
+ fontSize: "15px",
+ fontWeight: 400,
+ marginBlock: "10px",
+ marginInline: "40px",
+ string_id: "firefoxview-spotlight-promo-subtitle",
+ },
+ logo: { height: "48px" },
+ primary_button: {
+ label: {
+ string_id: "firefoxview-spotlight-promo-primarybutton",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "FEATURE_CALLOUT_1",
+ complete: false,
+ }),
+ },
+ },
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "firefoxview-spotlight-promo-secondarybutton",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ navigate: true,
+ },
+ },
+ },
+ },
+ ],
+ },
+ priority: 3,
+ trigger: {
+ id: "featureCalloutCheck",
+ },
+ frequency: {
+ // Add the highest possible cap to ensure impressions are recorded while allowing the Spotlight to sync across windows/tabs with Firefox View open
+ lifetime: 100,
+ },
+ targeting: `!inMr2022Holdback && source == "firefoxview" &&
+ !'browser.newtabpage.activity-stream.asrouter.providers.cfr'|preferenceIsUserSet &&
+ 'browser.newtabpage.activity-stream.asrouter.userprefs.cfr.features'|preferenceValue &&
+ ${matchCurrentScreenTargeting(
+ FIREFOX_VIEW_PREF,
+ "FIREFOX_VIEW_SPOTLIGHT"
+ )}`,
+ },
+ {
+ id: "FIREFOX_VIEW_FEATURE_TOUR_1_NO_CWS",
+ template: "feature_callout",
+ content: {
+ id: "FIREFOX_VIEW_FEATURE_TOUR",
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ {
+ id: "FEATURE_CALLOUT_1",
+ parent_selector: "#tab-pickup-container",
+ content: {
+ position: "callout",
+ arrow_position: "top",
+ title: {
+ string_id: "callout-firefox-view-tab-pickup-title",
+ },
+ subtitle: {
+ string_id: "callout-firefox-view-tab-pickup-subtitle",
+ },
+ logo: {
+ imageURL: "chrome://browser/content/callout-tab-pickup.svg",
+ darkModeImageURL:
+ "chrome://browser/content/callout-tab-pickup-dark.svg",
+ height: "128px",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-primary-advance-button-label",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "FEATURE_CALLOUT_2",
+ complete: false,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ EMPTY_SCREEN,
+ ],
+ },
+ priority: 3,
+ targeting: `!inMr2022Holdback && source == "firefoxview" && ${matchCurrentScreenTargeting(
+ FIREFOX_VIEW_PREF,
+ "FEATURE_CALLOUT_1"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "FIREFOX_VIEW_FEATURE_TOUR_2_NO_CWS",
+ template: "feature_callout",
+ content: {
+ id: "FIREFOX_VIEW_FEATURE_TOUR",
+ startScreen: 1,
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ EMPTY_SCREEN,
+ {
+ id: "FEATURE_CALLOUT_2",
+ parent_selector: "#recently-closed-tabs-container",
+ content: {
+ position: "callout",
+ arrow_position: "bottom",
+ title: {
+ string_id: "callout-firefox-view-recently-closed-title",
+ },
+ subtitle: {
+ string_id: "callout-firefox-view-recently-closed-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-primary-complete-button-label",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: FIREFOX_VIEW_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ ],
+ },
+ priority: 3,
+ targeting: `!inMr2022Holdback && source == "firefoxview" && ${matchCurrentScreenTargeting(
+ FIREFOX_VIEW_PREF,
+ "FEATURE_CALLOUT_2"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER",
+ template: "feature_callout",
+ content: {
+ id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER",
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ {
+ id: "FIREFOX_VIEW_TAB_PICKUP_REMINDER",
+ parent_selector: "#tab-pickup-container",
+ content: {
+ position: "callout",
+ arrow_position: "top",
+ title: {
+ string_id:
+ "continuous-onboarding-firefox-view-tab-pickup-title",
+ },
+ subtitle: {
+ string_id:
+ "continuous-onboarding-firefox-view-tab-pickup-subtitle",
+ },
+ logo: {
+ imageURL: "chrome://browser/content/callout-tab-pickup.svg",
+ darkModeImageURL:
+ "chrome://browser/content/callout-tab-pickup-dark.svg",
+ height: "128px",
+ },
+ primary_button: {
+ label: {
+ string_id: "mr1-onboarding-get-started-primary-button-label",
+ },
+ action: {
+ type: "CLICK_ELEMENT",
+ navigate: true,
+ data: {
+ selector:
+ "#tab-pickup-container button.primary:not(#error-state-button)",
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ ],
+ },
+ priority: 2,
+ targeting: `!inMr2022Holdback && source == "firefoxview" && "browser.firefox-view.view-count" | preferenceValue > 2
+ && (("identity.fxaccounts.enabled" | preferenceValue == false) || !(("services.sync.engine.tabs" | preferenceValue == true) && ("services.sync.username" | preferenceValue))) && (!messageImpressions.FIREFOX_VIEW_SPOTLIGHT[messageImpressions.FIREFOX_VIEW_SPOTLIGHT | length - 1] || messageImpressions.FIREFOX_VIEW_SPOTLIGHT[messageImpressions.FIREFOX_VIEW_SPOTLIGHT | length - 1] < currentDate|date - ${ONE_DAY_IN_MS})`,
+ frequency: {
+ lifetime: 1,
+ },
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "PDFJS_FEATURE_TOUR_1_A",
+ template: "feature_callout",
+ content: {
+ id: "PDFJS_FEATURE_TOUR",
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ {
+ id: "FEATURE_CALLOUT_1_A",
+ parent_selector: "hbox#browser",
+ content: {
+ position: "callout",
+ callout_position_override: {
+ top: "45px",
+ right: "55px",
+ },
+ arrow_position: "top-end",
+ title: {
+ string_id: "callout-pdfjs-edit-title",
+ },
+ subtitle: {
+ string_id: "callout-pdfjs-edit-body-a",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-pdfjs-edit-button",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "FEATURE_CALLOUT_2_A",
+ complete: false,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ EMPTY_SCREEN,
+ ],
+ },
+ priority: 1,
+ targeting: `${PDF_SOURCE} && ${matchCurrentScreenTargeting(
+ PDFJS_PREF,
+ "FEATURE_CALLOUT_1_A"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "PDFJS_FEATURE_TOUR_2_A",
+ template: "feature_callout",
+ content: {
+ id: "PDFJS_FEATURE_TOUR",
+ startScreen: 1,
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ EMPTY_SCREEN,
+ {
+ id: "FEATURE_CALLOUT_2_A",
+ parent_selector: "hbox#browser",
+ content: {
+ position: "callout",
+ callout_position_override: {
+ top: "45px",
+ right: "25px",
+ },
+ arrow_position: "top-end",
+ title: {
+ string_id: "callout-pdfjs-draw-title",
+ },
+ subtitle: {
+ string_id: "callout-pdfjs-draw-body-a",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-pdfjs-draw-button",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ ],
+ },
+ priority: 1,
+ targeting: `${PDF_SOURCE} && ${matchCurrentScreenTargeting(
+ PDFJS_PREF,
+ "FEATURE_CALLOUT_2_A"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "PDFJS_FEATURE_TOUR_1_B",
+ template: "feature_callout",
+ content: {
+ id: "PDFJS_FEATURE_TOUR",
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ {
+ id: "FEATURE_CALLOUT_1_B",
+ parent_selector: "hbox#browser",
+ content: {
+ position: "callout",
+ callout_position_override: {
+ top: "45px",
+ right: "55px",
+ },
+ arrow_position: "top-end",
+ title: {
+ string_id: "callout-pdfjs-edit-title",
+ },
+ subtitle: {
+ string_id: "callout-pdfjs-edit-body-b",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-pdfjs-edit-button",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "FEATURE_CALLOUT_2_B",
+ complete: false,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ EMPTY_SCREEN,
+ ],
+ },
+ priority: 1,
+ targeting: `${PDF_SOURCE} && ${matchCurrentScreenTargeting(
+ PDFJS_PREF,
+ "FEATURE_CALLOUT_1_B"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ {
+ id: "PDFJS_FEATURE_TOUR_2_B",
+ template: "feature_callout",
+ content: {
+ id: "PDFJS_FEATURE_TOUR",
+ startScreen: 1,
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: false,
+ disableHistoryUpdates: true,
+ screens: [
+ EMPTY_SCREEN,
+ {
+ id: "FEATURE_CALLOUT_2_B",
+ parent_selector: "hbox#browser",
+ content: {
+ position: "callout",
+ callout_position_override: {
+ top: "45px",
+ right: "25px",
+ },
+ arrow_position: "top-end",
+ title: {
+ string_id: "callout-pdfjs-draw-title",
+ },
+ subtitle: {
+ string_id: "callout-pdfjs-draw-body-b",
+ },
+ primary_button: {
+ label: {
+ string_id: "callout-pdfjs-draw-button",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ dismiss_button: {
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: PDFJS_PREF,
+ value: JSON.stringify({
+ screen: "",
+ complete: true,
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ ],
+ },
+ priority: 1,
+ targeting: `${PDF_SOURCE} && ${matchCurrentScreenTargeting(
+ PDFJS_PREF,
+ "FEATURE_CALLOUT_2_B"
+ )}`,
+ trigger: { id: "featureCalloutCheck" },
+ },
+ ];
+ messages = add24HourImpressionJEXLTargeting(
+ ["FIREFOX_VIEW_TAB_PICKUP_REMINDER"],
+ "FIREFOX_VIEW",
+ messages
+ );
+ return messages;
+};
+
+const FeatureCalloutMessages = {
+ getMessages() {
+ return MESSAGES();
+ },
+};
+
+const EXPORTED_SYMBOLS = ["FeatureCalloutMessages"];
diff --git a/browser/components/newtab/lib/FilterAdult.jsm b/browser/components/newtab/lib/FilterAdult.jsm
new file mode 100644
index 0000000000..2ee9c4c467
--- /dev/null
+++ b/browser/components/newtab/lib/FilterAdult.jsm
@@ -0,0 +1,3036 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "gFilterAdultEnabled",
+ "browser.newtabpage.activity-stream.filterAdult",
+ true
+);
+
+// Keep a Set of adult base domains for lookup (initialized at end of file)
+let gAdultSet;
+
+// Keep a hasher for repeated hashings
+let gCryptoHash = null;
+
+/**
+ * Run some text through md5 and return the base64 result.
+ */
+function md5Hash(text) {
+ // Lazily create a reusable hasher
+ if (gCryptoHash === null) {
+ gCryptoHash = Cc["@mozilla.org/security/hash;1"].createInstance(
+ Ci.nsICryptoHash
+ );
+ }
+
+ gCryptoHash.init(gCryptoHash.MD5);
+
+ // Convert the text to a byte array for hashing
+ gCryptoHash.update(
+ text.split("").map(c => c.charCodeAt(0)),
+ text.length
+ );
+
+ // Request the has result as ASCII base64
+ return gCryptoHash.finish(true);
+}
+
+const FilterAdult = {
+ /**
+ * Filter out any link objects that have a url with an adult base domain.
+ *
+ * @param {string[]} links
+ * An array of links to test.
+ * @returns {string[]}
+ * A filtered array without adult links.
+ */
+ filter(links) {
+ if (!lazy.gFilterAdultEnabled) {
+ return links;
+ }
+
+ return links.filter(({ url }) => {
+ try {
+ const uri = Services.io.newURI(url);
+ return !gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri)));
+ } catch (ex) {
+ return true;
+ }
+ });
+ },
+
+ /**
+ * Determine if the supplied url is an adult url or not.
+ *
+ * @param {string} url
+ * The url to test.
+ * @returns {boolean}
+ * True if it is an adult url.
+ */
+ isAdultUrl(url) {
+ if (!lazy.gFilterAdultEnabled) {
+ return false;
+ }
+ try {
+ const uri = Services.io.newURI(url);
+ return gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri)));
+ } catch (ex) {
+ return false;
+ }
+ },
+
+ /**
+ * For tests, adds a domain to the adult list.
+ */
+ addDomainToList(url) {
+ gAdultSet.add(
+ md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url)))
+ );
+ },
+
+ /**
+ * For tests, removes a domain to the adult list.
+ */
+ removeDomainFromList(url) {
+ gAdultSet.delete(
+ md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url)))
+ );
+ },
+};
+
+const EXPORTED_SYMBOLS = ["FilterAdult"];
+
+// These are md5 hashes of base domains to be filtered out. Originally from:
+// https://hg.mozilla.org/mozilla-central/log/default/browser/base/content/newtab/newTab.inadjacent.json
+gAdultSet = new Set([
+ "+/UCpAhZhz368iGioEO8aQ==",
+ "+1e7jvUo8f2/2l0TFrQqfA==",
+ "+1gcqAqaRZwCj5BGiZp3CA==",
+ "+25t/2lo0FUEtWYK8LdQZQ==",
+ "+8PiQt6O7pJI/nIvQpDaAg==",
+ "+CLf5witKkuOvPCulTlkqw==",
+ "+CvLiih/gf2ugXAF+LgWqw==",
+ "+DWs0vvFGt6d3mzdcsdsyA==",
+ "+H0Rglt/HnhZwdty2hsDHg==",
+ "+L1FDsr5VQtuYc2Is5QGjw==",
+ "+LJYVZl1iPrdMU3L5+nxZw==",
+ "+Mp+JIyO0XC5urvMyi3wvQ==",
+ "+NMUaQ7XPsAi0rk7tTT9wQ==",
+ "+NmjwjsPhGJh9bM10SFkLw==",
+ "+OERSmo7OQUUjudkccSMOA==",
+ "+OLntmlsMBBYPREPnS6iVw==",
+ "+OXdvbTxHtSoLg7bZMho4w==",
+ "+P5q4YD1Rr5SX26Xr+tzlw==",
+ "+PUVXkoTqHxJHO18z4KMfw==",
+ "+Pl0bSMBAdXpRIA+zE02JA==",
+ "+QosBAnSM2h4lsKuBlqEZw==",
+ "+S+WXgVDSU1oGmCzGwuT3g==",
+ "+SclwwY8R2RPrnX54Z+A6w==",
+ "+VfRcTBQ80KSeJRdg0cDfw==",
+ "+WpF8+poKmHPUBB4UYh/ig==",
+ "+YVxSyViJfrme/ENe1zA7A==",
+ "+YrqTEJlJCv0A2RHQ8tr1A==",
+ "+ZozWaPWw8ws1cE5DJACeg==",
+ "+aF4ilbjQbLpAuFXQEYMWQ==",
+ "+dBv88reDrjEz6a2xX3Hzw==",
+ "+dIEf5FBrHpkjmwUmGS6eg==",
+ "+edqJYGvcy1AH2mEjJtSIg==",
+ "+fcjH2kZKNj8quOytUk4nQ==",
+ "+gO0bg8LY+py2dLM1sM7Ag==",
+ "+gbitI/gpxebN/rK7qj8Fw==",
+ "+gpHnUj2GWocP74t5XWz4w==",
+ "+jVN/3ASc2O44sX6ab8/cg==",
+ "+mJLK+6qq8xFv7O/mbILTw==",
+ "+n0K7OB2ItzhySZ4rhUrMg==",
+ "+p8pofUlwn8vV6Rp6+sz9g==",
+ "+tuUmnRDRWVLA+1k0dcUvg==",
+ "+zBkeHF4P8vLzk1iO1Zn3Q==",
+ "//eHwmDOQRSrv+k9C/k3ZQ==",
+ "/2Chaw2M9DzsadFFkCu6WQ==",
+ "/2c4oNniwhL3z5IOngfggg==",
+ "/2jGyMekNu7U136K+2N3Jg==",
+ "/Bwpt5fllzDHq2Ul6v86fA==",
+ "/DJgKE9ouibewuZ2QEnk6w==",
+ "/DiUApY7cVp5W9o24rkgRA==",
+ "/FchS2nPezycB8Bcqc2dbg==",
+ "/FdZzSprPnNDPwbhV1C0Cg==",
+ "/FsJYFNe+7UvsSkiotNJEQ==",
+ "/G26n5Xoviqldr5sg/Jl3w==",
+ "/HU2+fBqfWTEuqINc0UZSA==",
+ "/IarsLzJB8bf0AupJJ+/Eg==",
+ "/KYZdUWrkfxSsIrp46xxow==",
+ "/MEOgAhwb7F0nBnV4tIRZA==",
+ "/MeHciFhvFzQsCIw39xIZA==",
+ "/Ph/6l/lFNVqxAje1+PgFA==",
+ "/SP6pOdYFzcAl2OL05z4uQ==",
+ "/TSsi/AwKHtP6kQaeReI3w==",
+ "/VnKh/NDv7y/bfO6CWsLaQ==",
+ "/XC/FmMIOdhMTPqmy4DfUA==",
+ "/XjB6c5fxFGcKVAQ4o+OMw==",
+ "/YuQw7oAF08KDptxJEBS9g==",
+ "/a+bLXOq02sa/s8h7PhUTg==",
+ "/a9O7kWeXa0le45ab3+nVw==",
+ "/c34NtdUZAHWIwGl3JM8Tw==",
+ "/cJ0Nn5YbXeUpOHMfWXNHQ==",
+ "/cdR1i5TuQvO+u3Ov3b0KQ==",
+ "/gi3UZmunVOIXhZSktZ8zQ==",
+ "/hFhjFGJx2wRfz6hyrIpvA==",
+ "/jDVt9dRIn+o4IQ1DPwbsg==",
+ "/jH6imhTPZ/tHI4gYz2+HA==",
+ "/kGxvyEokQsVz0xlKzCn2A==",
+ "/mFp3GFkGNLhx2CiDvJv4A==",
+ "/mrqas0eDX+sFUNJvCQY8g==",
+ "/n1RLTTVpygre1dl36PDwQ==",
+ "/ngbFuKIAVpdSwsA3VxvNw==",
+ "/p/aCTIhi1bU0/liuO/a2Q==",
+ "/u5W2Gab4GgCMIc4KTp2mg==",
+ "/wIZAye9h1TUiZmDW0ZmYA==",
+ "/wiA2ltAuWyBhIvQAYBTQw==",
+ "/y/jHHEpUu5TR+R2o96kXA==",
+ "/zFLRvi75UL8qvg+a6zqGg==",
+ "00TVKawojyqrJkC7YqT41Q==",
+ "022B0oiRMx8Xb4Af98mTvQ==",
+ "02im2RooJQ/9UfUrh5LO+A==",
+ "0G93AxGPVwmr66ZOleM90A==",
+ "0HN6MIGtkdzNPsrGs611xA==",
+ "0K4NBxqEa3RYpnrkrD/XjQ==",
+ "0L0FVcH5Dlj3oL8+e9Na7g==",
+ "0NrvBuyjcJ2q6yaHpz/FOA==",
+ "0ODJyWKJSfObo+FNdRQkkA==",
+ "0QB0OUW5x2JLHfrtmpZQ+w==",
+ "0QCQORCYfLuSbq94Sbt0bQ==",
+ "0QbH4oI8IjZ9BRcqRyvvDQ==",
+ "0QxPAqRF8inBuFEEzNmLjA==",
+ "0SkC/4PtnX1bMYgD6r6CLA==",
+ "0TxcYwG72dT7Tg+eG8pP1w==",
+ "0UeRwDID2RBIikInqFI7uw==",
+ "0VsaJHR0Ms8zegsCpAKoyg==",
+ "0Y6iiZjCwPDwD/CwJzfioQ==",
+ "0ZEC3hy411LkOhKblvTcqg==",
+ "0ZRGz+oj2infCAkuKKuHiQ==",
+ "0a4SafpDIe8V4FlFWYkMHw==",
+ "0b/xj6fd0x+aB8EB0LC4SA==",
+ "0bj069wXgEJbw7dpiPr8Tg==",
+ "0dIeIM5Zvm5nSVWLy94LWg==",
+ "0e8hM3E5tnABRyy29A8yFw==",
+ "0egBaMnAf0CQEXf1pCIKnA==",
+ "0fN+eHlbRS6mVZBbH/B9FQ==",
+ "0fnruVOCxEczscBuv4yL9A==",
+ "0fpe9E6m3eLp/5j5rLrz2Q==",
+ "0klouNfZRHFFpdHi4ZR2hA==",
+ "0nOg18ZJ/NicqVUz5Jr0Hg==",
+ "0ofMbUCA3/v5L8lHnX4S5w==",
+ "0p1jMr06OyBoXQuSLYN4aQ==",
+ "0p8YbEMxeb73HbAfvPLQRw==",
+ "0q+erphtrB+6HBnnYg7O6w==",
+ "0rTYcuVYdilO7zEfKrxY3A==",
+ "0rfG4gRugAwVP0i3AGVxxg==",
+ "0u+0WHr7WI6IlVBBgiRi6w==",
+ "0yJ7TQYzcp3DXVSvwavr+w==",
+ "1+A9FCGP3bZhk6gU3LQtNg==",
+ "1+XWdu4qCqLLVjqkKz3nmA==",
+ "1+qmrbC8c7MJ6pxmDMcKuA==",
+ "1/Hxu8M9N/oNwk8bCj4FNQ==",
+ "1/SGIab+NnizimUmNDC4wA==",
+ "1/ZheMsbojazxt31j/l3iA==",
+ "10OltdxPXOvfatJuwPVKbQ==",
+ "11FE2kknwYi2Qu0JUKMn3A==",
+ "11U5XEwfMI7avx014LfC8g==",
+ "16d+fhFlgayu3ttKVV/pbg==",
+ "16iT/jCcPDrJEfi2bE5F+Q==",
+ "18RKixTv12q3xoBLz6eKiA==",
+ "18ndtDM9UaNfBR1cr3SHdA==",
+ "19yQHaBemtlgo2QkU5M6jQ==",
+ "1AeReq55UQotRQVKJ66pmg==",
+ "1ApqwW7pE+XUB2Cs2M6y7g==",
+ "1B5gxGQSGzVKoNd5Ol4N7g==",
+ "1BjsijOzgHt/0i36ZGffoQ==",
+ "1C50kisi9nvyVJNfq2hOEQ==",
+ "1E3pMgAHOnHx3ALdNoHr8Q==",
+ "1EI9aa955ejNo1dJepcZJw==",
+ "1FSrgkUXgZot2CsmbAtkPw==",
+ "1Gpj4TPXhdPEI4zfQFsOCg==",
+ "1HDgfU7xU7LWO/BXsODZAQ==",
+ "1I+UVx3krrD4NhzO7dgfHQ==",
+ "1JI9bT92UzxI8txjhst9LQ==",
+ "1JRgSHnfAQFQtSkFTttkqQ==",
+ "1LPC0BzhJbepHTSAiZ3QTw==",
+ "1MIn73MLroxXirrb+vyg2Q==",
+ "1Oykse0jQVbuR3MvW5ot4A==",
+ "1Pmnur6TbZ9cmemvu0+dSA==",
+ "1PvTn90xwZJPoVfyT5/uIQ==",
+ "1QGhj9NONF2rC44UdO+Izw==",
+ "1RQZ2pWSxT+RKyhBigtSFg==",
+ "1Vtrv6QUAfiYQjlLTpNovg==",
+ "1WIi4I62GqkjDXOYqHWJfQ==",
+ "1Wc8jQlDSB4Dp32wkL2odw==",
+ "1X14kHeKwGmLeYqpe60XEA==",
+ "1YO9G8qAhLIu2rShvekedw==",
+ "1Ym0lyBJ9aFjhJb/GdUPvQ==",
+ "1b2uf+CdVjufqiVpUShvHw==",
+ "1buQEv2YlH/ljTgH0uJEtw==",
+ "1cj1Fpd3+UiBAOahEhsluA==",
+ "1d7RPHdZ9qzAbG3Vi9BdFA==",
+ "1dhq3ozNCx0o4dV1syLVDA==",
+ "1dsKN1nG6upj7kKTKuJWsQ==",
+ "1eCHcz4swFH+uRhiilOinQ==",
+ "1eRUCdIJe3YGD5jOMbkkOg==",
+ "1fztTtQWNMIMSAc5Hr6jMQ==",
+ "1gA65t5FiBTEgMELTQFUPQ==",
+ "1jBaRO8Bg5l6TH7qJ8EPiw==",
+ "1k8tL2xmGFVYMgKUcmDcEw==",
+ "1lCcQWGDePPYco4vYrA5vw==",
+ "1m1yD4L9A7Q1Ot+wCsrxJQ==",
+ "1mw6LfTiirFyfjejf8QNGA==",
+ "1nXByug2eKq0kR3H3VjnWQ==",
+ "1tpM0qgdo7JDFwvT0TD78g==",
+ "1vqRt79ukuvdJNyIlIag8Q==",
+ "1wBuHqS1ciup31WTfm3NPg==",
+ "1xWx5V3G9murZP7srljFmA==",
+ "1zDfWw5LdG20ClNP1HYxgw==",
+ "203EqmJI9Q4tWxTJaBdSzA==",
+ "23C4eh3yBb5n/RNZeTyJkA==",
+ "23d9B9Gz5kUOi1I//EYsSQ==",
+ "24H9q+E8pgCEdFS7JO5kzQ==",
+ "25w3ZRUzCvJwAVHYCIO5uw==",
+ "26+yXbqI+fmIZsYl4UhUzw==",
+ "26Wmdp6SkKN74W0/XPcnmA==",
+ "29EybnMEO95Ng4l/qK4NWQ==",
+ "2Ct+pLXrK6Ku1f4qehjurQ==",
+ "2D6yhuABiaFFoXz0Lh0C+w==",
+ "2DNbXVgesUa7PgYQ4zX5Lw==",
+ "2E41e0MgM3WhFx2oasIQeA==",
+ "2HHqeGRMfzf3RXwVybx+ZQ==",
+ "2Hc5oyl0AYRy2VzcDKy+VA==",
+ "2QQtKtBAm2AjJ5c0WQ6BQA==",
+ "2QS/6OBA1T01NlIbfkTYJg==",
+ "2RFaMPlSbVuoEqKXgkIa5A==",
+ "2SI4F7Vvde2yjzMLAwxOog==",
+ "2SwIiUwT4vRZPrg7+vZqDA==",
+ "2W6lz1Z7PhkvObEAg2XKJw==",
+ "2Wvk/kouEEOY0evUkQLhOQ==",
+ "2XrR2hjDEvx8MQpHk9dnjw==",
+ "2aDK0tGNgMLyxT+BQPDE8Q==",
+ "2aIx9UdMxxZWvrfeJ+DcTw==",
+ "2abfl3N46tznOpr+94VONQ==",
+ "2bsIpvnGcFhTCSrK9EW1FQ==",
+ "2hEzujfG3mR5uQJXbvOPTQ==",
+ "2j83jrPwPfYlpJJ2clEBYQ==",
+ "2ksediOVrh4asSBxKcudTg==",
+ "2melaInV0wnhBpiI3da6/A==",
+ "2nSTEYzLK77h5Rgyti+ULQ==",
+ "2os5s7j7Tl46ZmoZJH8FjA==",
+ "2rOkEVl90EPqfHOF5q2FYw==",
+ "2rhjiY0O0Lo36wTHjmlNyw==",
+ "2vm7g3rk1ACJOTCXkLB3zA==",
+ "2wesXiib76wM9sqRZ7JYwQ==",
+ "2ywo4t5PPSVUCWDwUlOVwQ==",
+ "3++dZXzZ6AFEz7hK+i5hww==",
+ "3+9nURtBK3FKn0J9DQDa3g==",
+ "3+zsjCi7TnJhti//YXK35w==",
+ "3/1puZTGSrD9qNKPGaUZww==",
+ "300hoYyMR/mk1mfWJxS8/w==",
+ "301utVPZ93AnPLYbsiJggw==",
+ "312g8iTB9oJgk/OqcgR7Cw==",
+ "342VOUOxoLHUqtHANt83Hw==",
+ "36XDmX6j542q+Oei1/x0gw==",
+ "37Nkh06O979nt7xzspOFyQ==",
+ "3AKEYQqpkfW7CZMFQZoxOw==",
+ "3AVYtcIv7A5mVbVnQMaCeA==",
+ "3BjLFon1Il0SsjxHE2A1LQ==",
+ "3CJbrUdW68E3Drhe4ahUnQ==",
+ "3EhLkC9NqD3A6ApV6idmgg==",
+ "3Ejtsqw3Iep/UQd0tXnSlg==",
+ "3FH4D31nKV13sC9RpRZFIg==",
+ "3Gg9N7vjAfQEYOtQKuF/Eg==",
+ "3HPOzIZxoaQAmWRy9OkoSg==",
+ "3JhnM6G4L06NHt31lR0zXA==",
+ "3L3KEBHhgDwH615w4OvgZA==",
+ "3Leu2Sc+YOntJFlrvhaXeg==",
+ "3P2aJxV8Trll2GH9ptElYA==",
+ "3RTtSaMp1TZegJo5gFtwwA==",
+ "3TbRZtFtsh9ez8hqZuTDeA==",
+ "3TjntNWtpG7VqBt3729L6Q==",
+ "3UBYBMejKInSbCHRoJJ7dg==",
+ "3UNJ37f+gnNyYk9yLFeoYA==",
+ "3WVBP9fyAiBPZAq3DpMwOQ==",
+ "3Wfj05vCLFAB9vII5AU9tw==",
+ "3WwITQML938W9+MUM56a3A==",
+ "3XyoREdvhmSbyvAbgw2y/A==",
+ "3Y4w0nETru3SiSVUMcWXqw==",
+ "3Y6/HqS1trYc9Dh778sefg==",
+ "3YXp1PmMldUjBz3hC6ItbA==",
+ "3djRJvkZk9O2bZeUTe+7xQ==",
+ "3go7bJ9WqH/PPUTjNP3q/Q==",
+ "3hVslsq98QCDIiO40JNOuA==",
+ "3iC21ByW/YVL+pSyppanWw==",
+ "3itfXtlLPRmPCSYaSvc39Q==",
+ "3j0kFUZ6g+yeeEljx+WXGg==",
+ "3jmCreW5ytSuGfmeLv7NfQ==",
+ "3jqsY8/xTWELmu/az3Daug==",
+ "3kREs/qaMX0AwFXN0LO5ow==",
+ "3ltw31yJuAl4VT6MieEXXw==",
+ "3nthUmLZ30HxQrzr2d7xFA==",
+ "3oMTbWf7Bv83KRlfjNWQZA==",
+ "3pi3aNVq1QNJmu1j0iyL0g==",
+ "3rbml1D0gfXnwOs5jRZ3gA==",
+ "3sNJJIx1NnjYcgJhjOLJOg==",
+ "3v09RHCPTLUztqapThYaHg==",
+ "3xw8+0/WU51Yz4TWIMK8mw==",
+ "3y5Xk65ShGvWFbQxcZaQAQ==",
+ "3yDD+xT8iRfUVdxcc7RxKw==",
+ "3yavzOJ1mM44pOSFLLszgA==",
+ "4+htiqjEz9oq0YcI/ErBVg==",
+ "40HzgVKYnqIb6NJhpSIF0A==",
+ "40gCrW4YWi+2lkqMSPKBPg==",
+ "41WEjhYUlG6jp2UPGj11eQ==",
+ "444F9T6Y7J67Y9sULG81qg==",
+ "46FCwqh+eMkf+czjhjworw==",
+ "46piyANQVvvLqcoMq5G8tQ==",
+ "49jZr/mEW6fvnyzskyN40w==",
+ "49z/15Nx9Og7dN9ebVqIzg==",
+ "4A+RHIw+aDzw0rSRYfbc7g==",
+ "4BkqgraeXY7yaI1FE07Evw==",
+ "4CfEP8TeMKX33ktwgifGgA==",
+ "4DIPP/yWRgRuFqVeqIyxMQ==",
+ "4FBBtWPvqJ3dv4w25tRHiQ==",
+ "4ID0PHTzIMZz2rQqDGBVfA==",
+ "4KJZPCE9NKTfzFxl76GWjg==",
+ "4LtQrahKXVtsbXrEzYU1zQ==",
+ "4LvQSicqsgxQFWauqlcEjw==",
+ "4NHQwbb3zWq2klqbT/pG6g==",
+ "4NP8EFFJyPcuQKnBSxzKgQ==",
+ "4PBaoeEwUj79njftnYYqLg==",
+ "4Qinl7cWmVeLJgah8bcNkw==",
+ "4SdHWowXgCpCDL28jEFpAw==",
+ "4TQkMnRsXBobbtnBmfPKnA==",
+ "4VR5LiXLew6Nyn91zH9L4w==",
+ "4WO6eT0Rh6sokb29zSJQnQ==",
+ "4WRdAjiUmOQg2MahsunjAg==",
+ "4WcFEswYU/HHQPw77DYnyA==",
+ "4XNUmgwxsqDYsNmPkgNQYQ==",
+ "4Xh/B3C16rrjbES+FM1W8g==",
+ "4ZFYKa7ZgvHyZLS6WpM8gA==",
+ "4aPU6053cfMLHgLwAZJRNg==",
+ "4ekt4m38G9m599xJCmhlug==",
+ "4erEA42TqGA9K4iFKkxMMA==",
+ "4ifNsmjYf1iOn2YpMfzihg==",
+ "4iiCq+HhC+hPMldNQMt0NA==",
+ "4itEKfbRCJvqlgKnyEdIOQ==",
+ "4jeOFKuKpCmMXUVJSh9y0g==",
+ "4kXlJNuT79XXf1HuuFOlHw==",
+ "4kj0S8XlmhHXoUP7dQItUw==",
+ "4mQVNv7FHj+/O6XFqWFt/Q==",
+ "4mig4AMLUw+T/ect9p4CfA==",
+ "4qMSNAxichi3ori/pR+o0w==",
+ "4rrSL6N0wyucuxeRELfAmw==",
+ "4u3eyKc+y3uRnkASrgBVUw==",
+ "4wnUAbPT3AHRJrPwTTEjyw==",
+ "4xojeUxTFmMLGm6jiMYh/Q==",
+ "4yEkKp2FYZ09mAhw2IcrrA==",
+ "4yVqq66iHYQjiTSxGgX2oA==",
+ "4yrFNgqWq17zVCyffULocA==",
+ "50jASqzGm4VyHJbFv8qVRA==",
+ "50xwiYvGQytEDyVgeeOnMg==",
+ "51yLpfEdvqXmtB6+q27/AQ==",
+ "520wTzrysiRi2Td92Zq0HQ==",
+ "53UccFNzMi9mKmdeD82vAw==",
+ "54XELlPm8gBvx8D5bN3aUg==",
+ "59ipbMH7cKBsF9bNf4PLeQ==",
+ "5CMadLqS2KWwwMCpzlDmLw==",
+ "5DDb7fFJQEb3XTc3YyOTjg==",
+ "5HovoyHtul8lXh+z8ywq9A==",
+ "5I/heFSQG/UpWGx0uhAqGQ==",
+ "5KOgetfZR+O2wHQSKt41BQ==",
+ "5LJqHFRyIwQKA4HbtqAYQQ==",
+ "5LuFDNKzMd2BzpWEIYO2Ww==",
+ "5M3dFrAOemzQ0MAbA8bI5w==",
+ "5N2oi2pB69NxeNt08yPLhw==",
+ "5NEP7Xt7ynj6xCzWzt21hQ==",
+ "5Nk2Z94DhlIdfG5HNgvBbQ==",
+ "5PfGtbH9fmVuNnq83xIIgQ==",
+ "5Q/Y2V0iSVTK8HE8JerEig==",
+ "5S5/asYfWjOwnzYpbK6JDw==",
+ "5SbwLDNT6sBOy6nONtUcTg==",
+ "5T39s5CtSrK5awMPUcEWJg==",
+ "5VO1inwXMvLDBQSOahT6rg==",
+ "5VY++KiWgo7jXSdFJsPN3A==",
+ "5Wcq+6hgnWsQZ/bojERpUw==",
+ "5Yrj6uevT8wHRyqqgnSfeg==",
+ "5dUry23poD+0wxZ3hH6WmA==",
+ "5eHStFN7wEmIE+uuRwIlPQ==",
+ "5eXpiczlRdmqMYSaodOUiQ==",
+ "5gGoDPTc/sOIDLngmlEq4A==",
+ "5jHgQF4SfO/zy9xy9t+9dw==",
+ "5jyuDp82Fux+B0+zlx8EXw==",
+ "5kvyy902llnYGQdn2Py04w==",
+ "5l6kDfjtZjkTZPJvNNOVFw==",
+ "5lfLJAk1L3QzGMML3fOuSw==",
+ "5m1ijXEW+4RTNGZsDA/rxQ==",
+ "5oD/aGqoakxaezq43x0Tvw==",
+ "5pje7qyz8BRsa8U4a4rmoA==",
+ "5pqqzC/YmRIMA9tMFPi7rg==",
+ "5r1ZsGkrzNQEpgt/gENibw==",
+ "5u2PdDcIY3RQgtchSGDCGg==",
+ "5ugVOraop5P5z5XLlYPJyQ==",
+ "5w/c9WkI/FA+4lOtdPxoww==",
+ "5w4FbRhWACP7k2WnNitiHg==",
+ "6+jhreeBLfw64tJ+Nhyipw==",
+ "600bwlyhcy754W1E6tuyYg==",
+ "600mjiWke4u0CDaSQKLOOg==",
+ "60suecbWRfexSh7C67RENA==",
+ "61V74uIjaSfZM8au1dxr1A==",
+ "62RHCbpGU8Hb+Ubn+SCTBg==",
+ "63OTPaKM0xCfJOy9EDto+Q==",
+ "64AA4jLHXc1Dp15aMaGVcA==",
+ "64QzHOYX0A9++FqRzZRHlQ==",
+ "64YsV2qeDxk2Q6WK/h7OqA==",
+ "65KhGKUBFQubRRIEdh9SwQ==",
+ "6706ncrH1OANFnaK6DUMqQ==",
+ "68jPYo3znYoU4uWI7FH3/g==",
+ "68nqDtXOuxF7DSw6muEZvg==",
+ "6ACvJNfryPSjGOK39ov8Qg==",
+ "6CjtF1S2Y6RCbhl7hMsD+g==",
+ "6G2bD3Y7qbGmfPqH9TqLFA==",
+ "6GXHGF62/+jZ7PfIBlMxZw==",
+ "6HGeEPyTAu9oiKhNVLjQnA==",
+ "6HnWgYNKohqhoa1tnjjU3A==",
+ "6M6QapJ5xtMXfiD3bMaiLA==",
+ "6NP81geiL14BeQW6TpLnUA==",
+ "6PzjncEw2wHZg7SP7SQk9w==",
+ "6QAtjOK9enNLRhcVa2iaTg==",
+ "6QUGE2S8oFYx4T4nW56cCw==",
+ "6W79FmpUN1ByNtv5IEXY4w==",
+ "6WhHPWlqEUqXC52rHGRHjA==",
+ "6XYqR2WvDzx4fWO7BIOTjA==",
+ "6Z9myGCF5ylWljgIYAmhqw==",
+ "6ZKmm7IW7IdWuVytLr68CQ==",
+ "6ZMs9vCzK9lsbS6eyzZlIA==",
+ "6b7ue29cBDsvmj1VSa5njw==",
+ "6c0iuya20Ys8BsvoI4iQaQ==",
+ "6cTETZ9iebhWl+4W5CB+YQ==",
+ "6dshA8knH5qqD+KmR/kdSQ==",
+ "6e8boFcyc8iF0/tHVje4eQ==",
+ "6erpZS36qZRXeZ9RN9L+kw==",
+ "6fWom3YoKvW6NIg6y9o9CQ==",
+ "6k2cuk0McTThSMW/QRHfjA==",
+ "6lVSzYUQ/r0ep4W2eCzFpg==",
+ "6leyDVmC5jglAa98NQ3+Hg==",
+ "6nwR+e9Qw0qp8qIwH9S/Mg==",
+ "6o5g9JfKLKQ2vBPqKs6kjg==",
+ "6rIWazDEWU5WPZHLkqznuQ==",
+ "6rqK8sjLPJUIp7ohkEwfZg==",
+ "6sBemZt4qY/TBwqk3YcLOQ==",
+ "6sNP0rzCCm3w976I2q2s/w==",
+ "6tfM6dx3R5TiVKaqYQjnCg==",
+ "6txm8z4/LGCH0cpaet/Hsg==",
+ "6uMF5i0b/xsk55DlPumT7A==",
+ "6uT7LZiWjLnnqnnSEW4e/Q==",
+ "6v3eTZtPYBfKFSjfOo2UaA==",
+ "6wkfN8hyKmKU6tG3YetCmw==",
+ "6z8CRivao3IMyV4p4gMh7g==",
+ "71w3aSvuh2mBLtdqJCN3wA==",
+ "734u4Y1R3u7UNUnD+wWUoA==",
+ "74FW/QYTzr/P1k6QwVHMcw==",
+ "778O1hdVKHLG2q9dycUS0Q==",
+ "78b8sDBp28zUlYPV5UTnYw==",
+ "79uTykH43voFC3XhHHUzKg==",
+ "7E6V6/zSjbtqraG7Umj+Jw==",
+ "7Ephy+mklG2Y3MFdqmXqlA==",
+ "7Eqzyb+Kep+dIahYJWNNxQ==",
+ "7GgNLBppgAKcgJCDSsRqOQ==",
+ "7J3FoFGuTIW36q0PZkgBiw==",
+ "7K8l6KoP0BH82/WMLntfrg==",
+ "7R5rFaXCxM3moIUtoCfM2g==",
+ "7Tauesu7bgs5lJmQROVFiQ==",
+ "7VHlLw20dWck+I8tCEZilA==",
+ "7W9aF7dxnL+E8lbS/F7brg==",
+ "7XRiYvytcwscemlxd9iXIQ==",
+ "7Y87wVJok20UfuwkGbXxLg==",
+ "7b0oo4+qphu6HRvJq6qkHQ==",
+ "7bM/pn4G7g7Zl6Xf1r62Lg==",
+ "7br49X11xc2GxQLSpZWjKQ==",
+ "7btpMFgeGkUsiTtsmNxGQA==",
+ "7cnUHeaPO8txZGGWHL9tKg==",
+ "7dz+W494zwU5sg63v5flCg==",
+ "7k5rBuh8FbTTI4TP87wBPQ==",
+ "7l0RMKbONGS/goW/M+gnMQ==",
+ "7mxU5fJl/c6dXss9H3vGcQ==",
+ "7nr3zyWL+HHtJhRrCPhYZA==",
+ "7p4NpnoNSQR7ISg+w+4yFg==",
+ "7pkUY2UzSbGnwLvyRrbxfA==",
+ "7sCJ4RxbxRqVnF4MBoKfuQ==",
+ "7w3b73nN/fIBvuLuGZDCYQ==",
+ "7w4PDRJxptG8HMe/ijL6cQ==",
+ "7wgT9WIiMVcrj48PVAMIgw==",
+ "7xDIG/80SnhgxAYPL9YJtg==",
+ "7xTKFcog69nTmMfr5qFUTA==",
+ "80C9TB9/XT1gGFfQDJxRoA==",
+ "80PCwYh4llIKAplcDvMj4g==",
+ "80UE+Ivby3nwplO/HA7cPw==",
+ "81ZH3SO0NrOO+xoR/Ngw1g==",
+ "81iQLU+YwxNwq4of6e9z7A==",
+ "81nkjWtpBhqhvOp6K8dcWg==",
+ "81pAhreEPxcKse+++h1qBg==",
+ "82hTTe1Nr4N2g7zwgGjxkw==",
+ "83ERX2XJV3ST4XwvN7YWCg==",
+ "83WGpQGWyt6mCV+emaomog==",
+ "83wtvSoSP9FVBsdWaiWfpA==",
+ "861mBNvjIkVgkBiocCUj/Q==",
+ "88PNi9+yn3Bp4/upgxtWGA==",
+ "88tB/HgUIUnqWXEX++b5Aw==",
+ "897ptlztTjr7yk+pk8MT0Q==",
+ "8AfCSZC0uasVON9Y/0P2Pw==",
+ "8B12CamjOGzJDnQ+RkUf4w==",
+ "8BLkvEkfnOizJq0OTCYGzw==",
+ "8CjmgWQSAAGcXX9kz3kssw==",
+ "8Cm19vJW8ivhFPy0oQXVNA==",
+ "8DtgIyYiNFqDc5qVrpFUng==",
+ "8GyPup4QAiolFJ9v80/Nkw==",
+ "8JVHFRwAd/SCLU0CRJYofg==",
+ "8LNNoHe6rEQyJ0ebl151Mw==",
+ "8M0kSvjn5KN8bjsMdUqKZQ==",
+ "8N3mhHt29FZDHn1P2WH1wQ==",
+ "8OFxXwnPmrogpNoueZlC4Q==",
+ "8QK7emHS6rAcAF5QQemW/A==",
+ "8RtLlzkGEiisy1v9Xo0sbw==",
+ "8VqeoQELbCs232+Mu+HblA==",
+ "8WU1vLKV1GhrL7oS9PpABg==",
+ "8ZBiwr842ZMKphlqmNngHw==",
+ "8ZFPMJJYVJHsfRpU4DigSg==",
+ "8ZqmPJDnQSOFXvNMRQYG2Q==",
+ "8c+lvG5sZNimvx9NKNH3ug==",
+ "8cXqZub6rjgJXmh1CYJBOg==",
+ "8dBIsHMEAk7aoArLZKDZtg==",
+ "8dUcSkd2qnX5lD9B+fUe+Q==",
+ "8dbyfox/isKLsnVjQNsEXg==",
+ "8fJLQeIHaTnJ8wGqUiKU6g==",
+ "8g08gjG/QtvAYer32xgNAg==",
+ "8hsfXqi4uiuL+bV1VrHqCw==",
+ "8iYdEleTXGM+Wc85/7vU9w==",
+ "8j9GVPiFdfIRm/+ho7hpoA==",
+ "8nOTDhFyZ8YUA4b6M5p84w==",
+ "8snljTGo/uICl9q0Hxy7/A==",
+ "8uP4HUnSodw88yoiWXOIcw==",
+ "8vLA9MOdmLTo3Qg+/2GzLA==",
+ "8vr+ERVrM99dp+IGnCWDGQ==",
+ "8ylI1AS3QJpAi3I/NLMYdg==",
+ "9+hjTVMQUsvVKs7Tmp52tg==",
+ "90dtIMq0ozJXezT2r79vMQ==",
+ "91+Yms6Oy/rP0rVjha5z9w==",
+ "91LQuW6bMSxl10J/UDX23A==",
+ "91SdBFJEZ65M+ixGaprY/A==",
+ "91VcAVv7YDzkC1XtluPigw==",
+ "91vfsZ7Lx9x5gqWTOdM4sg==",
+ "96ORaz1JRHY1Gk8H74+C2g==",
+ "99+SBN45LwKCPfrjUKRPmw==",
+ "9Bet5waJF5/ZvsYaHUVEjQ==",
+ "9DRHdyX8ECKHUoEsGuqR4Q==",
+ "9DtM1vls4rFTdrSnQ7uWXw==",
+ "9FdpxlIFu11qIPdO7WC5nw==",
+ "9Gkw+hvsR/tFY1cO89topg==",
+ "9J53kk+InE3CKa7cPyCXMw==",
+ "9JKIJrlQjhNSC46H3Cstcw==",
+ "9L6yLO93sRN70+3qq3ObfA==",
+ "9MDG0WeBPpjGJLEmUJgBWg==",
+ "9QFYrCXsGsInUb4SClS3cQ==",
+ "9RGIQ2qyevNbSSEF36xk/A==",
+ "9RXymE9kCkDvBzWGyMgIWA==",
+ "9SUOfKtfKmkGICJnvbIDMg==",
+ "9SgfpAY0UhNC6sYGus9GgQ==",
+ "9T7gB0ZkdWB0VpbKIXiujQ==",
+ "9TalxEyFgy6hFCM73hgb7Q==",
+ "9UhKmKtr4vMzXTEn74BEhg==",
+ "9W57pTzc572EvSURqwrRhw==",
+ "9Y1ZmfiHJd9vCiZ6KfO1xQ==",
+ "9aKH1u5+4lgYhhLztQ4KWA==",
+ "9ajIS45NTicqRANzRhDWFA==",
+ "9bAWYElyRN1oJ6eJwPtCtQ==",
+ "9cvHJmim9e0pOaoUEtiM6A==",
+ "9dbn0Kzwr9adCEfBJh78uQ==",
+ "9iB7+VwXRbi6HLkWyh9/kg==",
+ "9inw7xzbqAnZDKOl/MfCqA==",
+ "9jxA/t3TQx8dQ+FBsn/YCg==",
+ "9k17UqdR1HzlF7OBAjpREA==",
+ "9k1u/5TgPmXrsx3/NsYUhg==",
+ "9lLhHcrPWI4EsA4fHIIXuw==",
+ "9nMltdrrBmM5ESBY2FRjGA==",
+ "9oQ/SVNJ4Ye9lq8AaguGAQ==",
+ "9oUawSwUGOmb0sDn3XS6og==",
+ "9onh6QKp70glZk9cX3s34A==",
+ "9pdeedz1UZUlv8jPfPeZ1g==",
+ "9pk75mBzhmcdT+koHvgDlw==",
+ "9qWLbRLXWIBJUXYjYhY2pg==",
+ "9rL8nC/VbSqrvnUtH9WsxQ==",
+ "9reBKZ1Rp6xcdH1pFQacjw==",
+ "9s3ar9q32Y5A3tla5GW/2Q==",
+ "9sYLg75/hudZaBA3FrzKHw==",
+ "9tiibT8V9VwnPOErWGNT3w==",
+ "9vEgJVJLEfed6wJ7hBUGgQ==",
+ "9viAzLFGYYudBYFu7kFamg==",
+ "9vmJUS7WIVOlhMqwipAknQ==",
+ "9wUIeSgNN36SFxy8v2unVg==",
+ "9xIgKpZGqq0/OU6wM5ZSHw==",
+ "9xmtuClkFlpz/X5E9JBWBA==",
+ "A+DLpIlYyCb9DaarpLN76g==",
+ "A2ODff+ImIkreJtDPUVrlg==",
+ "A3dX2ShyL9+WOi6MNJBoYQ==",
+ "A6TLWhipfymkjPYq8kaoDQ==",
+ "AChOz8avRYsvxlbWcorQ3w==",
+ "AEpTVUQhIEJGlXJB6rS26A==",
+ "AFdelaqvxRj6T3YdLgCFyg==",
+ "AGd0rcLnQ0n+meYyJur1Pw==",
+ "AGoVLd0QPcXnTedT5T95JQ==",
+ "ALJWKUImVE40MbEooqsrng==",
+ "ALlGgVDO8So71ccX0D6u2g==",
+ "AMfL0rH+g8c0VqOUSgNzQw==",
+ "ARCWkHAnVgBOIkCDQ19ZuA==",
+ "ARKIvf4+zRF8eCvUITWPng==",
+ "ATmMzriwGLl+M3ppkfcZNA==",
+ "AUGmvZkpkKBry5bHZn4DJA==",
+ "AV/YJfdoDUdRcrXVwinhQg==",
+ "AVjwqrTBQH1VREuBlOyUOg==",
+ "AX1HxQKXD12Yv5HWi39aPQ==",
+ "AYxGETZs477n2sa1Ulu/RQ==",
+ "AZs3v4KJYxdi8T1gjVjI2Q==",
+ "AcKwfS8FRVqb72uSkDNY/Q==",
+ "AcbG0e6xN8pZfYAv7QJe1Q==",
+ "Af9j1naGtnZf0u1LyYmK1w==",
+ "AfVPdxD3FyfwwNrQnVNQ7A==",
+ "AgDJsaW0LkpGE65Kxk5+IA==",
+ "Ahpi9+nl13kPTdzL+jgqMw==",
+ "AiMtfedwGcddA+XYNc+21g==",
+ "AjHz9GkRTFPjrqBokCDzFw==",
+ "Ak3rlzEOds6ykivfg39xmw==",
+ "AkAes5oErTaJiGD2I4A1Pw==",
+ "AklOdt9/2//3ylUhWebHRw==",
+ "Al8+d/dlOA5BXsUc5GL8Tg==",
+ "Ao1Zc0h5AdSHtYt1caWZnQ==",
+ "AoN/pnK4KEUaGw4V9SFjpg==",
+ "ApiuEPWr8UjuRyJjsYZQBw==",
+ "AqHVaj3JcR44hnMzUPvVYg==",
+ "Ar1Eb/f/LtuIjXnnVPYQlA==",
+ "Ar9N1VYgE7riwmcrM3bA2Q==",
+ "AsAHrIkMgc3RRWnklY9lJw==",
+ "AvdeYb9XNOUFWiiz+XGfng==",
+ "AwPTZpC28NJQhf5fNiJuLA==",
+ "AxEjImKz4tMFieSo7m60Sg==",
+ "AyWlT+EGzIXc395zTlEU5Q==",
+ "B+TsxQZf0IiQrU8X9S4dsQ==",
+ "B0TaUQ6dKhPfSc5V/MjLEQ==",
+ "B1VVUbl8pU0Phyl1RYrmBg==",
+ "B6reUwMkQFaCHb9BYZExpw==",
+ "BA18GEAOOyVXO2yZt2U35w==",
+ "BAJ+/jbk2HyobezZyB9LiQ==",
+ "BB/R8oQOcoE4j63Hrh8ifg==",
+ "BB9PTlwKAWkExt3kKC/Wog==",
+ "BDNM1u/9mefjuW1YM2DuBg==",
+ "BDbfe/xa9Mz1lVD82ZYRGA==",
+ "BH+rkZWQjTp7au6vtll/CQ==",
+ "BL3buzSCV78rCXNEhUhuKQ==",
+ "BLJk9wA88z6e0IQNrWJIVw==",
+ "BLbTFLSb4mkxMaq4/B2khg==",
+ "BMOi5JmFUg5sCkbTTffXHw==",
+ "BMZB1FwvAuEqyrd0rZrEzw==",
+ "BPT4PQxeQcsZsUQl33VGmg==",
+ "BTiGLT6XdZIpFBc91IJY6g==",
+ "BV1moliPL15M14xkL+H1zw==",
+ "BW0A06zoQw7S+YMGaegT7g==",
+ "BXGlq54wIH6R3OdYfSSDRw==",
+ "BYpHADmEnzBsegdYTv8B5Q==",
+ "BYz52gYI/Z6AbYbjWefcEA==",
+ "BZTzHJGhzhs3mCXHDqMjnQ==",
+ "BaRwTrc5ulyKbW4+QqD0dw==",
+ "BhKO1s1O693Fjy1LItR/Jw==",
+ "BjfOelfc1IBgmUxMJFjlbQ==",
+ "BlCgDd7EYDIqnoAiKOXX6Q==",
+ "BophnnMszW5o+ywgb+3Qbw==",
+ "Bq82MoMcDjIo/exqd/6UoA==",
+ "BuDVDLl0OGdomEcr+73XhQ==",
+ "BuENxPg7JNrWXcCxBltOPg==",
+ "Bv4mNIC72KppYw/nHQxfpQ==",
+ "Bvk8NX4l6WktLcRDRKsK/A==",
+ "BwRA+tMtwEvth28IwpZx+w==",
+ "BxFP+4o6PSlGN78eSVT1pA==",
+ "BxsDnI8jXr4lBwDbyHaYXw==",
+ "Byhi4ymFqqH8uIeoMRvPug==",
+ "BzkNYH03gF/mQY71RwO3VA==",
+ "C+Ssp+v1r+00+qiTy2d7kA==",
+ "C4QEzQKGxyRi2rjwioHttA==",
+ "C65PZm8rZxJ6tTEb6d08Eg==",
+ "C7UaoIEXsVRxjeA0u99Qmw==",
+ "CBAGa5l95f3hVzNi6MPWeQ==",
+ "CCK+6Dr72G3WlNCzV7nmqw==",
+ "CDsanJz7e3r/eQe+ZYFeVQ==",
+ "CF1sAlhjDQY/KWOBnSSveA==",
+ "CHLHizLruvCrVi9chj9sXA==",
+ "CHsFJfsvZkPWDXkA6ZMsDQ==",
+ "CJoZn5wdTXbhrWO5LkiW0g==",
+ "CLPzjXKGGpJ0VrkSJp7wPQ==",
+ "CPDs+We/1wvsGdaiqxzeCQ==",
+ "CQ0PPwgdG3N6Ohfwx1C8xA==",
+ "CQpJFrpOvcQhsTXIlJli+Q==",
+ "CRiL6zpjfznhGXhCIbz8pQ==",
+ "CRmAj3JcasAb4iZ9ZbNIbw==",
+ "CT3ldhWpS1SEEmPtjejR/Q==",
+ "CT9g8mKsIN/VeHLSTFJcNQ==",
+ "CUCjG2UaEBmiYWQc6+AS1Q==",
+ "CUEueo8QXRxkfVdfNIk/gg==",
+ "CWBGcRFYwZ0va6115vV/oQ==",
+ "CX/N/lHckmAtHKysYtGdZA==",
+ "CXMKIdGvm60bgfsNc+Imvg==",
+ "CYJB3qy5GalPLAv1KGFEZA==",
+ "CZNoTy26VUQirvYxSPc/5A==",
+ "CZbd+UoTz0Qu1kkCS3k8Xg==",
+ "CazLJMJjQMeHhYLwXW7YNg==",
+ "Ci7sS7Yi1+IwAM3VMAB4ew==",
+ "CiiUeJ0LeWfm7+gmEmYXtg==",
+ "CkDIoAFLlIRXra78bxT/ZA==",
+ "CkZUmKBAGu0FLpgPDrybpw==",
+ "Cl1u5nGyXaoGyDmNdt38Bw==",
+ "CmBf5qchS1V3C2mS6Rl4bw==",
+ "CmVD6nh8b/04/6JV9SovlA==",
+ "CmkmWcMK4eqPBcRbdnQvhw==",
+ "CnIwpRVC2URVfoiymnsdYQ==",
+ "CoLvjQDQGldGDqRxfQo+WQ==",
+ "CrJDgdfzOea2M2hVedTrIg==",
+ "CsPkyTZADMnKcgSuNu1qxg==",
+ "CtDj/h2Q/lRey20G8dzSgA==",
+ "CuGIxWhRLN7AalafBZLCKQ==",
+ "Cv079ZF55RnbsDT27MOQIA==",
+ "Cz1G77hsDtAjpe0WzEgQog==",
+ "CzP13PM/mNpJcJg8JD3s6w==",
+ "CzSumIcYrZlxOUwUnLR2Zw==",
+ "CzWhuxwYbNB/Ffj/uSCtbw==",
+ "D09afzGpwCEH0EgZUSmIZA==",
+ "D0Qt9sRlMaPnOv1xaq+XUg==",
+ "D0W5F7gKMljoG5rlue1jrg==",
+ "D175i+2bZ7aWa4quSSkQpA==",
+ "D2JcY4zWwqaCKebLM8lPiQ==",
+ "D31ZticrjGWAO45l5hFh7A==",
+ "D5ibbo8UJMfFZ48RffuhgQ==",
+ "D5jaV+HtXkSpSxJPmaBDXg==",
+ "D66Suu3tWBD+eurBpPXfjA==",
+ "D7piVoB2NJlBxK5owyo4+g==",
+ "D7wN7b5u5PKkMaLJBP9Ksw==",
+ "DA+3fjr7mgpwf6BZcExj0w==",
+ "DB706G73NpBSRS8TKQOVZw==",
+ "DBKrdpCE0awppxST4o/zzg==",
+ "DCjgaGV5hgSVtFY5tcwkuA==",
+ "DCvI9byhw0wOFwF1uP6xIQ==",
+ "DDitrRSvovaiXe2nfAtp4g==",
+ "DEaZD/8aWV6+zkiLSVN/gA==",
+ "DG2Qe2DqPs5MkZPOqX363Q==",
+ "DJ+a37tCaGF5OgUhG+T0NA==",
+ "DJmrmNRKARzsTCKSMLmcNA==",
+ "DJoy1NSZZw87oxWGlNHhfg==",
+ "DJscTYNFPyPmTb57g/1w+Q==",
+ "DKApp/alXiaPSRNm3MfSuA==",
+ "DLzHkTjjuH6LpWHo2ITD0Q==",
+ "DMHmyn2U2n+UXxkqdvKpnA==",
+ "DO1/jfP/xBI9N0RJNqB2Rw==",
+ "DQJRsUwO1fOuGlkgJavcwQ==",
+ "DQQB/l55iPN9XcySieNX3A==",
+ "DQeib845UqBMEl96sqsaSg==",
+ "DQlZWBgdTCoYB1tJrNS5YQ==",
+ "DRiFNojs7wM8sfkWcmLnhQ==",
+ "DWKsPfKDAtfuwgmc2dKUNg==",
+ "DY0IolKTYlW+jbKLPAlYjQ==",
+ "DYWCPUq/hpjr6puBE7KBHg==",
+ "DbWQI3H2tcJsVJThszfHGA==",
+ "DdaT4JLC7U0EkF50LzIj9w==",
+ "DdiNGiOSoIZxrMrGNvqkXw==",
+ "DinJuuBX9OKsK5fUtcaTcQ==",
+ "DjHszpS8Dgocv3oQkW/VZQ==",
+ "DjeSrUoWW2QAZOAybeLGJg==",
+ "Dk0L/lQizPEb3Qud6VHb1Q==",
+ "DmxgZsQg+Qy1GP0fPkW3VA==",
+ "Dmyb+a7/QFsU4d2cVQsxDw==",
+ "DnF6TYSJxlc+cwdfevLYng==",
+ "Do3aqbRKtmlQI2fXtSZfxQ==",
+ "DoiItHSms0B9gYmunVbRkQ==",
+ "DqzWt1gfyu/e7RQl5zWnuQ==",
+ "Dt6hvhPJu94CJpiyJ5uUkg==",
+ "Dt8Q5ORzTmpPR2Wdk0k+Aw==",
+ "DuEKxykezAvyaFO2/5ZmKQ==",
+ "Dulw855DfgIwiK7hr3X8vg==",
+ "Duz/8Ebbd0w6oHwOs0Wnwg==",
+ "DwOTyyCoUfaSShHZx9u6xg==",
+ "DwP0MQf71VsqvAbAMtC3QQ==",
+ "DwrNdmU5VFFf3TwCCcptPA==",
+ "Dz90OhYEjpaJ/pxwg1Qxhg==",
+ "E+02smwQGBIxv42LIF2Y4Q==",
+ "E1CvxFbuu9AYW604mnpGTw==",
+ "E2LR1aZ3DcdCBuVT7BhReA==",
+ "E2v8Kk60qVpQ232YzjS2ow==",
+ "E3jMjAgXwvwR8PA53g4+PQ==",
+ "E4NtzxQruLcetC23zKVIng==",
+ "E4ojRDwGsIiyuxBuXHsKBA==",
+ "E8yMPK7W0SIGTK6gIqhxiQ==",
+ "E9IlDyULLdeaVUzN6eky8g==",
+ "E9ajQQMe02gyUiW3YLjO/A==",
+ "E9yeifEZtpqlD0N3pomnGw==",
+ "EATnlYm0p3h04cLAL95JgA==",
+ "EC0+iUdSZvmIEzipXgj7Gg==",
+ "EGLOaMe6Nvzs/cmb7pNpbg==",
+ "EJgedRYsZPc4cT9rlwaZhg==",
+ "EKU3OVlT4b/8j3MTBqpMNg==",
+ "ENFfP93LA257G6pXQkmIdg==",
+ "EUXQZwLgnDG+C8qxVoBNdw==",
+ "EXveRXjzsjh8zbbQY2pM9g==",
+ "EZVQGsXTZvht1qedRLF8bQ==",
+ "EbGG4X18upaiVQmPfwKytg==",
+ "EdvIAKdRAXj7e42mMlFOGQ==",
+ "Ee4A3lTMLQ7iDQ7b8QP8Qg==",
+ "EfXDc6h69aBPE6qsB+6+Ig==",
+ "Egs14xVbRWjfBBX7X5Z60g==",
+ "Ej7W3+67kCIng3yulXGpRQ==",
+ "ElTNyMR4Rg8ApKrPw88WPg==",
+ "Epm0d/DvXkOFeM4hoPCBrg==",
+ "EqMlrz1to7HG4GIFTPaehQ==",
+ "EqYq2aVOrdX5r7hBqUJP7g==",
+ "Err1mbWJud80JNsDEmXcYg==",
+ "EuGWtIbyKToOe6DN3NkVpQ==",
+ "Ev/xjTi7akYBI7IeZJ4Igw==",
+ "EvSB+rCggob2RBeXyDQRvQ==",
+ "Ex3x5HeDPhgO2S9jjCFy4g==",
+ "EyIsYQxgFa4huyo/Lomv7g==",
+ "EzjbinBHx3Wr08eXpH3HXA==",
+ "F50iXjRo1aSTr37GQQXuJA==",
+ "F58ktE4O0f7C9HdsXYm+lw==",
+ "F5FcNti7lUa9DyF2iEpBug==",
+ "F5bs0GGWBx9eBwcJJpXbqg==",
+ "F8l+Qd9TZgzV+r8G584lKA==",
+ "F8tEIT5EhcvLNRU5f0zlXQ==",
+ "FA+nK6mpFWdD0kLFcEdhxA==",
+ "FAXzjjIr8l1nsQFPpgxM/g==",
+ "FCLQocqxxhJeleARZ6kSPg==",
+ "FH5Z60RXXUiDk+dSZBxD3g==",
+ "FHvI0IVNvih8tC7JgzvCOw==",
+ "FI2WhaSMb3guFLe3e9il8Q==",
+ "FIOCTEbzb2+KMCnEdJ7jZw==",
+ "FL/j3GJBuXdAo54JYiWklQ==",
+ "FLvED9nB9FEl9LqPn7OOrA==",
+ "FN7oLGBQGHXXn5dLnr/ElA==",
+ "FNvQqYoe0s/SogpAB7Hr1Q==",
+ "FUQySDFodnRhr+NUsWt0KA==",
+ "FV/D5uSco+Iz8L+5t7E8SA==",
+ "FWphIPZMumqnXr1glnbK4w==",
+ "FXzaxi3nAXBc8WZfFElQeA==",
+ "FbxScyuRacAQkdQ034ShTA==",
+ "FcFcn4qmPse5mJCX5yNlsA==",
+ "FcKjlHKfQAGoovtpf+DxWQ==",
+ "Fd0c8f2eykUp9GYhqOcKoA==",
+ "Fd2fYFs8vtjws2kx1gf6Rw==",
+ "FeRovookFQIsXmHXUJhGOw==",
+ "FhthAO5IkMyW4dFwpFS7RA==",
+ "Fiy3hkcGZQjNKSQP9vRqyA==",
+ "FltEN+7NKvzt+XAktHpfHA==",
+ "FnVNxl5AFH1AieYru2ZG+A==",
+ "FoJZ61VrU8i084pAuoWhDQ==",
+ "FpWDTLTDmkUhH/Sgo+g1Gg==",
+ "FpgdsQ2OG+bVEy3AeuLXFQ==",
+ "FqWLkhWl0iiD/u2cp+XK9A==",
+ "FrTgaF5YZCNkyfR1kVzTLQ==",
+ "Ft2wXUokFdUf6d2Y/lwriw==",
+ "FtxpWdhEmC6MT61qQv4DGA==",
+ "FuWspiqu5g8Eeli5Az+BkA==",
+ "FxnbKnuDct4OWcnFMT/a5w==",
+ "Fz8EI+ZpYlbcttSHs5PfpA==",
+ "FzqIpOcTsckSNHExrl+9jg==",
+ "Fzuq+Wg7clo6DTujNrxsSA==",
+ "G+sGF13VXPH4Ih6XgFEXxg==",
+ "G/PA+kt0N+jXDVKjR/054A==",
+ "G0LChrb0OE5YFqsfTpIL1Q==",
+ "G0MlFNCbRjXk4ekcPO/chQ==",
+ "G2UponGde3/Z+9b2m9abpQ==",
+ "G37U8XTFyshfCs7qzFxATg==",
+ "G3PmmPGHaWHpPW30xQgm3Q==",
+ "G4qzBI1sFP2faN+tlRL/Bw==",
+ "G736AX070whraDxChqUrqw==",
+ "G7J/za99BFbAZH+Q+/B8WA==",
+ "G8LFBop8u6IIng+gQuVg3w==",
+ "GA8k6GQ20DGduVoC+gieRA==",
+ "GCYI9Dn1h3gOuueKc7pdKA==",
+ "GDMqfhPQN0PxfJPnK1Bb9A==",
+ "GF0lY77rx1NQzAsZpFtXIQ==",
+ "GF2yvI9UWf1WY7V7HXmKPA==",
+ "GFRJoPcXlkKSvJRuBOAYHQ==",
+ "GG8a3BlwGrYIwZH9j3cnPA==",
+ "GHEdXgGWOeOa6RuPMF0xXg==",
+ "GIHKW6plyLra0BmMOurFgA==",
+ "GKzs8mlnQQc58CyOBTlfIg==",
+ "GLDNTSwygNBmuFwCIm7HtA==",
+ "GLmWLXURlUOJ+PMjpWEXVA==",
+ "GLnS9wDCje7TOMvBX9jJVA==",
+ "GNak/LFeoHWlTdLW1iU4eg==",
+ "GNrMvNXQkW7PydlyJa+f1w==",
+ "GQJxu1SoMBH14KPV/G/KrQ==",
+ "GSWncBq4nwomZCBoxCULww==",
+ "GT6WUDXiheKAM7tPg3he9A==",
+ "GTNttXfMniNhrbhn92Aykg==",
+ "GUiinC3vgBjbQC2ybMrMNQ==",
+ "GW1Uaq622QamiiF24QUA0g==",
+ "GWwJ32SZqD5wldrXUdNTLA==",
+ "GdTanUprpE3X/YjJDPpkhQ==",
+ "Gdf4VEDLBrKJNQ8qzDsIyw==",
+ "GglPoW5fvr4JSM3Zv99oiA==",
+ "GhpJfRSWZigLg/azTssyVA==",
+ "Ghuj9hAyfehmYgebBktfgA==",
+ "GmC+0rNDMIR+YbUudoNUXw==",
+ "GnJKlRzmgKN9vWyGfMq3aA==",
+ "GncGQgmWpI/fZyb/6zaFCg==",
+ "GrSbnecYAC3j5gtoKntL0A==",
+ "Gt4/MMrLBErhbFjGbiNqQQ==",
+ "GzbeM7snhe+M+J7X+gAsQw==",
+ "H+NHjk/GJDh/GaNzMQSzjg==",
+ "H+yPRiooEh5J7lAJB4RZ7Q==",
+ "H0UMAUfHFQH92A2AXRCBKA==",
+ "H1NJEI+fvOQbI51kaNQQjQ==",
+ "H1y2iXVaQYwP0SakN6sa+Q==",
+ "H1zH9I8RwfEy5DGz3z+dHw==",
+ "H6HPFAcdHFbQUNrYnB74dA==",
+ "H6j2nPbBaxHecXruxiWYkA==",
+ "HBRzLacCVYfwUVGzrefZYg==",
+ "HCbHUfsTDl6+bxPjT57lrA==",
+ "HCu4ZMrcLMZbPXbTlWuvvQ==",
+ "HDxGhvdQwGh0aLRYEGFqnw==",
+ "HEcOaEd9zCoOVbEmroSvJg==",
+ "HEghmKg3GN60K7otpeNhaA==",
+ "HFCQEiZf7/SNc+oNSkkwlA==",
+ "HFHMGgfOeO0UPrray1G+Zw==",
+ "HGxe+5/kkh6R9GXzEOOFHA==",
+ "HHxn4iIQ7m0tF1rSd+BZBg==",
+ "HI4ZIE5s8ez8Rb+Mv39FxA==",
+ "HITIVoFoWNg04NExe13dNA==",
+ "HJYgUxFZ66fRT8Ka73RaUg==",
+ "HK0yf7F97bkf1VYCrEFoWA==",
+ "HK9xG03FjgCy8vSR+hx8+Q==",
+ "HLesnV3DL+FhWF3h6RXe8g==",
+ "HLxROy6fx/mLXFTDSX4eLA==",
+ "HMQarkPWOUDIg5+5ja2dBQ==",
+ "HMWOlMmzocOIiJ7yG1YaDQ==",
+ "HOi+vsGAae4vhr+lJ5ATnQ==",
+ "HPvYV94ufwiNHEImu4OYvQ==",
+ "HRF3WL/ue3/QlYyu7NUTrA==",
+ "HRWYX2XOdsOqYzCcqkwIyw==",
+ "HYylUirJRqLm+dkp39fSOQ==",
+ "HaHTsLzx7V3G1SFknXpGxA==",
+ "HaIRV9SNPRTPDOSX9sK/bg==",
+ "HaSc7MZphCMysTy2JbTJkw==",
+ "Hb+pdSavvJ9lUXkSVZW8Og==",
+ "HbT6W1Ssd3W7ApKzrmsbcg==",
+ "HbXv8InyZqFT7i3VrllBgg==",
+ "HdB7Se47cWjPgpJN0pZuiA==",
+ "HdXg64DBy5WcL5fRRiUVOg==",
+ "HeQbUuBM9sqfXFXRBDISSw==",
+ "HfvsiCQN/3mT0FabCU5ygQ==",
+ "HgIFX42oUdRPu7sKAXhNWg==",
+ "HhBHt5lQauNl7EZXpsDHJA==",
+ "HiAgt86AyznvbI2pnLalVQ==",
+ "HjlPM2FQWdILUXHalIhQ5w==",
+ "HjyxyL0db2hGDq2ZjwOOhg==",
+ "HkbdaMuDTPBDnt3wAn5RpQ==",
+ "Hm6MG6BXbAGURVJKWRM6ZA==",
+ "HnVfyqgJ+1xSsN4deTXcIA==",
+ "HoaBBw2aPCyhh0f5GxF+/Q==",
+ "Hs3vUOOs2TWQdQZHs+FaQQ==",
+ "Hst3yfyTB7yBUinvVzYROQ==",
+ "HtDXgMuF8PJ1haWk88S0Ew==",
+ "HuDuxs2KiGqmeyY1s1PjpQ==",
+ "HwLSUie8bzH+pOJT3XQFyg==",
+ "HxEU37uBMeiR5y8q/pM42g==",
+ "Hy1nqC40l5ItxumkIC2LAA==",
+ "I+wVQA+jpPTJ6xEsAlYucg==",
+ "I07W2eDQwe6DVsm1zHKM8A==",
+ "I5qDndyelK4Njv4YrX7S6w==",
+ "I9KNZC1tijiG1T72C4cVqQ==",
+ "IA1jmtfpYkz/E2wD0+27WA==",
+ "IADk81pIu8NIL/+9Fi94pA==",
+ "IAMInfSYb76GxDlAr1dsTg==",
+ "ICPdBCdONUqPwD5BXU5lrw==",
+ "IEz72W2/W8xBx5aCobUFOQ==",
+ "IHhyR6+5sZXTH+/NrghIPg==",
+ "IHyIeMad23fSDisblwyfpA==",
+ "IKgNa2oPaFVGYnOsL+GC5Q==",
+ "INNBBin5ePwTyhPIyndHHg==",
+ "IPLD9nT5EEYG9ioaSIYuuA==",
+ "ITYL3tDwddEdWSD6J6ULaA==",
+ "ITZ3P47ALS0JguFms6/cDA==",
+ "IUZ5aGpkJ9rLgSg6oAmMlw==",
+ "IUwVHH6+8/0c+nOrjclOWA==",
+ "IWZnTJ3Hb9qw9HAK/M9gTw==",
+ "IYIP2UBRyWetVfYLRsi1SQ==",
+ "IYIbEaErHoFBn8sTT9ICIQ==",
+ "IbN736G1Px5bsYqE5gW1JQ==",
+ "IdadoCPmSgHDHzn1zyf8Jw==",
+ "IdmcpJXyVDajzeiGZixhSA==",
+ "IhHyHbHGyQS+VawxteLP0w==",
+ "IhpXs1TK7itQ3uTzZPRP5Q==",
+ "IindlAnepkazs5DssBCPhA==",
+ "IjmLaf3stWDAwvjzNbJpQA==",
+ "Ily2MKoFI1zr5LxBy93EmQ==",
+ "Iqszlv4R49UevjGxIPMhIA==",
+ "IrDuBrVu1HWm0BthAHyOLQ==",
+ "Is3uxoSNqoIo5I15z6Z2UQ==",
+ "IshzWega6zr3979khNVFQQ==",
+ "It+K/RCYMOfNrDZxo7lbcA==",
+ "IwLbkL33z+LdTjaFYh93kg==",
+ "IwfeA6d0cT4nDTCCRhK+pA==",
+ "J/PNYu4y6ZMWFFXsAhaoow==",
+ "J/eAtAPswMELIj8K2ai+Xg==",
+ "J0NauydfKsACUUEpMhQg8A==",
+ "J1nYqJ7tIQK1+a/3sMXI/Q==",
+ "J2NFyb8cXEpZyxWDthYQiA==",
+ "J4MC9He6oqjOWsYQh9nl3Q==",
+ "J8v2f6hWFu8oLuwhOeoQjA==",
+ "JATLdpQm//SQnkyCfI5x7Q==",
+ "JBkbaBiorCtFq9M9lSUdMg==",
+ "JC8Q+8yOJ52NvtVeyHo68w==",
+ "JFFeXsFsMA59iNtZey7LAA==",
+ "JFHutgSe1/SlcYKIbNNYwQ==",
+ "JFi6N1PlrpKaYECOnI7GFg==",
+ "JGEy6VP3sz3LHiyT2UwNHQ==",
+ "JGeqHRQpf4No74aCs+YTfA==",
+ "JGx8sTyvr4bLREIhSqpFkw==",
+ "JHBjKpCgSgrNNACZW1W+1w==",
+ "JIC8R48jGVqro6wmG2KXIw==",
+ "JJJkp1TpuDx5wrua2Wml7g==",
+ "JJbzQ/trOeqQomsKXKwUpQ==",
+ "JKg64m6mU7C/CkTwVn4ASg==",
+ "JKmZqz9cUnj6eTsWnFaB0A==",
+ "JKphO0UYjFqcbPr6EeBuqg==",
+ "JLq/DrW2f26NaRwfpDXIEA==",
+ "JPxEncA4IkfBDvpjHsQzig==",
+ "JQf9UmutPh3tAnu7FDk3nA==",
+ "JSr/lqDej81xqUvd/O2s7w==",
+ "JSyhTcHLTfzHsPrxJyiVrA==",
+ "JSyq2MIuObPnEgEUDyALjQ==",
+ "JVSLiwurnCelNBiG2nflpQ==",
+ "JXCYeWjFqcdSf6QwB54G+A==",
+ "JYJvOZ4CHktLrYJyAbdOnA==",
+ "JZRjdJLgZ+S0ieWVDj8IJg==",
+ "Ja3ECL7ClwDrWMTdcSQ6Ug==",
+ "JaYQXntiyznQzrTlEeZMIw==",
+ "Jbxl8Nw1vlHO9rtu0q/Fpg==",
+ "Jcxjli2tcIAjCe+5LyvqdQ==",
+ "Je1UESovkBa9T6wS0hevLw==",
+ "JgXSPXDqaS1G9NqmJXZG0A==",
+ "JgxNrUlL8wutG04ogKFPvw==",
+ "JipruVZx4ban3Zo5nNM37g==",
+ "Jit0X0srSNFnn8Ymi1EY+g==",
+ "Jj4IrSVpqQnhFrzNvylSzA==",
+ "Jm862vBTCYbv/V4T1t46+Q==",
+ "JnE6BK0vpWIhNkaeaYNUzw==",
+ "JoATsk/aJH0UcDchFMksWA==",
+ "JquDByOmaQEpFb47ZJ4+JA==",
+ "JrKGKAKdjfAaYeQH8Y2ZRQ==",
+ "Js7g8Dr6XsnGURA4UNF0Ug==",
+ "Jt4Eg6MJn8O4Ph/K2LeSUA==",
+ "Ju4YwtPw+MKzpbC0wJsZow==",
+ "JvXTdChcE3AqMbFYTT3/wg==",
+ "JyIDGL1m/w+pQDOyyeYupA==",
+ "JyUJEnU6hJu8x2NCnGrYFw==",
+ "JzW+yhrjXW1ivKu3mUXPXg==",
+ "K1CGbMfhlhIuS0YHLG30PQ==",
+ "K1RL+tLjICBvMupe7QppIQ==",
+ "K1RgR6HR5uDEQgZ32TAFgA==",
+ "K2gk9zWGd0lJFRMQ1AjQ/Q==",
+ "K3NBEG8jJTJbSrYSOC3FKw==",
+ "K4VS+DDkTdBblG93l2eNkA==",
+ "K4yZNVoqHjXNhrZzz2gTew==",
+ "K5lhaAIZkGeP5rH2ebSJFw==",
+ "K8PVQhEJCEH1ghwOdztjRw==",
+ "K9A87aMlJC8XB9LuFM913g==",
+ "KCJJfgLe00+tjSfP6EBcUg==",
+ "KGI/cXVz6v6CfL8H6akcUQ==",
+ "KI7tQFYW38zYHOzkKp9/lQ==",
+ "KO2XVYyNZadcQv8aCNn5JA==",
+ "KOm8PTa+ICgDrgK9QxCJZw==",
+ "KOmdvm+wJuZ/nT/o1+xOuw==",
+ "KPh6TwYpspne4KZA6NyMbw==",
+ "KQw25X4LnQ9is+qdqfxo0w==",
+ "KR401XBdgCrtVDSaXqPEiA==",
+ "KSorNz/PLR/YYkxaj1fuqw==",
+ "KSumhnbKxMXQDkZIpDSWmQ==",
+ "KTjwL+qswa+Bid8xLdjMTg==",
+ "KXuFON8tMBizNkCC48ICLA==",
+ "KXvdjZ3rRKn60djPTCENGA==",
+ "KYuUNrkTvjUWQovw9dNakA==",
+ "Kh/J1NpDBGoyDU+Mrnnxkg==",
+ "KhUT2buOXavGCpcDOcbOYg==",
+ "KhrIIHfqXl9zGE9aGrkRVg==",
+ "Kj1QI+s9261S3lTtPKd9eg==",
+ "KjfL7YyVqmCJGBGDFdJ0gw==",
+ "KjnL3x+56r3M2pDj1pPihA==",
+ "KkXlgPJPen6HLxbNn5llBw==",
+ "KkwQL0DeUM3nPFfHb2ej+A==",
+ "KlY5TGg0pR/57TVX+ik1KQ==",
+ "KmcGEE0pacQ/HDUgjlt7Pg==",
+ "KodYHHN62zESrXUye7M01g==",
+ "Koiog/hpN7ew5kgJbty34A==",
+ "Kt6BTG1zdeBZ3nlVk+BZKQ==",
+ "KuNY8qAJBce+yUIluW8AYw==",
+ "KujFdhhgB9q4oJfjYMSsLg==",
+ "KyLQxi5UP+qOiyZl0PoHNQ==",
+ "KzWdWPP2gH0DoMYV4ndJRg==",
+ "Kzs+/IZJO8v4uIv9mlyJ2Q==",
+ "L+N/6geuokiLPPSDXM9Qkg==",
+ "L2D7G0btrwxl9V4dP3XM5Q==",
+ "L2IeUnATZHqOPcrnW2APbA==",
+ "L2RofFWDO0fVgSz4D2mtdw==",
+ "L3Jt5dHQpWQk74IAuDOL8g==",
+ "L4+C6I7ausPl6JbIbmozAg==",
+ "LATQEY7f47i77M6p11wjWA==",
+ "LCj4hI520tA685Sscq6uLw==",
+ "LCvz/h9hbouXCmdWDPGWqg==",
+ "LDuBcL5r3PUuzKKZ9x6Kfw==",
+ "LEVYAE54618FrlXkDN01Kw==",
+ "LFcpCtnSnsCPD2gT/RA+Zg==",
+ "LGwcvetzQ3QqKjNh5vA8vw==",
+ "LHQETSI5zsejvDaPpsO29g==",
+ "LJeLdqmriyAQp+QjZGFkdQ==",
+ "LJtRcR70ug6UHiuqbT6NGw==",
+ "LKyOFgUKKGUU/PxpFYMILw==",
+ "LMCZqd3UoF/kHHwzTdj7Tw==",
+ "LMEtzh0+J27+4zORfcjITw==",
+ "LPYFDbTEp5nGtG6uO8epSw==",
+ "LQttmX92SI94+hDNVd8Gtw==",
+ "LSN9GmT6LUHlCAMFqpuPIA==",
+ "LUWxfy4lfgB5wUrqCOUisw==",
+ "LWWfRqgtph1XrpxF4N64TA==",
+ "LWd0+N3M94n81qd346LfJQ==",
+ "LZAKplVoNjeQgfaHqkyEJA==",
+ "La0gzdbDyXUq6YAXeKPuJA==",
+ "LawT9ZygiVtBk0XJ+KkQgQ==",
+ "LbPp1oL0t3K2BAlIN+l8DA==",
+ "LblwOqNiciHmt2NXjd89tg==",
+ "LcF0OqPWrcpHby8RwXz1Yg==",
+ "LcoJBEPTlSsQwfuoKQUxEw==",
+ "LhqRc9oewY4XaaXTcnXIHQ==",
+ "Lo1xTCEWSxVuIGEbBEkVxA==",
+ "LoUv/f2lcWpjftzpdivMww==",
+ "LpoayYsTO8WLFLCSh2kf2w==",
+ "Lqel4GdU0ZkfoJVXI5WC/Q==",
+ "LqgzKxbI6WTMz0AMIDJR5w==",
+ "LsmsPokAwWNCuC74MaqFCQ==",
+ "Lt/pVD4TFRoiikmgAxEWEw==",
+ "Lu02ic/E94s42A14m7NGCA==",
+ "LyPXOoOPMieqINtX8C9Zag==",
+ "LyYPOZKm8bBegMr5NTSBfg==",
+ "M/cQja3uIk1im9++brbBOA==",
+ "M0ESOGwJ4WZ4Ons1ljP0bQ==",
+ "M20iX2sUfw5SXaZLZYlTaA==",
+ "M2JMnViESVHTZaru6LDM6w==",
+ "M2suCoFHJ5fh9oKEpUG3xA==",
+ "M55eersiJuN9v61r8DoAjQ==",
+ "M98hjSxCwvZ27aBaJTGozQ==",
+ "M9oqlPb63e0kZE0zWOm+JQ==",
+ "MArbGuIAGnw4+fw6mZIxaw==",
+ "MBjMU/17AXBK0tqyARZP5w==",
+ "MFeXfNZy6Q9wBfZmPQy3xg==",
+ "MI+HSMRh8KTW+Afiaxd/Fw==",
+ "MJ1FuK8PXcmnBAG9meU84A==",
+ "MK7AqlJIGqK2+K5mCvMXRQ==",
+ "ML7ipnY/g8mA1PUIju1j8Q==",
+ "MLHt6Ak288G0RGhCVaOeqA==",
+ "MLlVniZ08FHAS5xe+ZKRaA==",
+ "MMaegl2Md9s/wOx5o9564w==",
+ "MN94B0r5CNAF9sl3Kccdbw==",
+ "MOrAbuJTyGKPC6MgYJlx5Q==",
+ "MQYM3BT77i35LG9HcqxY2Q==",
+ "MQvAr+OOfnYnr/Il/2Ubkg==",
+ "MUkRa/PjeWMhbCTq43g6Aw==",
+ "MVoxyIA+emaulH8Oks8Weg==",
+ "MWcV03ULc0vSt/pFPYPvFA==",
+ "MbI04HlTGCoc/6WDejwtaQ==",
+ "MdvhC1cuXqni/0mtQlSOCw==",
+ "MeKXnEfxeuQu9t3r/qWvcw==",
+ "MfkyURTBfkNZwB+wZKjP4g==",
+ "Mj87ajJ/yR41XwAbFzJbcA==",
+ "Ml3mi1lGS1IspHp3dYYClg==",
+ "MlKWxeEh8404vXenBLq4bw==",
+ "MlOOZOwcRGIkifaktEq0aQ==",
+ "MnStiFQAr3QlaRZ02SYGaQ==",
+ "Mofqu40zMRrlcGRLS42eBw==",
+ "MpAwWMt7bcs4eL7hCSLudQ==",
+ "MqqDg9Iyt4k3vYVW5F+LDw==",
+ "Mr5mCtC53+wwmwujOU/fWw==",
+ "MrbEUlTagbesBNg0OemHpw==",
+ "MrxR3cJaDHp0t3jQNThEyg==",
+ "MsCloSmTFoBpm7XWYb+ueQ==",
+ "Muf2Eafcf9G3U2ZvQ9OgtQ==",
+ "MvMbvZNKbXFe2XdN+HtnpQ==",
+ "N+K1ibXAOyMWdfYctNDSZQ==",
+ "N/HgDydvaXuJvTCBhG/KtA==",
+ "N2KovXW14hN/6+iWa1Yv3g==",
+ "N2X7KWekNN+fMmwyXgKD5w==",
+ "N3YDSkBUqSmrmNvZZx4a1Q==",
+ "N4/mQFyhDpPzmihjFJJn6w==",
+ "N65PqIWiQeS082D6qpfrAg==",
+ "N7fHwb397tuQHtBz1P80ZQ==",
+ "N8dXCawxSBX40fgRRSDqlQ==",
+ "N9nD7BGEM7LDwWIMDB+rEQ==",
+ "NBmB/cQfS+ipERd7j9+oVg==",
+ "ND2hYtAIQGMxBF7o7+u7nQ==",
+ "ND9l4JWcncRaSLATsq0LVw==",
+ "NDZWIhhixq7NT8baJUR4VQ==",
+ "NGApiVkDSwzO45GT57GDQw==",
+ "NKGY0ANVZ0gnUtzVx1pKSw==",
+ "NKRzJndo2uXNiNppVnqy1g==",
+ "NMbAjbnuK7EkVeY3CQI5VA==",
+ "NN/ymVQNa17JOTGr6ki3eQ==",
+ "NOmu8oZc6CcKLu+Wfz2YOQ==",
+ "NQVQfN3nIg9ipHiFh4BvfQ==",
+ "NRyFx6jqO/oo9ojvbYzsAg==",
+ "NSrzwNlB0bde3ph8k6ZQcQ==",
+ "NZtcY8fIpSKPso/KA6ZfzA==",
+ "Nc5kiwXCAyjpzt43G5RF1A==",
+ "NdULoUDGhIolzw1PyYKV0A==",
+ "NdVyHoTbBhX6Umz/9vbi0g==",
+ "Ndx5LDiVyyTz/Fh3oBTgvA==",
+ "Nf9fbRHm844KZ2sqUjNgkA==",
+ "NfxVYc3RNWZwzh2RmfXpiA==",
+ "Ng5v/B9Z10TTfsDFQ/XrXQ==",
+ "NhZbSq0CjDNOAIvBHBM9zA==",
+ "NiQ/m4DZXUbpca9aZdzWAw==",
+ "NiawWuMBDo0Q3P2xK/vnLQ==",
+ "NjeDgQ1nzH1XGRnLNqCmSg==",
+ "NmQrsmb8PVP05qnSulPe5Q==",
+ "NmWmDxwK5FpKlZbo0Rt8RA==",
+ "NoX8lkY+kd2GPuGjp+s0tQ==",
+ "NquRbPn8fFQhBrUCQeRRoQ==",
+ "Nr4zGo5VUrjXbI8Lr4YVWQ==",
+ "Nsd+DfRX6L54xs+iWeMjCQ==",
+ "NtwqUO3SKZE/9MXLbTJo/g==",
+ "NuBYjwlxadAH+vLWYRZ3bg==",
+ "NvkR0inSzAdetpI4SOXGhw==",
+ "NvurnIHin4O+wNP7MnrZ1w==",
+ "NxSdT2+MUkQN49pyNO2bJw==",
+ "NyF+4VRog7etp90B9FuEjA==",
+ "O/EizzJSuFY8MpusBRn7Tg==",
+ "O1ckWUwuhD44MswpaD6/rw==",
+ "O209ftgvu0vSr0UZywRFXA==",
+ "O538ibsrI4gkE5tfwjxjmg==",
+ "O5N2yd+QQggPBinQ+zIhtQ==",
+ "O7JiE0bbp583G6ZWRGBcfw==",
+ "O839JUrR+JS30/nOp428QA==",
+ "OChiB4BzcRE8Qxilu6TgJg==",
+ "OEJ40VmMDYzc2ESEMontRA==",
+ "OERGn45uzfDfglzFFn6JAg==",
+ "OFLn4wun6lq484I7f6yEwg==",
+ "OGpsXRHlaN8BvZftxh1e7A==",
+ "OHJBT2SEv5b5NxBpiAf7oQ==",
+ "OIwtfdq37eQ0qoXuB2j7Hw==",
+ "OMO4pqzfcbQ11YO4nkTXfg==",
+ "OONAvFS/kmH7+vPhAGTNSg==",
+ "OOS6wQCJsXH8CsWEidB35A==",
+ "OVHqwV8oQMC5KSMzd5VemA==",
+ "OaNpzwshdHUZMphQXa6i8w==",
+ "Oc3BqTF3ZBW3xE0QsnFn/A==",
+ "OlpA9HsF8MBh7b45WZSSlg==",
+ "OlwHO6Sg2zIwsCOCRu0HiQ==",
+ "Omi2ZB9kdR1HrVP2nueQkA==",
+ "Omr+zPWVucPCSfkgOzLmSQ==",
+ "OnmvXbyT2BYsSDJYZhLScA==",
+ "OpC/sL320wl5anx6AVEL+A==",
+ "OpL+vHwPasW30s2E1TYgpA==",
+ "OrqJKjRndcZ8OjE3cSQv7g==",
+ "Otz/PgYOEZ1CQDW54FWJIQ==",
+ "OwArFF1hpdBupCkanpwT+Q==",
+ "OwIGvTh8FPFqa4ijNkguAw==",
+ "Owg8qCpjZa+PmbhZew6/sw==",
+ "OzFRv+PzPqTNmOnvZGoo5g==",
+ "OzH7jTcyeM7RPVFtBdakpQ==",
+ "OzMR5D2LriC5yrVd5hchnA==",
+ "P0Pc8owrqt6spdf7FgBFSw==",
+ "P14k+fyz0TG9yIPdojp52w==",
+ "P3y5MoXrkRTSLhCdLlnc4A==",
+ "P430CeF2MDkuq11YdjvV8A==",
+ "P5WPQc5NOaK7WQiRtFabkw==",
+ "P5fucOJhtcRIoElFJS4ffg==",
+ "P5wS+xB8srW4a5KDp/JVkA==",
+ "P7eMlOz9YUcJO+pJy0Kpkw==",
+ "P8lUiLFoL100c9YSQWYqDA==",
+ "PAlx9+U+yQCAc5Fi0BOG0w==",
+ "PBULPuFXb6V3Di713n3Gug==",
+ "PCOGl7GIqbizAKj/sZmlwQ==",
+ "PD+yHtJxZJ2XEvjIPIJHsQ==",
+ "PF0lpolQQXlpc3qTLMBk8w==",
+ "PHwJ5ZAqqftZ4ypr8H1qiQ==",
+ "PKtXc4x4DEjM45dnmPWzyg==",
+ "PMCWKgog/G+GFZcIruSONw==",
+ "PMvG4NqJP76kMRAup6TSZA==",
+ "PPa7BDMpRdxJdBxkuWCxKA==",
+ "PTAm/jGkie7OlgVOvPKpaA==",
+ "PTW+fhZq/ErxHqpM0DZwHQ==",
+ "PXC6ZpdMH0ATis/jGW12iA==",
+ "PaROi5U16Tk35p0EKX5JpA==",
+ "ParhxI6RtLETBSwB0vwChQ==",
+ "PbDVq2Iw1eeM8c2o/XYdTA==",
+ "PbnxuVerGwHyshkumqAARg==",
+ "Pc+u0MAzp4lndTz4m6oQ5w==",
+ "PcdBtV8pfKU0YbDpsjPgwg==",
+ "PcoVtZrS1x1Q+6nfm4f80w==",
+ "PdBgXFq5mBqNxgCiqaRnkw==",
+ "PeJS+mXnAA6jQ0WxybRQ8w==",
+ "PfkWkSbAxIt1Iso0znW0+Q==",
+ "PggVPQL5YKqSU/1asihcrg==",
+ "PibGJQNw7VHPTgqeCzGUGA==",
+ "Po0lhBfiMaXhl+vYh1D8gA==",
+ "PolhKCedOsplEcaX4hQ0YQ==",
+ "Pp1ZMxJ8yajdbfKM4HAQxA==",
+ "PqLCd/pwc+q5GkL6MB0jTg==",
+ "Pt3i49uweYVgWze3OjkjJA==",
+ "Pu9pEf+Tek3J+3jmQNqrKw==",
+ "Pv9FWQEDLKnG/9K9EIz4Gw==",
+ "PwvPBc+4L73xK22S9kTrdA==",
+ "PxReytUUn/BbxYTFMu1r2Q==",
+ "PybPZhJErbRTuAafrrkb3g==",
+ "Q0TJZxpn3jk67L7N+YDaNA==",
+ "Q1pdQadt12anX1QRmU2Y/A==",
+ "Q3TpCE+wnmH/1h/EPWsBtQ==",
+ "Q4bfQslDSqU64MOQbBQEUw==",
+ "Q6vGRQiNwoyz7bDETGvi5g==",
+ "Q7Df6zGwvb4rC+EtIKfaSw==",
+ "Q7teXmTHAC5qBy+t7ugf0w==",
+ "Q8RVI/kRbKuXa8HAQD7zUA==",
+ "QAz7FA+jpz9GgLvwdoNTEQ==",
+ "QCpzCTReHxGm5lcLsgwPCA==",
+ "QGYFMpkv37CS2wmyp42ppg==",
+ "QH36wzyIhh6I56Vnx79hRA==",
+ "QH3lAwOYBAJ0Fd5pULAZqw==",
+ "QIKjir/ppRyS63BwUcHWmw==",
+ "QJEbr3+42P9yiAfrekKdRQ==",
+ "QTz21WkhpPjfK8YoBrpo+w==",
+ "QV0OG5bpjrjku4AzDvp9yw==",
+ "QVwuN66yPajcjiRnVk/V8g==",
+ "QWURrsEgxbJ8MWcaRmOWqw==",
+ "Qc+XYy2qyWJ5VVwd2PExbw==",
+ "Qf7JFJJuuacSzl6djUT2EQ==",
+ "Qg1ubGl+orphvT990e5ZPA==",
+ "QiozlNcQCbqXtwItWExqJQ==",
+ "QmSBVvdk0tqH9RAicXq2zA==",
+ "QmcURiMzmVeUNaYPSOtTTg==",
+ "QoUC9nyK1BAzoUVnBLV2zw==",
+ "QoqHzpHDHTwQD5UF30NruQ==",
+ "QozQL0DTtr+PXNKifv6l6g==",
+ "Qrh7OEHjp80IW+YzQwzlJg==",
+ "QsquNcCZL9wv7oZFqm64vQ==",
+ "QtD35QhE8sAccPrDnhtQmQ==",
+ "Qv6wWP4PpycDGxe7EZNSCw==",
+ "QvYZxsLdu+3nV/WhY1DsYg==",
+ "Qx6rVv9Xj8CBjqikWI9KFA==",
+ "QyyiJ5I/OZC50o89fa5EmQ==",
+ "R+beucURp/H5jLs4kW6wmg==",
+ "R/y6+JJP8rzz1KITJ4qWBw==",
+ "R1TCCfgltnXBvt5AiUnCtQ==",
+ "R2OOV18CV/YpWL1xzr/VQg==",
+ "R2Use39If2C0FVBP7KDerA==",
+ "R36O31Pj8jn0AWSuqI7X2Q==",
+ "R3ijnutzvK6IKV3AKHQZSA==",
+ "R5oOM58zdbVxFSDQnNWqeA==",
+ "R6Me6sSGP5xpNI8R0xGOWw==",
+ "R6cO8GzYfOGTIi773jtkXw==",
+ "R81DX/5a7DYKkS4CU+TL+w==",
+ "R8FxgXWKBpEVbnl41+tWEw==",
+ "R8ULpSNu9FcCwXZM0QedSg==",
+ "R906Kxp2VFVR3VD+o6Vxcw==",
+ "R97chlspND/sE9/HMScXjQ==",
+ "RAAw14BA1ws5Wu/rU7oegw==",
+ "RAECgYZmcF4WxcFcZ4A0Ww==",
+ "RBMv0IxXEO3o7MnV47Bzow==",
+ "RClzwwKh51rbB4ekl99EZA==",
+ "RDgGGxTtcPvRg/5KRRlz4w==",
+ "REnDNe9mGfqVGZt+GdsmjQ==",
+ "RHKCMAqrPjvUYt13BVcmvw==",
+ "RHToSGASrwEmvzjX6VPvNQ==",
+ "RIVYGO2smx9rmRoDVYMPXw==",
+ "RIZYDgXqsIdTf9o2Tp/S7g==",
+ "RJJqFMeiCZHdsqs72J17MQ==",
+ "RKVDdE1AkILTFndYWi9wFg==",
+ "RM5CpIiB94Sqxi462G7caA==",
+ "RNK9G1hfuz3ETY/RmA9+aA==",
+ "RNdyt6ZRGvwYG5Ws3QTuEA==",
+ "ROSt+NlEoiPFtpRqKtDUrQ==",
+ "RQOlmzHwQKFpafKPJj0D8w==",
+ "RQywrOLZEKw9+kG6qTzr3g==",
+ "RUmhye56tQu9xXs4SRJpOQ==",
+ "RVD3Ij6sRwwxTUDAxwELtA==",
+ "RWI0HfpP7643OSEZR8kxzw==",
+ "RYkDwwng6eeffPHxt8iD9A==",
+ "RZTpYKxOAH9JgF1QFGN+hw==",
+ "RfSwpO/ywQx4lfgeYlBr2w==",
+ "RgtwfY5pTolKrUGT+6Pp6g==",
+ "RhcqXY4OsZlVVF7ZlkTeRw==",
+ "RiahBXX2JbPzt8baPiP/8g==",
+ "RkQK9S1ezo+dFYHQP57qrw==",
+ "RlNPyhgYOIn28R4vKCVtYA==",
+ "RnOXOygwJFqrD+DlM3R5Ew==",
+ "RnxOYPSQdHS6fw4KkDJtrA==",
+ "RppDe/WGt1Ed6Vqg1+cCkQ==",
+ "RqYpA5AY7mKPaSxoQfI1CA==",
+ "RrE3B3X/SJi3CqCUlTYwaw==",
+ "Rrq0ak9YexLqqbSD4SSXlw==",
+ "Rs8deApkoosIJSfX7NXtAA==",
+ "RuLeQHP1wHsxhdmYMcgtrQ==",
+ "RvXWAFwM+mUAPW1MjPBaHA==",
+ "Rvchz/xjcY9uKiDAkRBMmA==",
+ "Rww3qkF3kWSd+AaMT0kfdw==",
+ "RxmdoO8ak8y/HzMSIm+yBQ==",
+ "Ry3zgZ6KHrpNyb7+Tt2Pkw==",
+ "RzeH+G3gvuK1z+nJGYqARQ==",
+ "S+b37XhKRm8cDwRb1gSsKQ==",
+ "S2MAIYeDQeJ1pl9vhtYtUg==",
+ "S3VQa6DH+BdlSrxT/g6B5g==",
+ "S47hklz3Ow+n5aY6+qsCoA==",
+ "S4RvORcJ3m6WhnAgV4YfYA==",
+ "S4rFuiKLFKZ+cL7ldiTwpg==",
+ "S7Vjy/gOWp0HozPP1RUOZw==",
+ "S8jlvuYuankCnvIvMVMzmg==",
+ "S9L29U2P5K8wNW+sWbiH7w==",
+ "SCO9nQncEcyVXGCtx30Jdg==",
+ "SChDh/Np1HyTPWfICfE1uA==",
+ "SDi5+FoP9bMyKYp+vVv1XA==",
+ "SEGu+cSbeeeZg4xWwsSErQ==",
+ "SEIZhyguLoyH7So0p1KY0A==",
+ "SESKbGF35rjO64gktmLTWA==",
+ "SElc2+YVi3afE1eG1MI7dQ==",
+ "SFn78uklZfMtKoz2N0xDaQ==",
+ "SIuKH/Qediq0TyvqUF93HQ==",
+ "SM7E98MyViSSS9G0Pwzwyw==",
+ "SNPYH4r/J9vpciGN2ybP5Q==",
+ "SOdpdrk2ayeyv0xWdNuy9g==",
+ "SPGpjEJrpflv1hF0qsFlPw==",
+ "SPHU6ES1WVm0Mu2LB+YjrA==",
+ "SSKhl2L3Mvy93DcZulADtA==",
+ "SUAwMWLMml8uGqagz5oqhQ==",
+ "SVFbcjXbV7HRg+7jUrzpwg==",
+ "SVLHWPCCH7GPVCF7QApPbw==",
+ "SVuEYfQ9FGyVMo1672n0Yg==",
+ "SbMjjI8/P8B9a9H2G0wHEQ==",
+ "Scto+9TWxj1eZgvNKo+a9A==",
+ "SfwnYZCKP1iUJyU1yq4eKg==",
+ "SiSlasZ+6U2IZYogqr2UPg==",
+ "Slu3z535ijcs5kzDnR7kfA==",
+ "SmRWEzqddY9ucGAP5jXjAg==",
+ "Sr9c0ReRpkDYGAiqSy683g==",
+ "Srl4HivgHMxMOUHyM3jvNw==",
+ "StDtLMlCI75g4XC59mESEQ==",
+ "StoXC7TBzyRViPzytAlzyQ==",
+ "StpQm/cQF8cT0LFzKUhC5w==",
+ "SusSOsWNoAerAIMBVWHtfA==",
+ "Swjn3YkWgj0uxbZ1Idtk+A==",
+ "SzCGM8ypE58FLaR1+1ccxQ==",
+ "Szko0IPE7RX2+mfsWczrMg==",
+ "T/6gSz2HwWJDFIVrmcm8Ug==",
+ "T1pMWdoNDpIsHF8nKuOn2A==",
+ "T6LA+daQqRI38iDKZTdg1A==",
+ "T7waQc3PvTFr0yWGKmFQdQ==",
+ "T9WoUJNwp8h4Yydixbx6nA==",
+ "TA9WjiLAFgJubLN4StPwLw==",
+ "TAD0Lk95CD86vbwrcRogaQ==",
+ "TBQpcKq2huNC5OmI2wzRQw==",
+ "TDrq23VUdzEU/8L5i8jRJQ==",
+ "TGB+FIzzKnouLh5bAiVOQg==",
+ "THfzE2G2NVKKfO+A2TjeFw==",
+ "THs1r8ZEPChSGrrhrNTlsA==",
+ "TI90EuS/bHq/CAlX32UFXg==",
+ "TIKadc6FAaRWSQUg5OATgg==",
+ "TIWSM78m0RprwgPGK/e0JA==",
+ "TLJbasOoVO435E5NE5JDcA==",
+ "TNyvLixb03aP2f8cDozzfA==",
+ "TSGL3iQYUgVg/O9SBKP9EA==",
+ "TSPFvkgw6uLsJh66Ou0H9w==",
+ "TVlHoi8J7sOZ2Ti7Dm92cQ==",
+ "TXab/hqNGWaSK+fXAoB2bg==",
+ "TYlnrwgyeZoRgOpBYneRAg==",
+ "TZ3ATPOFjNqFGSKY3vP2Hw==",
+ "TZT86wXfzFffjt0f95UF5w==",
+ "TafM7nTE5d+tBpRCsb8TjQ==",
+ "TahqPgS7kEg+y6Df0HBASw==",
+ "TcFinyBrUoAEcLzWdFymow==",
+ "TcGhAJHRr7eMwGeFgpFBhg==",
+ "TcyyXrSsQsnz0gJ36w4Dxw==",
+ "TeBGJCqSqbzvljIh9viAqA==",
+ "TfHvdbl2M4deg65QKBTPng==",
+ "TfNHjSTV8w6Pg6+FaGlxvA==",
+ "TgWe70YalDPyyUz6n88ujg==",
+ "Tk5MAqd1gyHpkYi8ErlbWg==",
+ "TlJizlASbPtShZhkPww4UA==",
+ "Tm4zk2Lmg8w4ITMI31NfTA==",
+ "Tmx0suRHzlUK4FdBivwOwA==",
+ "Tp52d1NndiC9w3crFqFm9g==",
+ "TrLmfgwaNATh24eSrOT+pw==",
+ "TrWS+reCJ0vbrDNT5HDR9w==",
+ "Tu6w6DtX2RJJ3Ym3o3QAWw==",
+ "TuaG3wRdM9BWKAxh2UmAsg==",
+ "Tud+AMyuFkWYYZ73yoJGpQ==",
+ "Tug3eh+28ttyf+U7jfpg5w==",
+ "U+bB5NjFIuQr/Y5UpXHwxA==",
+ "U+oTpcjhc0E+6UjP11OE/Q==",
+ "U0KmEI6e5zJkaI4YJyA5Ew==",
+ "U49SfOBeqQV9wzsNkboi8Q==",
+ "U6VQghxOXsydh3Naa5Nz4A==",
+ "U9kE50Wq5/EHO03c5hE4Ug==",
+ "UAqf4owQ+EmrE45hBcUMEw==",
+ "UEMwF4kwgIGxGT4jrBhMPQ==",
+ "UHpge5Bldt9oPGo2oxnYvQ==",
+ "UIXytIHyVODxlrg+eQoARA==",
+ "UK+R+hAoVeZ4xvsoZjdWpw==",
+ "UNRlg6+CYVOt68NwgufGNA==",
+ "UNdKik7Vy23LjjPzEdzNsg==",
+ "UNt7CNMtltJWq8giDciGyA==",
+ "UP7NXAE0uxHRXUAWPhto0w==",
+ "UP9mmAKzeQqGhod7NCqzhg==",
+ "UPYR575ASaBSZIR3aX1IgQ==",
+ "UPzS4LR3p/h0u69+7YemrQ==",
+ "UQTQk5rrs6lEb1a+nkLwfg==",
+ "USCvrMEm/Wqeu9oX6FrgcQ==",
+ "USq1iF90eUv41QBebs3bhw==",
+ "UTmTgvl+vGiCDQpLXyVgOg==",
+ "UVEZPoH9cysC+17MKHFraw==",
+ "UXUNYEOffgW3AdBs7zTMFA==",
+ "UZoibx+y1YJy/uRSa9Oa2w==",
+ "Ua6aO6HwM+rY4sPR19CNFA==",
+ "UbABE6ECnjB+9YvblE9CYw==",
+ "UbSFw5jtyLk5MealqJw++A==",
+ "Ugt8HVC/aUzyWpiHd0gCOQ==",
+ "UgvtdE2eBZBUCAJG/6c0og==",
+ "Uh1mvZNGehK1AaI4a1auKQ==",
+ "Uje3Ild84sN41JEg3PEHDg==",
+ "UjmDFO7uzjl4RZDPeMeNyg==",
+ "Um1ftRBycvb+363a90Osog==",
+ "Umd+5fTcxa3mzRFDL9Z8Ww==",
+ "Uo+FIhw1mfjF6/M8cE1c/Q==",
+ "Uo1ebgsOxc3eDRds1ah3ag==",
+ "UreSZCIdDgloih8KLeX7gg==",
+ "UtLYUlQJ02oKcjNR3l+ktg==",
+ "Uudn69Kcv2CGz2FbfJSSEA==",
+ "UvC1WADanMrhT+gPp/yVqA==",
+ "Uw6Iw+TP9ZdZGm2b/DAmkg==",
+ "UwqBVd4Wfias4ElOjk2BzQ==",
+ "Uy4QI8D2y1bq/HDNItCtAw==",
+ "UymZUnEEQWVnLDdRemv+Tw==",
+ "UzPPFSXgeV7KW4CN5GIQXA==",
+ "V+QzdKh5gxTPp2yPC9ZNEg==",
+ "V/xG5QFyx1pihimKmAo8ZA==",
+ "V1fvtnJ0L3sluj9nI5KzRw==",
+ "V2P75JFB4Se9h7TCUMfeNA==",
+ "V5HEaY3v9agOhsbYOAZgJA==",
+ "V5HKdaTHjA8IzvHNd9C51g==",
+ "V6CRKrKezPwsRdbm0DJ2Yg==",
+ "V6zyoX6MERIybGhhULnZiw==",
+ "V7eji28JSg3vTi30BCS7gw==",
+ "V8m51xgUgywRoV6BGKUrgg==",
+ "V8q+xz4ljszLZMrOMOngug==",
+ "V9G1we3DOIQGKXjjPqIppQ==",
+ "V9vkAanK+Pkc4FGAokJsTA==",
+ "VAg/aU5nl72O+cdNuPRO4g==",
+ "VCL3xfPVCL5RjihQM59fgg==",
+ "VE4sLM5bKlLdk85sslxiLQ==",
+ "VGRCSrgGTkBNb8sve0fYnQ==",
+ "VH70dN82yPCRctmAHMfCig==",
+ "VI8pgqBZeGWNaxkuqQVe7g==",
+ "VIC7inSiqzM6v9VqtXDyCw==",
+ "VIkS30v268x+M1GCcq/A8A==",
+ "VJt2kPVBLEBpGpgvuv1oUw==",
+ "VK95g27ws2C6J2h/7rC2qA==",
+ "VOB+9Bcfu8aHKGdNO0iMRw==",
+ "VOvrzqiZ1EHw+ZzzTWtpsw==",
+ "VPa7DG6v7KnzMvtJPb88LQ==",
+ "VPqyIomYm7HbK5biVDvlpw==",
+ "VQIpquUqmeyt/q6OgxzduQ==",
+ "VRnx+kd6VdxChwsfbo1oeQ==",
+ "VUDsc9RMS1fSM43c+Jo9dQ==",
+ "VWNDBOtjiiI4uVNntOlu/A==",
+ "VWb8U4jF/Ic0+wpoXi/y/g==",
+ "VWy9lB5t4fNCp4O/4n8S4w==",
+ "VX+cVXV8p9i5EBTMoiQOQQ==",
+ "VXu4ARjq7DS2IR/gT24Pfw==",
+ "VZX1FnyC8NS2k3W+RGQm4g==",
+ "VaJc9vtYlqJbRPGb5Tf0ow==",
+ "VbCoGr8apEcN7xfdaVwVXw==",
+ "VbHoWmtiiPdABvkbt+3XKQ==",
+ "Vg2E5qEDfC+QxZTZDCu9yQ==",
+ "VhYGC8KYe5Up+UJ2OTLKUw==",
+ "Vik8tGNxO0xfdV0pFmmFDw==",
+ "ViweSJuNWbx5Lc49ETEs/A==",
+ "VjclDY8HN4fSpB263jsEiQ==",
+ "VllbOAjeW3Dpbj5lp2OSmA==",
+ "VoPth5hDHhkQcrQTxHXbuw==",
+ "VpmBstwR7qPVqPgKYQTA3g==",
+ "VsXEBIaMkVftkxt1kIh7TA==",
+ "Vu0E+IJXBnc25x4n41kQig==",
+ "VzQ1NwNv9btxUzxwVqvHQg==",
+ "VznvTPAAwAev+yhl9oZT0w==",
+ "W+M4BcYNmjj7xAximDGWsA==",
+ "W/0s1x3Qm+wN8DhROk6FrQ==",
+ "W/5ThNLu43uT1O+fg0Fzwg==",
+ "W04GeDh+Tk/I1S85KlozRA==",
+ "W2x0SBzSIsTRgyWUCOZ/lg==",
+ "W4CfeVp9mXgk04flryL7iA==",
+ "W4utAK3ws0zjiba/3i91YA==",
+ "W5now3RWSzzMDAxsHSl++Q==",
+ "W8bATujVUT80v2XGJTKXDg==",
+ "W8y32OLHihfeV0XFw7LmOg==",
+ "WADmxH7R6B4LR+W6HqQQ6A==",
+ "WBu0gJmmjVdVbjDmQOkU6w==",
+ "WGKFTWJac8uehn3N59yHJw==",
+ "WHutPin+uUEqtrA7L8878A==",
+ "WKehT4nGF2T7aKuzABDMlA==",
+ "WLsh3UF4WXdHwgnbKEwRlQ==",
+ "WLwpjgr9KzevuogoHZaVUw==",
+ "WN7lFJfw4lSnTCcbmt5nsg==",
+ "WNfDNaWUOqABQ6c6kR+eyw==",
+ "WQMffxULFKJ+bun6NrCURA==",
+ "WQznrwqvMhUlM3CzmbhAOQ==",
+ "WRjYdKdtnd1G9e/vFXCt0g==",
+ "WRoJMO0BCJyn5V6qnpUi4Q==",
+ "WTr3q/gDkmB4Zyj7Ly20+w==",
+ "WVhfn2yJZ43qCTu0TVWJwA==",
+ "WWN44lbUnEdHmxSfMCZc6w==",
+ "WY7mCUGvpXrC8gkBB46euw==",
+ "WbAdlac/PhYUq7J2+n5f+w==",
+ "Wd0dOs7eIMqW5wnILTQBtg==",
+ "WdCWezJU4JK43EOZ9YHVdg==",
+ "Wf2olJCYZRGTTZxZoBePuQ==",
+ "WjDqf1LyFyhdd8qkwWk+MA==",
+ "WkSJpxBa45XJRWWZFee7hw==",
+ "Wn+Vj4eiWx0WPUHr3nFbyA==",
+ "WnHK5ZQDR6Da5cGODXeo0A==",
+ "WrJMOuXSLKKzgmIDALkyNw==",
+ "WtT0QAERZSiIt2SFDiAizg==",
+ "WwraoO97OTalvavjUsqhxQ==",
+ "Wx9jh/teM0LJHrvTScssyQ==",
+ "WyCFB4+6lVtlzu3ExHAGbQ==",
+ "WzjvUJ4jZAEK7sBqw+m07A==",
+ "X/Gha4Ajjm/GStp/tv+Jvw==",
+ "X1PaCfEDScclLtOTiF5JUw==",
+ "X2Tawm2Cra6H7WtXi1Z4Qw==",
+ "X2YfnPXgF2VHVX95ZcBaxQ==",
+ "X4hrgqMIcApsjA9qOWBoCw==",
+ "X4kdXUuhcUqMSduqhfLpxA==",
+ "X4o0OkTz0ec70mzgwRfltA==",
+ "X6Ln4si8G5aKar52ZH/FEQ==",
+ "X6ulLp4noBgefQTsbuIbYQ==",
+ "X9QAaNjgiOeAWSphrGtyVw==",
+ "XA2hUgq3GVPpxtRYiqnclg==",
+ "XAq/C+XyR6m3uzzLlMWO5Q==",
+ "XEwOJG24eaEtAuBWtMxhwg==",
+ "XF/yncdoT4ruPeXCxEhl9Q==",
+ "XGAXhUFjORwKmAq9gGEcRg==",
+ "XHHEg/8KZioW/4/wgSEkbQ==",
+ "XHjrTLXkm/bBY/BewmJcCQ==",
+ "XJihma9zSRrXLC+T+VcFDA==",
+ "XLq/nWX8lQqjxsK9jlCqUg==",
+ "XOG1PYgqoG8gVLIbVLTQgg==",
+ "XSb71ae0v+yDxNF5HJXGbQ==",
+ "XTCcsVfEvqxnjc0K5PLcyw==",
+ "XV13yK0QypJXmgI+dj4KYw==",
+ "XV5MYe0Q7YMtoBD6/iMdSw==",
+ "XVVy3e6dTnO3HpgD6BtwQw==",
+ "XXFr0WUuGsH5nXPas7hR3Q==",
+ "Xconi1dtldH90Wou9swggw==",
+ "XddlSluOH6VkR7spFIFmdQ==",
+ "XdkxmYYooeDKzy7PXVigBQ==",
+ "XePy/hhnQwHXFeXUQQ55Vg==",
+ "XfBOCJwi2dezYzLe316ivw==",
+ "XfY+QUriCAA1+3QAsswdgg==",
+ "XgPHx2+ULpm14IOZU2lrDg==",
+ "XjjrIpsmATV/lyln4tPb+g==",
+ "Xo8ZjXOIoXlBjFCGdlPuZw==",
+ "XpGXh76RDgXC4qnTCsnNHA==",
+ "XqFSbgvgZn0CpaZoZiRauQ==",
+ "XqTK/2QuGWj50tGmiDxysA==",
+ "XqUO7ULEYhDOuT/I2J8BOA==",
+ "XqW7UBTobbV4lt1yfh0LZw==",
+ "XrFDomoH2qFjQ2jJ2yp9lA==",
+ "XsF7R12agx/KkRWl0TyXRA==",
+ "Xv0mNYedaBc57RrcbHr9OA==",
+ "XwKWd03sAz8MmvJEuN08xA==",
+ "Y1Nm3omeWX2MXaCjDDYnWQ==",
+ "Y1flEyZZAYxauMo4cmtJ1w==",
+ "Y26jxXvl79RcffH8O8b9Ew==",
+ "Y5KKN7t/v9JSxG/m1GMPSA==",
+ "Y5XR8Igvau/h+c1pRgKayg==",
+ "Y5iDQySR2c3MK7RPMCgSrw==",
+ "Y78dviyBS3Jq9zoRD5sZtQ==",
+ "Y7OofF9eUvp7qlpgdrzvkg==",
+ "Y7XpxIwsGK3Lm/7jX/rRmg==",
+ "Y7iDCWYrO1coopM3RZWIPg==",
+ "YA+zdEC+yEgFWRIgS1Eiqw==",
+ "YA0kMTJ82PYuLA4pkn4rfw==",
+ "YHM6NNHjmodv+G0mRLK7kw==",
+ "YK+q7uJObkQZvOwQ9hplMg==",
+ "YLz+HA6qIneP+4naavq44Q==",
+ "YNqIHCmBp/EbCgaPKJ7phw==",
+ "YPgMthbpcBN2CMkugV60hQ==",
+ "YVlRQHQglkbj3J2nHiP/Hw==",
+ "YXHQ3JI9+oca8pc/jMH6mA==",
+ "YZ39RIXpeLAhyMgmW2vfkQ==",
+ "YZt6HwCvdI5DRQqndA/hBQ==",
+ "YaUKOTyByjUvp1XaoLiW5Q==",
+ "YfbfE3WyYOW7083Y8sGfwQ==",
+ "YgVpC5d5V6K/BpOD663yQA==",
+ "YhLEPsi/TNyeUJw69SPYzQ==",
+ "Yig+Wh18VIqdsmwtwfoUQw==",
+ "Yjm5tSq1ejZn3aWqqysNvA==",
+ "YmaksRzoU+OwlpiEaBDYaQ==",
+ "YmjZJyNfHN5FaTL/HAm8ww==",
+ "YodhkayN5wsgPZEYN7/KNA==",
+ "YrEP9z2WPQ8l7TY1qWncDA==",
+ "YtZ8CYfnIpMd2FFA5fJ+1Q==",
+ "Yw4ztKv6yqxK9U1L0noFXg==",
+ "Yy2pPhITTmkEwoudXizHqQ==",
+ "YzTV0esAxBFVls3e0qRsnA==",
+ "Z+bsbVP91KrJvxrujBLrrQ==",
+ "Z0sjccxzKylgEiPCFBqPSA==",
+ "Z2MkqmpQXdlctCTCUDPyzw==",
+ "Z2rwGmVEMCY6nCfHO3qOzw==",
+ "Z5B+uOmPZbpbFWHpI9WhPw==",
+ "Z8T1b9RsUWf59D06MUrXCQ==",
+ "Z9bDWIgcq6XwMoU2ECDR5Q==",
+ "ZAQHWU6RMg4IadOxuaukyw==",
+ "ZCdad3AwhVArttapWFwT/Q==",
+ "ZH5Es/4lJ+D5KEkF1BVSGg==",
+ "ZIZx4MehWTVXPN9cVQBmyA==",
+ "ZItMIn1vhGqAlpDHclg0Ig==",
+ "ZJY+hujfd58mTKTdsmHoQQ==",
+ "ZJc7GV0Yb6MrXkpDVIuc8g==",
+ "ZKXxq9yr7NGBOHidht34uQ==",
+ "ZKeTDCboOgCptrjSfgu0xw==",
+ "ZKvox7BaQg4/p5jIX69Umw==",
+ "ZNrjP1fLdQpGykFXoLBNPw==",
+ "ZQ0ZnTsZKWxbRj7Tilh24Q==",
+ "ZQSDYgpsimK+lYGdXBWE/w==",
+ "ZRWyfXyXqAaOEjkzWl949Q==",
+ "ZRnR6i+5WKMRfs3BDRBCJg==",
+ "ZSmN8mmI9lDEHkJqBBg0Nw==",
+ "ZV8mEgJweIYk0/l0BFKetA==",
+ "ZVnErH1Si4u51QoT0OT7pA==",
+ "ZWXfE3uGU91WpPMGyknmqw==",
+ "ZXeMG5eqQpZO/SGKC4WQkA==",
+ "ZYW30FfgwHmW6nAbUGmwzA==",
+ "ZZImGypBWwYOAW43xDRWCQ==",
+ "ZaPsR9X77SNt7dLjMJUh8A==",
+ "ZbLVNTQSVZQWTNgC4ZGfQg==",
+ "ZcuIvc8fDI+2uF0I0uLiVA==",
+ "ZfRlID+pC1Rr4IY14jolMw==",
+ "ZgdpqFrVGiaHkh9o3rDszg==",
+ "ZgjifTVKmxOieco81gnccQ==",
+ "ZiJ/kJ9GneF3TIEm08lfvQ==",
+ "ZlBNHAiYsfaEEiPQ1z+rCA==",
+ "ZlOAnCLV1PkR0kb3E+Nfuw==",
+ "ZmVpw1TUVuT13Zw/MNI5hQ==",
+ "ZmblZauRqO5tGysY3/0kDw==",
+ "ZoNSxARrRiKZF5Wvpg7bew==",
+ "Zqd6+81TwYuiIgLrToFOTQ==",
+ "ZqjnqxZE/BjOUY0CMdVl0g==",
+ "ZqkmoGB0p5uT5J6XBGh7Tw==",
+ "ZrCezGLz38xKmzAom6yCTQ==",
+ "ZrCnZB/U/vcqEtI1cSvnww==",
+ "ZtWvgitOSRDWq7LAKYYd4Q==",
+ "ZtmnX24AwYAXHb2ZDC6MeQ==",
+ "ZuayB6IpbeITokKGVi9R5w==",
+ "ZvvxwDd0I6MsYd7aobjLUA==",
+ "ZyDh3vCQWzS5DI1zSasXWA==",
+ "ZybIEGf1Rn/26vlHmuMxhw==",
+ "ZydKlOpn2ySBW0G3uAqwuw==",
+ "ZygAjaN62XhW5smlLkks+Q==",
+ "Zyo0fzewcqXiKe2mAwKx5g==",
+ "ZyoaR1cMiKAsElmYZqKjLA==",
+ "Zz/5VMbw1TqwazReplvsEg==",
+ "ZzT5b0dYQXkQHTXySpWEaA==",
+ "ZzduJxTnXLD9EPKMn1LI4Q==",
+ "a/Y6IAVFv0ykRs9WD+ming==",
+ "a1aL8zQ+ie3YPogE3hyFFg==",
+ "a4EYNljinYTx9vb1VvUA6A==",
+ "a4rPqbDWiMivVzaRxvAj7g==",
+ "a5gZ5uuRrXEAjgaoh7PXAg==",
+ "a6IszND1m+6w+W+CvseC7g==",
+ "a6vem8n6WmRZAalDrHNP0g==",
+ "a7Pv1SOWYnkhIUC22dhdDA==",
+ "aD4QvtMlr8Lk/zZgZ6zIMg==",
+ "aEnHUfn7UE/Euh6jsMuZ7g==",
+ "aFJuE/s+Kbge4ppn+wulkA==",
+ "aIPde9CtyZrhbHLK740bfw==",
+ "aJFbBhYtMbTyMFBFIz/dTA==",
+ "aK9nybtiIBUvxgs1iQFgsw==",
+ "aLY2pCT0WfFO5EJyinLpPg==",
+ "aLh1XEUrfR9W82gzusKcOg==",
+ "aMa1yVA71/w6Uf1Szc9rMA==",
+ "aMmrAzoRWLOMPHhBuxczKg==",
+ "aN5x46Gw1VihRalwCt1CGg==",
+ "aOeJZUIZM9YWjIEokFPnzQ==",
+ "aRpdnrOyu5mWB1P5YMbvOA==",
+ "aRrcmH+Ud3mF1vEXcpEm4w==",
+ "aTWiWjyeSDVY/q8y9xc2zg==",
+ "aWZRql2IUPVe9hS3dxgVfQ==",
+ "aXqiibI6BpW3qilV6izHaQ==",
+ "aXrbsro7KLV8s4I4NMi4Eg==",
+ "aXs9qTEXLTkN956ch3pnOA==",
+ "aY6B28XdPnuYnbOy9uSP8A==",
+ "adJAjAFyR2ne1puEgRiH+g==",
+ "adT+OjEB2kqpeYi4kQ6FPg==",
+ "afMd/Hr3rYz/l7a3CfdDjg==",
+ "ahAbmGJZvUOXrcK6OydNGQ==",
+ "alJtvTAD7dH/zss/Ek1DMQ==",
+ "alqHQBz8V446EdzuVfeY5Q==",
+ "anyANMnNkUqr3JuPJz5Qzw==",
+ "apWEPWUvMC24Y+2vTSLXoA==",
+ "aqcOby9QyEbizPsgO3g0yw==",
+ "ash1r2J6B0PUxJe8P0otVQ==",
+ "asouSfUjJa8yfMG7BBe+fA==",
+ "auvG6kWMnhCMi7c7e9eHrw==",
+ "avFTp3rS6z5zxQUZQuaBHQ==",
+ "avZp5K7zJvRvJvpLSldNAw==",
+ "aw4CzX8pYbPVMuNrGCEcWg==",
+ "axEl7xXt/bwlvxKhI7hx4g==",
+ "ayBGGPEy++biljvGcwIjXA==",
+ "aySnrShOW4/xRSzl/dtSKQ==",
+ "ays5/F7JANIgPHN0vp2dqQ==",
+ "b06KGv5zDYsTxyTbQ9/eyA==",
+ "b0vZfEyuTja2JYMa20Rtbg==",
+ "b16O4LF7sVqB7aLU2f3F1A==",
+ "b3BQG9/9qDNC/bNSTBY/sQ==",
+ "b3q8kjHJPj9DWrz3yNgwjQ==",
+ "b4BoZmzVErvuynxirLxn0w==",
+ "b4aFwwcWMXsSdgS1AdFOXA==",
+ "b53qqLnrTBthRXmmnuXWvw==",
+ "b6rrRA0W247O+FfvDHbVCQ==",
+ "b85nxzs8xiHxaqezuDVWvg==",
+ "b8BZV1NfBdLi70ir4vYvZg==",
+ "bA2kaTpeXflTElTnQRp6GQ==",
+ "bBEndaOStXBpAK79FrgHaw==",
+ "bG+P+p34t/IJ1ubRiWg6IA==",
+ "bGGUhiG9SqJMHQWitXTcYQ==",
+ "bIk7Fa6SW7X18hfDjTKowg==",
+ "bJ1cZW7KsXmoLw0BcoppJg==",
+ "bJgsuw29cO2WozqsGZxl7w==",
+ "bK045TkBlz+/3+6n6Qwvrg==",
+ "bL2FuwsPT7a7oserJQnPcw==",
+ "bLEntCrCHFy9pg3T3gbBzg==",
+ "bLd38ZNkVeuhf0joEAxnBQ==",
+ "bLsStF0DDebpO+xulqGNtg==",
+ "bMWFvjM8eVezU1ZXKmdgqw==",
+ "bMb1ia0rElr2ZpZVhva0Jw==",
+ "bNDKcFu8T5Y6OoLSV+o/Sw==",
+ "bNq/hj0Cjt4lkLQeVxDVdQ==",
+ "bO55S58bqDiRWXSAIUGJKw==",
+ "bPRX2zl+K1S0iWAWUn1DZw==",
+ "bQ7J5mebp38rfP/fuqQOsg==",
+ "bQKkL+/KUCsAXlwwIH0N3w==",
+ "bTNRjJm+FfSQVfd56nNNqQ==",
+ "bUF0JIfS4uKd3JZj2xotLQ==",
+ "bUxQBaqKyvlSHcuRL9whjg==",
+ "bV9r7j2kNJpDCEM5E2339Q==",
+ "bWwtTFlhO3xEh/pdw0uWaQ==",
+ "bb/U8UynPHwczew/hxLQxw==",
+ "bbBsi6tXMVWyq3SDVTIXUg==",
+ "beSrliUu0BOadCWmx+yZyA==",
+ "bfUD03N2PRDT+MZ+WFVtow==",
+ "bhVbgJ4Do4v56D9mBuR/EA==",
+ "birqO8GOwGEI97zYaHyAuw==",
+ "bjLZ7ot/X/vWSVx4EYwMCg==",
+ "bkRdUHAksJZGzE1gugizYQ==",
+ "blygTgAHZJ3NzyAT33Bfww==",
+ "bs2QG8yYWxPzhtyMqO6u3A==",
+ "bsHIShcLS134C+dTxFQHyA==",
+ "bvbMJZMHScwjJALxEyGIyg==",
+ "bvyB6OEwhwCIfJ6KRhjnRw==",
+ "bz294kSG4egZnH2dJ8HwEg==",
+ "bzVeU2qM9zHuzf7cVIsSZw==",
+ "bzXXzQGZs8ustv0K4leklA==",
+ "c1wbFbN7AdUERO/xVPJlgw==",
+ "c3WVxyC5ZFtzGeQlH5Gw+w==",
+ "c5Tc7rTFXNJqYyc0ppW+Iw==",
+ "c5q/8n7Oeffv3B1snHM/lA==",
+ "c5ymZKqx/td1MiS2ERiz9A==",
+ "c6Yhwy/q3j7skXq52l36Ww==",
+ "cBBOQn7ZjxDku0CUrxq2ng==",
+ "cFFE2R4GztNoftYkqalqUQ==",
+ "cHSj5dpQ04h/WyefjABfmQ==",
+ "cHkOsVd80Rgwepeweq4S1g==",
+ "cLR0Ry4/N5swqga1R6QDMw==",
+ "cMo6l1EQESx1rIo+R4Vogg==",
+ "cNsC9bH30eM1EZS6IdEdtQ==",
+ "cSHSg9xJz/3F6kc+hKXkwg==",
+ "cT3PwwS6ALZA/na9NjtdzA==",
+ "cTvDd8okNUx0RCMer6O8sw==",
+ "cUyqCa7Oue934riyC17F8g==",
+ "cVhdRFuZaW/09CYPmtNv5g==",
+ "cWUg7AfqhiiEmBIu+ryImA==",
+ "cWdlhVZD7NWHUGte24tMjg==",
+ "cXpfd6Io6Glj2/QzrDMCvA==",
+ "ca+kx+kf7JuZ3pfYKDwFlg==",
+ "caepyBOAFu0MxbcXrGf6TA==",
+ "catI+QUNk3uJ+mUBY3bY8Q==",
+ "cbBXgB1WQ/i8Xul0bYY2fg==",
+ "ccK42Lm8Tsv73YMVZRwL6A==",
+ "cchuqe+CWCJpoakjHLvUfA==",
+ "ccmy4GVuX967KaQyycmO0w==",
+ "ccy3Ke2k4+evIw0agHlh3w==",
+ "cdWUm6uLNzR/knuj2x75eA==",
+ "cffrYrBX3UQhfX1TbAF+GQ==",
+ "cfh5VZFmIqJH/bKboDvtlA==",
+ "cgSEbLqqvDsNUyeA3ryJ6Q==",
+ "chwv4+xbEAa93PHg8q9zgQ==",
+ "ck86G8HsbXflyrK7MBntLg==",
+ "ckugAisBNX18eQz+EnEjjw==",
+ "cl4t9FXabQg7tbh1g7a0OA==",
+ "coGEgMVs2b314qrXMjNumQ==",
+ "cszpMdGbsbe6BygqMlnC9Q==",
+ "ctJYJegZhG42i+vnPFWAWw==",
+ "cu4ZluwohhfIYLkWp72pqA==",
+ "cuQslgfqD2VOMhAdnApHrA==",
+ "cvMJ714elj/HUh89a9lzOQ==",
+ "cvOg7N4DmTM+ok1NBLyBiQ==",
+ "cvZT1pvNbIL8TWg+SoTZdA==",
+ "cvrGmub2LoJ+FaM5HTPt9A==",
+ "cw1gBLtxH/m4H7dSM7yvFg==",
+ "cwBNvZc0u4bGABo88YUsVQ==",
+ "cxpZ4bloGv734LBf4NpVhA==",
+ "cxqHS4UbPolcYUwMMzgoOA==",
+ "czBWiYsQtNFrksWwoQxlOw==",
+ "d+ctfXU0j07rpRRzb5/HDA==",
+ "d/Wd3Ma1xYyoMByPQnA9Cw==",
+ "d0NBFiwGlQNclKObRtGVMQ==",
+ "d0VAZLbLcDUgLgIfT1GmVQ==",
+ "d0qvm3bl38rRCpYdWqolCQ==",
+ "d13Rj3NJdcat0K/kxlHLFw==",
+ "dAq8/1JSQf1f4QPLUitp0g==",
+ "dCDaYYrgASXPMGFRV0RCGg==",
+ "dChBe9QR29ObPFu/9PusLg==",
+ "dFSavcNwGd8OaLUdWq3sng==",
+ "dFetwmFw+D6bPMAZodUMZQ==",
+ "dG98w8MynOoX7aWmkvt+jg==",
+ "dGjcKAOGBd4gIjJq7fL+qQ==",
+ "dGrf9SWJ13+eWS6BtmKCNw==",
+ "dJHKDkfMFJeoULg7U4wwDQ==",
+ "dK2DU3t1ns+DWDwfBvH3SQ==",
+ "dL6n/JsK+Iq6UTbQuo/GOw==",
+ "dM9up4vKQV5LeX82j//1jQ==",
+ "dMRx4Mf6LrN64tiJuyWmDw==",
+ "dNTU+/2DdZyGGTdc+3KMhQ==",
+ "dNq2InSVDGnYXjkxPNPRxA==",
+ "dOS+mVCy3rFX9FvpkTxGXA==",
+ "dRFCIbVu0Y8XbjG5i+UFCQ==",
+ "dTMoNd6DDr1Tu8tuZWLudw==",
+ "dUx1REyXKiDFAABooqrKEA==",
+ "dVh/XMTUIx1nYN4q1iH1bA==",
+ "dXDPnL1ggEoBqR13aaW9HA==",
+ "dZg5w8rFETMp9SgW7m0gfg==",
+ "dZgMquvZmfLqP4EcFaWCiA==",
+ "daBhAvmE9shDgmciDAC5eg==",
+ "dhTevyxTYAuKbdLWhG47Kw==",
+ "dihDsG7+6aocG6M9BWrCzQ==",
+ "dmAfbd9F0OJHRAhNMEkRsA==",
+ "dml2gqLPsKpbIZ93zTXwCQ==",
+ "dnvatwSEcl73ROwcZ4bbIQ==",
+ "dpSTNOCPFHN5yGoMpl1EUA==",
+ "dqVw2q2nhCvTcW82MT7z0g==",
+ "drfODfDI6GyMW7hzkmzQvA==",
+ "dsueq9eygFXILDC7ZpamuA==",
+ "dtnE401dC0zRWU0S/QOTAg==",
+ "duRFqmvqF93uf/vWn8aOmg==",
+ "dxWv00FN/2Cgmgq9U3NVDQ==",
+ "e/nWuo5YalCAFKsoJmFyFA==",
+ "e2xLFVavnZIUUtxJx+qa1g==",
+ "e369ZIQjxMZJtopA//G55Q==",
+ "e4B3HmWjW+6hQzcOLru6Xg==",
+ "e5KCqQ/1GAyVMRNgQpYf6g==",
+ "e5l9ZiNWXglpw6nVCtO8JQ==",
+ "e5txnNRcGs2a9+mBFcF1Qg==",
+ "e9GqAEnk8XI5ix6kJuieNQ==",
+ "eAOEgF5N80A/oDVnlZYRAw==",
+ "eBapvE+hdyFTsZ0y5yrahg==",
+ "eC/RcoCVQBlXdE9WtcgXIw==",
+ "eCy/T+a8kXggn1L8SQwgvA==",
+ "eDWsx4isnr2xPveBOGc7Hw==",
+ "eDcyiPaB954q5cPXcuxAQw==",
+ "eFimq+LuHi42byKnBeqnZQ==",
+ "eFkXKRd2dwu/KWI5ZFpEzw==",
+ "eJDUejE/Ez/7kV+S74PDYg==",
+ "eJFIQh/TR7JriMzYiTw4Sg==",
+ "eJLrGwPRa6NgWiOrw1pA7w==",
+ "eJlcN+gJnqAnctbWSIO9uA==",
+ "eKQCVzLuzoCLcB4im8147A==",
+ "eLYKLr4labZeLiRrDJ9mnA==",
+ "ePlsM/iOMme2jEUYwi15ng==",
+ "eQ45Mvf5in9xKrP6/qjYbg==",
+ "eRwaYiog2DdlGQyaltCMJg==",
+ "eS/vTdSlMUnpmnl1PbHjyw==",
+ "eTMPXa60OTGjSPmvR4IgGw==",
+ "eV+RwWPiGEB+76bqvw+hbA==",
+ "eWgLAqJOU+fdn8raHb9HCw==",
+ "eXFOya6x5inTdGwJx/xtUQ==",
+ "eYAQWuWZX2346VMCD6s7/A==",
+ "eYE9No9sN5kUZ5ePEyS3+Q==",
+ "eddhS+FkXxiUnbPoCd5JJw==",
+ "edlXkskLx287vOBZ9+gVYg==",
+ "ehfPlu6YctzzpQmFiQDxGA==",
+ "ehwc2vvwNUAI7MxU4MWQZw==",
+ "ejfikwrSPMqEHjZAk3DMkA==",
+ "emVLJVzha7ui5OFHPJzeRQ==",
+ "enj9VEzLbmeOyYugTmdGfQ==",
+ "epY+dsm5EMoXnZCnO4WSHw==",
+ "es/L9iW8wsyLeC5S4Q8t+g==",
+ "eshD40tvOA6bXb0Fs/cH3A==",
+ "etRjRvfL/IwceY/IJ1tgzQ==",
+ "euxzbIq4vfGYoY3s1QmLcw==",
+ "evaWFoxZNQcRszIRnxqB+A==",
+ "ewPT4dM12nDWEDoRfiZZnA==",
+ "ewe/P3pJLYu/kMb5tpvVog==",
+ "ezsm4aFd6+DO9FUxz0A8Pg==",
+ "f/BjtP5fmFw2dRHgocbFlg==",
+ "f07bdNVAe9x+cAMdF1bByQ==",
+ "f09F7+1LRolRL5nZTcfKGA==",
+ "f0H/AFSx2KLZi9kVx5BAZg==",
+ "f1+fHgR5rDPsCZOzqrHM7Q==",
+ "f1Gs++Iilgq9GHukcnBG3w==",
+ "f1h+Vp+xmdZsZIziHrB2+g==",
+ "f5Xo7F1uaiM760Qbt978iw==",
+ "f6Ye5F0Lkn34uLVDCzogFQ==",
+ "f6iLrMpxKhFxIlfRsFAuew==",
+ "f9ywiGXsz+PuEsLTV3zIbQ==",
+ "fAKFfwlCOyhtdBK6yNnsNg==",
+ "fDOUzPTU2ndpbH0vgkgrJQ==",
+ "fFvXa1dbMoOOoWZdHxPGjw==",
+ "fHL+fHtDxhALZFb9W/uHuw==",
+ "fHNpW230mNib08aB7IM3XQ==",
+ "fKalNdhsyxTt1w08bv9fJA==",
+ "fM5uYpkvJFArnYiQ3MrQnA==",
+ "fO0+6TsjL+45p9mSsMRiIg==",
+ "fOARCnIg/foF/6tm7m9+3w==",
+ "fQS0jnQMnHBn7+JZWkiE/g==",
+ "fS471/rN4K2m10mUwGFuLg==",
+ "fSANOaHD0Koaqg7AoieY9A==",
+ "fU32wmMeD44UsFSqFY0wBA==",
+ "fU5ZZ1bIVsV+eXxOpGWo/Q==",
+ "fUAy3f9bAglLvZWvkO2Lug==",
+ "fVCRaPsTCKEVLkoF4y3zEw==",
+ "fW3QZyq5UixIA1mP6eWgqQ==",
+ "fX4G68hFL7DmEmjbWlCBJQ==",
+ "fY9VATklOvceDfHZDDk57A==",
+ "fZrj3wGQSt8RXv0ykJROcQ==",
+ "fbTm027Ms0/tEzbGnKZMDA==",
+ "fdqt93OrpG13KAJ5cASvkg==",
+ "fgXfRuqFfAu8qxbTi4bmhA==",
+ "fgdUFvQPb5h+Rqz8pzLsmw==",
+ "fhcbn9xE/6zobqQ2niSBgA==",
+ "fiv0DJivQeqUkrzDNlluRw==",
+ "fmC+85h5WBuk8fDEUWPjtQ==",
+ "fo3JL+2kPgDWfP+CCrFlFw==",
+ "foPAmiABJ3IXBoed2EgQXA==",
+ "foXSDEUwMhfHWJSmSejsQg==",
+ "fpXijBOM3Ai1RkmHven5Ww==",
+ "fsW2DaKYTCC7gswCT+ByQQ==",
+ "fsoXIbq0T0nmSpW8b+bj+g==",
+ "fsrX00onlGvfsuiCc35pGg==",
+ "ftsf2qztw3NC78ep/CZXWQ==",
+ "fv/PW8oexJYWf5De30fdLQ==",
+ "fvm0IQfnbfZFETg9v3z/Fg==",
+ "fxg/vQq9WPpmQsqQ4RFYaA==",
+ "fy54Milpa7KZH/zgrDmMXQ==",
+ "fzkmVWKhJsxyCwiqB/ULnQ==",
+ "g/z9yk94XaeBRFj4hqPzdw==",
+ "g0GbRp2hFVIdc7ct7Ky7ag==",
+ "g0aTR8aJ0uVy3YvGYu5xrw==",
+ "g0kHTNRI7x/lAsr92EEppw==",
+ "g0lWrzEYMntVIahC7i0O2g==",
+ "g1ELwsk6hQ+RAY1BH640Pg==",
+ "g2nh2xENCFOpHZfdEXnoQA==",
+ "g5EzTJ0KA4sO3+Opss3LMg==",
+ "g6udffWh7qUnSIo1Ldn3eA==",
+ "g6zSo8BvLuKqdmBFM1ejLA==",
+ "g8TcogVxHpw7uhgNFt5VCQ==",
+ "gAoV4BZYdW1Wm712YXOhWQ==",
+ "gB8wkuIzvuDAIhDtNT1gyA==",
+ "gBgJF0PiGEfcUnXF0RO7/w==",
+ "gC7gUwGumN7GNlWwfIOjJQ==",
+ "gDLjxT7vm07arF4SRX5/Vg==",
+ "gDxqUdxxeXDYhJk9zcrNyA==",
+ "gEHGeR2F82OgBeAlnYhRSw==",
+ "gFEnTI8os2BfRGqx9p5x8w==",
+ "gGLz3Ss+amU7y6JF09jq7A==",
+ "gICaI06E9scnisonpvqCsA==",
+ "gK7dhke5ChQzlYc/bcIkcg==",
+ "gR0sgItXIH8hE4FVs9Q07w==",
+ "gR3B8usSEb0NLos51BmJQg==",
+ "gTB2zM3RPm27mUQRXc/YRg==",
+ "gTnsH3IzALFscTZ1JkA9pw==",
+ "gU3gu8Y5CYVPqHrZmLYHbQ==",
+ "gUNP5w7ANJm257qjFxSJrA==",
+ "gW0oKhtQQ7BxozxUWw5XvQ==",
+ "gXlb7bbRqHXusTE5deolGA==",
+ "gYGQBLo5TdMyXks0LsZhsQ==",
+ "gYgCu/qUpXWryubJauuPNw==",
+ "gYnznEt9r97haD/j2Cko7g==",
+ "gYvdNJCDDQmNhtJ6NKSuTA==",
+ "gZNJ1Qq6OcnwXqc+jXzMLQ==",
+ "gZWTFt5CuLqMz6OhWL+hqQ==",
+ "gaEtlJtD6ZjF5Ftx0IFt0A==",
+ "gf1Ypna/Tt+TZ08Y+GcvGg==",
+ "gfhkPuMvjoC3CGcnOvki3Q==",
+ "gfnbviaVhKvv1UvlRGznww==",
+ "ggIfX1J4dX3xQoHnHUI7VA==",
+ "gglLMohmJDPRGMY1XKndjQ==",
+ "ghp8sWGKWw20S/z1tbTxFg==",
+ "ginkFyNVMwkZLE49AbfqfA==",
+ "gkrg0NR0iCaL7edq0vtewA==",
+ "glnqaRfwm6NxivtB2nySzw==",
+ "gnAIpoCyl3mQytLFgBEgGA==",
+ "gnez1VrH+UHT8C/SB9qGdA==",
+ "gnkadeCgjdmLdlu/AjBZJg==",
+ "goSgZ8N5UbT5NMnW3PjIlQ==",
+ "gqehq46BhFX2YLknuMv02w==",
+ "gsC/mWD8KFblxB0JxNuqJw==",
+ "gvvyX5ATi4q9NhnwxRxC8w==",
+ "gwyVIrTk5o0YMKQq4lpJ+Q==",
+ "gxwbqZDHLbQVqXjaq42BCg==",
+ "h+KRDKIvyVUBmRjv1LcCyg==",
+ "h0MH5NGFfChgmRJ3E/R3HQ==",
+ "h13Xuonj+0dD1xH86IhSyQ==",
+ "h1NNwMy0RjQmLloSw1hvdg==",
+ "h2B0ty0GobQhDnFqmKOpKQ==",
+ "h2cnQQF2/R3Mq2hWdDdrTg==",
+ "h3vYYI9yhpSZV2MQMJtwFQ==",
+ "h5HsEsObPuPFqREfynVblw==",
+ "h7Fc+eT/GuC8iWI+YTD0UQ==",
+ "hCzsi1yDv9ja5/o7t94j9Q==",
+ "hDGa2yLwNvgBd/v6mxmQaQ==",
+ "hDILjSpTLqJpiSSSGu445A==",
+ "hIABph+vhtSF5kkZQtOCTA==",
+ "hIJA+1QGuKEj+3ijniyBSQ==",
+ "hIjgi20+km+Ks23NJ4VQ6Q==",
+ "hJ8leLNuJ6DK5V8scnDaZQ==",
+ "hJSP7CostefBkJrwVEjKHA==",
+ "hK8KhTFcR06onlIJjTji/Q==",
+ "hKOsXOBoFTl/K4xE+RNHDA==",
+ "hN9bmMHfmnVBVr+7Ibd2Ng==",
+ "hNHqznsrIVRSQdII6crkww==",
+ "hP7dSa8lLn9KTE/Z0s4GVQ==",
+ "hPnPQOhz4QKhZi02KD6C+A==",
+ "hRxbdeniAVFgKUgB9Q3Y+g==",
+ "hSNZWNKUtDtMo6otkXA/DA==",
+ "hSkY45CeB6Ilvh0Io4W6cg==",
+ "hUWqqG1QwYgGC5uXJpCvJw==",
+ "hW9DJA1YCxHmVUAF7rhSmQ==",
+ "hWoxz5HhE50oYBNRoPp1JQ==",
+ "hY82j+sUQQRpCi6CCGea5A==",
+ "hZlX6qOfwxW5SPfqtRqaMw==",
+ "hdzol5dk//Q6tCm4+OndIA==",
+ "hf9HFxWRNX2ucH8FLS7ytA==",
+ "hfcH5Az2M7rp+EjtVpPwsg==",
+ "hiYg+aVzdBUDCG0CXz9kCw==",
+ "hkOBNoHbno2iNR7t3/d4vg==",
+ "hlMumZ7RJFpILuKs09ABtw==",
+ "hlu7os0KtAkpBTBV6D2jyQ==",
+ "hlvtFGW8r0PkbUAYXEM+Hw==",
+ "hnCUnoxofUiqQvrxl73M8w==",
+ "hq35Fjgvrcx6I9e6egWS4w==",
+ "hqeSvwu8eqA072iidlJBAw==",
+ "htDbVu1xGhCRd8qoMlBoMg==",
+ "htNVAogFakQkTX6GHoCVXg==",
+ "hv5GrLEIjPb4bGOi8RSO0w==",
+ "hvsZ5JmVevK1zclFYmxHaw==",
+ "hy303iin+Wm7JA6MeelwiQ==",
+ "i2sSvrTh/RdLJX0uKhbrew==",
+ "i42XumprV/aDT5R0HcmfIQ==",
+ "i6ZYpFwsyWyMJNgqUMSV1A==",
+ "i6r+mZfyhZyqlYv56o0H+w==",
+ "i8XXN7jcrmhnrOVDV8a2Hw==",
+ "i9IRqAqKjBTppsxtPB7rdw==",
+ "iANKiuMqWzrHSk9nbPe3bQ==",
+ "iCF+GWw9/YGQXsOOPAnPHQ==",
+ "iCnm5fPmSmxsIzuRK6osrA==",
+ "iFtadcw8v6betKka9yaJfg==",
+ "iGI9uqMoBBAjPszpxjZBWQ==",
+ "iGuY4VxcotHvMFXuXum7KA==",
+ "iGykaF+h4p46HhrWqL8Ffg==",
+ "iIWxFdolLcnXqIjPMg+5kQ==",
+ "iIm8c9uDotr87Aij+4vnMw==",
+ "iJ2nT8w8LuK11IXYqBK+YA==",
+ "iK0dWKHjVVexuXvMWJV9pg==",
+ "iPwX3SbbG9ez9HoHsrHbKw==",
+ "iQ304I1hmLZktA1d1cuOJA==",
+ "iS9wumBV5ktCTefFzKYfkA==",
+ "iSeH0JFSGK73F470Rhtesw==",
+ "iUsUCB0mfRsE9KPEQctIzw==",
+ "iVDd2Zk7vwmEh97LkOONpQ==",
+ "iWNlSnwrtCmVF89B+DZqOQ==",
+ "ibsb1ncaLZXAYgGkMO7tjQ==",
+ "ieEAgvK9LsWh2t6DsQOpWA==",
+ "ifZM0gBm9g9L09YlL+vXBg==",
+ "ifuJCv9ZA84Vz1FYAPsyEA==",
+ "ilBBNK/IV69xKTShvI94fQ==",
+ "imZ+mwiT22sW2M9alcUFfg==",
+ "inrUwXyKikpOW0y2Kl1wGw==",
+ "ionqS0piAOY2LeSReAz4zg==",
+ "ipPPjxpXHS1tcykXmrHPMQ==",
+ "irnD9K8bsT+up/JUrxPw6A==",
+ "iruDC5MeywV4yA8o1tw/KQ==",
+ "isep9d+Q7DEUf0W7CJJYzw==",
+ "itPtn+JaO4i7wz2wOPOmDQ==",
+ "iu5csar0IQQBOTgw5OvJwQ==",
+ "iujlt9fXcUXEYc+T2s5UjA==",
+ "iwKBOGDTFzV4aXgDGfyUkw==",
+ "izeyFvXOumNgVyLrbKW45g==",
+ "j+8/VARfbQSYhHzj0KPurQ==",
+ "j+lDhAnWAyso+1N8cm85hQ==",
+ "j4FBMnNfdBwx0VsDeTvhFg==",
+ "j8nMH8mK/0Aae7ZkqyPgdg==",
+ "j8to4gtSIRYpCogv2TESuQ==",
+ "jCgdKXsBCgf7giUKnr6paQ==",
+ "jEdanvXKyZdZJG6mj/3FWw==",
+ "jEqP0dyHKHiUjZ9dNNGTlQ==",
+ "jGHMJqbj6X1NdTDyWmXYAQ==",
+ "jHOoSl3ldFYr9YErEBnD3w==",
+ "jKJn4czwUl/6wtZklcMsSg==",
+ "jLI3XpVfjJ6IzrwOc4g9Pw==",
+ "jLkmUZ6fV56GfhC0nkh4GA==",
+ "jMZKSMP2THqwpWqJNJRWdw==",
+ "jNJQ6otieHBYIXA9LjXprg==",
+ "jNcMS2zX1iSZN9uYnb2EIg==",
+ "jOPdd330tB6+7C29a9wn0Q==",
+ "jQVlDU+HjZ2OHSDBidxX5A==",
+ "jQjyjWCEo9nWFjP4O8lehw==",
+ "jS0JuioLGAVaHdo/96JFoQ==",
+ "jTg9Y6EfpON4CRFOq0QovA==",
+ "jTmPbq+wh30+yJ/dRXk1cA==",
+ "jV/D2B11NLXZRH77sG9lBw==",
+ "jWsC7kdp2YmIZpfXGUimiA==",
+ "jZMDIu95ITTjaUX0pk4V5g==",
+ "jd6IpPJwOJW1otHKtKZ5Gw==",
+ "jdRzkUJrWxrqoyNH9paHfQ==",
+ "jdVMQqApseHH3fd91NFhxg==",
+ "jfegbZSZWkDoPulFomVntA==",
+ "jgNijyoj2JrQNSlUv4gk4A==",
+ "ji+1YHlRvzevs3q5Uw1gfA==",
+ "ji306HRiq965zb8EZD2uig==",
+ "jiV+b/1EFMnHG6J0hHpzBg==",
+ "jjNMPXbmpFNsCpWY0cv3eg==",
+ "jkUpkLoIXuu7aSH8ZghIAQ==",
+ "joDXdLpXvRjOqkRiYaD/Sw==",
+ "jon1y9yMEGfiIBjsDeeJdA==",
+ "jp5Em/0Ml4Txr1ptTUQjpg==",
+ "jpNUgFnanr9Sxvj2xbBXZw==",
+ "jpjpNjL1IKzJdGqWujhxCw==",
+ "jqPQ0aOuvOJte/ghI1RVng==",
+ "jrRH0aTUYCOpPLZwzwPRfQ==",
+ "jrfRznO0nAz6tZM1mHOKIA==",
+ "jt9Ocr9D8EwGRgrXVz//aQ==",
+ "jx7rpxbm1NaUMcE2ktg5sA==",
+ "jz7QlwxCIzysP39Cgro8jg==",
+ "k+IBS52XdOe5/hLp28ufnA==",
+ "k/Aou2Jmyh8Bu3k8/+ndsQ==",
+ "k/OVIllJvW6BefaLEPq7DA==",
+ "k/pBSWE2BvUsvJhA9Zl5uw==",
+ "k0XIjxp2vFG7sTrKcfAihA==",
+ "k1DPiH6NkOFXP/r3N12GyA==",
+ "k2KP9oPMnHmFlZO6u6tgyw==",
+ "k6OmSlaSZ5CB0i7SD9LczQ==",
+ "k8eZxqwxiN/ievXdLSEL/w==",
+ "kBAB2PSjXwqoQOXNrv80AA==",
+ "kFrRjz7Cf2KvLtz9X6oD+w==",
+ "kGeXrHEN6o7h5qJYcThCPw==",
+ "kHcBZXoxnFJ+GMwBZ/xhfQ==",
+ "kIGxCUxSlNgsKZ45Al1lWw==",
+ "kJdY3XEdJS/hyHdR+IN0GA==",
+ "kMUdiwM7WR8KGOucLK4Brw==",
+ "kNGIV3+jQmJlZDTXy1pnyA==",
+ "kRnBEH6ILR5GNSmjHYOclw==",
+ "kSUectNPXpXNg+tIveTFRw==",
+ "kTCHqcb3Cos51o8cL+MXcg==",
+ "kUhyc3G8Zvx8+q5q5nVEhw==",
+ "kUudvRfA33uJDzHIShQd3Q==",
+ "kWPUUi7x9kKKa6nJ+FDR5Q==",
+ "kZ/mZZg9YSDmk2rCGChYAg==",
+ "kZ0D191c/uv4YMG15yVLDw==",
+ "kZkmDatUOdIqs7GzH3nI1A==",
+ "ka7pMp8eSiv92WgAsz2vdA==",
+ "kcJ1acgBv6FtUhV8KuWoow==",
+ "kgKWQJJQKLUuD2VYKIKvxA==",
+ "kggaIvN2tlbZdZRI8S5Apw==",
+ "kgyUtd8MFe0tuuxDEUZA9w==",
+ "kh51WUI5TRnKhur6ZEpRTQ==",
+ "kj5WqpRCjWAfjM7ULMcuPQ==",
+ "kjWYVC7Eok2w2YT4rrI+IA==",
+ "kkbX+a00dfiTgbMI+aJpMg==",
+ "kly/2kE4/7ffbO34WTgoGg==",
+ "knYKU74onR6NkGVjQLezZg==",
+ "kq26VyDyJTH/eM6QvS2cMw==",
+ "kr8tw1+3NxoPExnAtTmfxg==",
+ "ksOFI9C7IrDNk4OP6SpPgw==",
+ "kuWGANwzNRpG4XmY7KjjNg==",
+ "kvAaIJb+aRAfKK104dxFAA==",
+ "kwlAQhR2jPMmfLTAwcmoxw==",
+ "kydoXVaNcx1peR5g6i588g==",
+ "kzGNkWh3fz27cZer4BspUQ==",
+ "kzTl7WH/JXsX1fqgnuTOgw==",
+ "kzXsrxWRnWhkA82LsLRYog==",
+ "kzYddqiMsY3EYrpxve2/CQ==",
+ "l+x2QhxG8wb5AQbcRxXlmA==",
+ "l0E0U/CJsyCVSTsXW4Fp+w==",
+ "l2NppPcweAtmA1V2CNdk2Q==",
+ "l2ZB9TvT68rn8AAN4MdxWw==",
+ "l2mAbuFF3QBIUILDODiUHQ==",
+ "l4ddTxbTCW5UmZW+KRmx6A==",
+ "l5f3I6osM9oxLRAwnUnc5A==",
+ "l6QHU5JsJExNoOnqxBPVbw==",
+ "l6Ssc04/CnsqUua9ELu2iQ==",
+ "l8/KMItWaW3n4g1Yot/rcQ==",
+ "lC5EumoIcctvxYqwELqIqw==",
+ "lFUq6PGk9dBRtUuiEW7Cug==",
+ "lHN2dn2cUKJ8ocVL3vEhUQ==",
+ "lJFPmPWcDzDp5B2S8Ad8AA==",
+ "lK2xe+OuPutp4os0ZAZx5w==",
+ "lM/EhwTsbivA7MDecaVTPw==",
+ "lMaO8Yf+6YNowGyhDkPhQA==",
+ "lMjip5hbCjkD9JQjuhewDg==",
+ "lNF8PvUIN02NattcGi5u4g==",
+ "lON3WM0uMJ30F8poBMvAjQ==",
+ "lOPJhHqCtMRFZfWMX/vFZQ==",
+ "lTE6u9G/RzvmbuAzq2J2/Q==",
+ "lV70RNlE++04G1KFB3BMXA==",
+ "lY+tivtsfvU0LJzBQ6itYQ==",
+ "lacCCRiWdquNm4YRO7FoKA==",
+ "leDlMcM+B1mDE8k5SWtUeg==",
+ "lf1fwA0YoWUZaEybE+LyMQ==",
+ "lfOLLyZNbsWQgHRhicr4ag==",
+ "lffapwUUgaQOIqLz2QPbAg==",
+ "lhAOM81Ej6YZYBu45pQYgg==",
+ "lizovLQxu6L9sbafNQuShQ==",
+ "lkl6XkrTMUpXi46dPxTPxg==",
+ "lkzFdvtBx5bV6xZO0cxK7g==",
+ "ll2M0QQzBsj5OFi02fv3Yg==",
+ "llOvGOUDVfX68jKnAlvVRA==",
+ "llujnWE17U8MIHmx4SbrSA==",
+ "lqhgbgEqROAdfzEnJ17eXA==",
+ "lsBTMnse2BgPS6wvPbe7JA==",
+ "luO1R8dUM9gy1E2lojRQoA==",
+ "luR/kvHLwA6tSdLeTM4TzA==",
+ "lwYQm2ynA3ik2gE1m11IEg==",
+ "lyfqic/AbEJbCiw+wA01FA==",
+ "lz+SeifYXxamOLs1FsFmSQ==",
+ "lzUQ1o7JAbdJYpmEqi6KnQ==",
+ "m+eh+ZqS74w2q0vejBkjaw==",
+ "m/Lp4U75AQyk9c8cX14HJg==",
+ "m06wctjNc3o7iyBHDMZs2w==",
+ "m3XYojKO+I6PXlVRUQBC3w==",
+ "m416yrrAlv+YPClGvGh+qQ==",
+ "m5JIUETVXcRza4VL4xlJbg==",
+ "m6get5wjq5j1i5abnpXuZQ==",
+ "m6srF+pMehggHB1tdoxlPg==",
+ "m9iuy4UtsjmyPzy6FTTZvw==",
+ "mAiD16zf+rCc7Qzxjd5buA==",
+ "mAzsVkijuqihhmhNTTz65g==",
+ "mDXHuOmI4ayjy2kLSHku1Q==",
+ "mI0eT4Rlr7QerMIngcu/ng==",
+ "mMLhjdWNnZ8zts9q+a2v3g==",
+ "mMfn8OaKBxtetweulho+xQ==",
+ "mNlYGAOPc6KIMW8ITyBzIg==",
+ "mNv2Q67zePjk/jbQuvkAFA==",
+ "mPk1IsU5DmDFA/Ym5+1ojw==",
+ "mPwCyD0yrIDonVi+fhXyEQ==",
+ "mS99D+CXhwyfVt8xJ+dJZA==",
+ "mSJF9dJnxZ15lTC6ilbJ2A==",
+ "mSstwJq7IkJ0JBJ5T8xDKg==",
+ "mTAqtg6oi0iytHQCaSVUsA==",
+ "mTLBkP+yGHsdk5g7zLjVUw==",
+ "mU4CqbAwpwqegxJaOz9ofQ==",
+ "mUek9NkXm8HiVhQ6YXiyzA==",
+ "mVT74Eht+gAowINoMKV7IQ==",
+ "mW6TCje9Zg2Ep7nzmDjSYQ==",
+ "mXBfDUt/sBW5OUZs2sihvw==",
+ "mXPtbPaoNAAlGmUMmJEWBQ==",
+ "mXZ4JeBwT2WJQL4a/Tm4jQ==",
+ "mXycPfF5zOvcj1p4hnikWw==",
+ "mc45FSMtzdw2PTcEBwHWPw==",
+ "md6zNd7ZBn3qArYqQz7/fw==",
+ "me61ST+JrXM5k3/a11gRAA==",
+ "meHzY9dIF7llDpFQo1gyMg==",
+ "miiOqnhtef1ODjFzMHnxjA==",
+ "mjFBVRJ7TgnJx+Q74xllPg==",
+ "mjQS8CpyGnsZIDOIEdYUxg==",
+ "mk1CKDah7EzDJEdhL22B7w==",
+ "mmRob7iyTkTLDu8ObmTPow==",
+ "mnalaO6xJucSiZ0+99r3Cg==",
+ "mpOtwBvle+nyY6lUBwTemw==",
+ "mpWNaUH9kn4WY26DWNAh3Q==",
+ "mr1qjhliRfl87wPOrJbFQg==",
+ "mrinv7KooPQPrLCNTRWCFg==",
+ "mrxlFD3FBqpSZr1kuuwxGg==",
+ "msstzxq++XO0AqNTmA7Bmg==",
+ "mxug34EekabLz0JynutfBg==",
+ "myzvc+2MfxGD9uuvZYdnqQ==",
+ "n+xYzfKmMoB3lWkdZ+D3rg==",
+ "n1M2dgFPpmaICP+JwxHUug==",
+ "n1ixvP7SfwYT3L2iWpJg6A==",
+ "n5GA+pA9mO/f4RN9NL9lNg==",
+ "n6QVaozMGniCO0PCwGQZ6w==",
+ "n7Bns42aTungqxKkRfQ5OQ==",
+ "n7KL1Kv027TSxBVwzt9qeA==",
+ "n7h9v2N1gOcvMuBEf8uThw==",
+ "nDAsSla+9XfAlQSPsXtzPA==",
+ "nE72uQToQFVLOzcu/nMjww==",
+ "nFBXCPeiwxK9mLXPScXzTA==",
+ "nFPDZGZowr3XXLmDVpo7hg==",
+ "nGzPc0kI/EduVjiK7bzM6Q==",
+ "nHTsDl0xeQPC5zNRnoa0Rw==",
+ "nHUpYmfV59fe3RWaXhPs3Q==",
+ "nL4iEd3b5v4Y9fHWDs+Lrw==",
+ "nMuMtK/Zkb3Xr34oFuX/Lg==",
+ "nNaGqigseHw30DaAhjBU3g==",
+ "nOiwBFnXxCBfPCHYITgqNg==",
+ "nR3ACzeVF5YcLX6Gj6AGyQ==",
+ "nULSbtw2dXbfVjZh33pDiA==",
+ "nUgYO7/oVNSX8fJqP2dbdg==",
+ "nVDxVhaa2o38gd1XJgE3aw==",
+ "nW3zZshjZEoM8KVJoVfnuQ==",
+ "nY/H7vThZ+dDxoPRyql+Cg==",
+ "neQoa8pvETr07blVMN3pgA==",
+ "nf8x+F03kOpMhsCSUWEhVg==",
+ "ng1Q0A7ljho3TUWWYl46sw==",
+ "nhAnHuCGXcYlqzOxrrEe1g==",
+ "nkbLVLvh3ClKED97+nH+7Q==",
+ "nkedTagkmf6YE4tEY+0fKw==",
+ "nknBKPgb7US42v8A0fTl/w==",
+ "nmD7fEU4u7/4+W/pkC4/0Q==",
+ "nqpKfidczdgrNaAyPi7BOQ==",
+ "nqtQI1bSM7DCO9P1jGV97Q==",
+ "nsnX3tKkN1elr18E31tXDw==",
+ "nvLEpj6ZZF3LWH3wUB6lKg==",
+ "nvUKoKfC6j8fz3gEDQrc/w==",
+ "nvmBgp0YlUrdZ05INsEE8Q==",
+ "nwtCsN1xEYaHvEOPzBv+qQ==",
+ "nx/U4Tode5ILux4DSR+QMg==",
+ "nxDGRpePV3H4NChn4eLwag==",
+ "nyaekSYTKzfSeSfPrB114Q==",
+ "nykEOLL/o7h0cs0yvdeT2g==",
+ "o+areESiXgSO0Lby56cBeg==",
+ "o+nYS4TqJc6XOiuUzEpC3A==",
+ "o/Y4U6rWfsUCXJ72p5CUGw==",
+ "o1uhaQg5/zfne84BFAINUQ==",
+ "o1zeXHJEKevURAAbUE/Vog==",
+ "o5XVEpdP4OXH0NEO4Yfc/A==",
+ "o64LDtKq/Fulf1PkVfFcyg==",
+ "o7y4zQXQAryST2cak4gVbw==",
+ "o9tdzmIu+3J/EYU4YWyTkA==",
+ "oAHVGBSJ2cf4dVnb/KEYmw==",
+ "oDca3JEdRb4vONT9GUUsaQ==",
+ "oFNMOKbQXcydxnp8fUNOHw==",
+ "oFanDWdePmmZN0xqwpUukA==",
+ "oGH7SMLI2/qjd9Vnhi3s0A==",
+ "oIU19xAvLJwQSZzIH577aA==",
+ "oIWwTbkVS5DDL47mY9/1KQ==",
+ "oKt57TPe4PogmsGssc3Cbg==",
+ "oLWWIn/2AbKRHnddr2og9g==",
+ "oMJLQTH1wW7LvOV0KRx/dw==",
+ "oNOI17POQCAkDwj6lJsYOA==",
+ "oONlXCW4aAqGczQ/bUllBw==",
+ "oPcxgoismve6+jXyIKK6AQ==",
+ "oPlhC4ebXdkIDazeMSn1fQ==",
+ "oQjugfjraFziga1BcwRLRA==",
+ "oR8rvIZoeoaZ/ufpo0htfQ==",
+ "oSnrpW4UmmVXtUGWqLq+tQ==",
+ "oUqO4HrBvkpSL781qAC9+w==",
+ "oVlG+0rjrg2tdFImxIeVBA==",
+ "oad5SwflzN0vfNcyEyF4EA==",
+ "obW3kzv2KBvuckU7F+tfjA==",
+ "ocRh5LR1ZIN9Johnht8fhQ==",
+ "ocpLRASvTgqfkY20YlVFHQ==",
+ "ocvA1/NbyxM0hanwwY6EiA==",
+ "odGhKtO4bDW5R8SYiI5yCg==",
+ "ogcuGHUZJkmv+vCz567a2g==",
+ "ohK6EftXOqBzIMI+5XnESw==",
+ "ojZY7Gi2QJXE/fp6Wy31iA==",
+ "ojf6uL85EuEYgLvHoGhUrw==",
+ "ojugpLIfzflgU2lonfdGxA==",
+ "ol9xhVTG9e1wNo50JdZbOA==",
+ "olTSlmirL9MFhKORiOKYkQ==",
+ "omAjyj1l6gyQAlBGfdxJTw==",
+ "onFcHOO1c3pDdfCb5N4WkQ==",
+ "oqlkgrYe9aCOwHXddxuyag==",
+ "oxoZP897lgMg/KLcZAtkAg==",
+ "oyYtf08AkWLR52bXm5+sKw==",
+ "ozVqYsmUueKifb4lDyVyrg==",
+ "p+bx+/WQWALXEBCTnIMr4w==",
+ "p/48hurJ1kh2FFPpyChzJg==",
+ "p/7qM5+Lwzw1/lIPY91YxQ==",
+ "p0eNK7zJd7D/HEGaVOrtrQ==",
+ "p2JPOX8yDQ0agG+tUyyT/g==",
+ "p3V7NfveB6cNxFW7+XQNeQ==",
+ "p48i7AfSSAyTdJSyHvOONw==",
+ "p73gSu4d+4T/ZNNkIv9Nlw==",
+ "p8W1LgFuW6JSOKjHkx3+aA==",
+ "pCQmlnn3BxhsV2GwqjRhXg==",
+ "pFKzcRHSUBqSMtkEJvrR1Q==",
+ "pGQEWJ38hb/ZYy2P1+FIuw==",
+ "pHo1O5zrCHCiLvopP2xaWw==",
+ "pHozgRyMiEmyzThtJnY4MQ==",
+ "pKaTI+TfcV3p/sxbd2e7YQ==",
+ "pT1raq2fChffFSIBX3fRiA==",
+ "pUfWmRXo70yGkUD/x5oIvA==",
+ "pVG1hL96/+hQ+58rJJy6/A==",
+ "pVgjGg4TeTNhKimyOu3AAw==",
+ "pW4gDKtVLj48gNz6V17QdA==",
+ "pZfn6IiG+V28fN8E2hawDQ==",
+ "pa8nkpAAzDKUldWjIvYMYg==",
+ "pcoBh5ic7baSD4TZWb3BSw==",
+ "pdPwUHauXOowaq9hpL2yFw==",
+ "pdaY6kZ8+QqkMOInvvACNA==",
+ "peMW+rpwmXrSwplVuB/gTA==",
+ "pfGcaa49SM3S6yJIPk/EJQ==",
+ "plXHHzA8X9QGwWzlJxhLRw==",
+ "pnJnBzAJlO4j3IRqcfmhkQ==",
+ "prCOYlboBnzmLEBG/OeVrQ==",
+ "prOsOG0adI4o+oz50moipw==",
+ "pulldyBt2sw6QDvTrCh6zw==",
+ "pv/m2mA/RJiEQu2Qyfv9RA==",
+ "pvXHwJ3dwf9GDzfDD9JI3g==",
+ "pw1jplCdTC+b0ThX0FXOjw==",
+ "pxuSWn1u+bHtRjyh2Z8veA==",
+ "pyrUqiZ98gVXxlXQNXv5fA==",
+ "pzC8Y0Vj9MPBy3YXR32z6w==",
+ "q/siBRjx6wNu+OTvpFKDwA==",
+ "q4z6A4l3nhX3smTmXr+Sig==",
+ "q5g3c8tnQTW2EjNfb2sukw==",
+ "q6LG0VzO1oxiogAAU63hyg==",
+ "q7m/EtZySBjZNBjQ5m1hKw==",
+ "q8YF9G2jqydAxSqwyyys5Q==",
+ "qA0sTaeNPNIiQbjIe1bOgQ==",
+ "qCPfJTR8ecTw6u6b1yHibA==",
+ "qE/h/Z+6buZWf+cmPdhxog==",
+ "qIFpKKwUmztsBpJgMaVvSg==",
+ "qIUJPanWmGzTD1XxvHp+6w==",
+ "qNOSm15bdkIDSc/iUr+UTQ==",
+ "qNyy6Fc0b8oOMWqqaliZ/w==",
+ "qO4HlyHMK5ygX+6HbwQe8w==",
+ "qOEIUWtGm5vx/+fg4tuazg==",
+ "qP1cCE4zsKGTPhjbcpczMw==",
+ "qQQwJ/aF87BbnLu3okXxaw==",
+ "qYHdgFAXhF/XcW4lxqfvWQ==",
+ "qYuo5vY8V3tZx41Kh9/4Dw==",
+ "qZ2q5j2gH3O56xqxkNhlIA==",
+ "qaTdVEeZ6S8NMOxfm+wOMA==",
+ "qcpeZWUlPllQYZU6mHVwUw==",
+ "qenHZKKlTUiEFv6goKM/Mw==",
+ "qkvEep4vvXhc2ZJ6R449Mg==",
+ "qngzBJbiTB4fivrdnE5gOg==",
+ "qnkFUlJ8QT322JuCI3LQgg==",
+ "qnsBdl050y9cUaWxbCczRw==",
+ "qnzWszsyJhYtx8wkMN6b1g==",
+ "qoK2keBg3hdbn7Q24kkVXg==",
+ "qpFJZqzkklby+u1UT3c1iA==",
+ "qt5CsMts2aD4lw/4Q6bHYQ==",
+ "qxALQrqHoDq9d91nU0DckA==",
+ "qyRmvxh8p4j4f+61c10ZFQ==",
+ "r/b5px/UImGNjT/X5sYjuA==",
+ "r0QffVKB9OD9yGsOtqzlhA==",
+ "r0hAwlS0mPZVfCSB+2G6uQ==",
+ "r1VGXWeqGeGbfKjigaAS+Q==",
+ "r2f2MyT+ww1g9uEBzdYI1w==",
+ "r36kVMpF+9J+sfI3GeGqow==",
+ "r3lQAYOYhwlLnDWQIunKqg==",
+ "r95wJtP5rsTExKMS7QhHcw==",
+ "rBt6L/KLT7eybxKt5wtFdg==",
+ "rCxoo4TP/+fupXMuIM0sDA==",
+ "rHagXw+CkF3uEWPWDKXvog==",
+ "rIMXaCaozDvrdpvpWvyZOQ==",
+ "rJ9qVn8/2nOxexWzqIHlcQ==",
+ "rJCuanCy51ydVD4nInf9IQ==",
+ "rKAQxu80Q8g1EEhW5Wh8tg==",
+ "rKb3TBM4EPx/RErFOFVCnQ==",
+ "rLZII1R6EGus+tYCiUtm6g==",
+ "rM/BOovNgnvebKMxZQdk7g==",
+ "rMm9bHK69h0fcMkMdGgeeA==",
+ "rOYeIcB+Rg5V6JG2k4zS2w==",
+ "rSvhrHyIlnIBlfNJqemEbw==",
+ "rTwJggSxTbwIYdp07ly0LA==",
+ "rUp5Mfc57+A8Q29SPcvH/Q==",
+ "rWliqgfZ3/uCRBOZ9sMmdA==",
+ "rXGWY/Gq+ZEsmvBHUfFMmQ==",
+ "rXSbbRABEf4Ymtda45w8Fw==",
+ "rXfWkabSPN+23Ei1bdxfmQ==",
+ "rXtGpN17Onx8LnccJnXwJQ==",
+ "rZKD8oJnIj5fSNGiccfcvA==",
+ "raKMXnnX6PFFsbloDqyVzQ==",
+ "raYifKqev8pASjjuV+UTKQ==",
+ "rcY4Ot40678ByCfqvGOGdg==",
+ "rdeftHE7gwAT67wwhCmkYQ==",
+ "rfPTskbnoh3hRJH6ZAzQRg==",
+ "rgcXxjx3pDLotH7TTfAoZw==",
+ "rh7bzsTQ1UZjG7amysr0Gg==",
+ "rhgtLQh0F9bRA6IllM7AGw==",
+ "ri4AOITPdB1YHyXV+5S51g==",
+ "rkeLYwMZ1/pW2EmIibALfA==",
+ "rlXt6zKE7DswUl0oWGOQUQ==",
+ "rqHKB91H3qVuQAm+Ym5cUA==",
+ "rqucO37p86LpzehR/asCSQ==",
+ "rs2QrN4qzAHCHhkcrAvIfA==",
+ "rtJdfki8fG6CB36CADp0QA==",
+ "rtd6mqFgGe98mqO0pFGbSw==",
+ "rueNryrchijjmWaA3kljYg==",
+ "rvE64KQGkVkbl07y7JwBqw==",
+ "rwplpbNJz0ADUHTmzAj15Q==",
+ "rwtF86ZAbWyKI6kLn4+KBw==",
+ "rxfACPLtKXbYua18l3WlUw==",
+ "rzj6mjHCcMEouL66083BAg==",
+ "s+eHg5K9zZ2Jozu5Oya9ZQ==",
+ "s/BZAhh1cTV3JCDUQsV8mA==",
+ "s2AKVTwrY65/SWqQxDGJQg==",
+ "s5+78jS4hQYrFtxqTW3g1Q==",
+ "s5RUHVRNAoKMuPR/Jkfc2Q==",
+ "s7iW1M6gkAMp+D/3jHY58w==",
+ "s8NpalwgPdHPla7Zi9FJ3w==",
+ "sBpytpE38xz0zYeT+0qc2A==",
+ "sC11Rf/mau3FG5SnON4+vQ==",
+ "sCLMrLjEUQ6P1L8tz90Kxg==",
+ "sEeblUmISi1HK4omrWuPTA==",
+ "sGLPmr568+SalaQr8SE/PA==",
+ "sLJrshdEANp0qk2xOUtTnQ==",
+ "sLdxIKap0ZfC3GpUk3gjog==",
+ "sNmW2b2Ud7dZi3qOF8O8EQ==",
+ "sQAxqWXeiu/Su0pnnXgI9A==",
+ "sQskMBELEq86o1SJGQqfzg==",
+ "sQzCwNDlRsSH7iB9cTbBcg==",
+ "sS6QcitMPdvUBLiMXkWQkw==",
+ "sWLcS+m4aWk31BiBF+vfJQ==",
+ "sXlFMSTBFnq0STHj6cS/8w==",
+ "sa2DECaqYH1z1/AFhpHi+g==",
+ "saEpnDGBSZWqeXSJm34eOA==",
+ "scCQPl0em2Zmv/RQYar60g==",
+ "sfIClgTMtZo9CM9MHaoqhQ==",
+ "sfowXUMdN2mCoBVrUzulZg==",
+ "sfte/o9vVNyida/yLvqADA==",
+ "siHwJx6EgeB1gBT9z/vTyw==",
+ "skrQRB9xbOsiSA19YgAdIQ==",
+ "snGTzo540cCqgBjxrfNpKw==",
+ "soBA65OmZdfBGJkBmY/4Iw==",
+ "spHVvA/pc7nF9Q4ON020+w==",
+ "spJI3xFUlpCDqzg0XCxopA==",
+ "sr3UXbMg5zzkRduFx/as7g==",
+ "sw+bmpzqsM4gEQtnqocQLQ==",
+ "swJhrPwllq5JORWiP5EkDA==",
+ "swsVVsPi/5aPFBGP+jmPIw==",
+ "syeBfQBUmkXNWCZ1GV8xSA==",
+ "t+bYn9UqrzKiuxAYGF7RLA==",
+ "t0WN8TwMLgi8UVEImoFXKg==",
+ "t2EkpUsLOEOsrnep0nZSmA==",
+ "t2vWMIh2BvfDSQaz5T1TZw==",
+ "t3Txxjq43e/CtQmfQTKwWg==",
+ "t5U+VMsTtlWAAWSW+00SfQ==",
+ "t5wh9JGSkQO78QoQoEqvXA==",
+ "t7HaNlXL16fVwjgSXmeOAQ==",
+ "t8pjhdyNJirkvYgWIO/eKg==",
+ "tBQDfy48FnIOZI04rxfdcA==",
+ "tFMJRXfWE9g78O1uBUxeqQ==",
+ "tFmWYH82I3zb+ymk5dhepA==",
+ "tG+rpfJBXlyGXxTmkceiKA==",
+ "tHDbi43e6k6uBgO0hA+Uiw==",
+ "tIqwBotg052wGBL65DZ+yA==",
+ "tJt6VDdAPEemBUvnoc4viA==",
+ "tOdlnsE3L3XCBDJRmb/OqA==",
+ "tOkYq1BZY152/7IJ6ZYKUg==",
+ "tU31r8zla146sqczdKXufg==",
+ "tVhXk9Ff3wAg56FbdNtcFg==",
+ "tVvWdA+JqH0HR2OlNVRoag==",
+ "tVw8U1AsslIFmQs4H1xshg==",
+ "tX8X8KoxUQ8atFSCxgwE1Q==",
+ "tXVb5f90k9l3e1oK2NGXog==",
+ "tXuu7YpZOuMLTv87NjKerA==",
+ "tY916jrSySzrL+YTcVmYKQ==",
+ "tYeIZjIm0tVEsYxH1iIiUQ==",
+ "tb5+2dmYALJibez1W4zXgA==",
+ "td7nDgTDmKPSODRusMcupw==",
+ "tdgI9v7cqJsgCAeW1Fii1A==",
+ "tdiTXKrkqxstDasT0D5BPA==",
+ "tejpAZp7y32SO2+o4OGvwQ==",
+ "tfgO55QqUyayjDfQh+Zo1Q==",
+ "tj2rWvF2Fl+XIccctj8Mhw==",
+ "tnUtJ/DQX9WaVJyTgemsUA==",
+ "tq5xUJt8GtjDIh1b48SthQ==",
+ "tr+U/vt+MIGXPRQYYWJfRg==",
+ "trjM81KANPZrg9iSThWx6Q==",
+ "tsiqwelcBAMU/HpLGBtMGw==",
+ "twPn6wTGqI0aR//0wP3xtA==",
+ "twjiDKJM7528oIu/el4Zbg==",
+ "tzV7ixFH37ze4zuLILTlfA==",
+ "u/QxrP1NOM/bOJlJlsi/jQ==",
+ "u2WQlcMxOACy6VbJXK4FwA==",
+ "u5cUPxM6/spLIV8VidPrAA==",
+ "uC2lzm7HaMAoczJO6Z/IhQ==",
+ "uChFnF0oCwARhAOz/d47eA==",
+ "uESeJe/nYrHCq4RQbrNpGA==",
+ "uExgqZkkJnZj252l5dKAGg==",
+ "uIkVijg7RPi/1j7c18G1qA==",
+ "uJZGw3IY2nCcdVeWW1geNQ==",
+ "uMq8cDVWFD+tpn8aeP8Pqg==",
+ "uNWFZlP7DA96sf+LWiAhtQ==",
+ "uNzpptKjihEfKRo5A1nWmw==",
+ "uO+uK1DntCxVRr1KttfUIw==",
+ "uOHrw37yF9oLLVd16nUpeg==",
+ "uOkMpYy/7DYYoethJdixfQ==",
+ "uPdjKJIGzN7pbGZDZdCGaA==",
+ "uPi8TsGY3vQsMVo/nsbgVQ==",
+ "uPm+cF4Jq08S5pQhYFjU8A==",
+ "uPnL9tboMZo0Kl2fe24CmA==",
+ "uQs79rbD/wEakMUxqMI48A==",
+ "uSIiF1r9F18avZczmlEuMQ==",
+ "uT6WRh5UpVdeABssoP2VTg==",
+ "uTA0XbiH3fTeVV7u5z0b3w==",
+ "uTHBqApdKOAgdwX3cjrCYQ==",
+ "uU1TX5DoDg6EcFKgFcn0GA==",
+ "uXuPA/2KJbb7ZX+NymN3dw==",
+ "uXvr6vi5kazZ9BCg2PWPJA==",
+ "uZ2gUA74/7Q33tI2TcGQlg==",
+ "ucLMWnNDSqE4NOCGWvcGWw==",
+ "udU65VtsvJspYmamiOsgXw==",
+ "ueODvMv/f9ZD8O0aIHn4sg==",
+ "ugY8rTtJkN4CXWMVcRZiZw==",
+ "uhT12XY79CtbwhcSfAmAXQ==",
+ "ulLuTZqhEDkX0EJ3xwRP9A==",
+ "ulpDxLeQnIRPnq6oaah2AA==",
+ "up2MVDi9ve+s83/nwNtZ7Q==",
+ "uqe3rFveJ2JIkcZQ3ZMXHQ==",
+ "uqp92lAqjec8UQYfyjaEZw==",
+ "ur9JDCVNwzSH4q4ngDlHNQ==",
+ "uu+ncs63SdQIvG6z4r7Q3Q==",
+ "uuiJ+yB7JLDh2ulthM0mjg==",
+ "uvKYnKE01D5r7kR9UQyo5A==",
+ "uvzmRcvgepW6mZbMfYgcNw==",
+ "uwA6N5LptSXqIBkTO0Jd7Q==",
+ "uwGivY3/C9WK+dirRPJZ4A==",
+ "uzEgwx1iAXAvWPKSVwYSeQ==",
+ "uzkNhmo2d08tv5AmnyqkoQ==",
+ "v/PshI6JjkL9nojLlMNfhg==",
+ "v0Bvws1WYVoEgDt8xmVKew==",
+ "v1AWe5qb5y3vSKFb7ADeEw==",
+ "v4xIYrfPGILEbD/LwVDDzA==",
+ "v6jZicMNM3ysm3U5xu0HoQ==",
+ "v7BrkRmK0FfWSHunTRHQFQ==",
+ "vCekQ2nOQKiN/q8Be/qwZg==",
+ "vFFzkWgGyw6OPADONtEojQ==",
+ "vFox1d3llOeBeCUZGvTy0A==",
+ "vFtC0B2oe1gck28JOM1dyg==",
+ "vGKknndb4j6VTV8DxeT4fQ==",
+ "vHGjRRSlZHJIliCwIkCAmQ==",
+ "vHVXsAMQqc0qp7HA5Q+YkA==",
+ "vHmQUl4WHXs1E/Shh+TeyA==",
+ "vIORTYSHFIXk5E2NyIvWcQ==",
+ "vMuaLvAntJB5o7lmt/kVXA==",
+ "vOJ55zFdgPPauPyFYBf01w==",
+ "vRgkZZGVN7YZrlml0vxrKA==",
+ "vSKsa0JhLCe9QFZKkcj58Q==",
+ "vTAmgfq3GxL4+ubXpzwk5w==",
+ "vUC0HlTTHj6qNHwfviDtAw==",
+ "vUE8Iw3NyWXURpXyoNJdaw==",
+ "vWn9OPnrJgfPavg4D6T/HQ==",
+ "vX7RIhatQeXAMr1+OjzhZw==",
+ "vZtL0yWpSIA+9v8i23bZSg==",
+ "vb6Agwzk4JG0Nn7qRPPFMQ==",
+ "vbyiKeDCQ4q9dDRI1Q0Ong==",
+ "vg3jozLXEmAnmJwdfcEN0g==",
+ "vhdFtKVH4bVatb4n8KzeXw==",
+ "vjrSYGUpeKOtJ2cNgLFg2g==",
+ "vljJciS+uuIvL7XXm5688g==",
+ "vmqfGJE6r4yDahtU/HLrxw==",
+ "vnOJ3e9Zd4wPx8PX7QgZzQ==",
+ "voO3krg4sdy4Iu+MZEr8+g==",
+ "vqYHQ3MnHrAIAr1QHwfIag==",
+ "vsRNZx4thFFFPneubKq1Fw==",
+ "vvEH5A39TTe1AOC11rRCLA==",
+ "vvh9vAIrXjIwLVkuJb5oDQ==",
+ "vwno3vugCvt6ooT3CD4qIQ==",
+ "w+jzM0I5DRzoUiLS/9QIMQ==",
+ "w0PKdssv+Zc5J/BbphoxpA==",
+ "w1zN28mSrI/gqHsgs4ME3A==",
+ "w3G+qXXqqKi8F5s+qvkBUg==",
+ "w5N/aHbtOIKzcvG3GlMjGA==",
+ "wDiGoFEfIVEDyyc4VpwhWQ==",
+ "wEJDulZafLuXCvcqBYioFQ==",
+ "wHA+D5cObfV3kGORCdEknw==",
+ "wI7JrSPQwYHpv2lRsQu9nQ==",
+ "wIfvvLKC61gOpsddUFjVog==",
+ "wJ4uCrl4DPg70ltw1dZO3w==",
+ "wJKFMqh6MGctWfasjHrPEg==",
+ "wJpepvmtQQ3sz3tVFDnFqw==",
+ "wK6Srd83eLigZ11Q20XGrg==",
+ "wM8tnXO4PDlLVHspZFcjYw==",
+ "wMOE/pEKVIklE75xjt6b6w==",
+ "wMum67lfk5E1ohUObJgrOg==",
+ "wMyJLQJdmrC2TSeFkIuSvQ==",
+ "wOc4TbwQGUwOC1B3BEZ4OQ==",
+ "wOhbpTzmFla8R0kI9OiHaA==",
+ "wPhJcp7U7IVX83szbIOOxQ==",
+ "wQKL8Ga6JQkpZ7yymDkC3w==",
+ "wR2Gxb07nkaPcZHlEjr8iA==",
+ "wRqaDZVHHurp5whOQ1kDbQ==",
+ "wTO49YX/ePHMWtcoxUAHpw==",
+ "wUYhs4j3W9nIywu1HIv2JA==",
+ "wVfSZYjMjbTsD2gaSbwuqQ==",
+ "wX2URK6eDDHeEOF3cgPgHA==",
+ "wX70jKLKJApHnhyK0r6t3A==",
+ "wajwXfWz2J+O+NVaj6j2UQ==",
+ "wc+8ohFWgOF4VlSYiZIGwQ==",
+ "wdRyYjaM11VmqkkxV/5bsA==",
+ "wfwuxn+Vja1DNwiDwL2pcQ==",
+ "wgH1GlUxWi6/yLLFzE76uQ==",
+ "who8uUamlHWHXnBf7dwy4A==",
+ "wlWxtQDJ+siGhN2fJn3qtw==",
+ "wnfYUctNK+UPwefX5y4/Rw==",
+ "wpZqFkKafFpLcykN2IISqg==",
+ "wqUJ1Gq1Yz2cXFkbcCmzHQ==",
+ "wqWqe0KRjZlUIrGgEOG9Mg==",
+ "wrewZ0hoHODf7qmoGcOd7g==",
+ "wsp+vmW8sEqXYVURd/gjHA==",
+ "wt+qDLU38kzNU75ZYi3Hbw==",
+ "wtyAZIfhomcHe9dLbYoSvA==",
+ "wux5Y8AipBnc5tJapTzgEQ==",
+ "wv4NC9CIpwuGf/nOQYe/oA==",
+ "wxkb8evGEaGf/rg/1XUWiA==",
+ "wy/Z8505o4sVovk4UuBp1A==",
+ "wyqmQGB6vgRVrYtmB2vB7w==",
+ "wyx5mnUMgP5wjykjAfTO7w==",
+ "x+8rwkqKCv0juoT5m1A4eg==",
+ "x/BIDm6TKMhqu/gtb3kGyw==",
+ "x/MpsQvziUpW40nNUHDS5Q==",
+ "x0eIHCvQLd2jdDaXwSWTYQ==",
+ "x1A74vg/hwwjAx6GrkU8zw==",
+ "x2NpqNnqRihktNzpxmepkQ==",
+ "x2nSgcTjA3oGgI8mMgiqjw==",
+ "x5lyMArsv1MuJmEFlWCnNw==",
+ "x5zMDuW66467ofgL3spLUQ==",
+ "x6M66krXSi0EhppwmDmsxA==",
+ "x6lNRGgJcRxgKTlzhc1WPg==",
+ "x8kRVzohTdhkryvYeMvkMw==",
+ "x9TIZ9Ua++3BX+MpjgTuWA==",
+ "x9VwDdFPp/rJ+SF16ooWYg==",
+ "xAAipGfHTGTjp9Qk1MR8RQ==",
+ "xJi0T+psHOXMivSOVpMWeQ==",
+ "xLm/bJBonpTs0PwsF0DvRg==",
+ "xMIHeno2qj3V8q9H1xezeg==",
+ "xNilc7UOu1kyP0+nK5MrLw==",
+ "xPe76nHyHmald6kmMQsKdg==",
+ "xQpYjaAmrQudWgsdu24J0A==",
+ "xTizUioizbMQxD0T6fy/EQ==",
+ "xUXEE7OBBCudsQnuj5ycOA==",
+ "xWYecfzAtXT9WyQ8NYY/hw==",
+ "xX6atcCApI08oVLjjLteLg==",
+ "xYD8jrCDmuQna+p1ebnKDQ==",
+ "xbBxUP9JyY0wDgHDipBHeg==",
+ "xdCCdP8SNBOK3IsX6PiPQA==",
+ "xdmY+qyoxxuRZa9kuNpDEg==",
+ "xfYZ6qhWNBqqJ0PdWRjOwA==",
+ "xfjBQk3CrNjhufdPIhr91A==",
+ "xiFlcSfa/gnPiO+LwbixcQ==",
+ "xiyRfVG0EfBA+rCk+tgWRQ==",
+ "xjA21QjNdThLW3VV7SCnrg==",
+ "xjTMO2mvtpvwQrounD4e8g==",
+ "xktOghh1S9nIX6fXWnT+Ug==",
+ "xmGgK3W5y+oCd0K2u8XjZQ==",
+ "xmsYnsJq78/f9xuKuQ2pBQ==",
+ "xoPSM86Se+1hHX0y3hhdkw==",
+ "xs8J3cesq7lDhP/dNltqOw==",
+ "xsCZVhCk2qJmOqvUjK3Y8Q==",
+ "xsf0m31Am0W9eLhopAkfnA==",
+ "xukOAM0QVsA72qEy0yku9A==",
+ "xvipmmwKdYt4eoKvvRnjEg==",
+ "xweGAZf+Yb3TtwR/sGmGIA==",
+ "xzGzN5Hhbh0m/KezjNvXbQ==",
+ "y+1I05LDAYJ09tKMs3zW6g==",
+ "y+cl1/Knb9MZPz8nBB0M+w==",
+ "y/e3HSdg7T19FanRpJ7+7Q==",
+ "y1J+o6DC2sETFsySgpDZyA==",
+ "y2JOIoIiT9cV1VxplZPraQ==",
+ "y2Tn2gmhKs5WKc01ce74rg==",
+ "y4/HohCJxtt+cT7nLJB08w==",
+ "y4Y4mSSTw/WrIdRpktc5Hw==",
+ "y4iBxAMn/KzMmaWShdYiIw==",
+ "y4mfEDerrhaqApDdhP5vjA==",
+ "y7yS9x3yshVhMpDbQtfYOQ==",
+ "yCu+DVU/ceMTOZ5h/7wQTg==",
+ "yD3Dd4ToRrl53k/2NSCJiw==",
+ "yDrAd1ot38soBk7zKdnT8A==",
+ "yKLLiqzxfrCsr6+Rm6kx1Q==",
+ "yKrsKX4/1B1C0TyvciNz5w==",
+ "yL1DwlIIREPuyuCFULi0uw==",
+ "yLAhLNezvqVHmN1SfMRrPw==",
+ "yOE90OHQdyOfrAgwDvn2gA==",
+ "yPIeWcW8+3HjDagegrN8bw==",
+ "yQCLV9IoPyXEOaj3IdFMWw==",
+ "yQmNZnp/JZywbBiZs3gecA==",
+ "yS/yMnJDHW0iaOsbj4oPTg==",
+ "yTVJKBn72RjakMBXDoBKHg==",
+ "yTgN5xFIdz1MzFS6xMl5uQ==",
+ "yU3N0HMSP5etuHPNrVkZtg==",
+ "yV3IbbTWAbHMhMGVvgb/ZQ==",
+ "yYBIS9PZbKo7Gram7IXWPA==",
+ "yYVW07lOZHdgtX42xJONIA==",
+ "yYmnM/WOgi+48Rw7foGyXA==",
+ "yYp4iuI5f/y/l1AEJxYolQ==",
+ "ybpTgPr3SjJ12Rj5lC/IMA==",
+ "ycjv4XkS5O7zcF3sqq9MwQ==",
+ "yctId8ltkl3+xqi9bj+RqA==",
+ "ydVj2odhergi+2zGUwK4/A==",
+ "yf06Slv9l3IZEjVqvxP2aA==",
+ "yfAaL0MMtSXPQ37pBdmHxQ==",
+ "yhI5jHlfFJxu4eV5VJO2zQ==",
+ "yhRi5M9Etuu9HSu4d24i3w==",
+ "yhexr/OFKfZl0o3lS70e4w==",
+ "ylA6sU7Kaf9fMNIx1+sIlw==",
+ "ymtA8EMPMgmMcimWZZ0A1Q==",
+ "ynaj4XjU27b7XbqPyxI8Ig==",
+ "yqQPU4jT9XvRABZgNQXjgg==",
+ "yqtj8GfLaUHYv/BsdjxIVw==",
+ "ysRQ+7Aq7eVLOp88KnFVMA==",
+ "ytDXLDBqWiU1w3sTurYmaw==",
+ "yteeQr3ub2lDXgLziZV+DQ==",
+ "yxCyBXqGWA735JEyljDP7Q==",
+ "z+1oDVy8GJ5u/UDF+bIQdA==",
+ "z/e5M2lE9qh3bzB97jZCKA==",
+ "z0BU//aSjYHAkGGk3ZSGNg==",
+ "z20AAnvj7WsfJeOu3vemlA==",
+ "z3L2BNjQOMOfTVBUxcpnRA==",
+ "z4Bft++f72QeDh4PWGr/sw==",
+ "z4oKy2wKH+sbNSgGjbdHGw==",
+ "z5DveTu377UW8IHnsiUGZg==",
+ "z920R8eahJPiTsifrPYdxA==",
+ "z9cd+Qj+ueX34Zf3997MNQ==",
+ "zCRZgVsHbQZcVMHd9pGD3A==",
+ "zCpibjrZOA3FQ4lYt0WoVA==",
+ "zDSQ3NJuUGkVOlvVCATRwA==",
+ "zDUZCzQesFjO1JI3PwDjfg==",
+ "zEzWZ6l7EKoVUxvk/l78Mw==",
+ "zJ7ScHNxr2leCDNNcuDApA==",
+ "zNLlWGW/aKBhUwQZ4DZWoQ==",
+ "zVupSPz7cD0v/mD/eUIIjg==",
+ "zZtYkKU50PPEj6qSbO5/Sw==",
+ "za4rzveYVMFe3Gw531DQJQ==",
+ "zaqyy3GaJ7cp8qDoLJWcTw==",
+ "zbjXhZaeyMfdTb2zxvmRMg==",
+ "zeELfk015D5krExLKRUYtg==",
+ "zeHF6fdeqcOId3fRUGscRw==",
+ "zgEyxj/sCs63O98sZS94Yw==",
+ "zi04Yc01ZheuFAQc59E45A==",
+ "zirOtGUXeRL22ezfotZfQg==",
+ "zm+z+OOyHhljV2TjA3U9zw==",
+ "zrZWcqQsUE3ocWE0fG+SOA==",
+ "ztULoqHvCOE6qV7ocqa4/w==",
+ "zwQ/3MzTJ9rfBmrANIh14w==",
+ "zwY6tCjjya/bgrYaCncaag==",
+ "zxsSqovedB3HT99jVblCnQ==",
+ "zyA9f5J7mw5InjhcfeumAQ==",
+]);
diff --git a/browser/components/newtab/lib/HighlightsFeed.jsm b/browser/components/newtab/lib/HighlightsFeed.jsm
new file mode 100644
index 0000000000..c8e3dd9b61
--- /dev/null
+++ b/browser/components/newtab/lib/HighlightsFeed.jsm
@@ -0,0 +1,357 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const { shortURL } = ChromeUtils.import(
+ "resource://activity-stream/lib/ShortURL.jsm"
+);
+const { SectionsManager } = ChromeUtils.import(
+ "resource://activity-stream/lib/SectionsManager.jsm"
+);
+const {
+ TOP_SITES_DEFAULT_ROWS,
+ TOP_SITES_MAX_SITES_PER_ROW,
+} = ChromeUtils.import("resource://activity-stream/common/Reducers.jsm");
+const { Dedupe } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Dedupe.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "FilterAdult",
+ "resource://activity-stream/lib/FilterAdult.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "LinksCache",
+ "resource://activity-stream/lib/LinksCache.jsm"
+);
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Screenshots",
+ "resource://activity-stream/lib/Screenshots.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PageThumbs",
+ "resource://gre/modules/PageThumbs.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "DownloadsManager",
+ "resource://activity-stream/lib/DownloadsManager.jsm"
+);
+
+const HIGHLIGHTS_MAX_LENGTH = 16;
+const MANY_EXTRA_LENGTH =
+ HIGHLIGHTS_MAX_LENGTH * 5 +
+ TOP_SITES_DEFAULT_ROWS * TOP_SITES_MAX_SITES_PER_ROW;
+const SECTION_ID = "highlights";
+const SYNC_BOOKMARKS_FINISHED_EVENT = "weave:engine:sync:applied";
+const BOOKMARKS_RESTORE_SUCCESS_EVENT = "bookmarks-restore-success";
+const BOOKMARKS_RESTORE_FAILED_EVENT = "bookmarks-restore-failed";
+const RECENT_DOWNLOAD_THRESHOLD = 36 * 60 * 60 * 1000;
+
+class HighlightsFeed {
+ constructor() {
+ this.dedupe = new Dedupe(this._dedupeKey);
+ this.linksCache = new lazy.LinksCache(
+ lazy.NewTabUtils.activityStreamLinks,
+ "getHighlights",
+ ["image"]
+ );
+ lazy.PageThumbs.addExpirationFilter(this);
+ this.downloadsManager = new lazy.DownloadsManager();
+ }
+
+ _dedupeKey(site) {
+ // Treat bookmarks, pocket, and downloaded items as un-dedupable, otherwise show one of a url
+ return (
+ site &&
+ (site.pocket_id || site.type === "bookmark" || site.type === "download"
+ ? {}
+ : site.url)
+ );
+ }
+
+ init() {
+ Services.obs.addObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT);
+ Services.obs.addObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT);
+ Services.obs.addObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT);
+ SectionsManager.onceInitialized(this.postInit.bind(this));
+ }
+
+ postInit() {
+ SectionsManager.enableSection(SECTION_ID, true /* isStartup */);
+ this.fetchHighlights({ broadcast: true, isStartup: true });
+ this.downloadsManager.init(this.store);
+ }
+
+ uninit() {
+ SectionsManager.disableSection(SECTION_ID);
+ lazy.PageThumbs.removeExpirationFilter(this);
+ Services.obs.removeObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT);
+ Services.obs.removeObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT);
+ Services.obs.removeObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT);
+ }
+
+ observe(subject, topic, data) {
+ // When we receive a notification that a sync has happened for bookmarks,
+ // or Places finished importing or restoring bookmarks, refresh highlights
+ const manyBookmarksChanged =
+ (topic === SYNC_BOOKMARKS_FINISHED_EVENT && data === "bookmarks") ||
+ topic === BOOKMARKS_RESTORE_SUCCESS_EVENT ||
+ topic === BOOKMARKS_RESTORE_FAILED_EVENT;
+ if (manyBookmarksChanged) {
+ this.fetchHighlights({ broadcast: true });
+ }
+ }
+
+ filterForThumbnailExpiration(callback) {
+ const state = this.store
+ .getState()
+ .Sections.find(section => section.id === SECTION_ID);
+
+ callback(
+ state && state.initialized
+ ? state.rows.reduce((acc, site) => {
+ // Screenshots call in `fetchImage` will search for preview_image_url or
+ // fallback to URL, so we prevent both from being expired.
+ acc.push(site.url);
+ if (site.preview_image_url) {
+ acc.push(site.preview_image_url);
+ }
+ return acc;
+ }, [])
+ : []
+ );
+ }
+
+ /**
+ * Chronologically sort highlights of all types except 'visited'. Then just append
+ * the rest at the end of highlights.
+ * @param {Array} pages The full list of links to order.
+ * @return {Array} A sorted array of highlights
+ */
+ _orderHighlights(pages) {
+ const splitHighlights = { chronologicalCandidates: [], visited: [] };
+ for (let page of pages) {
+ if (page.type === "history") {
+ splitHighlights.visited.push(page);
+ } else {
+ splitHighlights.chronologicalCandidates.push(page);
+ }
+ }
+
+ return splitHighlights.chronologicalCandidates
+ .sort((a, b) => a.date_added < b.date_added)
+ .concat(splitHighlights.visited);
+ }
+
+ /**
+ * Refresh the highlights data for content.
+ * @param {bool} options.broadcast Should the update be broadcasted.
+ */
+ async fetchHighlights(options = {}) {
+ // If TopSites are enabled we need them for deduping, so wait for
+ // TOP_SITES_UPDATED. We also need the section to be registered to update
+ // state, so wait for postInit triggered by SectionsManager initializing.
+ if (
+ (!this.store.getState().TopSites.initialized &&
+ this.store.getState().Prefs.values["feeds.system.topsites"] &&
+ this.store.getState().Prefs.values["feeds.topsites"]) ||
+ !this.store.getState().Sections.length
+ ) {
+ return;
+ }
+
+ // We broadcast when we want to force an update, so get fresh links
+ if (options.broadcast) {
+ this.linksCache.expire();
+ }
+
+ // Request more than the expected length to allow for items being removed by
+ // deduping against Top Sites or multiple history from the same domain, etc.
+ const manyPages = await this.linksCache.request({
+ numItems: MANY_EXTRA_LENGTH,
+ excludeBookmarks: !this.store.getState().Prefs.values[
+ "section.highlights.includeBookmarks"
+ ],
+ excludeHistory: !this.store.getState().Prefs.values[
+ "section.highlights.includeVisited"
+ ],
+ excludePocket: !this.store.getState().Prefs.values[
+ "section.highlights.includePocket"
+ ],
+ });
+
+ if (
+ this.store.getState().Prefs.values["section.highlights.includeDownloads"]
+ ) {
+ // We only want 1 download that is less than 36 hours old, and the file currently exists
+ let results = await this.downloadsManager.getDownloads(
+ RECENT_DOWNLOAD_THRESHOLD,
+ { numItems: 1, onlySucceeded: true, onlyExists: true }
+ );
+ if (results.length) {
+ // We only want 1 download, the most recent one
+ manyPages.push({
+ ...results[0],
+ type: "download",
+ });
+ }
+ }
+
+ const orderedPages = this._orderHighlights(manyPages);
+
+ // Remove adult highlights if we need to
+ const checkedAdult = lazy.FilterAdult.filter(orderedPages);
+
+ // Remove any Highlights that are in Top Sites already
+ const [, deduped] = this.dedupe.group(
+ this.store.getState().TopSites.rows,
+ checkedAdult
+ );
+
+ // Keep all "bookmark"s and at most one (most recent) "history" per host
+ const highlights = [];
+ const hosts = new Set();
+ for (const page of deduped) {
+ const hostname = shortURL(page);
+ // Skip this history page if we already something from the same host
+ if (page.type === "history" && hosts.has(hostname)) {
+ continue;
+ }
+
+ // If we already have the image for the card, use that immediately. Else
+ // asynchronously fetch the image. NEVER fetch a screenshot for downloads
+ if (!page.image && page.type !== "download") {
+ this.fetchImage(page, options.isStartup);
+ }
+
+ // Adjust the type for 'history' items that are also 'bookmarked' when we
+ // want to include bookmarks
+ if (
+ page.type === "history" &&
+ page.bookmarkGuid &&
+ this.store.getState().Prefs.values[
+ "section.highlights.includeBookmarks"
+ ]
+ ) {
+ page.type = "bookmark";
+ }
+
+ // We want the page, so update various fields for UI
+ Object.assign(page, {
+ hasImage: page.type !== "download", // Downloads do not have an image - all else types fall back to a screenshot
+ hostname,
+ type: page.type,
+ pocket_id: page.pocket_id,
+ });
+
+ // Add the "bookmark", "pocket", or not-skipped "history"
+ highlights.push(page);
+ hosts.add(hostname);
+
+ // Remove internal properties that might be updated after dispatch
+ delete page.__sharedCache;
+
+ // Skip the rest if we have enough items
+ if (highlights.length === HIGHLIGHTS_MAX_LENGTH) {
+ break;
+ }
+ }
+
+ const { initialized } = this.store
+ .getState()
+ .Sections.find(section => section.id === SECTION_ID);
+ // Broadcast when required or if it is the first update.
+ const shouldBroadcast = options.broadcast || !initialized;
+
+ SectionsManager.updateSection(
+ SECTION_ID,
+ { rows: highlights },
+ shouldBroadcast,
+ options.isStartup
+ );
+ }
+
+ /**
+ * Fetch an image for a given highlight and update the card with it. If no
+ * image is available then fallback to fetching a screenshot.
+ */
+ fetchImage(page, isStartup = false) {
+ // Request a screenshot if we don't already have one pending
+ const { preview_image_url: imageUrl, url } = page;
+ return lazy.Screenshots.maybeCacheScreenshot(
+ page,
+ imageUrl || url,
+ "image",
+ image => {
+ SectionsManager.updateSectionCard(
+ SECTION_ID,
+ url,
+ { image },
+ true,
+ isStartup
+ );
+ }
+ );
+ }
+
+ onAction(action) {
+ // Relay the downloads actions to DownloadsManager - it is a child of HighlightsFeed
+ this.downloadsManager.onAction(action);
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ break;
+ case at.SYSTEM_TICK:
+ case at.TOP_SITES_UPDATED:
+ this.fetchHighlights({
+ broadcast: false,
+ isStartup: !!action.meta?.isStartup,
+ });
+ break;
+ case at.PREF_CHANGED:
+ // Update existing pages when the user changes what should be shown
+ if (action.data.name.startsWith("section.highlights.include")) {
+ this.fetchHighlights({ broadcast: true });
+ }
+ break;
+ case at.PLACES_HISTORY_CLEARED:
+ case at.PLACES_LINK_BLOCKED:
+ case at.DOWNLOAD_CHANGED:
+ case at.POCKET_LINK_DELETED_OR_ARCHIVED:
+ this.fetchHighlights({ broadcast: true });
+ break;
+ case at.PLACES_LINKS_CHANGED:
+ case at.PLACES_SAVED_TO_POCKET:
+ this.linksCache.expire();
+ this.fetchHighlights({ broadcast: false });
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = [
+ "HighlightsFeed",
+ "SECTION_ID",
+ "MANY_EXTRA_LENGTH",
+ "SYNC_BOOKMARKS_FINISHED_EVENT",
+ "BOOKMARKS_RESTORE_SUCCESS_EVENT",
+ "BOOKMARKS_RESTORE_FAILED_EVENT",
+];
diff --git a/browser/components/newtab/lib/InfoBar.jsm b/browser/components/newtab/lib/InfoBar.jsm
new file mode 100644
index 0000000000..7bdcd41316
--- /dev/null
+++ b/browser/components/newtab/lib/InfoBar.jsm
@@ -0,0 +1,179 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ RemoteL10n: "resource://activity-stream/lib/RemoteL10n.jsm",
+});
+
+class InfoBarNotification {
+ constructor(message, dispatch) {
+ this._dispatch = dispatch;
+ this.dispatchUserAction = this.dispatchUserAction.bind(this);
+ this.buttonCallback = this.buttonCallback.bind(this);
+ this.infobarCallback = this.infobarCallback.bind(this);
+ this.message = message;
+ this.notification = null;
+ }
+
+ /**
+ * Show the infobar notification and send an impression ping
+ *
+ * @param {object} browser Browser reference for the currently selected tab
+ */
+ showNotification(browser) {
+ let { content } = this.message;
+ let { gBrowser } = browser.ownerGlobal;
+ let doc = gBrowser.ownerDocument;
+ let notificationContainer;
+ if (content.type === "global") {
+ notificationContainer = browser.ownerGlobal.gNotificationBox;
+ } else {
+ notificationContainer = gBrowser.getNotificationBox(browser);
+ }
+
+ let priority = content.priority || notificationContainer.PRIORITY_SYSTEM;
+
+ this.notification = notificationContainer.appendNotification(
+ this.message.id,
+ {
+ label: this.formatMessageConfig(doc, content.text),
+ image: content.icon || "chrome://branding/content/icon64.png",
+ priority,
+ eventCallback: this.infobarCallback,
+ },
+ content.buttons.map(b => this.formatButtonConfig(b))
+ );
+
+ this.addImpression();
+ }
+
+ formatMessageConfig(doc, content) {
+ let docFragment = doc.createDocumentFragment();
+ // notificationbox will only `appendChild` for documentFragments
+ docFragment.appendChild(
+ lazy.RemoteL10n.createElement(doc, "span", { content })
+ );
+
+ return docFragment;
+ }
+
+ formatButtonConfig(button) {
+ let btnConfig = { callback: this.buttonCallback, ...button };
+ // notificationbox will set correct data-l10n-id attributes if passed in
+ // using the l10n-id key. Otherwise the `button.label` text is used.
+ if (button.label.string_id) {
+ btnConfig["l10n-id"] = button.label.string_id;
+ }
+
+ return btnConfig;
+ }
+
+ addImpression() {
+ // Record an impression in ASRouter for frequency capping
+ this._dispatch({ type: "IMPRESSION", data: this.message });
+ // Send a user impression telemetry ping
+ this.sendUserEventTelemetry("IMPRESSION");
+ }
+
+ /**
+ * Called when one of the infobar buttons is clicked
+ */
+ buttonCallback(notificationBox, btnDescription, target) {
+ this.dispatchUserAction(
+ btnDescription.action,
+ target.ownerGlobal.gBrowser.selectedBrowser
+ );
+ let isPrimary = target.classList.contains("primary");
+ let eventName = isPrimary
+ ? "CLICK_PRIMARY_BUTTON"
+ : "CLICK_SECONDARY_BUTTON";
+ this.sendUserEventTelemetry(eventName);
+ }
+
+ dispatchUserAction(action, selectedBrowser) {
+ this._dispatch({ type: "USER_ACTION", data: action }, selectedBrowser);
+ }
+
+ /**
+ * Called when interacting with the toolbar (but not through the buttons)
+ */
+ infobarCallback(eventType) {
+ if (eventType === "removed") {
+ this.notification = null;
+ // eslint-disable-next-line no-use-before-define
+ InfoBar._activeInfobar = null;
+ } else if (this.notification) {
+ this.sendUserEventTelemetry("DISMISSED");
+ this.notification = null;
+ // eslint-disable-next-line no-use-before-define
+ InfoBar._activeInfobar = null;
+ }
+ }
+
+ sendUserEventTelemetry(event) {
+ const ping = {
+ message_id: this.message.id,
+ event,
+ };
+ this._dispatch({
+ type: "INFOBAR_TELEMETRY",
+ data: { action: "infobar_user_event", ...ping },
+ });
+ }
+}
+
+const InfoBar = {
+ _activeInfobar: null,
+
+ maybeLoadCustomElement(win) {
+ if (!win.customElements.get("remote-text")) {
+ Services.scriptloader.loadSubScript(
+ "resource://activity-stream/data/custom-elements/paragraph.js",
+ win
+ );
+ }
+ },
+
+ maybeInsertFTL(win) {
+ win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl");
+ win.MozXULElement.insertFTLIfNeeded(
+ "browser/defaultBrowserNotification.ftl"
+ );
+ },
+
+ showInfoBarMessage(browser, message, dispatch) {
+ // Prevent stacking multiple infobars
+ if (this._activeInfobar) {
+ return null;
+ }
+
+ const win = browser.ownerGlobal;
+
+ if (lazy.PrivateBrowsingUtils.isWindowPrivate(win)) {
+ return null;
+ }
+
+ this.maybeLoadCustomElement(win);
+ this.maybeInsertFTL(win);
+
+ let notification = new InfoBarNotification(message, dispatch);
+ notification.showNotification(browser);
+ this._activeInfobar = true;
+
+ return notification;
+ },
+};
+
+const EXPORTED_SYMBOLS = ["InfoBar"];
diff --git a/browser/components/newtab/lib/LinksCache.jsm b/browser/components/newtab/lib/LinksCache.jsm
new file mode 100644
index 0000000000..c0a73cc18e
--- /dev/null
+++ b/browser/components/newtab/lib/LinksCache.jsm
@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const EXPORTED_SYMBOLS = ["LinksCache"];
+
+// This should be slightly less than SYSTEM_TICK_INTERVAL as timer
+// comparisons are too exact while the async/await functionality will make the
+// last recorded time a little bit later. This causes the comparasion to skip
+// updates.
+// It should be 10% less than SYSTEM_TICK to update at least once every 5 mins.
+// https://github.com/mozilla/activity-stream/pull/3695#discussion_r144678214
+const EXPIRATION_TIME = 4.5 * 60 * 1000; // 4.5 minutes
+
+/**
+ * Cache link results from a provided object property and refresh after some
+ * amount of time has passed. Allows for migrating data from previously cached
+ * links to the new links with the same url.
+ */
+class LinksCache {
+ /**
+ * Create a links cache for a given object property.
+ *
+ * @param {object} linkObject Object containing the link property
+ * @param {string} linkProperty Name of property on object to access
+ * @param {array} properties Optional properties list to migrate to new links.
+ * @param {function} shouldRefresh Optional callback receiving the old and new
+ * options to refresh even when not expired.
+ */
+ constructor(
+ linkObject,
+ linkProperty,
+ properties = [],
+ shouldRefresh = () => {}
+ ) {
+ this.clear();
+
+ // Allow getting links from both methods and array properties
+ this.linkGetter = options => {
+ const ret = linkObject[linkProperty];
+ return typeof ret === "function" ? ret.call(linkObject, options) : ret;
+ };
+
+ // Always migrate the shared cache data in addition to any custom properties
+ this.migrateProperties = ["__sharedCache", ...properties];
+ this.shouldRefresh = shouldRefresh;
+ }
+
+ /**
+ * Clear the cached data.
+ */
+ clear() {
+ this.cache = Promise.resolve([]);
+ this.lastOptions = {};
+ this.expire();
+ }
+
+ /**
+ * Force the next request to update the cache.
+ */
+ expire() {
+ delete this.lastUpdate;
+ }
+
+ /**
+ * Request data and update the cache if necessary.
+ *
+ * @param {object} options Optional data to pass to the underlying method.
+ * @returns {promise(array)} Links array with objects that can be modified.
+ */
+ async request(options = {}) {
+ // Update the cache if the data has been expired
+ const now = Date.now();
+ if (
+ this.lastUpdate === undefined ||
+ now > this.lastUpdate + EXPIRATION_TIME ||
+ // Allow custom rules around refreshing based on options
+ this.shouldRefresh(this.lastOptions, options)
+ ) {
+ // Update request state early so concurrent requests can refer to it
+ this.lastOptions = options;
+ this.lastUpdate = now;
+
+ // Save a promise before awaits, so other requests wait for correct data
+ // eslint-disable-next-line no-async-promise-executor
+ this.cache = new Promise(async (resolve, reject) => {
+ try {
+ // Allow fast lookup of old links by url that might need to migrate
+ const toMigrate = new Map();
+ for (const oldLink of await this.cache) {
+ if (oldLink) {
+ toMigrate.set(oldLink.url, oldLink);
+ }
+ }
+
+ // Update the cache with migrated links without modifying source objects
+ resolve(
+ (await this.linkGetter(options)).map(link => {
+ // Keep original array hole positions
+ if (!link) {
+ return link;
+ }
+
+ // Migrate data to the new link copy if we have an old link
+ const newLink = Object.assign({}, link);
+ const oldLink = toMigrate.get(newLink.url);
+ if (oldLink) {
+ for (const property of this.migrateProperties) {
+ const oldValue = oldLink[property];
+ if (oldValue !== undefined) {
+ newLink[property] = oldValue;
+ }
+ }
+ } else {
+ // Share data among link copies and new links from future requests
+ newLink.__sharedCache = {};
+ }
+ // Provide a helper to update the cached link
+ newLink.__sharedCache.updateLink = (property, value) => {
+ newLink[property] = value;
+ };
+
+ return newLink;
+ })
+ );
+ } catch (error) {
+ reject(error);
+ }
+ });
+ }
+
+ // Provide a shallow copy of the cached link objects for callers to modify
+ return (await this.cache).map(link => link && Object.assign({}, link));
+ }
+}
diff --git a/browser/components/newtab/lib/MomentsPageHub.jsm b/browser/components/newtab/lib/MomentsPageHub.jsm
new file mode 100644
index 0000000000..e37d1df8b1
--- /dev/null
+++ b/browser/components/newtab/lib/MomentsPageHub.jsm
@@ -0,0 +1,174 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ setInterval: "resource://gre/modules/Timer.sys.mjs",
+ clearInterval: "resource://gre/modules/Timer.sys.mjs",
+});
+
+// Frequency at which to check for new messages
+const SYSTEM_TICK_INTERVAL = 5 * 60 * 1000;
+const HOMEPAGE_OVERRIDE_PREF = "browser.startup.homepage_override.once";
+
+// For the "reach" event of Messaging Experiments
+const REACH_EVENT_CATEGORY = "messaging_experiments";
+const REACH_EVENT_METHOD = "reach";
+// Note it's not "moments-page" as Telemetry Events only accepts understores
+// for the event `object`
+const REACH_EVENT_OBJECT = "moments_page";
+
+class _MomentsPageHub {
+ constructor() {
+ this.id = "moments-page-hub";
+ this.state = {};
+ this.checkHomepageOverridePref = this.checkHomepageOverridePref.bind(this);
+ this._initialized = false;
+ }
+
+ async init(
+ waitForInitialized,
+ { handleMessageRequest, addImpression, blockMessageById, sendTelemetry }
+ ) {
+ if (this._initialized) {
+ return;
+ }
+
+ this._initialized = true;
+ this._handleMessageRequest = handleMessageRequest;
+ this._addImpression = addImpression;
+ this._blockMessageById = blockMessageById;
+ this._sendTelemetry = sendTelemetry;
+
+ // Need to wait for ASRouter to initialize before trying to fetch messages
+ await waitForInitialized;
+
+ this.messageRequest({
+ triggerId: "momentsUpdate",
+ template: "update_action",
+ });
+
+ const _intervalId = lazy.setInterval(
+ () => this.checkHomepageOverridePref(),
+ SYSTEM_TICK_INTERVAL
+ );
+ this.state = { _intervalId };
+ }
+
+ _sendPing(ping) {
+ this._sendTelemetry({
+ type: "MOMENTS_PAGE_TELEMETRY",
+ data: { action: "moments_user_event", ...ping },
+ });
+ }
+
+ sendUserEventTelemetry(message) {
+ this._sendPing({
+ message_id: message.id,
+ bucket_id: message.id,
+ event: "MOMENTS_PAGE_SET",
+ });
+ }
+
+ /**
+ * If we don't have `expire` defined with the message it could be because
+ * it depends on user dependent parameters. Since the message matched
+ * targeting we calculate `expire` based on the current timestamp and the
+ * `expireDelta` which defines for how long it should be available.
+ * @param expireDelta {number} - Offset in milliseconds from the current date
+ */
+ getExpirationDate(expireDelta) {
+ return Date.now() + expireDelta;
+ }
+
+ executeAction(message) {
+ const { id, data } = message.content.action;
+ switch (id) {
+ case "moments-wnp":
+ const { url, expireDelta } = data;
+ let { expire } = data;
+ if (!expire) {
+ expire = this.getExpirationDate(expireDelta);
+ }
+ // In order to reset this action we can dispatch a new message that
+ // will overwrite the prev value with an expiration date from the past.
+ Services.prefs.setStringPref(
+ HOMEPAGE_OVERRIDE_PREF,
+ JSON.stringify({ message_id: message.id, url, expire })
+ );
+ // Add impression and block immediately after taking the action
+ this.sendUserEventTelemetry(message);
+ this._addImpression(message);
+ this._blockMessageById(message.id);
+ break;
+ }
+ }
+
+ _recordReachEvent(message) {
+ const extra = { branches: message.branchSlug };
+ Services.telemetry.recordEvent(
+ REACH_EVENT_CATEGORY,
+ REACH_EVENT_METHOD,
+ REACH_EVENT_OBJECT,
+ message.experimentSlug,
+ extra
+ );
+ }
+
+ async messageRequest({ triggerId, template }) {
+ const telemetryObject = { triggerId };
+ TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ const messages = await this._handleMessageRequest({
+ triggerId,
+ template,
+ returnAll: true,
+ });
+ TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+
+ // Record the "reach" event for all the messages with `forReachEvent`,
+ // only execute action for the first message without forReachEvent.
+ const nonReachMessages = [];
+ for (const message of messages) {
+ if (message.forReachEvent) {
+ if (!message.forReachEvent.sent) {
+ this._recordReachEvent(message);
+ message.forReachEvent.sent = true;
+ }
+ } else {
+ nonReachMessages.push(message);
+ }
+ }
+ if (nonReachMessages.length) {
+ this.executeAction(nonReachMessages[0]);
+ }
+ }
+
+ /**
+ * Pref is set via Remote Settings message. We want to continously
+ * monitor new messages that come in to ensure the one with the
+ * highest priority is set.
+ */
+ checkHomepageOverridePref() {
+ this.messageRequest({
+ triggerId: "momentsUpdate",
+ template: "update_action",
+ });
+ }
+
+ uninit() {
+ lazy.clearInterval(this.state._intervalId);
+ this.state = {};
+ this._initialized = false;
+ }
+}
+
+/**
+ * ToolbarBadgeHub - singleton instance of _ToolbarBadgeHub that can initiate
+ * message requests and render messages.
+ */
+const MomentsPageHub = new _MomentsPageHub();
+
+const EXPORTED_SYMBOLS = ["_MomentsPageHub", "MomentsPageHub"];
diff --git a/browser/components/newtab/lib/NewTabInit.jsm b/browser/components/newtab/lib/NewTabInit.jsm
new file mode 100644
index 0000000000..f1a7f0d142
--- /dev/null
+++ b/browser/components/newtab/lib/NewTabInit.jsm
@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+/**
+ * NewTabInit - A placeholder for now. This will send a copy of the state to all
+ * newly opened tabs.
+ */
+class NewTabInit {
+ constructor() {
+ this._repliedEarlyTabs = new Map();
+ }
+
+ reply(target) {
+ // Skip this reply if we already replied to an early tab
+ if (this._repliedEarlyTabs.get(target)) {
+ return;
+ }
+
+ const action = {
+ type: at.NEW_TAB_INITIAL_STATE,
+ data: this.store.getState(),
+ };
+ this.store.dispatch(ac.AlsoToOneContent(action, target));
+
+ // Remember that this early tab has already gotten a rehydration response in
+ // case it thought we lost its initial REQUEST and asked again
+ if (this._repliedEarlyTabs.has(target)) {
+ this._repliedEarlyTabs.set(target, true);
+ }
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.NEW_TAB_STATE_REQUEST:
+ this.reply(action.meta.fromTarget);
+ break;
+ case at.NEW_TAB_INIT:
+ // Initialize data for early tabs that might REQUEST twice
+ if (action.data.simulated) {
+ this._repliedEarlyTabs.set(action.data.portID, false);
+ }
+ break;
+ case at.NEW_TAB_UNLOAD:
+ // Clean up for any tab (no-op if not an early tab)
+ this._repliedEarlyTabs.delete(action.meta.fromTarget);
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["NewTabInit"];
diff --git a/browser/components/newtab/lib/OnboardingMessageProvider.jsm b/browser/components/newtab/lib/OnboardingMessageProvider.jsm
new file mode 100644
index 0000000000..9438c8660c
--- /dev/null
+++ b/browser/components/newtab/lib/OnboardingMessageProvider.jsm
@@ -0,0 +1,1120 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+/* globals Localization */
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+const { FeatureCalloutMessages } = ChromeUtils.import(
+ "resource://activity-stream/lib/FeatureCalloutMessages.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ BrowserUtils: "resource://gre/modules/BrowserUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+ ShellService: "resource:///modules/ShellService.jsm",
+});
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "usesFirefoxSync",
+ "services.sync.username"
+);
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "mobileDevices",
+ "services.sync.clients.devices.mobile",
+ 0
+);
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "hidePrivatePin",
+ "browser.startup.upgradeDialog.pinPBM.disabled",
+ false
+);
+
+const L10N = new Localization([
+ "branding/brand.ftl",
+ "browser/branding/brandings.ftl",
+ "browser/branding/sync-brand.ftl",
+ "browser/newtab/onboarding.ftl",
+]);
+
+const HOMEPAGE_PREF = "browser.startup.homepage";
+const NEWTAB_PREF = "browser.newtabpage.enabled";
+
+const BASE_MESSAGES = () => [
+ {
+ id: "FXA_ACCOUNTS_BADGE",
+ template: "toolbar_badge",
+ content: {
+ delay: 10000, // delay for 10 seconds
+ target: "fxa-toolbar-menu-button",
+ },
+ targeting: "false",
+ trigger: { id: "toolbarBadgeUpdate" },
+ },
+ {
+ id: "PROTECTIONS_PANEL_1",
+ template: "protections_panel",
+ content: {
+ title: { string_id: "cfr-protections-panel-header" },
+ body: { string_id: "cfr-protections-panel-body" },
+ link_text: { string_id: "cfr-protections-panel-link-text" },
+ cta_url: `${Services.urlFormatter.formatURLPref(
+ "app.support.baseURL"
+ )}etp-promotions?as=u&utm_source=inproduct`,
+ cta_type: "OPEN_URL",
+ },
+ trigger: { id: "protectionsPanelOpen" },
+ },
+ {
+ id: "CFR_FIREFOX_VIEW",
+ groups: ["cfr"],
+ template: "cfr_doorhanger",
+ //If Firefox View button has been moved to the overflow menu, we want to change the anchor element
+ content: {
+ bucket_id: "CFR_FIREFOX_VIEW",
+ anchor_id: "firefox-view-button",
+ alt_anchor_id: "nav-bar-overflow-button",
+ layout: "icon_and_message",
+ icon: "chrome://browser/content/cfr-lightning.svg",
+ icon_dark_theme: "chrome://browser/content/cfr-lightning-dark.svg",
+ icon_class: "cfr-doorhanger-small-icon",
+ heading_text: {
+ string_id: "firefoxview-cfr-header-v2",
+ },
+ text: {
+ string_id: "firefoxview-cfr-body-v2",
+ },
+ buttons: {
+ primary: {
+ label: {
+ string_id: "firefoxview-cfr-primarybutton",
+ },
+ action: {
+ type: "OPEN_FIREFOX_VIEW",
+ navigate: true,
+ },
+ },
+ secondary: [
+ {
+ label: {
+ string_id: "firefoxview-cfr-secondarybutton",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ ],
+ },
+ skip_address_bar_notifier: true,
+ },
+ frequency: {
+ lifetime: 1,
+ },
+ trigger: {
+ id: "nthTabClosed",
+ },
+ // Avoid breaking existing tests that close tabs for now.
+ targeting: `!inMr2022Holdback && fxViewButtonAreaType != null && (currentDate|date - profileAgeCreated) / 86400000 >= 2 && tabsClosedCount >= 3 && 'browser.firefox-view.view-count'|preferenceValue == 0 && !'browser.newtabpage.activity-stream.asrouter.providers.cfr'|preferenceIsUserSet`,
+ },
+ {
+ id: "FX_MR_106_UPGRADE",
+ template: "spotlight",
+ targeting: "true",
+ content: {
+ template: "multistage",
+ id: "FX_MR_106_UPGRADE",
+ transitions: true,
+ modal: "tab",
+ screens: [
+ {
+ id: "UPGRADE_PIN_FIREFOX",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-155px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-pin-image-alt",
+ },
+ progress_bar: "true",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-pintaskbar.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-existing-pin-header",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-existing-pin-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id: "mr2022-onboarding-pin-primary-button-label",
+ },
+ action: {
+ navigate: true,
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ },
+ checkbox: {
+ label: {
+ string_id: "mr2022-onboarding-existing-pin-checkbox-label",
+ },
+ defaultValue: true,
+ action: {
+ type: "MULTI_ACTION",
+ navigate: true,
+ data: {
+ actions: [
+ {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ data: {
+ privatePin: true,
+ },
+ },
+ {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ ],
+ },
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-secondary-skip-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ has_arrow_icon: true,
+ },
+ },
+ },
+ {
+ id: "UPGRADE_SET_DEFAULT",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-60px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-default-image-alt",
+ },
+ progress_bar: "true",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-settodefault.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-set-default-title",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-set-default-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id: "mr2022-onboarding-set-default-primary-button-label",
+ },
+ action: {
+ navigate: true,
+ type: "SET_DEFAULT_BROWSER",
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-secondary-skip-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ has_arrow_icon: true,
+ },
+ },
+ },
+ {
+ id: "UPGRADE_IMPORT_SETTINGS",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-42px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-import-image-alt",
+ },
+ progress_bar: "true",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-import.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-import-header",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-import-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id:
+ "mr2022-onboarding-import-primary-button-label-no-attribution",
+ },
+ action: {
+ type: "SHOW_MIGRATION_WIZARD",
+ data: {},
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-secondary-skip-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ has_arrow_icon: true,
+ },
+ },
+ },
+ {
+ id: "UPGRADE_MOBILE_DOWNLOAD",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-160px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-mobile-download-image-alt",
+ },
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-mobilecrosspromo.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ progress_bar: true,
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-mobile-download-title",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-mobile-download-subtitle",
+ },
+ hero_image: {
+ url:
+ "chrome://activity-stream/content/data/content/assets/mobile-download-qr-existing-user.svg",
+ },
+ cta_paragraph: {
+ text: {
+ string_id: "mr2022-onboarding-mobile-download-cta-text",
+ string_name: "download-label",
+ },
+ action: {
+ type: "OPEN_URL",
+ data: {
+ args:
+ "https://www.mozilla.org/firefox/mobile/get-app/?utm_medium=firefox-desktop&utm_source=onboarding-modal&utm_campaign=mr2022&utm_content=existing-global",
+ where: "tab",
+ },
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-secondary-skip-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ has_arrow_icon: true,
+ },
+ },
+ },
+ {
+ id: "UPGRADE_PIN_PRIVATE_WINDOW",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-100px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-pin-private-image-alt",
+ },
+ progress_bar: "true",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-pinprivate.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-upgrade-onboarding-pin-private-window-header",
+ },
+ subtitle: {
+ string_id:
+ "mr2022-upgrade-onboarding-pin-private-window-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id:
+ "mr2022-upgrade-onboarding-pin-private-window-primary-button-label",
+ },
+ action: {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ data: {
+ privatePin: true,
+ },
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-secondary-skip-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ has_arrow_icon: true,
+ },
+ },
+ },
+ {
+ id: "UPGRADE_DATA_RECOMMENDATION",
+ content: {
+ position: "split",
+ split_narrow_bkg_position: "-80px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-privacy-segmentation-image-alt",
+ },
+ progress_bar: "true",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-privacysegmentation.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-privacy-segmentation-title",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-privacy-segmentation-subtitle",
+ },
+ cta_paragraph: {
+ text: {
+ string_id: "mr2022-onboarding-privacy-segmentation-text-cta",
+ },
+ },
+ primary_button: {
+ label: {
+ string_id:
+ "mr2022-onboarding-privacy-segmentation-button-primary-label",
+ },
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: "browser.dataFeatureRecommendations.enabled",
+ value: true,
+ },
+ },
+ navigate: true,
+ },
+ },
+ additional_button: {
+ label: {
+ string_id:
+ "mr2022-onboarding-privacy-segmentation-button-secondary-label",
+ },
+ style: "secondary",
+ action: {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: "browser.dataFeatureRecommendations.enabled",
+ value: false,
+ },
+ },
+ navigate: true,
+ },
+ },
+ },
+ },
+ {
+ id: "UPGRADE_GRATITUDE",
+ content: {
+ position: "split",
+ progress_bar: "true",
+ split_narrow_bkg_position: "-228px",
+ image_alt_text: {
+ string_id: "mr2022-onboarding-gratitude-image-alt",
+ },
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/mr-gratitude.svg') var(--mr-secondary-position) no-repeat var(--mr-screen-background-color)",
+ logo: {},
+ title: {
+ string_id: "mr2022-onboarding-gratitude-title",
+ },
+ subtitle: {
+ string_id: "mr2022-onboarding-gratitude-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id: "mr2022-onboarding-gratitude-primary-button-label",
+ },
+ action: {
+ type: "OPEN_FIREFOX_VIEW",
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr2022-onboarding-gratitude-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ {
+ id: "FX_100_UPGRADE",
+ template: "spotlight",
+ targeting: "false",
+ content: {
+ template: "multistage",
+ id: "FX_100_UPGRADE",
+ transitions: true,
+ screens: [
+ {
+ id: "UPGRADE_PIN_FIREFOX",
+ content: {
+ logo: {
+ imageURL:
+ "chrome://activity-stream/content/data/content/assets/heart.webp",
+ height: "73px",
+ },
+ has_noodles: true,
+ title: {
+ fontSize: "36px",
+ string_id: "fx100-upgrade-thanks-header",
+ },
+ title_style: "fancy shine",
+ background:
+ "url('chrome://activity-stream/content/data/content/assets/confetti.svg') top / 100% no-repeat var(--in-content-page-background)",
+ subtitle: {
+ string_id: "fx100-upgrade-thanks-keep-body",
+ },
+ primary_button: {
+ label: {
+ string_id: "fx100-thank-you-pin-primary-button-label",
+ },
+ action: {
+ navigate: true,
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr1-onboarding-set-default-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ {
+ id: "PB_NEWTAB_FOCUS_PROMO",
+ type: "default",
+ template: "pb_newtab",
+ groups: ["pbNewtab"],
+ content: {
+ infoBody: "fluent:about-private-browsing-info-description-simplified",
+ infoEnabled: true,
+ infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitle: "",
+ infoTitleEnabled: false,
+ promoEnabled: true,
+ promoType: "FOCUS",
+ promoHeader: "fluent:about-private-browsing-focus-promo-header-c",
+ promoImageLarge: "chrome://browser/content/assets/focus-promo.png",
+ promoLinkText: "fluent:about-private-browsing-focus-promo-cta",
+ promoLinkType: "button",
+ promoSectionStyle: "below-search",
+ promoTitle: "fluent:about-private-browsing-focus-promo-text-c",
+ promoTitleEnabled: true,
+ promoButton: {
+ action: {
+ type: "SHOW_SPOTLIGHT",
+ data: {
+ content: {
+ id: "FOCUS_PROMO",
+ template: "multistage",
+ modal: "tab",
+ backdrop: "transparent",
+ screens: [
+ {
+ id: "DEFAULT_MODAL_UI",
+ content: {
+ logo: {
+ imageURL:
+ "chrome://browser/content/assets/focus-logo.svg",
+ height: "48px",
+ },
+ title: {
+ string_id: "spotlight-focus-promo-title",
+ },
+ subtitle: {
+ string_id: "spotlight-focus-promo-subtitle",
+ },
+ dismiss_button: {
+ action: {
+ navigate: true,
+ },
+ },
+ ios: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fus%2Fapp%2Ffirefox-focus-privacy-browser%2Fid1055677337",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ android: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.focus",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ tiles: {
+ type: "mobile_downloads",
+ data: {
+ QR_code: {
+ image_url:
+ "chrome://browser/content/assets/focus-qr-code.svg",
+ alt_text: {
+ string_id: "spotlight-focus-promo-qr-code",
+ },
+ },
+ marketplace_buttons: ["ios", "android"],
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ },
+ },
+ },
+ priority: 2,
+ frequency: {
+ custom: [
+ {
+ cap: 3,
+ period: 604800000, // Max 3 per week
+ },
+ ],
+ lifetime: 12,
+ },
+ // Exclude the next 2 messages: 1) Klar for en 2) Klar for de
+ targeting:
+ "!(region in [ 'DE', 'AT', 'CH'] && localeLanguageCode == 'en') && localeLanguageCode != 'de'",
+ },
+ {
+ id: "PB_NEWTAB_KLAR_PROMO",
+ type: "default",
+ template: "pb_newtab",
+ groups: ["pbNewtab"],
+ content: {
+ infoBody: "fluent:about-private-browsing-info-description-simplified",
+ infoEnabled: true,
+ infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitle: "",
+ infoTitleEnabled: false,
+ promoEnabled: true,
+ promoType: "FOCUS",
+ promoHeader: "fluent:about-private-browsing-focus-promo-header-c",
+ promoImageLarge: "chrome://browser/content/assets/focus-promo.png",
+ promoLinkText: "Download Firefox Klar",
+ promoLinkType: "button",
+ promoSectionStyle: "below-search",
+ promoTitle:
+ "Firefox Klar clears your history every time while blocking ads and trackers.",
+ promoTitleEnabled: true,
+ promoButton: {
+ action: {
+ type: "SHOW_SPOTLIGHT",
+ data: {
+ content: {
+ id: "KLAR_PROMO",
+ template: "multistage",
+ modal: "tab",
+ backdrop: "transparent",
+ screens: [
+ {
+ id: "DEFAULT_MODAL_UI",
+ order: 0,
+ content: {
+ logo: {
+ imageURL:
+ "chrome://browser/content/assets/focus-logo.svg",
+ height: "48px",
+ },
+ title: "Get Firefox Klar",
+ subtitle: {
+ string_id: "spotlight-focus-promo-subtitle",
+ },
+ dismiss_button: {
+ action: {
+ navigate: true,
+ },
+ },
+ ios: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fde%2Fapp%2Fklar-by-firefox%2Fid1073435754",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ android: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.klar",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ tiles: {
+ type: "mobile_downloads",
+ data: {
+ QR_code: {
+ image_url:
+ "chrome://browser/content/assets/klar-qr-code.svg",
+ alt_text: "Scan the QR code to get Firefox Klar",
+ },
+ marketplace_buttons: ["ios", "android"],
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ },
+ },
+ },
+ priority: 2,
+ frequency: {
+ custom: [
+ {
+ cap: 3,
+ period: 604800000, // Max 3 per week
+ },
+ ],
+ lifetime: 12,
+ },
+ targeting: "region in [ 'DE', 'AT', 'CH'] && localeLanguageCode == 'en'",
+ },
+ {
+ id: "PB_NEWTAB_KLAR_PROMO_DE",
+ type: "default",
+ template: "pb_newtab",
+ groups: ["pbNewtab"],
+ content: {
+ infoBody: "fluent:about-private-browsing-info-description-simplified",
+ infoEnabled: true,
+ infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitle: "",
+ infoTitleEnabled: false,
+ promoEnabled: true,
+ promoType: "FOCUS",
+ promoHeader: "fluent:about-private-browsing-focus-promo-header-c",
+ promoImageLarge: "chrome://browser/content/assets/focus-promo.png",
+ promoLinkText: "fluent:about-private-browsing-focus-promo-cta",
+ promoLinkType: "button",
+ promoSectionStyle: "below-search",
+ promoTitle: "fluent:about-private-browsing-focus-promo-text-c",
+ promoTitleEnabled: true,
+ promoButton: {
+ action: {
+ type: "SHOW_SPOTLIGHT",
+ data: {
+ content: {
+ id: "FOCUS_PROMO",
+ template: "multistage",
+ modal: "tab",
+ backdrop: "transparent",
+ screens: [
+ {
+ id: "DEFAULT_MODAL_UI",
+ content: {
+ logo: {
+ imageURL:
+ "chrome://browser/content/assets/focus-logo.svg",
+ height: "48px",
+ },
+ title: {
+ string_id: "spotlight-focus-promo-title",
+ },
+ subtitle: {
+ string_id: "spotlight-focus-promo-subtitle",
+ },
+ dismiss_button: {
+ action: {
+ navigate: true,
+ },
+ },
+ ios: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fde%2Fapp%2Fklar-by-firefox%2Fid1073435754",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ android: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/a8bxj8j?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.klar",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ tiles: {
+ type: "mobile_downloads",
+ data: {
+ QR_code: {
+ image_url:
+ "chrome://browser/content/assets/klar-qr-code.svg",
+ alt_text: {
+ string_id: "spotlight-focus-promo-qr-code",
+ },
+ },
+ marketplace_buttons: ["ios", "android"],
+ },
+ },
+ },
+ },
+ ],
+ },
+ },
+ },
+ },
+ },
+ priority: 2,
+ frequency: {
+ custom: [
+ {
+ cap: 3,
+ period: 604800000, // Max 3 per week
+ },
+ ],
+ lifetime: 12,
+ },
+ targeting: "localeLanguageCode == 'de'",
+ },
+ {
+ id: "PB_NEWTAB_INFO_SECTION",
+ template: "pb_newtab",
+ content: {
+ promoEnabled: false,
+ infoEnabled: true,
+ infoIcon: "",
+ infoTitle: "",
+ infoBody: "fluent:about-private-browsing-info-description-private-window",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitleEnabled: false,
+ },
+ targeting: "true",
+ },
+ {
+ id: "PB_NEWTAB_PIN_PROMO",
+ template: "pb_newtab",
+ type: "default",
+ groups: ["pbNewtab"],
+ content: {
+ infoBody: "fluent:about-private-browsing-info-description-simplified",
+ infoEnabled: true,
+ infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitle: "",
+ infoTitleEnabled: false,
+ promoEnabled: true,
+ promoType: "PIN",
+ promoHeader: "fluent:about-private-browsing-pin-promo-header",
+ promoImageLarge:
+ "chrome://browser/content/assets/private-promo-asset.svg",
+ promoLinkText: "fluent:about-private-browsing-pin-promo-link-text",
+ promoLinkType: "button",
+ promoSectionStyle: "below-search",
+ promoTitle: "fluent:about-private-browsing-pin-promo-title",
+ promoTitleEnabled: true,
+ promoButton: {
+ action: {
+ type: "MULTI_ACTION",
+ data: {
+ actions: [
+ {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: "browser.privateWindowSeparation.enabled",
+ value: true,
+ },
+ },
+ },
+ {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ data: {
+ privatePin: true,
+ },
+ },
+ {
+ type: "BLOCK_MESSAGE",
+ data: {
+ id: "PB_NEWTAB_PIN_PROMO",
+ },
+ },
+ {
+ type: "OPEN_ABOUT_PAGE",
+ data: { args: "privatebrowsing", where: "current" },
+ },
+ ],
+ },
+ },
+ },
+ },
+ priority: 3,
+ frequency: {
+ custom: [
+ {
+ cap: 3,
+ period: 604800000, // Max 3 per week
+ },
+ ],
+ lifetime: 12,
+ },
+ targeting: "!inMr2022Holdback && doesAppNeedPrivatePin",
+ },
+];
+
+// Eventually, move Feature Callout messages to their own provider
+const ONBOARDING_MESSAGES = () =>
+ BASE_MESSAGES().concat(FeatureCalloutMessages.getMessages());
+
+const OnboardingMessageProvider = {
+ async getExtraAttributes() {
+ const [header, button_label] = await L10N.formatMessages([
+ { id: "onboarding-welcome-header" },
+ { id: "onboarding-start-browsing-button-label" },
+ ]);
+ return { header: header.value, button_label: button_label.value };
+ },
+ async getMessages() {
+ const messages = await this.translateMessages(await ONBOARDING_MESSAGES());
+ return messages;
+ },
+ async getUntranslatedMessages() {
+ // This is helpful for jsonSchema testing - since we are localizing in the provider
+ const messages = await ONBOARDING_MESSAGES();
+ return messages;
+ },
+ async translateMessages(messages) {
+ let translatedMessages = [];
+ for (const msg of messages) {
+ let translatedMessage = { ...msg };
+
+ // If the message has no content, do not attempt to translate it
+ if (!translatedMessage.content) {
+ translatedMessages.push(translatedMessage);
+ continue;
+ }
+
+ // Translate any secondary buttons separately
+ if (msg.content.secondary_button) {
+ const [secondary_button_string] = await L10N.formatMessages([
+ { id: msg.content.secondary_button.label.string_id },
+ ]);
+ translatedMessage.content.secondary_button.label =
+ secondary_button_string.value;
+ }
+ if (msg.content.header) {
+ const [header_string] = await L10N.formatMessages([
+ { id: msg.content.header.string_id },
+ ]);
+ translatedMessage.content.header = header_string.value;
+ }
+ translatedMessages.push(translatedMessage);
+ }
+ return translatedMessages;
+ },
+ async _doesAppNeedPin(privateBrowsing = false) {
+ const needPin = await lazy.ShellService.doesAppNeedPin(privateBrowsing);
+ return needPin;
+ },
+ async _doesAppNeedDefault() {
+ let checkDefault = Services.prefs.getBoolPref(
+ "browser.shell.checkDefaultBrowser",
+ false
+ );
+ let isDefault = await lazy.ShellService.isDefaultBrowser();
+ return checkDefault && !isDefault;
+ },
+ _shouldShowPrivacySegmentationScreen() {
+ // Fall back to pref: browser.privacySegmentation.preferences.show
+ return lazy.NimbusFeatures.majorRelease2022.getVariable(
+ "feltPrivacyShowPreferencesSection"
+ );
+ },
+ _doesHomepageNeedReset() {
+ return (
+ Services.prefs.prefHasUserValue(HOMEPAGE_PREF) ||
+ Services.prefs.prefHasUserValue(NEWTAB_PREF)
+ );
+ },
+
+ async getUpgradeMessage() {
+ let message = (await OnboardingMessageProvider.getMessages()).find(
+ ({ id }) => id === "FX_MR_106_UPGRADE"
+ );
+
+ let { content } = message;
+ // Helper to find screens and remove them where applicable.
+ function removeScreens(check) {
+ const { screens } = content;
+ for (let i = 0; i < screens?.length; i++) {
+ if (check(screens[i])) {
+ screens.splice(i--, 1);
+ }
+ }
+ }
+
+ // Helper to prepare mobile download screen content
+ function prepareMobileDownload() {
+ let mobileContent = content.screens.find(
+ screen => screen.id === "UPGRADE_MOBILE_DOWNLOAD"
+ )?.content;
+
+ if (!mobileContent) {
+ return;
+ }
+ if (!lazy.BrowserUtils.sendToDeviceEmailsSupported()) {
+ // If send to device emails are not supported for a user's locale,
+ // remove the send to device link and update the screen text
+ delete mobileContent.cta_paragraph.action;
+ mobileContent.cta_paragraph.text = {
+ string_id: "mr2022-onboarding-no-mobile-download-cta-text",
+ };
+ }
+ // Update CN specific QRCode url
+ if (AppConstants.isChinaRepack()) {
+ mobileContent.hero_image.url = `${mobileContent.hero_image.url.slice(
+ 0,
+ mobileContent.hero_image.url.indexOf(".svg")
+ )}-cn.svg`;
+ }
+ }
+
+ let pinScreen = content.screens?.find(
+ screen => screen.id === "UPGRADE_PIN_FIREFOX"
+ );
+ const needPin = await this._doesAppNeedPin();
+ const needDefault = await this._doesAppNeedDefault();
+ const needPrivatePin =
+ !lazy.hidePrivatePin && (await this._doesAppNeedPin(true));
+ const showSegmentation = this._shouldShowPrivacySegmentationScreen();
+
+ //If a user has Firefox as default remove import screen
+ if (!needDefault) {
+ removeScreens(screen => screen.id?.startsWith("UPGRADE_IMPORT_SETTINGS"));
+ }
+
+ // If already pinned, convert "pin" screen to "welcome" with desired action.
+ let removeDefault = !needDefault;
+ // If user doesn't need pin, update screen to set "default" or "get started" configuration
+ if (!needPin && pinScreen) {
+ // don't need to show the checkbox
+ delete pinScreen.content.checkbox;
+
+ removeDefault = true;
+ let primary = pinScreen.content.primary_button;
+ if (needDefault) {
+ pinScreen.id = "UPGRADE_ONLY_DEFAULT";
+ pinScreen.content.subtitle = {
+ string_id: "mr2022-onboarding-existing-set-default-only-subtitle",
+ };
+ primary.label.string_id =
+ "mr2022-onboarding-set-default-primary-button-label";
+
+ // The "pin" screen will now handle "default" so remove other "default."
+ primary.action.type = "SET_DEFAULT_BROWSER";
+ } else {
+ pinScreen.id = "UPGRADE_GET_STARTED";
+ pinScreen.content.subtitle = {
+ string_id: "mr2022-onboarding-get-started-primary-subtitle",
+ };
+ primary.label = {
+ string_id: "mr2022-onboarding-get-started-primary-button-label",
+ };
+ delete primary.action.type;
+ }
+ }
+
+ // If a user has Firefox private pinned remove pin private window screen
+ // We also remove standalone pin private window screen if a user doesn't have
+ // Firefox pinned in which case the option is shown as checkbox with UPGRADE_PIN_FIREFOX screen
+ if (!needPrivatePin || needPin) {
+ removeScreens(screen =>
+ screen.id?.startsWith("UPGRADE_PIN_PRIVATE_WINDOW")
+ );
+ }
+
+ if (!showSegmentation) {
+ removeScreens(screen =>
+ screen.id?.startsWith("UPGRADE_DATA_RECOMMENDATION")
+ );
+ }
+
+ //If privatePin, remove checkbox from pinscreen
+ if (!needPrivatePin) {
+ delete content.screens?.find(
+ screen => screen.id === "UPGRADE_PIN_FIREFOX"
+ )?.content?.checkbox;
+ }
+
+ if (removeDefault) {
+ removeScreens(screen => screen.id?.startsWith("UPGRADE_SET_DEFAULT"));
+ }
+
+ // Remove mobile download screen if user has sync enabled
+ if (lazy.usesFirefoxSync && lazy.mobileDevices > 0) {
+ removeScreens(screen => screen.id === "UPGRADE_MOBILE_DOWNLOAD");
+ } else {
+ prepareMobileDownload();
+ }
+
+ return message;
+ },
+};
+
+const EXPORTED_SYMBOLS = ["OnboardingMessageProvider"];
diff --git a/browser/components/newtab/lib/PageEventManager.jsm b/browser/components/newtab/lib/PageEventManager.jsm
new file mode 100644
index 0000000000..58dc076829
--- /dev/null
+++ b/browser/components/newtab/lib/PageEventManager.jsm
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/**
+ * Methods for setting up and tearing down page event listeners. These are used
+ * to dismiss Feature Callouts when the callout's anchor element is clicked.
+ */
+class PageEventManager {
+ /**
+ * A set of parameters defining a page event listener.
+ * @typedef {Object} PageEventListenerParams
+ * @property {String} type Event type string e.g. `click`
+ * @property {String} selectors Target selector, e.g. `tag.class, #id[attr]`
+ * @property {PageEventListenerOptions} [options] addEventListener options
+ *
+ * @typedef {Object} PageEventListenerOptions
+ * @property {Boolean} [capture] Use event capturing phase?
+ * @property {Boolean} [once] Remove listener after first event?
+ * @property {Boolean} [preventDefault] Inverted value for `passive` option
+ */
+
+ /**
+ * Maps event listener params to their abort controllers.
+ * @type {Map<PageEventListenerParams, AbortController>}
+ */
+ _listeners = new Map();
+
+ /**
+ * @param {Document} doc The document to look for event targets in
+ */
+ constructor(doc) {
+ this.doc = doc;
+ }
+
+ /**
+ * Adds a page event listener.
+ * @param {PageEventListenerParams} params
+ * @param {Function} callback Function to call when event is triggered
+ */
+ on(params, callback) {
+ if (this._listeners.has(params)) {
+ return;
+ }
+ const { type, selectors, options = {} } = params;
+ const controller = new AbortController();
+ const opt = {
+ capture: !!options.capture,
+ passive: !options.preventDefault,
+ signal: controller.signal,
+ };
+ const targets = this.doc.querySelectorAll(selectors);
+ for (const target of targets) {
+ target.addEventListener(type, callback, opt);
+ }
+ this._listeners.set(params, controller);
+ }
+
+ /**
+ * Removes a page event listener.
+ * @param {PageEventListenerParams} params
+ */
+ off(params) {
+ const controller = this._listeners.get(params);
+ if (!controller) {
+ return;
+ }
+ controller.abort();
+ this._listeners.delete(params);
+ }
+
+ /**
+ * Adds a page event listener that is removed after the first event.
+ * @param {PageEventListenerParams} params
+ * @param {Function} callback Function to call when event is triggered
+ */
+ once(params, callback) {
+ const wrappedCallback = (...args) => {
+ this.off(params);
+ callback(...args);
+ };
+ this.on(params, wrappedCallback);
+ }
+
+ /**
+ * Removes all page event listeners.
+ */
+ clear() {
+ for (const controller of this._listeners.values()) {
+ controller.abort();
+ }
+ this._listeners.clear();
+ }
+}
+
+const EXPORTED_SYMBOLS = ["PageEventManager"];
diff --git a/browser/components/newtab/lib/PanelTestProvider.jsm b/browser/components/newtab/lib/PanelTestProvider.jsm
new file mode 100644
index 0000000000..896f0995b2
--- /dev/null
+++ b/browser/components/newtab/lib/PanelTestProvider.jsm
@@ -0,0 +1,782 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const TWO_DAYS = 2 * 24 * 3600 * 1000;
+
+const MESSAGES = () => [
+ {
+ id: "WNP_THANK_YOU",
+ template: "update_action",
+ content: {
+ action: {
+ id: "moments-wnp",
+ data: {
+ url:
+ "https://www.mozilla.org/%LOCALE%/etc/firefox/retention/thank-you-a/",
+ expireDelta: TWO_DAYS,
+ },
+ },
+ },
+ trigger: { id: "momentsUpdate" },
+ },
+ {
+ id: "WHATS_NEW_FINGERPRINTER_COUNTER_ALT",
+ template: "whatsnew_panel_message",
+ order: 6,
+ content: {
+ bucket_id: "WHATS_NEW_72",
+ published_date: 1574776601000,
+ title: "Title",
+ icon_url:
+ "chrome://activity-stream/content/data/content/assets/protection-report-icon.png",
+ icon_alt: { string_id: "cfr-badge-reader-label-newfeature" },
+ body: "Message body",
+ link_text: "Click here",
+ cta_url: "about:blank",
+ cta_type: "OPEN_PROTECTION_REPORT",
+ },
+ targeting: `firefoxVersion >= 72`,
+ trigger: { id: "whatsNewPanelOpened" },
+ },
+ {
+ id: "WHATS_NEW_70_1",
+ template: "whatsnew_panel_message",
+ order: 3,
+ content: {
+ bucket_id: "WHATS_NEW_70_1",
+ published_date: 1560969794394,
+ title: "Protection Is Our Focus",
+ icon_url:
+ "chrome://activity-stream/content/data/content/assets/whatsnew-send-icon.png",
+ icon_alt: "Firefox Send Logo",
+ body:
+ "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.",
+ cta_url: "https://blog.mozilla.org/",
+ cta_type: "OPEN_URL",
+ },
+ targeting: `firefoxVersion > 69`,
+ trigger: { id: "whatsNewPanelOpened" },
+ },
+ {
+ id: "WHATS_NEW_70_2",
+ template: "whatsnew_panel_message",
+ order: 1,
+ content: {
+ bucket_id: "WHATS_NEW_70_1",
+ published_date: 1560969794394,
+ title: "Another thing new in Firefox 70",
+ body:
+ "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.",
+ link_text: "Learn more on our blog",
+ cta_url: "https://blog.mozilla.org/",
+ cta_type: "OPEN_URL",
+ },
+ targeting: `firefoxVersion > 69`,
+ trigger: { id: "whatsNewPanelOpened" },
+ },
+ {
+ id: "WHATS_NEW_SEARCH_SHORTCUTS_84",
+ template: "whatsnew_panel_message",
+ order: 2,
+ content: {
+ bucket_id: "WHATS_NEW_SEARCH_SHORTCUTS_84",
+ published_date: 1560969794394,
+ title: "Title",
+ icon_url: "chrome://global/skin/icons/check.svg",
+ icon_alt: "",
+ body: "Message content",
+ cta_url:
+ "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/search-shortcuts",
+ cta_type: "OPEN_URL",
+ link_text: "Click here",
+ },
+ targeting: "firefoxVersion >= 84",
+ trigger: {
+ id: "whatsNewPanelOpened",
+ },
+ },
+ {
+ id: "WHATS_NEW_PIONEER_82",
+ template: "whatsnew_panel_message",
+ order: 1,
+ content: {
+ bucket_id: "WHATS_NEW_PIONEER_82",
+ published_date: 1603152000000,
+ title: "Put your data to work for a better internet",
+ body:
+ "Contribute your data to Mozilla's Pioneer program to help researchers understand pressing technology issues like misinformation, data privacy, and ethical AI.",
+ cta_url: "about:blank",
+ cta_where: "tab",
+ cta_type: "OPEN_ABOUT_PAGE",
+ link_text: "Join Pioneer",
+ },
+ targeting: "firefoxVersion >= 82",
+ trigger: {
+ id: "whatsNewPanelOpened",
+ },
+ },
+ {
+ id: "WHATS_NEW_MEDIA_SESSION_82",
+ template: "whatsnew_panel_message",
+ order: 3,
+ content: {
+ bucket_id: "WHATS_NEW_MEDIA_SESSION_82",
+ published_date: 1603152000000,
+ title: "Title",
+ body: "Message content",
+ cta_url:
+ "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/media-keyboard-control",
+ cta_type: "OPEN_URL",
+ link_text: "Click here",
+ },
+ targeting: "firefoxVersion >= 82",
+ trigger: {
+ id: "whatsNewPanelOpened",
+ },
+ },
+ {
+ id: "WHATS_NEW_69_1",
+ template: "whatsnew_panel_message",
+ order: 1,
+ content: {
+ bucket_id: "WHATS_NEW_69_1",
+ published_date: 1557346235089,
+ title: "Something new in Firefox 69",
+ body:
+ "The New Enhanced Tracking Protection, gives you the best level of protection and performance. Discover how this version is the safest version of firefox ever made.",
+ link_text: "Learn more on our blog",
+ cta_url: "https://blog.mozilla.org/",
+ cta_type: "OPEN_URL",
+ },
+ targeting: `firefoxVersion > 68`,
+ trigger: { id: "whatsNewPanelOpened" },
+ },
+ {
+ id: "PERSONALIZED_CFR_MESSAGE",
+ template: "cfr_doorhanger",
+ groups: ["cfr"],
+ content: {
+ layout: "icon_and_message",
+ category: "cfrFeatures",
+ bucket_id: "PERSONALIZED_CFR_MESSAGE",
+ notification_text: "Personalized CFR Recommendation",
+ heading_text: { string_id: "cfr-doorhanger-bookmark-fxa-header" },
+ info_icon: {
+ label: {
+ attributes: {
+ tooltiptext: { string_id: "cfr-doorhanger-fxa-close-btn-tooltip" },
+ },
+ },
+ sumo_path: "https://example.com",
+ },
+ text: { string_id: "cfr-doorhanger-bookmark-fxa-body" },
+ icon: "chrome://branding/content/icon64.png",
+ icon_class: "cfr-doorhanger-large-icon",
+ persistent_doorhanger: true,
+ buttons: {
+ primary: {
+ label: { string_id: "cfr-doorhanger-milestone-ok-button" },
+ action: {
+ type: "OPEN_URL",
+ data: {
+ args:
+ "https://send.firefox.com/login/?utm_source=activity-stream&entrypoint=activity-stream-cfr-pdf",
+ where: "tabshifted",
+ },
+ },
+ },
+ secondary: [
+ {
+ label: { string_id: "cfr-doorhanger-extension-cancel-button" },
+ action: { type: "CANCEL" },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-never-show-recommendation",
+ },
+ },
+ {
+ label: {
+ string_id: "cfr-doorhanger-extension-manage-settings-button",
+ },
+ action: {
+ type: "OPEN_PREFERENCES_PAGE",
+ data: { category: "general-cfrfeatures" },
+ },
+ },
+ ],
+ },
+ },
+ targeting: "scores.PERSONALIZED_CFR_MESSAGE.score > scoreThreshold",
+ trigger: {
+ id: "openURL",
+ patterns: ["*://*/*.pdf"],
+ },
+ },
+ {
+ id: "SPOTLIGHT_MESSAGE_93",
+ template: "spotlight",
+ groups: ["panel-test-provider"],
+ content: {
+ template: "logo-and-content",
+ logo: {
+ imageURL: "chrome://browser/content/logos/vpn-promo-logo.svg",
+ },
+ body: {
+ title: {
+ label: {
+ string_id: "spotlight-public-wifi-vpn-header",
+ },
+ },
+ text: {
+ label: {
+ string_id: "spotlight-public-wifi-vpn-body",
+ },
+ },
+ primary: {
+ label: {
+ string_id: "spotlight-public-wifi-vpn-primary-button",
+ },
+ action: {
+ type: "OPEN_URL",
+ data: {
+ args: "https://www.mozilla.org/en-US/products/vpn/",
+ where: "tabshifted",
+ },
+ },
+ },
+ secondary: {
+ label: {
+ string_id: "spotlight-public-wifi-vpn-link",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ },
+ },
+ frequency: { lifetime: 3 },
+ trigger: { id: "defaultBrowserCheck" },
+ },
+ {
+ id: "BETTER_INTERNET_GLOBAL_ROLLOUT",
+ groups: ["eco"],
+ content: {
+ template: "logo-and-content",
+ logo: {
+ imageURL:
+ "chrome://activity-stream/content/data/content/assets/remote/mountain.svg",
+ size: "115px",
+ },
+ body: {
+ title: {
+ label: {
+ string_id: "spotlight-better-internet-header",
+ },
+ size: "22px",
+ },
+ text: {
+ label: {
+ string_id: "spotlight-better-internet-body",
+ },
+ size: "16px",
+ },
+ primary: {
+ label: {
+ string_id: "spotlight-pin-primary-button",
+ },
+ action: {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ },
+ secondary: {
+ label: {
+ string_id: "spotlight-pin-secondary-button",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ },
+ },
+ trigger: {
+ id: "defaultBrowserCheck",
+ },
+ template: "spotlight",
+ frequency: {
+ lifetime: 1,
+ },
+ targeting:
+ "userMonthlyActivity|length >= 1 && userMonthlyActivity|length <= 6 && doesAppNeedPin",
+ },
+ {
+ id: "PEACE_OF_MIND_GLOBAL_ROLLOUT",
+ groups: ["eco"],
+ content: {
+ template: "logo-and-content",
+ logo: {
+ imageURL:
+ "chrome://activity-stream/content/data/content/assets/remote/umbrella.png",
+ size: "115px",
+ },
+ body: {
+ title: {
+ label: {
+ string_id: "spotlight-peace-mind-header",
+ },
+ size: "22px",
+ },
+ text: {
+ label: {
+ string_id: "spotlight-peace-mind-body",
+ },
+ size: "15px",
+ },
+ primary: {
+ label: {
+ string_id: "spotlight-pin-primary-button",
+ },
+ action: {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ },
+ secondary: {
+ label: {
+ string_id: "spotlight-pin-secondary-button",
+ },
+ action: {
+ type: "CANCEL",
+ },
+ },
+ },
+ },
+ trigger: {
+ id: "defaultBrowserCheck",
+ },
+ template: "spotlight",
+ frequency: {
+ lifetime: 1,
+ },
+ targeting:
+ "userMonthlyActivity|length >= 7 && userMonthlyActivity|length <= 13 && doesAppNeedPin",
+ },
+ {
+ id: "MULTISTAGE_SPOTLIGHT_MESSAGE",
+ groups: ["panel-test-provider"],
+ template: "spotlight",
+ content: {
+ id: "control",
+ template: "multistage",
+ backdrop: "transparent",
+ transitions: true,
+ screens: [
+ {
+ id: "AW_PIN_FIREFOX",
+ content: {
+ has_noodles: true,
+ title: {
+ string_id: "mr1-onboarding-pin-header",
+ },
+ logo: {
+ imageURL: "chrome://browser/content/callout-tab-pickup.svg",
+ darkModeImageURL:
+ "chrome://browser/content/callout-tab-pickup-dark.svg",
+ reducedMotionImageURL:
+ "chrome://browser/content/callout-colorways.svg",
+ darkModeReducedMotionImageURL:
+ "chrome://browser/content/callout-colorways-dark.svg",
+ alt: "sample alt text",
+ },
+ hero_text: {
+ string_id: "mr1-welcome-screen-hero-text",
+ },
+ help_text: {
+ text: {
+ string_id: "mr1-onboarding-welcome-image-caption",
+ },
+ },
+ primary_button: {
+ label: {
+ string_id: "mr1-onboarding-pin-primary-button-label",
+ },
+ action: {
+ navigate: true,
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr1-onboarding-set-default-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ {
+ id: "AW_SET_DEFAULT",
+ content: {
+ has_noodles: true,
+ logo: {
+ imageURL: "chrome://browser/content/logos/vpn-promo-logo.svg",
+ height: "100px",
+ },
+ title: {
+ fontSize: "36px",
+ fontWeight: 276,
+ string_id: "mr1-onboarding-default-header",
+ },
+ subtitle: {
+ string_id: "mr1-onboarding-default-subtitle",
+ },
+ primary_button: {
+ label: {
+ string_id: "mr1-onboarding-default-primary-button-label",
+ },
+ action: {
+ navigate: true,
+ type: "SET_DEFAULT_BROWSER",
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr1-onboarding-set-default-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ {
+ id: "BACKGROUND_IMAGE",
+ content: {
+ background:
+ "url(chrome://activity-stream/content/data/content/assets/proton-bkg.avif) no-repeat center/cover",
+ text_color: "light",
+ progress_bar: true,
+ logo: {
+ imageURL:
+ "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/a3c640c8-7594-4bb2-bc18-8b4744f3aaf2.gif",
+ },
+ title: "A dialog with a background image",
+ subtitle:
+ "The text color is configurable and a progress bar style step indicator is used",
+ primary_button: {
+ label: "Continue",
+ action: {
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr1-onboarding-set-default-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ {
+ id: "BACKGROUND_COLOR",
+ content: {
+ background: "white",
+ progress_bar: true,
+ logo: {
+ height: "200px",
+ imageURL: "",
+ },
+ title: {
+ fontSize: "36px",
+ fontWeight: 276,
+ raw: "Peace of mind.",
+ },
+ title_style: "fancy shine",
+ text_color: "dark",
+ subtitle: "Using progress bar style step indicator",
+ primary_button: {
+ label: "Continue",
+ action: {
+ navigate: true,
+ },
+ },
+ secondary_button: {
+ label: {
+ string_id: "mr1-onboarding-set-default-secondary-button-label",
+ },
+ action: {
+ navigate: true,
+ },
+ },
+ },
+ },
+ ],
+ },
+ frequency: { lifetime: 3 },
+ trigger: { id: "defaultBrowserCheck" },
+ },
+ {
+ id: "PB_FOCUS_PROMO",
+ groups: ["panel-test-provider"],
+ template: "spotlight",
+ content: {
+ template: "multistage",
+ backdrop: "transparent",
+ screens: [
+ {
+ id: "PBM_FIREFOX_FOCUS",
+ order: 0,
+ content: {
+ logo: {
+ imageURL: "chrome://browser/content/assets/focus-logo.svg",
+ height: "48px",
+ },
+ title: {
+ string_id: "spotlight-focus-promo-title",
+ },
+ subtitle: {
+ string_id: "spotlight-focus-promo-subtitle",
+ },
+ dismiss_button: {
+ action: {
+ navigate: true,
+ },
+ },
+ ios: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fapps.apple.com%2Fus%2Fapp%2Ffirefox-focus-privacy-browser%2Fid1055677337",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ android: {
+ action: {
+ data: {
+ args:
+ "https://app.adjust.com/167k4ih?campaign=firefox-desktop&adgroup=pb&creative=focus-omc172&redirect=https%3A%2F%2Fplay.google.com%2Fstore%2Fapps%2Fdetails%3Fid%3Dorg.mozilla.focus",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ email_link: {
+ action: {
+ data: {
+ args: "https://mozilla.org",
+ where: "tabshifted",
+ },
+ type: "OPEN_URL",
+ navigate: true,
+ },
+ },
+ tiles: {
+ type: "mobile_downloads",
+ data: {
+ QR_code: {
+ image_url:
+ "chrome://browser/content/assets/focus-qr-code.svg",
+ alt_text: {
+ string_id: "spotlight-focus-promo-qr-code",
+ },
+ },
+ email: {
+ link_text: "Email yourself a link",
+ },
+ marketplace_buttons: ["ios", "android"],
+ },
+ },
+ },
+ },
+ ],
+ },
+ trigger: { id: "defaultBrowserCheck" },
+ },
+ {
+ id: "PB_NEWTAB_VPN_PROMO",
+ template: "pb_newtab",
+ content: {
+ promoEnabled: true,
+ promoType: "VPN",
+ infoEnabled: true,
+ infoBody: "fluent:about-private-browsing-info-description-private-window",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitleEnabled: false,
+ promoLinkType: "button",
+ promoLinkText: "fluent:about-private-browsing-prominent-cta",
+ promoSectionStyle: "below-search",
+ promoHeader: "fluent:about-private-browsing-get-privacy",
+ promoTitle: "fluent:about-private-browsing-hide-activity-1",
+ promoTitleEnabled: true,
+ promoImageLarge: "chrome://browser/content/assets/moz-vpn.svg",
+ promoButton: {
+ action: {
+ type: "OPEN_URL",
+ data: {
+ args: "https://vpn.mozilla.org/",
+ },
+ },
+ },
+ },
+ groups: ["panel-test-provider"],
+ targeting: "region != 'CN' && !hasActiveEnterprisePolicies",
+ frequency: { lifetime: 3 },
+ },
+ {
+ id: "PB_PIN_PROMO",
+ template: "pb_newtab",
+ groups: ["pbNewtab"],
+ content: {
+ infoBody: "fluent:about-private-browsing-info-description-simplified",
+ infoEnabled: true,
+ infoIcon: "chrome://global/skin/icons/indicator-private-browsing.svg",
+ infoLinkText: "fluent:about-private-browsing-learn-more-link",
+ infoTitle: "",
+ infoTitleEnabled: false,
+ promoEnabled: true,
+ promoType: "PIN",
+ promoHeader: "Private browsing freedom in one click",
+ promoImageLarge:
+ "chrome://browser/content/assets/private-promo-asset.svg",
+ promoLinkText: "Pin To Taskbar",
+ promoLinkType: "button",
+ promoSectionStyle: "below-search",
+ promoTitle:
+ "No saved cookies or history, right from your desktop. Browse like no one’s watching.",
+ promoTitleEnabled: true,
+ promoButton: {
+ action: {
+ type: "MULTI_ACTION",
+ data: {
+ actions: [
+ {
+ type: "SET_PREF",
+ data: {
+ pref: {
+ name: "browser.privateWindowSeparation.enabled",
+ value: true,
+ },
+ },
+ },
+ {
+ type: "PIN_FIREFOX_TO_TASKBAR",
+ },
+ {
+ type: "BLOCK_MESSAGE",
+ data: {
+ id: "PB_PIN_PROMO",
+ },
+ },
+ {
+ type: "OPEN_ABOUT_PAGE",
+ data: { args: "privatebrowsing", where: "current" },
+ },
+ ],
+ },
+ },
+ },
+ },
+ priority: 3,
+ frequency: {
+ custom: [
+ {
+ cap: 3,
+ period: 604800000, // Max 3 per week
+ },
+ ],
+ lifetime: 12,
+ },
+ targeting:
+ "region != 'CN' && !hasActiveEnterprisePolicies && doesAppNeedPin",
+ },
+ {
+ id: "TEST_TOAST_NOTIFICATION1",
+ weight: 100,
+ template: "toast_notification",
+ content: {
+ title: {
+ string_id: "cfr-doorhanger-bookmark-fxa-header",
+ },
+ body: "Body",
+ image_url:
+ "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/a3c640c8-7594-4bb2-bc18-8b4744f3aaf2.gif",
+ launch_url: "https://mozilla.org",
+ requireInteraction: true,
+ actions: [
+ {
+ action: "dismiss",
+ title: "Dismiss",
+ windowsSystemActivationType: true,
+ },
+ {
+ action: "snooze",
+ title: "Snooze",
+ windowsSystemActivationType: true,
+ },
+ { action: "callback", title: "Callback" },
+ ],
+ tag: "test_toast_notification",
+ },
+ groups: ["panel-test-provider"],
+ targeting: "!hasActiveEnterprisePolicies",
+ trigger: { id: "backgroundTaskMessage" },
+ frequency: { lifetime: 3 },
+ },
+ {
+ id: "MR2022_BACKGROUND_UPDATE_TOAST_NOTIFICATION",
+ weight: 100,
+ template: "toast_notification",
+ content: {
+ title: {
+ string_id: "mr2022-background-update-toast-title",
+ },
+ body: {
+ string_id: "mr2022-background-update-toast-text",
+ },
+ image_url:
+ "https://firefox-settings-attachments.cdn.mozilla.net/main-workspace/ms-images/673d2808-e5d8-41b9-957e-f60d53233b97.png",
+ requireInteraction: true,
+ actions: [
+ {
+ action: "open",
+ title: {
+ string_id: "mr2022-background-update-toast-primary-button-label",
+ },
+ },
+ {
+ action: "snooze",
+ windowsSystemActivationType: true,
+ title: {
+ string_id: "mr2022-background-update-toast-secondary-button-label",
+ },
+ },
+ ],
+ tag: "mr2022_background_update",
+ },
+ groups: ["panel-test-provider"],
+ targeting: "!hasActiveEnterprisePolicies",
+ trigger: { id: "backgroundTaskMessage" },
+ frequency: { lifetime: 3 },
+ },
+];
+
+const PanelTestProvider = {
+ getMessages() {
+ return Promise.resolve(
+ MESSAGES().map(message => ({
+ ...message,
+ targeting: `providerCohorts.panel_local_testing == "SHOW_TEST"`,
+ }))
+ );
+ },
+};
+
+const EXPORTED_SYMBOLS = ["PanelTestProvider"];
diff --git a/browser/components/newtab/lib/PersistentCache.jsm b/browser/components/newtab/lib/PersistentCache.jsm
new file mode 100644
index 0000000000..ac6ac5c73d
--- /dev/null
+++ b/browser/components/newtab/lib/PersistentCache.jsm
@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/**
+ * A file (disk) based persistent cache of a JSON serializable object.
+ */
+class PersistentCache {
+ /**
+ * Create a cache object based on a name.
+ *
+ * @param {string} name Name of the cache. It will be used to create the filename.
+ * @param {boolean} preload (optional). Whether the cache should be preloaded from file. Defaults to false.
+ */
+ constructor(name, preload = false) {
+ this.name = name;
+ this._filename = `activity-stream.${name}.json`;
+ if (preload) {
+ this._load();
+ }
+ }
+
+ /**
+ * Set a value to be cached with the specified key.
+ *
+ * @param {string} key The cache key.
+ * @param {object} value The data to be cached.
+ */
+ async set(key, value) {
+ const data = await this._load();
+ data[key] = value;
+ await this._persist(data);
+ }
+
+ /**
+ * Get a value from the cache.
+ *
+ * @param {string} key (optional) The cache key. If not provided, we return the full cache.
+ * @returns {object} The cached data.
+ */
+ async get(key) {
+ const data = await this._load();
+ return key ? data[key] : data;
+ }
+
+ /**
+ * Load the cache into memory if it isn't already.
+ */
+ _load() {
+ return (
+ this._cache ||
+ // eslint-disable-next-line no-async-promise-executor
+ (this._cache = new Promise(async (resolve, reject) => {
+ let filepath;
+ try {
+ filepath = PathUtils.join(PathUtils.localProfileDir, this._filename);
+ } catch (error) {
+ reject(error);
+ return;
+ }
+
+ let data = {};
+ try {
+ data = await IOUtils.readJSON(filepath);
+ } catch (error) {
+ if (
+ // isInstance() is not available in node unit test. It should be safe to use instanceof as it's directly from IOUtils.
+ // eslint-disable-next-line mozilla/use-isInstance
+ !(error instanceof DOMException) ||
+ error.name !== "NotFoundError"
+ ) {
+ console.error(
+ `Failed to parse ${this._filename}: ${error.message}`
+ );
+ }
+ }
+
+ resolve(data);
+ }))
+ );
+ }
+
+ /**
+ * Persist the cache to file.
+ */
+ async _persist(data) {
+ const filepath = PathUtils.join(PathUtils.localProfileDir, this._filename);
+ await IOUtils.writeJSON(filepath, data, {
+ tmpPath: `${filepath}.tmp`,
+ });
+ }
+}
+
+const EXPORTED_SYMBOLS = ["PersistentCache"];
diff --git a/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm
new file mode 100644
index 0000000000..cc625076ba
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// We load this into a worker using importScripts, and in tests using import.
+// We use var to avoid name collision errors.
+// eslint-disable-next-line no-var
+var EXPORTED_SYMBOLS = ["NaiveBayesTextTagger"];
+
+const NaiveBayesTextTagger = class NaiveBayesTextTagger {
+ constructor(model, toksToTfIdfVector) {
+ this.model = model;
+ this.toksToTfIdfVector = toksToTfIdfVector;
+ }
+
+ /**
+ * Determines if the tokenized text belongs to class according to binary naive Bayes
+ * classifier. Returns an object containing the class label ("label"), and
+ * the log probability ("logProb") that the text belongs to that class. If
+ * the positive class is more likely, then "label" is the positive class
+ * label. If the negative class is matched, then "label" is set to null.
+ */
+ tagTokens(tokens) {
+ let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs);
+
+ let bestLogProb = null;
+ let bestClassId = -1;
+ let bestClassLabel = null;
+ let logSumExp = 0.0; // will be P(x). Used to create a proper probability
+ for (let classId = 0; classId < this.model.classes.length; classId++) {
+ let classModel = this.model.classes[classId];
+ let classLogProb = classModel.log_prior;
+
+ // dot fv with the class model
+ for (let pair of Object.values(fv)) {
+ let [termId, tfidf] = pair;
+ classLogProb += tfidf * classModel.feature_log_probs[termId];
+ }
+
+ if (bestLogProb === null || classLogProb > bestLogProb) {
+ bestLogProb = classLogProb;
+ bestClassId = classId;
+ }
+ logSumExp += Math.exp(classLogProb);
+ }
+
+ // now normalize the probability by dividing by P(x)
+ logSumExp = Math.log(logSumExp);
+ bestLogProb -= logSumExp;
+ if (bestClassId === this.model.positive_class_id) {
+ bestClassLabel = this.model.positive_class_label;
+ } else {
+ bestClassLabel = null;
+ }
+
+ let confident =
+ bestClassId === this.model.positive_class_id &&
+ bestLogProb > this.model.positive_class_threshold_log_prob;
+ return {
+ label: bestClassLabel,
+ logProb: bestLogProb,
+ confident,
+ };
+ }
+};
diff --git a/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm
new file mode 100644
index 0000000000..639c92b6e4
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm
@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// We load this into a worker using importScripts, and in tests using import.
+// We use var to avoid name collision errors.
+// eslint-disable-next-line no-var
+var EXPORTED_SYMBOLS = ["NmfTextTagger"];
+
+const NmfTextTagger = class NmfTextTagger {
+ constructor(model, toksToTfIdfVector) {
+ this.model = model;
+ this.toksToTfIdfVector = toksToTfIdfVector;
+ }
+
+ /**
+ * A multiclass classifier that scores tokenized text against several classes through
+ * inference of a nonnegative matrix factorization of TF-IDF vectors and
+ * class labels. Returns a map of class labels as string keys to scores.
+ * (Higher is more confident.) All classes get scored, so it is up to
+ * consumer of this data determine what classes are most valuable.
+ */
+ tagTokens(tokens) {
+ let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs);
+ let fve = Object.values(fv);
+
+ // normalize by the sum of the vector
+ let sum = 0.0;
+ for (let pair of fve) {
+ // eslint-disable-next-line prefer-destructuring
+ sum += pair[1];
+ }
+ for (let i = 0; i < fve.length; i++) {
+ // eslint-disable-next-line prefer-destructuring
+ fve[i][1] /= sum;
+ }
+
+ // dot the document with each topic vector so that we can transform it into
+ // the latent space
+ let toksInLatentSpace = [];
+ for (let topicVect of this.model.topic_word) {
+ let fvDotTwv = 0;
+ // dot fv with each topic word vector
+ for (let pair of fve) {
+ let [termId, tfidf] = pair;
+ fvDotTwv += tfidf * topicVect[termId];
+ }
+ toksInLatentSpace.push(fvDotTwv);
+ }
+
+ // now project toksInLatentSpace back into class space
+ let predictions = {};
+ Object.keys(this.model.document_topic).forEach(topic => {
+ let score = 0;
+ for (let i = 0; i < toksInLatentSpace.length; i++) {
+ score += toksInLatentSpace[i] * this.model.document_topic[topic][i];
+ }
+ predictions[topic] = score;
+ });
+
+ return predictions;
+ }
+};
diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm
new file mode 100644
index 0000000000..5812666bc9
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm
@@ -0,0 +1,292 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "RemoteSettings",
+ "resource://services-settings/remote-settings.js"
+);
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Utils",
+ "resource://services-settings/Utils.jsm"
+);
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+});
+
+const { BasePromiseWorker } = ChromeUtils.import(
+ "resource://gre/modules/PromiseWorker.jsm"
+);
+
+const RECIPE_NAME = "personality-provider-recipe";
+const MODELS_NAME = "personality-provider-models";
+
+class PersonalityProvider {
+ constructor(modelKeys) {
+ this.modelKeys = modelKeys;
+ this.onSync = this.onSync.bind(this);
+ this.setup();
+ }
+
+ setScores(scores) {
+ this.scores = scores || {};
+ this.interestConfig = this.scores.interestConfig;
+ this.interestVector = this.scores.interestVector;
+ }
+
+ get personalityProviderWorker() {
+ if (this._personalityProviderWorker) {
+ return this._personalityProviderWorker;
+ }
+
+ this._personalityProviderWorker = new BasePromiseWorker(
+ "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorker.js"
+ );
+
+ return this._personalityProviderWorker;
+ }
+
+ get baseAttachmentsURL() {
+ // Returning a promise, so we can have an async getter.
+ return this._getBaseAttachmentsURL();
+ }
+
+ async _getBaseAttachmentsURL() {
+ if (this._baseAttachmentsURL) {
+ return this._baseAttachmentsURL;
+ }
+ const server = lazy.Utils.SERVER_URL;
+ const serverInfo = await (
+ await fetch(`${server}/`, {
+ credentials: "omit",
+ })
+ ).json();
+ const {
+ capabilities: {
+ attachments: { base_url },
+ },
+ } = serverInfo;
+ this._baseAttachmentsURL = base_url;
+ return this._baseAttachmentsURL;
+ }
+
+ setup() {
+ this.setupSyncAttachment(RECIPE_NAME);
+ this.setupSyncAttachment(MODELS_NAME);
+ }
+
+ teardown() {
+ this.teardownSyncAttachment(RECIPE_NAME);
+ this.teardownSyncAttachment(MODELS_NAME);
+ if (this._personalityProviderWorker) {
+ this._personalityProviderWorker.terminate();
+ }
+ }
+
+ setupSyncAttachment(collection) {
+ lazy.RemoteSettings(collection).on("sync", this.onSync);
+ }
+
+ teardownSyncAttachment(collection) {
+ lazy.RemoteSettings(collection).off("sync", this.onSync);
+ }
+
+ onSync(event) {
+ this.personalityProviderWorker.post("onSync", [event]);
+ }
+
+ /**
+ * Gets contents of the attachment if it already exists on file,
+ * and if not attempts to download it.
+ */
+ getAttachment(record) {
+ return this.personalityProviderWorker.post("getAttachment", [record]);
+ }
+
+ /**
+ * Returns a Recipe from remote settings to be consumed by a RecipeExecutor.
+ * A Recipe is a set of instructions on how to processes a RecipeExecutor.
+ */
+ async getRecipe() {
+ if (!this.recipes || !this.recipes.length) {
+ const result = await lazy.RemoteSettings(RECIPE_NAME).get();
+ this.recipes = await Promise.all(
+ result.map(async record => ({
+ ...(await this.getAttachment(record)),
+ recordKey: record.key,
+ }))
+ );
+ }
+ return this.recipes[0];
+ }
+
+ /**
+ * Grabs a slice of browse history for building a interest vector
+ */
+ async fetchHistory(columns, beginTimeSecs, endTimeSecs) {
+ let sql = `SELECT url, title, visit_count, frecency, last_visit_date, description
+ FROM moz_places
+ WHERE last_visit_date >= ${beginTimeSecs * 1000000}
+ AND last_visit_date < ${endTimeSecs * 1000000}`;
+ columns.forEach(requiredColumn => {
+ sql += ` AND IFNULL(${requiredColumn}, '') <> ''`;
+ });
+ sql += " LIMIT 30000";
+
+ const { activityStreamProvider } = lazy.NewTabUtils;
+ const history = await activityStreamProvider.executePlacesQuery(sql, {
+ columns,
+ params: {},
+ });
+
+ return history;
+ }
+
+ /**
+ * Handles setup and metrics of history fetch.
+ */
+ async getHistory() {
+ let endTimeSecs = new Date().getTime() / 1000;
+ let beginTimeSecs = endTimeSecs - this.interestConfig.history_limit_secs;
+ if (
+ !this.interestConfig ||
+ !this.interestConfig.history_required_fields ||
+ !this.interestConfig.history_required_fields.length
+ ) {
+ return [];
+ }
+ let history = await this.fetchHistory(
+ this.interestConfig.history_required_fields,
+ beginTimeSecs,
+ endTimeSecs
+ );
+
+ return history;
+ }
+
+ async setBaseAttachmentsURL() {
+ await this.personalityProviderWorker.post("setBaseAttachmentsURL", [
+ await this.baseAttachmentsURL,
+ ]);
+ }
+
+ async setInterestConfig() {
+ this.interestConfig = this.interestConfig || (await this.getRecipe());
+ await this.personalityProviderWorker.post("setInterestConfig", [
+ this.interestConfig,
+ ]);
+ }
+
+ async setInterestVector() {
+ await this.personalityProviderWorker.post("setInterestVector", [
+ this.interestVector,
+ ]);
+ }
+
+ async fetchModels() {
+ const models = await lazy.RemoteSettings(MODELS_NAME).get();
+ return this.personalityProviderWorker.post("fetchModels", [models]);
+ }
+
+ async generateTaggers() {
+ await this.personalityProviderWorker.post("generateTaggers", [
+ this.modelKeys,
+ ]);
+ }
+
+ async generateRecipeExecutor() {
+ await this.personalityProviderWorker.post("generateRecipeExecutor");
+ }
+
+ async createInterestVector() {
+ const history = await this.getHistory();
+
+ const interestVectorResult = await this.personalityProviderWorker.post(
+ "createInterestVector",
+ [history]
+ );
+
+ return interestVectorResult;
+ }
+
+ async init(callback) {
+ await this.setBaseAttachmentsURL();
+ await this.setInterestConfig();
+ if (!this.interestConfig) {
+ return;
+ }
+
+ // We always generate a recipe executor, no cache used here.
+ // This is because the result of this is an object with
+ // functions (taggers) so storing it in cache is not possible.
+ // Thus we cannot use it to rehydrate anything.
+ const fetchModelsResult = await this.fetchModels();
+ // If this fails, log an error and return.
+ if (!fetchModelsResult.ok) {
+ return;
+ }
+ await this.generateTaggers();
+ await this.generateRecipeExecutor();
+
+ // If we don't have a cached vector, create a new one.
+ if (!this.interestVector) {
+ const interestVectorResult = await this.createInterestVector();
+ // If that failed, log an error and return.
+ if (!interestVectorResult.ok) {
+ return;
+ }
+ this.interestVector = interestVectorResult.interestVector;
+ }
+
+ // This happens outside the createInterestVector call above,
+ // because create can be skipped if rehydrating from cache.
+ // In that case, the interest vector is provided and not created, so we just set it.
+ await this.setInterestVector();
+
+ this.initialized = true;
+ if (callback) {
+ callback();
+ }
+ }
+
+ async calculateItemRelevanceScore(pocketItem) {
+ if (!this.initialized) {
+ return pocketItem.item_score || 1;
+ }
+ const itemRelevanceScore = await this.personalityProviderWorker.post(
+ "calculateItemRelevanceScore",
+ [pocketItem]
+ );
+ if (!itemRelevanceScore) {
+ return -1;
+ }
+ const { scorableItem, rankingVector } = itemRelevanceScore;
+ // Put the results on the item for debugging purposes.
+ pocketItem.scorableItem = scorableItem;
+ pocketItem.rankingVector = rankingVector;
+ return rankingVector.score;
+ }
+
+ /**
+ * Returns an object holding the personalization scores of this provider instance.
+ */
+ getScores() {
+ return {
+ // We cannot return taggers here.
+ // What we return here goes into persistent cache, and taggers have functions on it.
+ // If we attempted to save taggers into persistent cache, it would store it to disk,
+ // and the next time we load it, it would start thowing function is not defined.
+ interestConfig: this.interestConfig,
+ interestVector: this.interestVector,
+ };
+ }
+}
+
+const EXPORTED_SYMBOLS = ["PersonalityProvider"];
diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js
new file mode 100644
index 0000000000..68bc97ee77
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorker.js
@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env mozilla/chrome-worker */
+
+"use strict";
+
+// Order of these are important.
+/* import-globals-from /toolkit/components/workerloader/require.js */
+/* import-globals-from Tokenize.jsm */
+/* import-globals-from NaiveBayesTextTagger.jsm */
+/* import-globals-from NmfTextTagger.jsm */
+/* import-globals-from RecipeExecutor.jsm */
+/* import-globals-from PersonalityProviderWorkerClass.jsm */
+importScripts(
+ "resource://gre/modules/workers/require.js",
+ "resource://activity-stream/lib/PersonalityProvider/Tokenize.jsm",
+ "resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.jsm",
+ "resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.jsm",
+ "resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.jsm",
+ "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm"
+);
+
+const PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js");
+
+const personalityProviderWorker = new PersonalityProviderWorker();
+
+// This is boiler plate worker stuff that connects it to the main thread PromiseWorker.
+const worker = new PromiseWorker.AbstractWorker();
+worker.dispatch = function(method, args = []) {
+ return personalityProviderWorker[method](...args);
+};
+worker.postMessage = function(message, ...transfers) {
+ self.postMessage(message, ...transfers);
+};
+worker.close = function() {
+ self.close();
+};
+
+self.addEventListener("message", msg => worker.handleMessage(msg));
+self.addEventListener("unhandledrejection", function(error) {
+ throw error.reason;
+});
diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm
new file mode 100644
index 0000000000..e761f827d2
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm
@@ -0,0 +1,311 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// PersonalityProviderWorker.js imports the following scripts before this.
+/* import-globals-from Tokenize.jsm */
+/* import-globals-from NaiveBayesTextTagger.jsm */
+/* import-globals-from NmfTextTagger.jsm */
+/* import-globals-from RecipeExecutor.jsm */
+
+// We load this into a worker using importScripts, and in tests using import.
+// We use var to avoid name collision errors.
+// eslint-disable-next-line no-var
+var EXPORTED_SYMBOLS = ["PersonalityProviderWorker"];
+
+// A helper function to create a hash out of a file.
+async function _getFileHash(filepath) {
+ const data = await IOUtils.read(filepath);
+ // File is an instance of Uint8Array
+ const digest = await crypto.subtle.digest("SHA-256", data);
+ const uint8 = new Uint8Array(digest);
+ // return the two-digit hexadecimal code for a byte
+ const toHex = b => b.toString(16).padStart(2, "0");
+ return Array.from(uint8, toHex).join("");
+}
+
+/**
+ * V2 provider builds and ranks an interest profile (also called an “interest vector”) off the browse history.
+ * This allows Firefox to classify pages into topics, by examining the text found on the page.
+ * It does this by looking at the history text content, title, and description.
+ */
+const PersonalityProviderWorker = class PersonalityProviderWorker {
+ async getPersonalityProviderDir() {
+ const personalityProviderDir = PathUtils.join(
+ await PathUtils.getLocalProfileDir(),
+ "personality-provider"
+ );
+
+ // Cache this so we don't need to await again.
+ this.getPersonalityProviderDir = () =>
+ Promise.resolve(personalityProviderDir);
+ return personalityProviderDir;
+ }
+
+ setBaseAttachmentsURL(url) {
+ this.baseAttachmentsURL = url;
+ }
+
+ setInterestConfig(interestConfig) {
+ this.interestConfig = interestConfig;
+ }
+
+ setInterestVector(interestVector) {
+ this.interestVector = interestVector;
+ }
+
+ onSync(event) {
+ const {
+ data: { created, updated, deleted },
+ } = event;
+ // Remove every removed attachment.
+ const toRemove = deleted.concat(updated.map(u => u.old));
+ toRemove.forEach(record => this.deleteAttachment(record));
+
+ // Download every new/updated attachment.
+ const toDownload = created.concat(updated.map(u => u.new));
+ // maybeDownloadAttachment is async but we don't care inside onSync.
+ toDownload.forEach(record => this.maybeDownloadAttachment(record));
+ }
+
+ /**
+ * Attempts to download the attachment, but only if it doesn't already exist.
+ */
+ async maybeDownloadAttachment(record, retries = 3) {
+ const {
+ attachment: { filename, hash, size },
+ } = record;
+ await IOUtils.makeDirectory(await this.getPersonalityProviderDir());
+ const localFilePath = PathUtils.join(
+ await this.getPersonalityProviderDir(),
+ filename
+ );
+
+ let retry = 0;
+ while (
+ retry++ < retries &&
+ // exists is an issue for perf because I might not need to call it.
+ (!(await IOUtils.exists(localFilePath)) ||
+ (await IOUtils.stat(localFilePath)).size !== size ||
+ (await _getFileHash(localFilePath)) !== hash)
+ ) {
+ await this._downloadAttachment(record);
+ }
+ }
+
+ /**
+ * Downloads the attachment to disk assuming the dir already exists
+ * and any existing files matching the filename are clobbered.
+ */
+ async _downloadAttachment(record) {
+ const {
+ attachment: { location, filename },
+ } = record;
+ const remoteFilePath = this.baseAttachmentsURL + location;
+ const localFilePath = PathUtils.join(
+ await this.getPersonalityProviderDir(),
+ filename
+ );
+
+ const xhr = new XMLHttpRequest();
+ // Set false here for a synchronous request, because we're in a worker.
+ xhr.open("GET", remoteFilePath, false);
+ xhr.setRequestHeader("Accept-Encoding", "gzip");
+ xhr.responseType = "arraybuffer";
+ xhr.withCredentials = false;
+ xhr.send(null);
+
+ if (xhr.status !== 200) {
+ console.error(`Failed to fetch ${remoteFilePath}: ${xhr.statusText}`);
+ return;
+ }
+
+ const buffer = xhr.response;
+ const bytes = new Uint8Array(buffer);
+
+ await IOUtils.write(localFilePath, bytes, {
+ tmpPath: `${localFilePath}.tmp`,
+ });
+ }
+
+ async deleteAttachment(record) {
+ const {
+ attachment: { filename },
+ } = record;
+ await IOUtils.makeDirectory(await this.getPersonalityProviderDir());
+ const path = PathUtils.join(
+ await this.getPersonalityProviderDir(),
+ filename
+ );
+
+ await IOUtils.remove(path, { ignoreAbsent: true });
+ // Cleanup the directory if it is empty, do nothing if it is not empty.
+ try {
+ await IOUtils.remove(await this.getPersonalityProviderDir(), {
+ ignoreAbsent: true,
+ });
+ } catch (e) {
+ // This is likely because the directory is not empty, so we don't care.
+ }
+ }
+
+ /**
+ * Gets contents of the attachment if it already exists on file,
+ * and if not attempts to download it.
+ */
+ async getAttachment(record) {
+ const {
+ attachment: { filename },
+ } = record;
+ const filepath = PathUtils.join(
+ await this.getPersonalityProviderDir(),
+ filename
+ );
+
+ try {
+ await this.maybeDownloadAttachment(record);
+ return await IOUtils.readJSON(filepath);
+ } catch (error) {
+ console.error(`Failed to load ${filepath}: ${error.message}`);
+ }
+ return {};
+ }
+
+ async fetchModels(models) {
+ this.models = await Promise.all(
+ models.map(async record => ({
+ ...(await this.getAttachment(record)),
+ recordKey: record.key,
+ }))
+ );
+ if (!this.models.length) {
+ return {
+ ok: false,
+ };
+ }
+ return {
+ ok: true,
+ };
+ }
+
+ generateTaggers(modelKeys) {
+ if (!this.taggers) {
+ let nbTaggers = [];
+ let nmfTaggers = {};
+
+ for (let model of this.models) {
+ if (!modelKeys.includes(model.recordKey)) {
+ continue;
+ }
+ if (model.model_type === "nb") {
+ nbTaggers.push(new NaiveBayesTextTagger(model, toksToTfIdfVector));
+ } else if (model.model_type === "nmf") {
+ nmfTaggers[model.parent_tag] = new NmfTextTagger(
+ model,
+ toksToTfIdfVector
+ );
+ }
+ }
+ this.taggers = { nbTaggers, nmfTaggers };
+ }
+ }
+
+ /**
+ * Sets and generates a Recipe Executor.
+ * A Recipe Executor is a set of actions that can be consumed by a Recipe.
+ * The Recipe determines the order and specifics of which the actions are called.
+ */
+ generateRecipeExecutor() {
+ const recipeExecutor = new RecipeExecutor(
+ this.taggers.nbTaggers,
+ this.taggers.nmfTaggers,
+ tokenize
+ );
+ this.recipeExecutor = recipeExecutor;
+ }
+
+ /**
+ * Examines the user's browse history and returns an interest vector that
+ * describes the topics the user frequently browses.
+ */
+ createInterestVector(history) {
+ let interestVector = {};
+
+ for (let historyRec of history) {
+ let ivItem = this.recipeExecutor.executeRecipe(
+ historyRec,
+ this.interestConfig.history_item_builder
+ );
+ if (ivItem === null) {
+ continue;
+ }
+ interestVector = this.recipeExecutor.executeCombinerRecipe(
+ interestVector,
+ ivItem,
+ this.interestConfig.interest_combiner
+ );
+ if (interestVector === null) {
+ return null;
+ }
+ }
+
+ const finalResult = this.recipeExecutor.executeRecipe(
+ interestVector,
+ this.interestConfig.interest_finalizer
+ );
+
+ return {
+ ok: true,
+ interestVector: finalResult,
+ };
+ }
+
+ /**
+ * Calculates a score of a Pocket item when compared to the user's interest
+ * vector. Returns the score. Higher scores are better. Assumes this.interestVector
+ * is populated.
+ */
+ calculateItemRelevanceScore(pocketItem) {
+ const { personalization_models } = pocketItem;
+ let scorableItem;
+
+ // If the server provides some models, we can just use them,
+ // and skip generating them.
+ if (personalization_models && Object.keys(personalization_models).length) {
+ scorableItem = {
+ id: pocketItem.id,
+ item_tags: personalization_models,
+ item_score: pocketItem.item_score,
+ item_sort_id: 1,
+ };
+ } else {
+ scorableItem = this.recipeExecutor.executeRecipe(
+ pocketItem,
+ this.interestConfig.item_to_rank_builder
+ );
+ if (scorableItem === null) {
+ return null;
+ }
+ }
+
+ // We're doing a deep copy on an object.
+ let rankingVector = JSON.parse(JSON.stringify(this.interestVector));
+
+ Object.keys(scorableItem).forEach(key => {
+ rankingVector[key] = scorableItem[key];
+ });
+
+ rankingVector = this.recipeExecutor.executeRecipe(
+ rankingVector,
+ this.interestConfig.item_ranker
+ );
+
+ if (rankingVector === null) {
+ return null;
+ }
+
+ return { scorableItem, rankingVector };
+ }
+};
diff --git a/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm
new file mode 100644
index 0000000000..9dbf8b802d
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm
@@ -0,0 +1,1126 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// We load this into a worker using importScripts, and in tests using import.
+// We use var to avoid name collision errors.
+// eslint-disable-next-line no-var
+var EXPORTED_SYMBOLS = ["RecipeExecutor"];
+
+/**
+ * RecipeExecutor is the core feature engineering pipeline for the in-browser
+ * personalization work. These pipelines are called "recipes". A recipe is an
+ * array of objects that define a "step" in the recipe. A step is simply an
+ * object with a field "function" that specifies what is being done in the step
+ * along with other fields that are semantically defined for that step.
+ *
+ * There are two types of recipes "builder" recipes and "combiner" recipes. Builder
+ * recipes mutate an object until it matches some set of critera. Combiner
+ * recipes take two objects, (a "left" and a "right"), and specify the steps
+ * to merge the right object into the left object.
+ *
+ * A short nonsense example recipe is:
+ * [ {"function": "get_url_domain", "path_length": 1, "field": "url", "dest": "url_domain"},
+ * {"function": "nb_tag", "fields": ["title", "description"]},
+ * {"function": "conditionally_nmf_tag", "fields": ["title", "description"]} ]
+ *
+ * Recipes are sandboxed by the fact that the step functions must be explicitly
+ * allowed. Functions allowed for builder recipes are specifed in the
+ * RecipeExecutor.ITEM_BUILDER_REGISTRY, while combiner functions are allowed
+ * in RecipeExecutor.ITEM_COMBINER_REGISTRY .
+ */
+const RecipeExecutor = class RecipeExecutor {
+ constructor(nbTaggers, nmfTaggers, tokenize) {
+ this.ITEM_BUILDER_REGISTRY = {
+ nb_tag: this.naiveBayesTag,
+ conditionally_nmf_tag: this.conditionallyNmfTag,
+ accept_item_by_field_value: this.acceptItemByFieldValue,
+ tokenize_url: this.tokenizeUrl,
+ get_url_domain: this.getUrlDomain,
+ tokenize_field: this.tokenizeField,
+ copy_value: this.copyValue,
+ keep_top_k: this.keepTopK,
+ scalar_multiply: this.scalarMultiply,
+ elementwise_multiply: this.elementwiseMultiply,
+ vector_multiply: this.vectorMultiply,
+ scalar_add: this.scalarAdd,
+ vector_add: this.vectorAdd,
+ make_boolean: this.makeBoolean,
+ allow_fields: this.allowFields,
+ filter_by_value: this.filterByValue,
+ l2_normalize: this.l2Normalize,
+ prob_normalize: this.probNormalize,
+ set_default: this.setDefault,
+ lookup_value: this.lookupValue,
+ copy_to_map: this.copyToMap,
+ scalar_multiply_tag: this.scalarMultiplyTag,
+ apply_softmax_tags: this.applySoftmaxTags,
+ };
+ this.ITEM_COMBINER_REGISTRY = {
+ combiner_add: this.combinerAdd,
+ combiner_max: this.combinerMax,
+ combiner_collect_values: this.combinerCollectValues,
+ };
+ this.nbTaggers = nbTaggers;
+ this.nmfTaggers = nmfTaggers;
+ this.tokenize = tokenize;
+ }
+
+ /**
+ * Determines the type of a field. Valid types are:
+ * string
+ * number
+ * array
+ * map (strings to anything)
+ */
+ _typeOf(data) {
+ let t = typeof data;
+ if (t === "object") {
+ if (data === null) {
+ return "null";
+ }
+ if (Array.isArray(data)) {
+ return "array";
+ }
+ return "map";
+ }
+ return t;
+ }
+
+ /**
+ * Returns a scalar, either because it was a constant, or by
+ * looking it up from the item. Allows for a default value if the lookup
+ * fails.
+ */
+ _lookupScalar(item, k, dfault) {
+ if (this._typeOf(k) === "number") {
+ return k;
+ } else if (
+ this._typeOf(k) === "string" &&
+ k in item &&
+ this._typeOf(item[k]) === "number"
+ ) {
+ return item[k];
+ }
+ return dfault;
+ }
+
+ /**
+ * Simply appends all the strings from a set fields together. If the field
+ * is a list, then the cells of the list are append.
+ */
+ _assembleText(item, fields) {
+ let textArr = [];
+ for (let field of fields) {
+ if (field in item) {
+ let type = this._typeOf(item[field]);
+ if (type === "string") {
+ textArr.push(item[field]);
+ } else if (type === "array") {
+ for (let ele of item[field]) {
+ textArr.push(String(ele));
+ }
+ } else {
+ textArr.push(String(item[field]));
+ }
+ }
+ }
+ return textArr.join(" ");
+ }
+
+ /**
+ * Runs the naive bayes text taggers over a set of text fields. Stores the
+ * results in new fields:
+ * nb_tags: a map of text strings to probabilites
+ * nb_tokens: the tokenized text that was tagged
+ *
+ * Config:
+ * fields: an array containing a list of fields to concatenate and tag
+ */
+ naiveBayesTag(item, config) {
+ let text = this._assembleText(item, config.fields);
+ let tokens = this.tokenize(text);
+ let tags = {};
+ let extended_tags = {};
+
+ for (let nbTagger of this.nbTaggers) {
+ let result = nbTagger.tagTokens(tokens);
+ if (result.label !== null && result.confident) {
+ extended_tags[result.label] = result;
+ tags[result.label] = Math.exp(result.logProb);
+ }
+ }
+ item.nb_tags = tags;
+ item.nb_tags_extended = extended_tags;
+ item.nb_tokens = tokens;
+ return item;
+ }
+
+ /**
+ * Selectively runs NMF text taggers depending on which tags were found
+ * by the naive bayes taggers. Writes the results in into new fields:
+ * nmf_tags_parent_weights: map of pareent tags to probabilites of those parent tags
+ * nmf_tags: map of strings to maps of strings to probabilities
+ * nmf_tags_parent map of child tags to parent tags
+ *
+ * Config:
+ * Not configurable
+ */
+ conditionallyNmfTag(item, config) {
+ let nestedNmfTags = {};
+ let parentTags = {};
+ let parentWeights = {};
+
+ if (!("nb_tags" in item) || !("nb_tokens" in item)) {
+ return null;
+ }
+
+ Object.keys(item.nb_tags).forEach(parentTag => {
+ let nmfTagger = this.nmfTaggers[parentTag];
+ if (nmfTagger !== undefined) {
+ nestedNmfTags[parentTag] = {};
+ parentWeights[parentTag] = item.nb_tags[parentTag];
+ let nmfTags = nmfTagger.tagTokens(item.nb_tokens);
+ Object.keys(nmfTags).forEach(nmfTag => {
+ nestedNmfTags[parentTag][nmfTag] = nmfTags[nmfTag];
+ parentTags[nmfTag] = parentTag;
+ });
+ }
+ });
+
+ item.nmf_tags = nestedNmfTags;
+ item.nmf_tags_parent = parentTags;
+ item.nmf_tags_parent_weights = parentWeights;
+
+ return item;
+ }
+
+ /**
+ * Checks a field's value against another value (either from another field
+ * or a constant). If the test passes, then the item is emitted, otherwise
+ * the pipeline is aborted.
+ *
+ * Config:
+ * field Field to read the value to test. Left side of operator.
+ * op one of ==, !=, <, <=, >, >=
+ * rhsValue Constant value to compare against. Right side of operator.
+ * rhsField Field to read value to compare against. Right side of operator.
+ *
+ * NOTE: rhsValue takes precidence over rhsField.
+ */
+ acceptItemByFieldValue(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let rhs = null;
+ if ("rhsValue" in config) {
+ rhs = config.rhsValue;
+ } else if ("rhsField" in config && config.rhsField in item) {
+ rhs = item[config.rhsField];
+ }
+ if (rhs === null) {
+ return null;
+ }
+
+ if (
+ // eslint-disable-next-line eqeqeq
+ (config.op === "==" && item[config.field] == rhs) ||
+ // eslint-disable-next-line eqeqeq
+ (config.op === "!=" && item[config.field] != rhs) ||
+ (config.op === "<" && item[config.field] < rhs) ||
+ (config.op === "<=" && item[config.field] <= rhs) ||
+ (config.op === ">" && item[config.field] > rhs) ||
+ (config.op === ">=" && item[config.field] >= rhs)
+ ) {
+ return item;
+ }
+
+ return null;
+ }
+
+ /**
+ * Splits a URL into text-like tokens.
+ *
+ * Config:
+ * field Field containing a URL
+ * dest Field to write the tokens to as an array of strings
+ *
+ * NOTE: Any initial 'www' on the hostname is removed.
+ */
+ tokenizeUrl(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+
+ let url = new URL(item[config.field]);
+ let domain = url.hostname;
+ if (domain.startsWith("www.")) {
+ domain = domain.substring(4);
+ }
+ let toks = this.tokenize(domain);
+ let pathToks = this.tokenize(
+ decodeURIComponent(url.pathname.replace(/\+/g, " "))
+ );
+ for (let tok of pathToks) {
+ toks.push(tok);
+ }
+ for (let pair of url.searchParams.entries()) {
+ let k = this.tokenize(decodeURIComponent(pair[0].replace(/\+/g, " ")));
+ for (let tok of k) {
+ toks.push(tok);
+ }
+ if (pair[1] !== null && pair[1] !== "") {
+ let v = this.tokenize(decodeURIComponent(pair[1].replace(/\+/g, " ")));
+ for (let tok of v) {
+ toks.push(tok);
+ }
+ }
+ }
+ item[config.dest] = toks;
+
+ return item;
+ }
+
+ /**
+ * Gets the hostname (minus any initial "www." along with the left most
+ * directories on the path.
+ *
+ * Config:
+ * field Field containing the URL
+ * dest Field to write the array of strings to
+ * path_length OPTIONAL (DEFAULT: 0) Number of leftmost subdirectories to include
+ */
+ getUrlDomain(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+
+ let url = new URL(item[config.field]);
+ let domain = url.hostname.toLocaleLowerCase();
+ if (domain.startsWith("www.")) {
+ domain = domain.substring(4);
+ }
+ item[config.dest] = domain;
+ let pathLength = 0;
+ if ("path_length" in config) {
+ pathLength = config.path_length;
+ }
+ if (pathLength > 0) {
+ item[config.dest] += url.pathname
+ .toLocaleLowerCase()
+ .split("/")
+ .slice(0, pathLength + 1)
+ .join("/");
+ }
+
+ return item;
+ }
+
+ /**
+ * Splits a field into tokens.
+ * Config:
+ * field Field containing a string to tokenize
+ * dest Field to write the array of strings to
+ */
+ tokenizeField(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+
+ item[config.dest] = this.tokenize(item[config.field]);
+
+ return item;
+ }
+
+ /**
+ * Deep copy from one field to another.
+ * Config:
+ * src Field to read from
+ * dest Field to write to
+ */
+ copyValue(item, config) {
+ if (!(config.src in item)) {
+ return null;
+ }
+
+ item[config.dest] = JSON.parse(JSON.stringify(item[config.src]));
+
+ return item;
+ }
+
+ /**
+ * Converts a field containing a map of strings to a map of strings
+ * to numbers, to a map of strings to numbers containing at most k elements.
+ * This operation is performed by first, promoting all the subkeys up one
+ * level, and then taking the top (or bottom) k values.
+ *
+ * Config:
+ * field Points to a map of strings to a map of strings to numbers
+ * k Maximum number of items to keep
+ * descending OPTIONAL (DEFAULT: True) Sorts score in descending order
+ * (i.e. keeps maximum)
+ */
+ keepTopK(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let k = this._lookupScalar(item, config.k, 1048576);
+ let descending = !("descending" in config) || config.descending !== false;
+
+ // we can't sort by the values in the map, so we have to convert this
+ // to an array, and then sort.
+ let sortable = [];
+ Object.keys(item[config.field]).forEach(outerKey => {
+ let innerType = this._typeOf(item[config.field][outerKey]);
+ if (innerType === "map") {
+ Object.keys(item[config.field][outerKey]).forEach(innerKey => {
+ sortable.push({
+ key: innerKey,
+ value: item[config.field][outerKey][innerKey],
+ });
+ });
+ } else {
+ sortable.push({ key: outerKey, value: item[config.field][outerKey] });
+ }
+ });
+
+ sortable.sort((a, b) => {
+ if (descending) {
+ return b.value - a.value;
+ }
+ return a.value - b.value;
+ });
+
+ // now take the top k
+ let newMap = {};
+ let i = 0;
+ for (let pair of sortable) {
+ if (i >= k) {
+ break;
+ }
+ newMap[pair.key] = pair.value;
+ i++;
+ }
+ item[config.field] = newMap;
+
+ return item;
+ }
+
+ /**
+ * Scalar multiplies a vector by some constant
+ *
+ * Config:
+ * field Points to:
+ * a map of strings to numbers
+ * an array of numbers
+ * a number
+ * k Either a number, or a string. If it's a number then This
+ * is the scalar value to multiply by. If it's a string,
+ * the value in the pointed to field is used.
+ * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric
+ * value is found, then use this value.
+ */
+ scalarMultiply(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let k = this._lookupScalar(item, config.k, config.dfault);
+
+ let fieldType = this._typeOf(item[config.field]);
+ if (fieldType === "number") {
+ item[config.field] *= k;
+ } else if (fieldType === "array") {
+ for (let i = 0; i < item[config.field].length; i++) {
+ item[config.field][i] *= k;
+ }
+ } else if (fieldType === "map") {
+ Object.keys(item[config.field]).forEach(key => {
+ item[config.field][key] *= k;
+ });
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Elementwise multiplies either two maps or two arrays together, storing
+ * the result in left. If left and right are of the same type, results in an
+ * error.
+ *
+ * Maps are special case. For maps the left must be a nested map such as:
+ * { k1: { k11: 1, k12: 2}, k2: { k21: 3, k22: 4 } } and right needs to be
+ * simple map such as: { k1: 5, k2: 6} . The operation is then to mulitply
+ * every value of every right key, to every value every subkey where the
+ * parent keys match. Using the previous examples, the result would be:
+ * { k1: { k11: 5, k12: 10 }, k2: { k21: 18, k22: 24 } } .
+ *
+ * Config:
+ * left
+ * right
+ */
+ elementwiseMultiply(item, config) {
+ if (!(config.left in item) || !(config.right in item)) {
+ return null;
+ }
+ let leftType = this._typeOf(item[config.left]);
+ if (leftType !== this._typeOf(item[config.right])) {
+ return null;
+ }
+ if (leftType === "array") {
+ if (item[config.left].length !== item[config.right].length) {
+ return null;
+ }
+ for (let i = 0; i < item[config.left].length; i++) {
+ item[config.left][i] *= item[config.right][i];
+ }
+ } else if (leftType === "map") {
+ Object.keys(item[config.left]).forEach(outerKey => {
+ let r = 0.0;
+ if (outerKey in item[config.right]) {
+ r = item[config.right][outerKey];
+ }
+ Object.keys(item[config.left][outerKey]).forEach(innerKey => {
+ item[config.left][outerKey][innerKey] *= r;
+ });
+ });
+ } else if (leftType === "number") {
+ item[config.left] *= item[config.right];
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Vector multiplies (i.e. dot products) two vectors and stores the result in
+ * third field. Both vectors must either by maps, or arrays of numbers with
+ * the same length.
+ *
+ * Config:
+ * left A field pointing to either a map of strings to numbers,
+ * or an array of numbers
+ * right A field pointing to either a map of strings to numbers,
+ * or an array of numbers
+ * dest The field to store the dot product.
+ */
+ vectorMultiply(item, config) {
+ if (!(config.left in item) || !(config.right in item)) {
+ return null;
+ }
+
+ let leftType = this._typeOf(item[config.left]);
+ if (leftType !== this._typeOf(item[config.right])) {
+ return null;
+ }
+
+ let destVal = 0.0;
+ if (leftType === "array") {
+ if (item[config.left].length !== item[config.right].length) {
+ return null;
+ }
+ for (let i = 0; i < item[config.left].length; i++) {
+ destVal += item[config.left][i] * item[config.right][i];
+ }
+ } else if (leftType === "map") {
+ Object.keys(item[config.left]).forEach(key => {
+ if (key in item[config.right]) {
+ destVal += item[config.left][key] * item[config.right][key];
+ }
+ });
+ } else {
+ return null;
+ }
+
+ item[config.dest] = destVal;
+ return item;
+ }
+
+ /**
+ * Adds a constant value to all elements in the field. Mathematically,
+ * this is the same as taking a 1-vector, scalar multiplying it by k,
+ * and then vector adding it to a field.
+ *
+ * Config:
+ * field A field pointing to either a map of strings to numbers,
+ * or an array of numbers
+ * k Either a number, or a string. If it's a number then This
+ * is the scalar value to multiply by. If it's a string,
+ * the value in the pointed to field is used.
+ * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric
+ * value is found, then use this value.
+ */
+ scalarAdd(item, config) {
+ let k = this._lookupScalar(item, config.k, config.dfault);
+ if (!(config.field in item)) {
+ return null;
+ }
+
+ let fieldType = this._typeOf(item[config.field]);
+ if (fieldType === "array") {
+ for (let i = 0; i < item[config.field].length; i++) {
+ item[config.field][i] += k;
+ }
+ } else if (fieldType === "map") {
+ Object.keys(item[config.field]).forEach(key => {
+ item[config.field][key] += k;
+ });
+ } else if (fieldType === "number") {
+ item[config.field] += k;
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Adds two vectors together and stores the result in left.
+ *
+ * Config:
+ * left A field pointing to either a map of strings to numbers,
+ * or an array of numbers
+ * right A field pointing to either a map of strings to numbers,
+ * or an array of numbers
+ */
+ vectorAdd(item, config) {
+ if (!(config.left in item)) {
+ return this.copyValue(item, { src: config.right, dest: config.left });
+ }
+ if (!(config.right in item)) {
+ return null;
+ }
+
+ let leftType = this._typeOf(item[config.left]);
+ if (leftType !== this._typeOf(item[config.right])) {
+ return null;
+ }
+ if (leftType === "array") {
+ if (item[config.left].length !== item[config.right].length) {
+ return null;
+ }
+ for (let i = 0; i < item[config.left].length; i++) {
+ item[config.left][i] += item[config.right][i];
+ }
+ return item;
+ } else if (leftType === "map") {
+ Object.keys(item[config.right]).forEach(key => {
+ let v = 0;
+ if (key in item[config.left]) {
+ v = item[config.left][key];
+ }
+ item[config.left][key] = v + item[config.right][key];
+ });
+ return item;
+ }
+
+ return null;
+ }
+
+ /**
+ * Converts a vector from real values to boolean integers. (i.e. either 1/0
+ * or 1/-1).
+ *
+ * Config:
+ * field Field containing either a map of strings to numbers or
+ * an array of numbers to convert.
+ * threshold OPTIONAL (DEFAULT: 0) Values above this will be replaced
+ * with 1.0. Those below will be converted to 0.
+ * keep_negative OPTIONAL (DEFAULT: False) If true, values below the
+ * threshold will be converted to -1 instead of 0.
+ */
+ makeBoolean(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let threshold = this._lookupScalar(item, config.threshold, 0.0);
+ let type = this._typeOf(item[config.field]);
+ if (type === "array") {
+ for (let i = 0; i < item[config.field].length; i++) {
+ if (item[config.field][i] > threshold) {
+ item[config.field][i] = 1.0;
+ } else if (config.keep_negative) {
+ item[config.field][i] = -1.0;
+ } else {
+ item[config.field][i] = 0.0;
+ }
+ }
+ } else if (type === "map") {
+ Object.keys(item[config.field]).forEach(key => {
+ let value = item[config.field][key];
+ if (value > threshold) {
+ item[config.field][key] = 1.0;
+ } else if (config.keep_negative) {
+ item[config.field][key] = -1.0;
+ } else {
+ item[config.field][key] = 0.0;
+ }
+ });
+ } else if (type === "number") {
+ let value = item[config.field];
+ if (value > threshold) {
+ item[config.field] = 1.0;
+ } else if (config.keep_negative) {
+ item[config.field] = -1.0;
+ } else {
+ item[config.field] = 0.0;
+ }
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Removes all keys from the item except for the ones specified.
+ *
+ * fields An array of strings indicating the fields to keep
+ */
+ allowFields(item, config) {
+ let newItem = {};
+ for (let ele of config.fields) {
+ if (ele in item) {
+ newItem[ele] = item[ele];
+ }
+ }
+ return newItem;
+ }
+
+ /**
+ * Removes all keys whose value does not exceed some threshold.
+ *
+ * Config:
+ * field Points to a map of strings to numbers
+ * threshold Values must exceed this value, otherwise they are removed.
+ */
+ filterByValue(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let threshold = this._lookupScalar(item, config.threshold, 0.0);
+ let filtered = {};
+ Object.keys(item[config.field]).forEach(key => {
+ let value = item[config.field][key];
+ if (value > threshold) {
+ filtered[key] = value;
+ }
+ });
+ item[config.field] = filtered;
+
+ return item;
+ }
+
+ /**
+ * Rewrites a field so that its values are now L2 normed.
+ *
+ * Config:
+ * field Points to a map of strings to numbers, or an array of numbers
+ */
+ l2Normalize(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let data = item[config.field];
+ let type = this._typeOf(data);
+ if (type === "array") {
+ let norm = 0.0;
+ for (let datum of data) {
+ norm += datum * datum;
+ }
+ norm = Math.sqrt(norm);
+ if (norm !== 0) {
+ for (let i = 0; i < data.length; i++) {
+ data[i] /= norm;
+ }
+ }
+ } else if (type === "map") {
+ let norm = 0.0;
+ Object.keys(data).forEach(key => {
+ norm += data[key] * data[key];
+ });
+ norm = Math.sqrt(norm);
+ if (norm !== 0) {
+ Object.keys(data).forEach(key => {
+ data[key] /= norm;
+ });
+ }
+ } else {
+ return null;
+ }
+
+ item[config.field] = data;
+
+ return item;
+ }
+
+ /**
+ * Rewrites a field so that all of its values sum to 1.0
+ *
+ * Config:
+ * field Points to a map of strings to numbers, or an array of numbers
+ */
+ probNormalize(item, config) {
+ if (!(config.field in item)) {
+ return null;
+ }
+ let data = item[config.field];
+ let type = this._typeOf(data);
+ if (type === "array") {
+ let norm = 0.0;
+ for (let datum of data) {
+ norm += datum;
+ }
+ if (norm !== 0) {
+ for (let i = 0; i < data.length; i++) {
+ data[i] /= norm;
+ }
+ }
+ } else if (type === "map") {
+ let norm = 0.0;
+ Object.keys(item[config.field]).forEach(key => {
+ norm += item[config.field][key];
+ });
+ if (norm !== 0) {
+ Object.keys(item[config.field]).forEach(key => {
+ item[config.field][key] /= norm;
+ });
+ }
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Stores a value, if it is not already present
+ *
+ * Config:
+ * field field to write to if it is missing
+ * value value to store in that field
+ */
+ setDefault(item, config) {
+ let val = this._lookupScalar(item, config.value, config.value);
+ if (!(config.field in item)) {
+ item[config.field] = val;
+ }
+
+ return item;
+ }
+
+ /**
+ * Selctively promotes an value from an inner map up to the outer map
+ *
+ * Config:
+ * haystack Points to a map of strings to values
+ * needle Key inside the map we should promote up
+ * dest Where we should write the value of haystack[needle]
+ */
+ lookupValue(item, config) {
+ if (config.haystack in item && config.needle in item[config.haystack]) {
+ item[config.dest] = item[config.haystack][config.needle];
+ }
+
+ return item;
+ }
+
+ /**
+ * Demotes a field into a map
+ *
+ * Config:
+ * src Field to copy
+ * dest_map Points to a map
+ * dest_key Key inside dest_map to copy src to
+ */
+ copyToMap(item, config) {
+ if (config.src in item) {
+ if (!(config.dest_map in item)) {
+ item[config.dest_map] = {};
+ }
+ item[config.dest_map][config.dest_key] = item[config.src];
+ }
+
+ return item;
+ }
+
+ /**
+ * Config:
+ * field Points to a string to number map
+ * k Scalar to multiply the values by
+ * log_scale Boolean, if true, then the values will be transformed
+ * by a logrithm prior to multiplications
+ */
+ scalarMultiplyTag(item, config) {
+ let EPSILON = 0.000001;
+ if (!(config.field in item)) {
+ return null;
+ }
+ let k = this._lookupScalar(item, config.k, 1);
+ let type = this._typeOf(item[config.field]);
+ if (type === "map") {
+ Object.keys(item[config.field]).forEach(parentKey => {
+ Object.keys(item[config.field][parentKey]).forEach(key => {
+ let v = item[config.field][parentKey][key];
+ if (config.log_scale) {
+ v = Math.log(v + EPSILON);
+ }
+ item[config.field][parentKey][key] = v * k;
+ });
+ });
+ } else {
+ return null;
+ }
+
+ return item;
+ }
+
+ /**
+ * Independently applies softmax across all subtags.
+ *
+ * Config:
+ * field Points to a map of strings with values being another map of strings
+ */
+ applySoftmaxTags(item, config) {
+ let type = this._typeOf(item[config.field]);
+ if (type !== "map") {
+ return null;
+ }
+
+ let abort = false;
+ let softmaxSum = {};
+ Object.keys(item[config.field]).forEach(tag => {
+ if (this._typeOf(item[config.field][tag]) !== "map") {
+ abort = true;
+ return;
+ }
+ if (abort) {
+ return;
+ }
+ softmaxSum[tag] = 0;
+ Object.keys(item[config.field][tag]).forEach(subtag => {
+ if (this._typeOf(item[config.field][tag][subtag]) !== "number") {
+ abort = true;
+ return;
+ }
+ let score = item[config.field][tag][subtag];
+ softmaxSum[tag] += Math.exp(score);
+ });
+ });
+ if (abort) {
+ return null;
+ }
+
+ Object.keys(item[config.field]).forEach(tag => {
+ Object.keys(item[config.field][tag]).forEach(subtag => {
+ item[config.field][tag][subtag] =
+ Math.exp(item[config.field][tag][subtag]) / softmaxSum[tag];
+ });
+ });
+
+ return item;
+ }
+
+ /**
+ * Vector adds a field and stores the result in left.
+ *
+ * Config:
+ * field The field to vector add
+ */
+ combinerAdd(left, right, config) {
+ if (!(config.field in right)) {
+ return left;
+ }
+ let type = this._typeOf(right[config.field]);
+ if (!(config.field in left)) {
+ if (type === "map") {
+ left[config.field] = {};
+ } else if (type === "array") {
+ left[config.field] = [];
+ } else if (type === "number") {
+ left[config.field] = 0;
+ } else {
+ return null;
+ }
+ }
+ if (type !== this._typeOf(left[config.field])) {
+ return null;
+ }
+ if (type === "map") {
+ Object.keys(right[config.field]).forEach(key => {
+ if (!(key in left[config.field])) {
+ left[config.field][key] = 0;
+ }
+ left[config.field][key] += right[config.field][key];
+ });
+ } else if (type === "array") {
+ for (let i = 0; i < right[config.field].length; i++) {
+ if (i < left[config.field].length) {
+ left[config.field][i] += right[config.field][i];
+ } else {
+ left[config.field].push(right[config.field][i]);
+ }
+ }
+ } else if (type === "number") {
+ left[config.field] += right[config.field];
+ } else {
+ return null;
+ }
+
+ return left;
+ }
+
+ /**
+ * Stores the maximum value of the field in left.
+ *
+ * Config:
+ * field The field to vector add
+ */
+ combinerMax(left, right, config) {
+ if (!(config.field in right)) {
+ return left;
+ }
+ let type = this._typeOf(right[config.field]);
+ if (!(config.field in left)) {
+ if (type === "map") {
+ left[config.field] = {};
+ } else if (type === "array") {
+ left[config.field] = [];
+ } else if (type === "number") {
+ left[config.field] = 0;
+ } else {
+ return null;
+ }
+ }
+ if (type !== this._typeOf(left[config.field])) {
+ return null;
+ }
+ if (type === "map") {
+ Object.keys(right[config.field]).forEach(key => {
+ if (
+ !(key in left[config.field]) ||
+ right[config.field][key] > left[config.field][key]
+ ) {
+ left[config.field][key] = right[config.field][key];
+ }
+ });
+ } else if (type === "array") {
+ for (let i = 0; i < right[config.field].length; i++) {
+ if (i < left[config.field].length) {
+ if (left[config.field][i] < right[config.field][i]) {
+ left[config.field][i] = right[config.field][i];
+ }
+ } else {
+ left[config.field].push(right[config.field][i]);
+ }
+ }
+ } else if (type === "number") {
+ if (left[config.field] < right[config.field]) {
+ left[config.field] = right[config.field];
+ }
+ } else {
+ return null;
+ }
+
+ return left;
+ }
+
+ /**
+ * Associates a value in right with another value in right. This association
+ * is then stored in a map in left.
+ *
+ * For example: If a sequence of rights is:
+ * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 41 }
+ * { 'tags': {}, 'url_domain': 'mbusa.com/mercedes', 'time': 21 }
+ * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 34 }
+ *
+ * Then assuming a 'sum' operation, left can build a map that would look like:
+ * {
+ * 'maseratiusa.com/maserati': 75,
+ * 'mbusa.com/mercedes': 21,
+ * }
+ *
+ * Fields:
+ * left_field field in the left to store / update the map
+ * right_key_field Field in the right to use as a key
+ * right_value_field Field in the right to use as a value
+ * operation One of "sum", "max", "overwrite", "count"
+ */
+ combinerCollectValues(left, right, config) {
+ let op;
+ if (config.operation === "sum") {
+ op = (a, b) => a + b;
+ } else if (config.operation === "max") {
+ op = (a, b) => (a > b ? a : b);
+ } else if (config.operation === "overwrite") {
+ op = (a, b) => b;
+ } else if (config.operation === "count") {
+ op = (a, b) => a + 1;
+ } else {
+ return null;
+ }
+ if (!(config.left_field in left)) {
+ left[config.left_field] = {};
+ }
+ if (
+ !(config.right_key_field in right) ||
+ !(config.right_value_field in right)
+ ) {
+ return left;
+ }
+
+ let key = right[config.right_key_field];
+ let rightValue = right[config.right_value_field];
+ let leftValue = 0.0;
+ if (key in left[config.left_field]) {
+ leftValue = left[config.left_field][key];
+ }
+
+ left[config.left_field][key] = op(leftValue, rightValue);
+
+ return left;
+ }
+
+ /**
+ * Executes a recipe. Returns an object on success, or null on failure.
+ */
+ executeRecipe(item, recipe) {
+ let newItem = item;
+ if (recipe) {
+ for (let step of recipe) {
+ let op = this.ITEM_BUILDER_REGISTRY[step.function];
+ if (op === undefined) {
+ return null;
+ }
+ newItem = op.call(this, newItem, step);
+ if (newItem === null) {
+ break;
+ }
+ }
+ }
+ return newItem;
+ }
+
+ /**
+ * Executes a recipe. Returns an object on success, or null on failure.
+ */
+ executeCombinerRecipe(item1, item2, recipe) {
+ let newItem1 = item1;
+ for (let step of recipe) {
+ let op = this.ITEM_COMBINER_REGISTRY[step.function];
+ if (op === undefined) {
+ return null;
+ }
+ newItem1 = op.call(this, newItem1, item2, step);
+ if (newItem1 === null) {
+ break;
+ }
+ }
+
+ return newItem1;
+ }
+};
diff --git a/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm b/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm
new file mode 100644
index 0000000000..94835557a6
--- /dev/null
+++ b/browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm
@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// We load this into a worker using importScripts, and in tests using import.
+// We use var to avoid name collision errors.
+// eslint-disable-next-line no-var
+var EXPORTED_SYMBOLS = ["tokenize", "toksToTfIdfVector"];
+
+// Unicode specifies certain mnemonics for code pages and character classes.
+// They call them "character properties" https://en.wikipedia.org/wiki/Unicode_character_property .
+// These mnemonics are have been adopted by many regular expression libraries,
+// however the standard Javascript regexp system doesn't support unicode
+// character properties, so we have to define these ourself.
+//
+// Each of these sections contains the characters values / ranges for specific
+// character property: Whitespace, Symbol (S), Punctuation (P), Number (N),
+// Mark (M), and Letter (L).
+const UNICODE_SPACE =
+ "\x20\xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000";
+const UNICODE_SYMBOL =
+ "\\x24\\x2B\x3C-\x3E\\x5E\x60\\x7C\x7E\xA2-\xA6\xA8\xA9\xAC\xAE-\xB1\xB4\xB8\xD7\xF7\u02C2-\u02C5\u02D2-\u02DF\u02E5-\u02EB\u02ED\u02EF-\u02FF\u0375\u0384\u0385\u03F6\u0482\u058D-\u058F\u0606-\u0608\u060B\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09F2\u09F3\u09FA\u09FB\u0AF1\u0B70\u0BF3-\u0BFA\u0C7F\u0D4F\u0D79\u0E3F\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u17DB\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u1FBD\u1FBF-\u1FC1\u1FCD-\u1FCF\u1FDD-\u1FDF\u1FED-\u1FEF\u1FFD\u1FFE\u2044\u2052\u207A-\u207C\u208A-\u208C\u20A0-\u20BE\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116-\u2118\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u2140-\u2144\u214A-\u214D\u214F\u218A\u218B\u2190-\u2307\u230C-\u2328\u232B-\u23FE\u2400-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u2767\u2794-\u27C4\u27C7-\u27E5\u27F0-\u2982\u2999-\u29D7\u29DC-\u29FB\u29FE-\u2B73\u2B76-\u2B95\u2B98-\u2BB9\u2BBD-\u2BC8\u2BCA-\u2BD1\u2BEC-\u2BEF\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u309B\u309C\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA700-\uA716\uA720\uA721\uA789\uA78A\uA828-\uA82B\uA836-\uA839\uAA77-\uAA79\uAB5B\uFB29\uFBB2-\uFBC1\uFDFC\uFDFD\uFE62\uFE64-\uFE66\uFE69\uFF04\uFF0B\uFF1C-\uFF1E\uFF3E\uFF40\uFF5C\uFF5E\uFFE0-\uFFE6\uFFE8-\uFFEE\uFFFC\uFFFD";
+const UNICODE_PUNCT =
+ "\x21-\x23\x25-\\x2A\x2C-\x2F\x3A\x3B\\x3F\x40\\x5B-\\x5D\x5F\\x7B\x7D\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E44\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65";
+
+const UNICODE_NUMBER =
+ "0-9\xB2\xB3\xB9\xBC-\xBE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D58-\u0D5E\u0D66-\u0D78\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19";
+const UNICODE_MARK =
+ "\u0300-\u036F\u0483-\u0489\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED\u0711\u0730-\u074A\u07A6-\u07B0\u07EB-\u07F3\u0816-\u0819\u081B-\u0823\u0825-\u0827\u0829-\u082D\u0859-\u085B\u08D4-\u08E1\u08E3-\u0903\u093A-\u093C\u093E-\u094F\u0951-\u0957\u0962\u0963\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7\u09C8\u09CB-\u09CD\u09D7\u09E2\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A70\u0A71\u0A75\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AE2\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B56\u0B57\u0B62\u0B63\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7\u0C00-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C62\u0C63\u0C81-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CE2\u0CE3\u0D01-\u0D03\u0D3E-\u0D44\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D62\u0D63\u0D82\u0D83\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2\u0DF3\u0E31\u0E34-\u0E3A\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB\u0EBC\u0EC8-\u0ECD\u0F18\u0F19\u0F35\u0F37\u0F39\u0F3E\u0F3F\u0F71-\u0F84\u0F86\u0F87\u0F8D-\u0F97\u0F99-\u0FBC\u0FC6\u102B-\u103E\u1056-\u1059\u105E-\u1060\u1062-\u1064\u1067-\u106D\u1071-\u1074\u1082-\u108D\u108F\u109A-\u109D\u135D-\u135F\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17B4-\u17D3\u17DD\u180B-\u180D\u1885\u1886\u18A9\u1920-\u192B\u1930-\u193B\u1A17-\u1A1B\u1A55-\u1A5E\u1A60-\u1A7C\u1A7F\u1AB0-\u1ABE\u1B00-\u1B04\u1B34-\u1B44\u1B6B-\u1B73\u1B80-\u1B82\u1BA1-\u1BAD\u1BE6-\u1BF3\u1C24-\u1C37\u1CD0-\u1CD2\u1CD4-\u1CE8\u1CED\u1CF2-\u1CF4\u1CF8\u1CF9\u1DC0-\u1DF5\u1DFB-\u1DFF\u20D0-\u20F0\u2CEF-\u2CF1\u2D7F\u2DE0-\u2DFF\u302A-\u302F\u3099\u309A\uA66F-\uA672\uA674-\uA67D\uA69E\uA69F\uA6F0\uA6F1\uA802\uA806\uA80B\uA823-\uA827\uA880\uA881\uA8B4-\uA8C5\uA8E0-\uA8F1\uA926-\uA92D\uA947-\uA953\uA980-\uA983\uA9B3-\uA9C0\uA9E5\uAA29-\uAA36\uAA43\uAA4C\uAA4D\uAA7B-\uAA7D\uAAB0\uAAB2-\uAAB4\uAAB7\uAAB8\uAABE\uAABF\uAAC1\uAAEB-\uAAEF\uAAF5\uAAF6\uABE3-\uABEA\uABEC\uABED\uFB1E\uFE00-\uFE0F\uFE20-\uFE2F";
+const UNICODE_LETTER =
+ "A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u08B6-\u08BD\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C80-\u1C88\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AE\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC";
+
+const REGEXP_SPLITS = new RegExp(
+ `[${UNICODE_SPACE}${UNICODE_SYMBOL}${UNICODE_PUNCT}]+`
+);
+// Match all token characters, so okay for regex to split multiple code points
+// eslint-disable-next-line no-misleading-character-class
+const REGEXP_ALPHANUMS = new RegExp(
+ `^[${UNICODE_NUMBER}${UNICODE_MARK}${UNICODE_LETTER}]+$`
+);
+
+/**
+ * Downcases the text, and splits it into consecutive alphanumeric characters.
+ * This is locale aware, and so will not strip accents. This uses "word
+ * breaks", and os is not appropriate for languages without them
+ * (e.g. Chinese).
+ */
+function tokenize(text) {
+ return text
+ .toLocaleLowerCase()
+ .split(REGEXP_SPLITS)
+ .filter(tok => tok.match(REGEXP_ALPHANUMS));
+}
+
+/**
+ * Converts a sequence of tokens into an L2 normed TF-IDF. Any terms that are
+ * not preindexed (i.e. does have a computed inverse document frequency) will
+ * be dropped.
+ */
+function toksToTfIdfVector(tokens, vocab_idfs) {
+ let tfidfs = {};
+
+ // calcualte the term frequencies
+ for (let tok of tokens) {
+ if (!(tok in vocab_idfs)) {
+ continue;
+ }
+ if (!(tok in tfidfs)) {
+ tfidfs[tok] = [vocab_idfs[tok][0], 1];
+ } else {
+ tfidfs[tok][1]++;
+ }
+ }
+
+ // now multiply by the log inverse document frequencies, then take
+ // the L2 norm of this.
+ let l2Norm = 0.0;
+ Object.keys(tfidfs).forEach(tok => {
+ tfidfs[tok][1] *= vocab_idfs[tok][1];
+ l2Norm += tfidfs[tok][1] * tfidfs[tok][1];
+ });
+ l2Norm = Math.sqrt(l2Norm);
+ Object.keys(tfidfs).forEach(tok => {
+ tfidfs[tok][1] /= l2Norm;
+ });
+
+ return tfidfs;
+}
diff --git a/browser/components/newtab/lib/PlacesFeed.jsm b/browser/components/newtab/lib/PlacesFeed.jsm
new file mode 100644
index 0000000000..adeb189179
--- /dev/null
+++ b/browser/components/newtab/lib/PlacesFeed.jsm
@@ -0,0 +1,615 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const {
+ actionCreators: ac,
+ actionTypes: at,
+ actionUtils: au,
+} = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { shortURL } = ChromeUtils.import(
+ "resource://activity-stream/lib/ShortURL.jsm"
+);
+const { AboutNewTab } = ChromeUtils.import(
+ "resource:///modules/AboutNewTab.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+ PartnerLinkAttribution: "resource:///modules/PartnerLinkAttribution.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "pktApi",
+ "chrome://pocket/content/pktApi.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "ExperimentAPI",
+ "resource://nimbus/ExperimentAPI.jsm"
+);
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+const LINK_BLOCKED_EVENT = "newtab-linkBlocked";
+const PLACES_LINKS_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for places links changed events
+
+// The pref to store the blocked sponsors of the sponsored Top Sites.
+// The value of this pref is an array (JSON serialized) of hostnames of the
+// blocked sponsors.
+const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors";
+
+/**
+ * Observer - a wrapper around history/bookmark observers to add the QueryInterface.
+ */
+class Observer {
+ constructor(dispatch, observerInterface) {
+ this.dispatch = dispatch;
+ this.QueryInterface = ChromeUtils.generateQI([
+ observerInterface,
+ "nsISupportsWeakReference",
+ ]);
+ }
+}
+
+/**
+ * BookmarksObserver - observes events from PlacesUtils.bookmarks
+ */
+class BookmarksObserver extends Observer {
+ constructor(dispatch) {
+ super(dispatch, Ci.nsINavBookmarkObserver);
+ this.skipTags = true;
+ }
+
+ // Empty functions to make xpconnect happy.
+ // Disabled due to performance cost, see Issue 3203 /
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1392267.
+ onItemChanged() {}
+}
+
+/**
+ * PlacesObserver - observes events from PlacesUtils.observers
+ */
+class PlacesObserver extends Observer {
+ constructor(dispatch) {
+ super(dispatch, Ci.nsINavBookmarkObserver);
+ this.handlePlacesEvent = this.handlePlacesEvent.bind(this);
+ }
+
+ handlePlacesEvent(events) {
+ const removedPages = [];
+ const removedBookmarks = [];
+
+ for (const {
+ itemType,
+ source,
+ dateAdded,
+ guid,
+ title,
+ url,
+ isRemovedFromStore,
+ isTagging,
+ type,
+ } of events) {
+ switch (type) {
+ case "history-cleared":
+ this.dispatch({ type: at.PLACES_HISTORY_CLEARED });
+ break;
+ case "page-removed":
+ if (isRemovedFromStore) {
+ removedPages.push(url);
+ }
+ break;
+ case "bookmark-added":
+ // Skips items that are not bookmarks (like folders), about:* pages or
+ // default bookmarks, added when the profile is created.
+ if (
+ isTagging ||
+ itemType !== lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK ||
+ source === lazy.PlacesUtils.bookmarks.SOURCES.IMPORT ||
+ source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE ||
+ source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP ||
+ source === lazy.PlacesUtils.bookmarks.SOURCES.SYNC ||
+ (!url.startsWith("http://") && !url.startsWith("https://"))
+ ) {
+ return;
+ }
+
+ this.dispatch({ type: at.PLACES_LINKS_CHANGED });
+ this.dispatch({
+ type: at.PLACES_BOOKMARK_ADDED,
+ data: {
+ bookmarkGuid: guid,
+ bookmarkTitle: title,
+ dateAdded: dateAdded * 1000,
+ url,
+ },
+ });
+ break;
+ case "bookmark-removed":
+ if (
+ isTagging ||
+ (itemType === lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK &&
+ source !== lazy.PlacesUtils.bookmarks.SOURCES.IMPORT &&
+ source !== lazy.PlacesUtils.bookmarks.SOURCES.RESTORE &&
+ source !==
+ lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP &&
+ source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC)
+ ) {
+ removedBookmarks.push(url);
+ }
+ break;
+ }
+ }
+
+ if (removedPages.length || removedBookmarks.length) {
+ this.dispatch({ type: at.PLACES_LINKS_CHANGED });
+ }
+
+ if (removedPages.length) {
+ this.dispatch({
+ type: at.PLACES_LINKS_DELETED,
+ data: { urls: removedPages },
+ });
+ }
+
+ if (removedBookmarks.length) {
+ this.dispatch({
+ type: at.PLACES_BOOKMARKS_REMOVED,
+ data: { urls: removedBookmarks },
+ });
+ }
+ }
+}
+
+class PlacesFeed {
+ constructor() {
+ this.placesChangedTimer = null;
+ this.customDispatch = this.customDispatch.bind(this);
+ this.bookmarksObserver = new BookmarksObserver(this.customDispatch);
+ this.placesObserver = new PlacesObserver(this.customDispatch);
+ }
+
+ addObservers() {
+ // NB: Directly get services without importing the *BIG* PlacesUtils module
+ Cc["@mozilla.org/browser/nav-bookmarks-service;1"]
+ .getService(Ci.nsINavBookmarksService)
+ .addObserver(this.bookmarksObserver, true);
+ lazy.PlacesUtils.observers.addListener(
+ ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"],
+ this.placesObserver.handlePlacesEvent
+ );
+
+ Services.obs.addObserver(this, LINK_BLOCKED_EVENT);
+ }
+
+ /**
+ * setTimeout - A custom function that creates an nsITimer that can be cancelled
+ *
+ * @param {func} callback A function to be executed after the timer expires
+ * @param {int} delay The time (in ms) the timer should wait before the function is executed
+ */
+ setTimeout(callback, delay) {
+ let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+ timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT);
+ return timer;
+ }
+
+ customDispatch(action) {
+ // If we are changing many links at once, delay this action and only dispatch
+ // one action at the end
+ if (action.type === at.PLACES_LINKS_CHANGED) {
+ if (this.placesChangedTimer) {
+ this.placesChangedTimer.delay = PLACES_LINKS_CHANGED_DELAY_TIME;
+ } else {
+ this.placesChangedTimer = this.setTimeout(() => {
+ this.placesChangedTimer = null;
+ this.store.dispatch(ac.OnlyToMain(action));
+ }, PLACES_LINKS_CHANGED_DELAY_TIME);
+ }
+ } else {
+ this.store.dispatch(ac.BroadcastToContent(action));
+ }
+ }
+
+ removeObservers() {
+ if (this.placesChangedTimer) {
+ this.placesChangedTimer.cancel();
+ this.placesChangedTimer = null;
+ }
+ lazy.PlacesUtils.bookmarks.removeObserver(this.bookmarksObserver);
+ lazy.PlacesUtils.observers.removeListener(
+ ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"],
+ this.placesObserver.handlePlacesEvent
+ );
+ Services.obs.removeObserver(this, LINK_BLOCKED_EVENT);
+ }
+
+ /**
+ * observe - An observer for the LINK_BLOCKED_EVENT.
+ * Called when a link is blocked.
+ * Links can be blocked outside of newtab,
+ * which is why we need to listen to this
+ * on such a generic level.
+ *
+ * @param {null} subject
+ * @param {str} topic The name of the event
+ * @param {str} value The data associated with the event
+ */
+ observe(subject, topic, value) {
+ if (topic === LINK_BLOCKED_EVENT) {
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PLACES_LINK_BLOCKED,
+ data: { url: value },
+ })
+ );
+ }
+ }
+
+ /**
+ * Open a link in a desired destination defaulting to action's event.
+ */
+ openLink(action, where = "", isPrivate = false) {
+ const params = {
+ private: isPrivate,
+ targetBrowser: action._target.browser,
+ fromChrome: false, // This ensure we maintain user preference for how to open new tabs.
+ globalHistoryOptions: {
+ triggeringSponsoredURL: action.data.sponsored_tile_id
+ ? action.data.url
+ : undefined,
+ },
+ };
+
+ // Always include the referrer (even for http links) if we have one
+ const { event, referrer, typedBonus } = action.data;
+ if (referrer) {
+ const ReferrerInfo = Components.Constructor(
+ "@mozilla.org/referrer-info;1",
+ "nsIReferrerInfo",
+ "init"
+ );
+ params.referrerInfo = new ReferrerInfo(
+ Ci.nsIReferrerInfo.UNSAFE_URL,
+ true,
+ Services.io.newURI(referrer)
+ );
+ }
+
+ // Pocket gives us a special reader URL to open their stories in
+ const urlToOpen =
+ action.data.type === "pocket" ? action.data.open_url : action.data.url;
+
+ try {
+ let uri = Services.io.newURI(urlToOpen);
+ if (!["http", "https"].includes(uri.scheme)) {
+ throw new Error(
+ `Can't open link using ${uri.scheme} protocol from the new tab page.`
+ );
+ }
+ } catch (e) {
+ console.error(e);
+ return;
+ }
+
+ // Mark the page as typed for frecency bonus before opening the link
+ if (typedBonus) {
+ lazy.PlacesUtils.history.markPageAsTyped(Services.io.newURI(urlToOpen));
+ }
+
+ const win = action._target.browser.ownerGlobal;
+ win.openTrustedLinkIn(
+ urlToOpen,
+ where || win.whereToOpenLink(event),
+ params
+ );
+
+ // If there's an original URL e.g. using the unprocessed %YYYYMMDDHH% tag,
+ // add a visit for that so it may become a frecent top site.
+ if (action.data.original_url) {
+ lazy.PlacesUtils.history.insert({
+ url: action.data.original_url,
+ visits: [{ transition: lazy.PlacesUtils.history.TRANSITION_TYPED }],
+ });
+ }
+ }
+
+ async saveToPocket(site, browser) {
+ const sendToPocket = lazy.NimbusFeatures.pocketNewtab.getVariable(
+ "sendToPocket"
+ );
+ // An experiment to send the user directly to Pocket's signup page.
+ if (sendToPocket && !lazy.pktApi.isUserLoggedIn()) {
+ const pocketNewtabExperiment = lazy.ExperimentAPI.getExperiment({
+ featureId: "pocketNewtab",
+ });
+ const pocketSiteHost = Services.prefs.getStringPref(
+ "extensions.pocket.site"
+ ); // getpocket.com
+ let utmSource = "firefox_newtab_save_button";
+ // We want to know if the user is in a Pocket newtab related experiment.
+ let utmCampaign = pocketNewtabExperiment?.slug;
+ let utmContent = pocketNewtabExperiment?.branch?.slug;
+
+ const url = new URL(`https://${pocketSiteHost}/signup`);
+ url.searchParams.append("utm_source", utmSource);
+ if (utmCampaign && utmContent) {
+ url.searchParams.append("utm_campaign", utmCampaign);
+ url.searchParams.append("utm_content", utmContent);
+ }
+
+ const win = browser.ownerGlobal;
+ win.openTrustedLinkIn(url.href, "tab");
+ return;
+ }
+
+ const { url, title } = site;
+ try {
+ let data = await lazy.NewTabUtils.activityStreamLinks.addPocketEntry(
+ url,
+ title,
+ browser
+ );
+ if (data) {
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PLACES_SAVED_TO_POCKET,
+ data: {
+ url,
+ open_url: data.item.open_url,
+ title,
+ pocket_id: data.item.item_id,
+ },
+ })
+ );
+ }
+ } catch (err) {
+ console.error(err);
+ }
+ }
+
+ /**
+ * Deletes an item from a user's saved to Pocket feed
+ * @param {int} itemID
+ * The unique ID given by Pocket for that item; used to look the item up when deleting
+ */
+ async deleteFromPocket(itemID) {
+ try {
+ await lazy.NewTabUtils.activityStreamLinks.deletePocketEntry(itemID);
+ this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED });
+ } catch (err) {
+ console.error(err);
+ }
+ }
+
+ /**
+ * Archives an item from a user's saved to Pocket feed
+ * @param {int} itemID
+ * The unique ID given by Pocket for that item; used to look the item up when archiving
+ */
+ async archiveFromPocket(itemID) {
+ try {
+ await lazy.NewTabUtils.activityStreamLinks.archivePocketEntry(itemID);
+ this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED });
+ } catch (err) {
+ console.error(err);
+ }
+ }
+
+ /**
+ * Sends an attribution request for Top Sites interactions.
+ * @param {object} data
+ * Attribution paramters from a Top Site.
+ */
+ makeAttributionRequest(data) {
+ let args = Object.assign(
+ {
+ campaignID: Services.prefs.getStringPref(
+ "browser.partnerlink.campaign.topsites"
+ ),
+ },
+ data
+ );
+ lazy.PartnerLinkAttribution.makeRequest(args);
+ }
+
+ async fillSearchTopSiteTerm({ _target, data }) {
+ const searchEngine = await Services.search.getEngineByAlias(data.label);
+ _target.browser.ownerGlobal.gURLBar.search(data.label, {
+ searchEngine,
+ searchModeEntry: "topsites_newtab",
+ });
+ }
+
+ _getDefaultSearchEngine(isPrivateWindow) {
+ return Services.search[
+ isPrivateWindow ? "defaultPrivateEngine" : "defaultEngine"
+ ];
+ }
+
+ handoffSearchToAwesomebar(action) {
+ const { _target, data, meta } = action;
+ const searchEngine = this._getDefaultSearchEngine(
+ lazy.PrivateBrowsingUtils.isBrowserPrivate(_target.browser)
+ );
+ const urlBar = _target.browser.ownerGlobal.gURLBar;
+ let isFirstChange = true;
+
+ const newtabSession = AboutNewTab.activityStream.store.feeds
+ .get("feeds.telemetry")
+ ?.sessions.get(au.getPortIdOfSender(action));
+ if (!data || !data.text) {
+ urlBar.setHiddenFocus();
+ } else {
+ urlBar.handoff(data.text, searchEngine, newtabSession?.session_id);
+ isFirstChange = false;
+ }
+
+ const checkFirstChange = () => {
+ // Check if this is the first change since we hidden focused. If it is,
+ // remove hidden focus styles, prepend the search alias and hide the
+ // in-content search.
+ if (isFirstChange) {
+ isFirstChange = false;
+ urlBar.removeHiddenFocus(true);
+ urlBar.handoff("", searchEngine, newtabSession?.session_id);
+ this.store.dispatch(
+ ac.OnlyToOneContent({ type: at.DISABLE_SEARCH }, meta.fromTarget)
+ );
+ urlBar.removeEventListener("compositionstart", checkFirstChange);
+ urlBar.removeEventListener("paste", checkFirstChange);
+ }
+ };
+
+ const onKeydown = ev => {
+ // Check if the keydown will cause a value change.
+ if (ev.key.length === 1 && !ev.altKey && !ev.ctrlKey && !ev.metaKey) {
+ checkFirstChange();
+ }
+ // If the Esc button is pressed, we are done. Show in-content search and cleanup.
+ if (ev.key === "Escape") {
+ onDone(); // eslint-disable-line no-use-before-define
+ }
+ };
+
+ const onDone = ev => {
+ // We are done. Show in-content search again and cleanup.
+ this.store.dispatch(
+ ac.OnlyToOneContent({ type: at.SHOW_SEARCH }, meta.fromTarget)
+ );
+
+ const forceSuppressFocusBorder = ev?.type === "mousedown";
+ urlBar.removeHiddenFocus(forceSuppressFocusBorder);
+
+ urlBar.removeEventListener("keydown", onKeydown);
+ urlBar.removeEventListener("mousedown", onDone);
+ urlBar.removeEventListener("blur", onDone);
+ urlBar.removeEventListener("compositionstart", checkFirstChange);
+ urlBar.removeEventListener("paste", checkFirstChange);
+ };
+
+ urlBar.addEventListener("keydown", onKeydown);
+ urlBar.addEventListener("mousedown", onDone);
+ urlBar.addEventListener("blur", onDone);
+ urlBar.addEventListener("compositionstart", checkFirstChange);
+ urlBar.addEventListener("paste", checkFirstChange);
+ }
+
+ /**
+ * Add the hostnames of the given urls to the Top Sites sponsor blocklist.
+ *
+ * @param {array} urls
+ * An array of the objects structured as `{ url }`
+ */
+ addToBlockedTopSitesSponsors(urls) {
+ const blockedPref = JSON.parse(
+ Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
+ );
+ const merged = new Set([...blockedPref, ...urls.map(url => shortURL(url))]);
+
+ Services.prefs.setStringPref(
+ TOP_SITES_BLOCKED_SPONSORS_PREF,
+ JSON.stringify([...merged])
+ );
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ // Briefly avoid loading services for observing for better startup timing
+ Services.tm.dispatchToMainThread(() => this.addObservers());
+ break;
+ case at.UNINIT:
+ this.removeObservers();
+ break;
+ case at.ABOUT_SPONSORED_TOP_SITES: {
+ const url = `${Services.urlFormatter.formatURLPref(
+ "app.support.baseURL"
+ )}sponsor-privacy`;
+ const win = action._target.browser.ownerGlobal;
+ win.openTrustedLinkIn(url, "tab");
+ break;
+ }
+ case at.BLOCK_URL: {
+ if (action.data) {
+ let sponsoredTopSites = [];
+ action.data.forEach(site => {
+ const { url, pocket_id, isSponsoredTopSite } = site;
+ lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id });
+ if (isSponsoredTopSite) {
+ sponsoredTopSites.push({ url });
+ }
+ });
+ if (sponsoredTopSites.length) {
+ this.addToBlockedTopSitesSponsors(sponsoredTopSites);
+ }
+ }
+ break;
+ }
+ case at.BOOKMARK_URL:
+ lazy.NewTabUtils.activityStreamLinks.addBookmark(
+ action.data,
+ action._target.browser.ownerGlobal
+ );
+ break;
+ case at.DELETE_BOOKMARK_BY_ID:
+ lazy.NewTabUtils.activityStreamLinks.deleteBookmark(action.data);
+ break;
+ case at.DELETE_HISTORY_URL: {
+ const { url, forceBlock, pocket_id } = action.data;
+ lazy.NewTabUtils.activityStreamLinks.deleteHistoryEntry(url);
+ if (forceBlock) {
+ lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id });
+ }
+ break;
+ }
+ case at.OPEN_NEW_WINDOW:
+ this.openLink(action, "window");
+ break;
+ case at.OPEN_PRIVATE_WINDOW:
+ this.openLink(action, "window", true);
+ break;
+ case at.SAVE_TO_POCKET:
+ this.saveToPocket(action.data.site, action._target.browser);
+ break;
+ case at.DELETE_FROM_POCKET:
+ this.deleteFromPocket(action.data.pocket_id);
+ break;
+ case at.ARCHIVE_FROM_POCKET:
+ this.archiveFromPocket(action.data.pocket_id);
+ break;
+ case at.FILL_SEARCH_TERM:
+ this.fillSearchTopSiteTerm(action);
+ break;
+ case at.HANDOFF_SEARCH_TO_AWESOMEBAR:
+ this.handoffSearchToAwesomebar(action);
+ break;
+ case at.OPEN_LINK: {
+ this.openLink(action);
+ break;
+ }
+ case at.PARTNER_LINK_ATTRIBUTION:
+ this.makeAttributionRequest(action.data);
+ break;
+ }
+ }
+}
+
+// Exported for testing only
+PlacesFeed.BookmarksObserver = BookmarksObserver;
+PlacesFeed.PlacesObserver = PlacesObserver;
+
+const EXPORTED_SYMBOLS = ["PlacesFeed"];
diff --git a/browser/components/newtab/lib/PrefsFeed.jsm b/browser/components/newtab/lib/PrefsFeed.jsm
new file mode 100644
index 0000000000..d76df6e70f
--- /dev/null
+++ b/browser/components/newtab/lib/PrefsFeed.jsm
@@ -0,0 +1,273 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { Prefs } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamPrefs.jsm"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+ "resource://gre/modules/AppConstants.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+ Region: "resource://gre/modules/Region.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+class PrefsFeed {
+ constructor(prefMap) {
+ this._prefMap = prefMap;
+ this._prefs = new Prefs();
+ this.onExperimentUpdated = this.onExperimentUpdated.bind(this);
+ this.onPocketExperimentUpdated = this.onPocketExperimentUpdated.bind(this);
+ }
+
+ onPrefChanged(name, value) {
+ const prefItem = this._prefMap.get(name);
+ if (prefItem) {
+ this.store.dispatch(
+ ac[prefItem.skipBroadcast ? "OnlyToMain" : "BroadcastToContent"]({
+ type: at.PREF_CHANGED,
+ data: { name, value },
+ })
+ );
+ }
+ }
+
+ _setStringPref(values, key, defaultValue) {
+ this._setPref(values, key, defaultValue, Services.prefs.getStringPref);
+ }
+
+ _setBoolPref(values, key, defaultValue) {
+ this._setPref(values, key, defaultValue, Services.prefs.getBoolPref);
+ }
+
+ _setIntPref(values, key, defaultValue) {
+ this._setPref(values, key, defaultValue, Services.prefs.getIntPref);
+ }
+
+ _setPref(values, key, defaultValue, getPrefFunction) {
+ let value = getPrefFunction(
+ `browser.newtabpage.activity-stream.${key}`,
+ defaultValue
+ );
+ values[key] = value;
+ this._prefMap.set(key, { value });
+ }
+
+ /**
+ * Handler for when experiment data updates.
+ */
+ onExperimentUpdated(event, reason) {
+ const value = lazy.NimbusFeatures.newtab.getAllVariables() || {};
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PREF_CHANGED,
+ data: {
+ name: "featureConfig",
+ value,
+ },
+ })
+ );
+ }
+
+ /**
+ * Handler for Pocket specific experiment data updates.
+ */
+ onPocketExperimentUpdated(event, reason) {
+ const value = lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {};
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PREF_CHANGED,
+ data: {
+ name: "pocketConfig",
+ value,
+ },
+ })
+ );
+ }
+
+ init() {
+ this._prefs.observeBranch(this);
+ lazy.NimbusFeatures.newtab.onUpdate(this.onExperimentUpdated);
+ lazy.NimbusFeatures.pocketNewtab.onUpdate(this.onPocketExperimentUpdated);
+
+ this._storage = this.store.dbStorage.getDbTable("sectionPrefs");
+
+ // Get the initial value of each activity stream pref
+ const values = {};
+ for (const name of this._prefMap.keys()) {
+ values[name] = this._prefs.get(name);
+ }
+
+ // These are not prefs, but are needed to determine stuff in content that can only be
+ // computed in main process
+ values.isPrivateBrowsingEnabled = lazy.PrivateBrowsingUtils.enabled;
+ values.platform = AppConstants.platform;
+
+ // Save the geo pref if we have it
+ if (lazy.Region.home) {
+ values.region = lazy.Region.home;
+ this.geo = values.region;
+ } else if (this.geo !== "") {
+ // Watch for geo changes and use a dummy value for now
+ Services.obs.addObserver(this, lazy.Region.REGION_TOPIC);
+ this.geo = "";
+ }
+
+ // Get the firefox accounts url for links and to send firstrun metrics to.
+ values.fxa_endpoint = Services.prefs.getStringPref(
+ "browser.newtabpage.activity-stream.fxaccounts.endpoint",
+ "https://accounts.firefox.com"
+ );
+
+ // Get the firefox update channel with values as default, nightly, beta or release
+ values.appUpdateChannel = Services.prefs.getStringPref(
+ "app.update.channel",
+ ""
+ );
+
+ // Read the pref for search shortcuts top sites experiment from firefox.js and store it
+ // in our internal list of prefs to watch
+ let searchTopSiteExperimentPrefValue = Services.prefs.getBoolPref(
+ "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts"
+ );
+ values[
+ "improvesearch.topSiteSearchShortcuts"
+ ] = searchTopSiteExperimentPrefValue;
+ this._prefMap.set("improvesearch.topSiteSearchShortcuts", {
+ value: searchTopSiteExperimentPrefValue,
+ });
+
+ values.mayHaveSponsoredTopSites = Services.prefs.getBoolPref(
+ "browser.topsites.useRemoteSetting"
+ );
+
+ // Read the pref for search hand-off from firefox.js and store it
+ // in our internal list of prefs to watch
+ let handoffToAwesomebarPrefValue = Services.prefs.getBoolPref(
+ "browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar"
+ );
+ values["improvesearch.handoffToAwesomebar"] = handoffToAwesomebarPrefValue;
+ this._prefMap.set("improvesearch.handoffToAwesomebar", {
+ value: handoffToAwesomebarPrefValue,
+ });
+
+ // Read the pref for the cached default engine name from firefox.js and
+ // store it in our internal list of prefs to watch
+ let placeholderPrefValue = Services.prefs.getStringPref(
+ "browser.urlbar.placeholderName",
+ ""
+ );
+ values["urlbar.placeholderName"] = placeholderPrefValue;
+ this._prefMap.set("urlbar.placeholderName", {
+ value: placeholderPrefValue,
+ });
+
+ // Add experiment values and default values
+ values.featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {};
+ values.pocketConfig =
+ lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {};
+ this._setBoolPref(values, "logowordmark.alwaysVisible", false);
+ this._setBoolPref(values, "feeds.section.topstories", false);
+ this._setBoolPref(values, "discoverystream.enabled", false);
+ this._setBoolPref(
+ values,
+ "discoverystream.sponsored-collections.enabled",
+ false
+ );
+ this._setBoolPref(values, "discoverystream.isCollectionDismissible", false);
+ this._setBoolPref(values, "discoverystream.hardcoded-basic-layout", false);
+ this._setBoolPref(values, "discoverystream.personalization.enabled", false);
+ this._setBoolPref(values, "discoverystream.personalization.override");
+ this._setStringPref(
+ values,
+ "discoverystream.personalization.modelKeys",
+ ""
+ );
+ this._setStringPref(values, "discoverystream.spocs-endpoint", "");
+ this._setStringPref(values, "discoverystream.spocs-endpoint-query", "");
+ this._setStringPref(values, "newNewtabExperience.colors", "");
+
+ // Set the initial state of all prefs in redux
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PREFS_INITIAL_VALUES,
+ data: values,
+ meta: {
+ isStartup: true,
+ },
+ })
+ );
+ }
+
+ uninit() {
+ this.removeListeners();
+ }
+
+ removeListeners() {
+ this._prefs.ignoreBranch(this);
+ lazy.NimbusFeatures.newtab.off(this.onExperimentUpdated);
+ lazy.NimbusFeatures.pocketNewtab.off(this.onPocketExperimentUpdated);
+ if (this.geo === "") {
+ Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC);
+ }
+ }
+
+ async _setIndexedDBPref(id, value) {
+ const name = id === "topsites" ? id : `feeds.section.${id}`;
+ try {
+ await this._storage.set(name, value);
+ } catch (e) {
+ console.error("Could not set section preferences.");
+ }
+ }
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case lazy.Region.REGION_TOPIC:
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.PREF_CHANGED,
+ data: { name: "region", value: lazy.Region.home },
+ })
+ );
+ break;
+ }
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ case at.CLEAR_PREF:
+ Services.prefs.clearUserPref(this._prefs._branchStr + action.data.name);
+ break;
+ case at.SET_PREF:
+ this._prefs.set(action.data.name, action.data.value);
+ break;
+ case at.UPDATE_SECTION_PREFS:
+ this._setIndexedDBPref(action.data.id, action.data.value);
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["PrefsFeed"];
diff --git a/browser/components/newtab/lib/RecommendationProvider.jsm b/browser/components/newtab/lib/RecommendationProvider.jsm
new file mode 100644
index 0000000000..b25e2f4185
--- /dev/null
+++ b/browser/components/newtab/lib/RecommendationProvider.jsm
@@ -0,0 +1,133 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// Use XPCOMUtils.defineLazyModuleGetters to make the test harness keeps working
+// after bug 1608279.
+//
+// The test harness's workaround for "lazy getter on a plain object" is to
+// set the `lazy` object's prototype to the global object, inside the lazy
+// getter API.
+//
+// ChromeUtils.defineModuleGetter is converted into a static import declaration
+// by babel-plugin-jsm-to-esmodules, and it doesn't work for the following
+// 2 reasons:
+//
+// * There's no other lazy getter API call in this file, and the workaround
+// above stops working
+// * babel-plugin-jsm-to-esmodules ignores the first parameter of the lazy
+// getter API, and the result is wrong
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const lazy = {};
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ PersonalityProvider:
+ "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm",
+});
+
+const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const PREF_PERSONALIZATION_MODEL_KEYS =
+ "discoverystream.personalization.modelKeys";
+const PREF_PERSONALIZATION = "discoverystream.personalization.enabled";
+
+// The main purpose of this class is to handle interactions with the recommendation provider.
+// A recommendation provider scores a list of stories, currently this is a personality provider.
+// So all calls to the provider, anything involved with the setup of the provider,
+// accessing prefs for the provider, or updaing devtools with provider state, is contained in here.
+class RecommendationProvider {
+ setProvider(scores) {
+ // A provider is already set. This can happen when new stories come in
+ // and we need to update their scores.
+ // We can use the existing one, a fresh one is created after startup.
+ // Using the existing one might be a bit out of date,
+ // but it's fine for now. We can rely on restarts for updates.
+ // See bug 1629931 for improvements to this.
+ if (this.provider) {
+ return;
+ }
+ // At this point we've determined we can successfully create a v2 personalization provider.
+ this.provider = new lazy.PersonalityProvider(this.modelKeys);
+ this.provider.setScores(scores);
+ }
+
+ /*
+ * This calls any async initialization that's required,
+ * and then signals to devtools when that's done.
+ */
+ async init() {
+ if (this.provider && this.provider.init) {
+ await this.provider.init();
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_PERSONALIZATION_INIT,
+ })
+ );
+ }
+ }
+
+ get modelKeys() {
+ if (!this._modelKeys) {
+ this._modelKeys = this.store.getState().Prefs.values[
+ PREF_PERSONALIZATION_MODEL_KEYS
+ ];
+ }
+
+ return this._modelKeys;
+ }
+
+ getScores() {
+ return this.provider.getScores();
+ }
+
+ async calculateItemRelevanceScore(item) {
+ if (this.provider) {
+ const scoreResult = await this.provider.calculateItemRelevanceScore(item);
+ if (scoreResult === 0 || scoreResult) {
+ item.score = scoreResult;
+ }
+ }
+ }
+
+ teardown() {
+ if (this.provider && this.provider.teardown) {
+ // This removes any in memory listeners if available.
+ this.provider.teardown();
+ }
+ }
+
+ resetState() {
+ this._modelKeys = null;
+ this.provider = null;
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.DISCOVERY_STREAM_CONFIG_CHANGE:
+ this.teardown();
+ this.resetState();
+ break;
+ case at.PREF_CHANGED:
+ switch (action.data.name) {
+ case PREF_PERSONALIZATION_MODEL_KEYS:
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.DISCOVERY_STREAM_CONFIG_RESET,
+ })
+ );
+ break;
+ }
+ break;
+ case at.DISCOVERY_STREAM_PERSONALIZATION_TOGGLE:
+ let enabled = this.store.getState().Prefs.values[PREF_PERSONALIZATION];
+
+ this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION, !enabled));
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["RecommendationProvider"];
diff --git a/browser/components/newtab/lib/RemoteImages.jsm b/browser/components/newtab/lib/RemoteImages.jsm
new file mode 100644
index 0000000000..4e048bac63
--- /dev/null
+++ b/browser/components/newtab/lib/RemoteImages.jsm
@@ -0,0 +1,609 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { JSONFile } = ChromeUtils.importESModule(
+ "resource://gre/modules/JSONFile.sys.mjs"
+);
+const { PromiseUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/PromiseUtils.sys.mjs"
+);
+const { RemoteSettings } = ChromeUtils.import(
+ "resource://services-settings/remote-settings.js"
+);
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Downloader",
+ "resource://services-settings/Attachments.jsm"
+);
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "KintoHttpClient",
+ "resource://services-common/kinto-http-client.js"
+);
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Utils",
+ "resource://services-settings/Utils.jsm"
+);
+
+const RS_MAIN_BUCKET = "main";
+const RS_COLLECTION = "ms-images";
+const RS_DOWNLOAD_MAX_RETRIES = 2;
+
+const REMOTE_IMAGES_PATH = PathUtils.join(
+ PathUtils.localProfileDir,
+ "settings",
+ RS_MAIN_BUCKET,
+ RS_COLLECTION
+);
+const REMOTE_IMAGES_DB_PATH = PathUtils.join(REMOTE_IMAGES_PATH, "db.json");
+
+const IMAGE_EXPIRY_DURATION = 30 * 24 * 60 * 60; // 30 days in seconds.
+
+const PREFETCH_FINISHED_TOPIC = "remote-images:prefetch-finished";
+
+/**
+ * Inspectors for FxMS messages.
+ *
+ * Each member is the name of a FxMS template (spotlight, infobar, etc.) and
+ * corresponds to a function that accepts a message and returns all record IDs
+ * for remote images.
+ */
+const MessageInspectors = {
+ spotlight(message) {
+ if (
+ message.content.template === "logo-and-content" &&
+ message.content.logo?.imageId
+ ) {
+ return [message.content.logo.imageId];
+ }
+ return [];
+ },
+};
+
+class _RemoteImages {
+ #dbPromise;
+
+ #fetching;
+
+ constructor() {
+ this.#dbPromise = null;
+ this.#fetching = new Map();
+
+ RemoteSettings(RS_COLLECTION).on("sync", () => this.#onSync());
+
+ // Ensure we migrate all our images to a JSONFile database.
+ this.withDb(() => {});
+ }
+
+ /**
+ * Load the database from disk.
+ *
+ * If the database does not yet exist, attempt a migration from legacy Remote
+ * Images (i.e., image files in |REMOTE_IMAGES_PATH|).
+ *
+ * @returns {Promise<JSONFile>} A promise that resolves with the database
+ * instance.
+ */
+ async #loadDb() {
+ let db;
+
+ if (!(await IOUtils.exists(REMOTE_IMAGES_DB_PATH))) {
+ db = await this.#migrate();
+ } else {
+ db = new JSONFile({ path: REMOTE_IMAGES_DB_PATH });
+ await db.load();
+ }
+
+ return db;
+ }
+
+ /**
+ * Reset the RemoteImages database
+ *
+ * NB: This is only meant to be used by unit tests.
+ *
+ * @returns {Promise<void>} A promise that resolves when the database has been
+ * reset.
+ */
+ reset() {
+ return this.withDb(async db => {
+ // We must reset |#dbPromise| *before* awaiting because if we do not, then
+ // another function could call withDb() while we are awaiting and get a
+ // promise that will resolve to |db| instead of getting null and forcing a
+ // db reload.
+ this.#dbPromise = null;
+ await db.finalize();
+ });
+ }
+
+ /*
+ * Execute |fn| with the RemoteSettings database.
+ *
+ * This ensures that only one caller can have a handle to the database at any
+ * given time (unless it is leaked through assignment from within |fn|). This
+ * prevents re-entrancy issues with multiple calls to cleanup() and calling
+ * cleanup while loading images.
+ *
+ * @param fn The function to call with the database.
+ */
+ async withDb(fn) {
+ const dbPromise = this.#dbPromise ?? this.#loadDb();
+
+ const { resolve, promise } = PromiseUtils.defer();
+ // NB: Update |#dbPromise| before awaiting anything so that the next call to
+ // |withDb()| will see the new value of |#dbPromise|.
+ this.#dbPromise = promise;
+
+ const db = await dbPromise;
+
+ try {
+ return await fn(db);
+ } finally {
+ resolve(db);
+ }
+ }
+
+ /**
+ * Patch a reference to a remote image in a message with a blob URL.
+ *
+ * @param message The remote image reference to be patched.
+ * @param replaceWith The property name that will be used to store the blob
+ * URL on |message|.
+ *
+ * @return A promise that resolves with an unloading function for the patched
+ * URL, or rejects with an error.
+ *
+ * If the message isn't patched (because there isn't a remote image)
+ * then the promise will resolve to null.
+ */
+ async patchMessage(message, replaceWith = "imageURL") {
+ if (!!message && !!message.imageId) {
+ const { imageId } = message;
+ const urls = await this.load(imageId);
+
+ if (urls.size) {
+ const blobURL = urls.get(imageId);
+
+ delete message.imageId;
+ message[replaceWith] = blobURL;
+
+ return () => this.unload(urls);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Load remote images.
+ *
+ * If the images have not been previously downloaded, then they will be
+ * downloaded from RemoteSettings.
+ *
+ * @param {...string} imageIds The image IDs to load.
+ *
+ * @returns {object} An object mapping image Ids to blob: URLs.
+ * If an image could not be loaded, it will not be present
+ * in the returned object.
+ *
+ * After the caller is finished with the images, they must call
+ * |RemoteImages.unload()| on the object.
+ */
+ load(...imageIds) {
+ return this.withDb(async db => {
+ // Deduplicate repeated imageIds by using a Map.
+ const urls = new Map(imageIds.map(key => [key, undefined]));
+
+ await Promise.all(
+ Array.from(urls.keys()).map(async imageId => {
+ try {
+ urls.set(imageId, await this.#loadImpl(db, imageId));
+ } catch (e) {
+ console.error(`Could not load image ID ${imageId}: ${e}`);
+ urls.delete(imageId);
+ }
+ })
+ );
+
+ return urls;
+ });
+ }
+
+ async #loadImpl(db, imageId) {
+ const recordId = this.#getRecordId(imageId);
+
+ // If we are pre-fetching an image, we can piggy-back on that request.
+ if (this.#fetching.has(imageId)) {
+ const { record, arrayBuffer } = await this.#fetching.get(imageId);
+ return new Blob([arrayBuffer], { type: record.data.attachment.mimetype });
+ }
+
+ let blob;
+ if (db.data.images[recordId]) {
+ // We have previously fetched this image, we can load it from disk.
+ try {
+ blob = await this.#readFromDisk(db, recordId);
+ } catch (e) {
+ if (
+ !(
+ e instanceof Components.Exception &&
+ e.name === "NS_ERROR_FILE_NOT_FOUND"
+ )
+ ) {
+ throw e;
+ }
+ }
+
+ // Fall back to downloading if we cannot read it from disk.
+ }
+
+ if (typeof blob === "undefined") {
+ blob = await this.#download(db, recordId);
+ }
+
+ return URL.createObjectURL(blob);
+ }
+
+ /**
+ * Unload URLs returned by RemoteImages
+ *
+ * @param {Map<string, string>} urls The result of calling |RemoteImages.load()|.
+ **/
+ unload(urls) {
+ for (const url of urls.keys()) {
+ URL.revokeObjectURL(url);
+ }
+ }
+
+ #onSync() {
+ // This is OK to run while pre-fetches are ocurring. Pre-fetches don't check
+ // if there is a new version available, so there will be no race between
+ // syncing an updated image and pre-fetching
+ return this.withDb(async db => {
+ await this.#cleanup(db);
+
+ const recordsById = await RemoteSettings(RS_COLLECTION)
+ .db.list()
+ .then(records =>
+ Object.assign({}, ...records.map(record => ({ [record.id]: record })))
+ );
+
+ await Promise.all(
+ Object.values(db.data.images)
+ .filter(
+ entry => recordsById[entry.recordId]?.attachment.hash !== entry.hash
+ )
+ .map(entry => this.#download(db, entry.recordId, { fetchOnly: true }))
+ );
+ });
+ }
+
+ forceCleanup() {
+ return this.withDb(db => this.#cleanup(db));
+ }
+
+ /**
+ * Clean up all files that haven't been touched in 30d.
+ *
+ * @returns {Promise<undefined>} A promise that resolves once cleanup has
+ * finished.
+ */
+ async #cleanup(db) {
+ // This may run while background fetches are happening. However, that
+ // doesn't matter because those images will definitely not be expired.
+ const now = Date.now();
+ await Promise.all(
+ Object.values(db.data.images)
+ .filter(entry => now - entry.lastLoaded >= IMAGE_EXPIRY_DURATION)
+ .map(entry => {
+ const path = PathUtils.join(REMOTE_IMAGES_PATH, entry.recordId);
+ delete db.data.images[entry.recordId];
+
+ return IOUtils.remove(path).catch(e => {
+ console.error(
+ `Could not remove remote image ${entry.recordId}: ${e}`
+ );
+ });
+ })
+ );
+
+ db.saveSoon();
+ }
+
+ /**
+ * Return the record ID from an image ID.
+ *
+ * Prior to Firefox 101, imageIds were of the form ${recordId}.${extension} so
+ * that we could infer the mimetype.
+ *
+ * @returns The RemoteSettings record ID.
+ */
+ #getRecordId(imageId) {
+ const idx = imageId.lastIndexOf(".");
+ if (idx === -1) {
+ return imageId;
+ }
+ return imageId.substring(0, idx);
+ }
+
+ /**
+ * Read the image from disk
+ *
+ * @param {JSONFile} db The RemoteImages database.
+ * @param {string} recordId The record ID of the image.
+ *
+ * @returns A promise that resolves to a blob, or rejects with an Error.
+ */
+ async #readFromDisk(db, recordId) {
+ const path = PathUtils.join(REMOTE_IMAGES_PATH, recordId);
+
+ try {
+ const blob = await File.createFromFileName(path, {
+ type: db.data.images[recordId].mimetype,
+ });
+ db.data.images[recordId].lastLoaded = Date.now();
+
+ return blob;
+ } catch (e) {
+ // If we cannot read the file from disk, delete the entry.
+ delete db.data.images[recordId];
+
+ throw e;
+ } finally {
+ db.saveSoon();
+ }
+ }
+
+ /**
+ * Download an image from RemoteSettings.
+ *
+ * @param {JSONFile} db The RemoteImages database.
+ * @param {string} recordId The record ID of the image.
+ * @param {object} options Options for downloading the image.
+ * @param {boolean} options.fetchOnly Whether or not to only fetch the image.
+ *
+ * @returns If |fetchOnly| is true, a promise that resolves to undefined.
+ * If |fetchOnly| is false, a promise that resolves to a Blob of the
+ * image data.
+ */
+ async #download(db, recordId, { fetchOnly = false } = {}) {
+ // It is safe to call #unsafeDownload here because we hold the db while the
+ // entire download runs.
+ const { record, arrayBuffer } = await this.#unsafeDownload(recordId);
+ const { mimetype, hash } = record.data.attachment;
+
+ if (fetchOnly) {
+ Object.assign(db.data.images[recordId], { mimetype, hash });
+ } else {
+ db.data.images[recordId] = {
+ recordId,
+ mimetype,
+ hash,
+ lastLoaded: Date.now(),
+ };
+ }
+
+ db.saveSoon();
+
+ if (fetchOnly) {
+ return undefined;
+ }
+
+ return new Blob([arrayBuffer], { type: record.data.attachment.mimetype });
+ }
+
+ /**
+ * Download an image *without* holding a handle to the database.
+ *
+ * @param {string} recordId The record ID of the image to download
+ *
+ * @returns A promise that resolves to the RemoteSettings record and the
+ * downloaded ArrayBuffer.
+ */
+ async #unsafeDownload(recordId) {
+ const client = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL);
+
+ const record = await client
+ .bucket(RS_MAIN_BUCKET)
+ .collection(RS_COLLECTION)
+ .getRecord(recordId);
+
+ const downloader = new lazy.Downloader(RS_MAIN_BUCKET, RS_COLLECTION);
+ const arrayBuffer = await downloader.downloadAsBytes(record.data, {
+ retries: RS_DOWNLOAD_MAX_RETRIES,
+ });
+
+ const path = PathUtils.join(REMOTE_IMAGES_PATH, recordId);
+
+ // Cache to disk.
+ //
+ // We do not await this promise because any other attempt to interact with
+ // the file via IOUtils will have to synchronize via the IOUtils event queue
+ // anyway.
+ //
+ // This is OK to do without holding the db because cleanup will not touch
+ // this image.
+ IOUtils.write(path, new Uint8Array(arrayBuffer));
+
+ return { record, arrayBuffer };
+ }
+
+ /**
+ * Prefetch images for the given messages.
+ *
+ * This will only acquire the db handle when we need to handle internal state
+ * so that other consumers can interact with RemoteImages while pre-fetches
+ * are happening.
+ *
+ * NB: This function is not intended to be awaited so that it can run the
+ * fetches in the background.
+ *
+ * @param {object[]} messages The FxMS messages to prefetch images for.
+ */
+ async prefetchImagesFor(messages) {
+ // Collect the list of record IDs from the message, if we have an inspector
+ // for it.
+ const recordIds = messages
+ .filter(
+ message =>
+ message.template && Object.hasOwn(MessageInspectors, message.template)
+ )
+ .flatMap(message => MessageInspectors[message.template](message))
+ .map(imageId => this.#getRecordId(imageId));
+
+ // If we find some messages, grab the db lock and queue the downloads of
+ // each.
+ if (recordIds.length) {
+ const promises = await this.withDb(
+ db =>
+ new Map(
+ recordIds.reduce((entries, recordId) => {
+ const promise = this.#beginPrefetch(db, recordId);
+
+ // If we already have the image, #beginPrefetching will return
+ // null instead of a promise.
+ if (promise !== null) {
+ this.#fetching.set(recordId, promise);
+ entries.push([recordId, promise]);
+ }
+
+ return entries;
+ }, [])
+ )
+ );
+
+ // We have dropped db lock and the fetches will continue in the background.
+ // If we do not drop the lock here, nothing can interact with RemoteImages
+ // while we are pre-fetching.
+ //
+ // As each prefetch request finishes, they will individually grab the db
+ // lock (inside #finishPrefetch or #handleFailedPrefetch) to update
+ // internal state.
+ const prefetchesFinished = Array.from(promises.entries()).map(
+ ([recordId, promise]) =>
+ promise.then(
+ result => this.#finishPrefetch(result),
+ () => this.#handleFailedPrefetch(recordId)
+ )
+ );
+
+ // Wait for all prefetches to finish before we send our notification.
+ await Promise.all(prefetchesFinished);
+
+ Services.obs.notifyObservers(null, PREFETCH_FINISHED_TOPIC);
+ }
+ }
+
+ /**
+ * Ensure the image for the given record ID has a database entry.
+ * Begin pre-fetching the requested image if we do not already have it locally.
+ *
+ * @param {JSONFile} db The database.
+ * @param {string} recordId The record ID of the image.
+ *
+ * @returns If the image is already cached locally, null is returned.
+ * Otherwise, a promise that resolves to an object including the
+ * recordId, the Remote Settings record, and the ArrayBuffer of the
+ * downloaded file.
+ */
+ #beginPrefetch(db, recordId) {
+ if (!Object.hasOwn(db.data.images, recordId)) {
+ // We kick off the download while we hold the db (so we can record the
+ // promise in #fetches), but we do not ensure that the download completes
+ // while we hold it.
+ //
+ // It is safe to call #unsafeDownload here and let the promises resolve
+ // outside this function because we record the recordId and promise in
+ // #fetching so any concurrent request to load the same image will re-use
+ // that promise and not trigger a second download (and therefore IO).
+ const promise = this.#unsafeDownload(recordId);
+ this.#fetching.set(recordId, promise);
+
+ return promise;
+ }
+
+ return null;
+ }
+
+ /**
+ * Finish prefetching an image.
+ *
+ * @param {object} options
+ * @param {object} options.record The Remote Settings record.
+ */
+ #finishPrefetch({ record }) {
+ return this.withDb(db => {
+ const { id: recordId } = record.data;
+ const { mimetype, hash } = record.data.attachment;
+
+ this.#fetching.delete(recordId);
+
+ db.data.images[recordId] = {
+ recordId,
+ mimetype,
+ hash,
+ lastLoaded: Date.now(),
+ };
+
+ db.saveSoon();
+ });
+ }
+
+ /**
+ * Remove the prefetch entry for a fetch that failed.
+ */
+ #handleFailedPrefetch(recordId) {
+ return this.withDb(db => {
+ this.#fetching.delete(recordId);
+ });
+ }
+
+ /**
+ * Migrate from a file-based store to an index-based store.
+ */
+ async #migrate() {
+ let children;
+ try {
+ children = await IOUtils.getChildren(REMOTE_IMAGES_PATH);
+
+ // Delete all previously cached entries.
+ await Promise.all(
+ children.map(async path => {
+ try {
+ await IOUtils.remove(path);
+ } catch (e) {
+ console.error(`RemoteImages could not delete ${path}: ${e}`);
+ }
+ })
+ );
+ } catch (e) {
+ if (!(DOMException.isInstance(e) && e.name === "NotFoundError")) {
+ throw e;
+ }
+ }
+
+ await IOUtils.makeDirectory(REMOTE_IMAGES_PATH);
+ const db = new JSONFile({ path: REMOTE_IMAGES_DB_PATH });
+ db.data = {
+ version: 1,
+ images: {},
+ };
+ db.saveSoon();
+ return db;
+ }
+}
+
+const RemoteImages = new _RemoteImages();
+
+const EXPORTED_SYMBOLS = [
+ "RemoteImages",
+ "REMOTE_IMAGES_PATH",
+ "REMOTE_IMAGES_DB_PATH",
+];
diff --git a/browser/components/newtab/lib/RemoteL10n.jsm b/browser/components/newtab/lib/RemoteL10n.jsm
new file mode 100644
index 0000000000..6c96e954b9
--- /dev/null
+++ b/browser/components/newtab/lib/RemoteL10n.jsm
@@ -0,0 +1,252 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/**
+ * The downloaded Fluent file is located in this sub-directory of the local
+ * profile directory.
+ */
+const USE_REMOTE_L10N_PREF =
+ "browser.newtabpage.activity-stream.asrouter.useRemoteL10n";
+
+/**
+ * All supported locales for remote l10n
+ *
+ * This is used by ASRouter.jsm to check if the locale is supported before
+ * issuing the request for remote fluent files to RemoteSettings.
+ *
+ * Note:
+ * * this is generated based on "browser/locales/all-locales" as l10n doesn't
+ * provide an API to fetch that list
+ *
+ * * this list doesn't include "en-US", though "en-US" is well supported and
+ * `_RemoteL10n.isLocaleSupported()` will handle it properly
+ */
+const ALL_LOCALES = new Set([
+ "ach",
+ "af",
+ "an",
+ "ar",
+ "ast",
+ "az",
+ "be",
+ "bg",
+ "bn",
+ "bo",
+ "br",
+ "brx",
+ "bs",
+ "ca",
+ "ca-valencia",
+ "cak",
+ "ckb",
+ "cs",
+ "cy",
+ "da",
+ "de",
+ "dsb",
+ "el",
+ "en-CA",
+ "en-GB",
+ "eo",
+ "es-AR",
+ "es-CL",
+ "es-ES",
+ "es-MX",
+ "et",
+ "eu",
+ "fa",
+ "ff",
+ "fi",
+ "fr",
+ "fy-NL",
+ "ga-IE",
+ "gd",
+ "gl",
+ "gn",
+ "gu-IN",
+ "he",
+ "hi-IN",
+ "hr",
+ "hsb",
+ "hu",
+ "hy-AM",
+ "hye",
+ "ia",
+ "id",
+ "is",
+ "it",
+ "ja",
+ "ja-JP-mac",
+ "ka",
+ "kab",
+ "kk",
+ "km",
+ "kn",
+ "ko",
+ "lij",
+ "lo",
+ "lt",
+ "ltg",
+ "lv",
+ "meh",
+ "mk",
+ "mr",
+ "ms",
+ "my",
+ "nb-NO",
+ "ne-NP",
+ "nl",
+ "nn-NO",
+ "oc",
+ "pa-IN",
+ "pl",
+ "pt-BR",
+ "pt-PT",
+ "rm",
+ "ro",
+ "ru",
+ "scn",
+ "si",
+ "sk",
+ "sl",
+ "son",
+ "sq",
+ "sr",
+ "sv-SE",
+ "szl",
+ "ta",
+ "te",
+ "th",
+ "tl",
+ "tr",
+ "trs",
+ "uk",
+ "ur",
+ "uz",
+ "vi",
+ "wo",
+ "xh",
+ "zh-CN",
+ "zh-TW",
+]);
+
+class _RemoteL10n {
+ constructor() {
+ this._l10n = null;
+ }
+
+ createElement(doc, elem, options = {}) {
+ let node;
+ if (options.content && options.content.string_id) {
+ node = doc.createElement("remote-text");
+ } else {
+ node = doc.createElementNS("http://www.w3.org/1999/xhtml", elem);
+ }
+ if (options.classList) {
+ node.classList.add(options.classList);
+ }
+ this.setString(node, options);
+
+ return node;
+ }
+
+ // If `string_id` is present it means we are relying on fluent for translations.
+ // Otherwise, we have a vanilla string.
+ setString(el, { content, attributes = {} }) {
+ if (content && content.string_id) {
+ for (let [fluentId, value] of Object.entries(attributes)) {
+ el.setAttribute(`fluent-variable-${fluentId}`, value);
+ }
+ el.setAttribute("fluent-remote-id", content.string_id);
+ } else {
+ el.textContent = content;
+ }
+ }
+
+ /**
+ * Creates a new DOMLocalization instance with the Fluent file from Remote Settings.
+ *
+ * Note: it will use the local Fluent file in any of following cases:
+ * * the remote Fluent file is not available
+ * * it was told to use the local Fluent file
+ */
+ _createDOML10n() {
+ /* istanbul ignore next */
+ let useRemoteL10n = Services.prefs.getBoolPref(USE_REMOTE_L10N_PREF, true);
+ if (useRemoteL10n && !L10nRegistry.getInstance().hasSource("cfr")) {
+ const appLocale = Services.locale.appLocaleAsBCP47;
+ const l10nFluentDir = PathUtils.join(
+ Services.dirsvc.get("ProfLD", Ci.nsIFile).path,
+ "settings",
+ "main",
+ "ms-language-packs"
+ );
+ let cfrIndexedFileSource = new L10nFileSource(
+ "cfr",
+ "app",
+ [appLocale],
+ `file://${l10nFluentDir}/`,
+ {
+ addResourceOptions: {
+ allowOverrides: true,
+ },
+ },
+ [`file://${l10nFluentDir}/browser/newtab/asrouter.ftl`]
+ );
+ L10nRegistry.getInstance().registerSources([cfrIndexedFileSource]);
+ } else if (!useRemoteL10n && L10nRegistry.getInstance().hasSource("cfr")) {
+ L10nRegistry.getInstance().removeSources(["cfr"]);
+ }
+
+ return new DOMLocalization(
+ [
+ "browser/newtab/asrouter.ftl",
+ "browser/branding/brandings.ftl",
+ "browser/branding/sync-brand.ftl",
+ "branding/brand.ftl",
+ "browser/defaultBrowserNotification.ftl",
+ ],
+ false
+ );
+ }
+
+ get l10n() {
+ if (!this._l10n) {
+ this._l10n = this._createDOML10n();
+ }
+ return this._l10n;
+ }
+
+ reloadL10n() {
+ this._l10n = null;
+ }
+
+ isLocaleSupported(locale) {
+ return locale === "en-US" || ALL_LOCALES.has(locale);
+ }
+
+ /**
+ * Format given `localizableText`.
+ *
+ * Format `localizableText` if it is an object using any `string_id` field,
+ * otherwise return `localizableText` unmodified.
+ *
+ * @param {object|string} `localizableText` to format.
+ * @return {string} formatted text.
+ */
+ async formatLocalizableText(localizableText) {
+ if (typeof localizableText !== "string") {
+ // It's more useful to get an error than passing through an object without
+ // a `string_id` field.
+ let value = await this.l10n.formatValue(localizableText.string_id);
+ return value;
+ }
+ return localizableText;
+ }
+}
+
+const RemoteL10n = new _RemoteL10n();
+
+const EXPORTED_SYMBOLS = ["RemoteL10n", "_RemoteL10n"];
diff --git a/browser/components/newtab/lib/Screenshots.jsm b/browser/components/newtab/lib/Screenshots.jsm
new file mode 100644
index 0000000000..556e2e6aa2
--- /dev/null
+++ b/browser/components/newtab/lib/Screenshots.jsm
@@ -0,0 +1,144 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const EXPORTED_SYMBOLS = ["Screenshots"];
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "BackgroundPageThumbs",
+ "resource://gre/modules/BackgroundPageThumbs.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PageThumbs",
+ "resource://gre/modules/PageThumbs.jsm"
+);
+ChromeUtils.defineESModuleGetters(lazy, {
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+});
+
+const GREY_10 = "#F9F9FA";
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "gPrivilegedAboutProcessEnabled",
+ "browser.tabs.remote.separatePrivilegedContentProcess",
+ false
+);
+
+const Screenshots = {
+ /**
+ * Get a screenshot / thumbnail for a url. Either returns the disk cached
+ * image or initiates a background request for the url.
+ *
+ * @param url {string} The url to get a thumbnail
+ * @return {Promise} Resolves a custom object or null if failed
+ */
+ async getScreenshotForURL(url) {
+ try {
+ await lazy.BackgroundPageThumbs.captureIfMissing(url, {
+ backgroundColor: GREY_10,
+ });
+
+ // The privileged about content process is able to use the moz-page-thumb
+ // protocol, so if it's enabled, send that down.
+ if (lazy.gPrivilegedAboutProcessEnabled) {
+ return lazy.PageThumbs.getThumbnailURL(url);
+ }
+
+ // Otherwise, for normal content processes, we fallback to using
+ // Blob URIs for the screenshots.
+ const imgPath = lazy.PageThumbs.getThumbnailPath(url);
+
+ const filePathResponse = await fetch(`file://${imgPath}`);
+ const fileContents = await filePathResponse.blob();
+
+ // Check if the file is empty, which indicates there isn't actually a
+ // thumbnail, so callers can show a failure state.
+ if (fileContents.size === 0) {
+ return null;
+ }
+
+ return { path: imgPath, data: fileContents };
+ } catch (err) {
+ console.error(`getScreenshot(${url}) failed: ${err}`);
+ }
+
+ // We must have failed to get the screenshot, so persist the failure by
+ // storing an empty file. Future calls will then skip requesting and return
+ // failure, so do the same thing here. The empty file should not expire with
+ // the usual filtering process to avoid repeated background requests, which
+ // can cause unwanted high CPU, network and memory usage - Bug 1384094
+ try {
+ await lazy.PageThumbs._store(url, url, null, true);
+ } catch (err) {
+ // Probably failed to create the empty file, but not much more we can do.
+ }
+ return null;
+ },
+
+ /**
+ * Checks if all the open windows are private browsing windows. If so, we do not
+ * want to collect screenshots. If there exists at least 1 non-private window,
+ * we are ok to collect screenshots.
+ */
+ _shouldGetScreenshots() {
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ if (!lazy.PrivateBrowsingUtils.isWindowPrivate(win)) {
+ // As soon as we encounter 1 non-private window, screenshots are fair game.
+ return true;
+ }
+ }
+ return false;
+ },
+
+ /**
+ * Conditionally get a screenshot for a link if there's no existing pending
+ * screenshot. Updates the cached link's desired property with the result.
+ *
+ * @param link {object} Link object to update
+ * @param url {string} Url to get a screenshot of
+ * @param property {string} Name of property on object to set
+ @ @param onScreenshot {function} Callback for when the screenshot loads
+ */
+ async maybeCacheScreenshot(link, url, property, onScreenshot) {
+ // If there are only private windows open, do not collect screenshots
+ if (!this._shouldGetScreenshots()) {
+ return;
+ }
+ // __sharedCache may not exist yet for links from default top sites that
+ // don't have a default tippy top icon.
+ if (!link.__sharedCache) {
+ link.__sharedCache = {
+ updateLink(prop, val) {
+ link[prop] = val;
+ },
+ };
+ }
+ const cache = link.__sharedCache;
+ // Nothing to do if we already have a pending screenshot or
+ // if a previous request failed and returned null.
+ if (cache.fetchingScreenshot || link[property] !== undefined) {
+ return;
+ }
+
+ // Save the promise to the cache so other links get it immediately
+ cache.fetchingScreenshot = this.getScreenshotForURL(url);
+
+ // Clean up now that we got the screenshot
+ const screenshot = await cache.fetchingScreenshot;
+ delete cache.fetchingScreenshot;
+
+ // Update the cache for future links and call back for existing content
+ cache.updateLink(property, screenshot);
+ onScreenshot(screenshot);
+ },
+};
diff --git a/browser/components/newtab/lib/SearchShortcuts.jsm b/browser/components/newtab/lib/SearchShortcuts.jsm
new file mode 100644
index 0000000000..926681feca
--- /dev/null
+++ b/browser/components/newtab/lib/SearchShortcuts.jsm
@@ -0,0 +1,76 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// List of sites we match against Topsites in order to identify sites
+// that should be converted to search Topsites
+const SEARCH_SHORTCUTS = [
+ { keyword: "@amazon", shortURL: "amazon", url: "https://amazon.com" },
+ { keyword: "@\u767E\u5EA6", shortURL: "baidu", url: "https://baidu.com" },
+ { keyword: "@google", shortURL: "google", url: "https://google.com" },
+ {
+ keyword: "@\u044F\u043D\u0434\u0435\u043A\u0441",
+ shortURL: "yandex",
+ url: "https://yandex.com",
+ },
+];
+
+// These can be added via the editor but will not be added organically
+const CUSTOM_SEARCH_SHORTCUTS = [
+ ...SEARCH_SHORTCUTS,
+ { keyword: "@bing", shortURL: "bing", url: "https://bing.com" },
+ {
+ keyword: "@duckduckgo",
+ shortURL: "duckduckgo",
+ url: "https://duckduckgo.com",
+ },
+ { keyword: "@ebay", shortURL: "ebay", url: "https://ebay.com" },
+ { keyword: "@twitter", shortURL: "twitter", url: "https://twitter.com" },
+ {
+ keyword: "@wikipedia",
+ shortURL: "wikipedia",
+ url: "https://wikipedia.org",
+ },
+];
+
+// Note: you must add the activity stream branch to the beginning of this if using outside activity stream
+const SEARCH_SHORTCUTS_EXPERIMENT = "improvesearch.topSiteSearchShortcuts";
+const SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF =
+ "improvesearch.topSiteSearchShortcuts.searchEngines";
+const SEARCH_SHORTCUTS_HAVE_PINNED_PREF =
+ "improvesearch.topSiteSearchShortcuts.havePinned";
+
+function getSearchProvider(candidateShortURL) {
+ return (
+ SEARCH_SHORTCUTS.filter(match => candidateShortURL === match.shortURL)[0] ||
+ null
+ );
+}
+
+// Get the search form URL for a given search keyword. This allows us to pick
+// different tippytop icons for the different variants. Sush as yandex.com vs. yandex.ru.
+// See more details in bug 1643523.
+async function getSearchFormURL(keyword) {
+ const engine = await Services.search.getEngineByAlias(keyword);
+ return engine?.wrappedJSObject._searchForm;
+}
+
+// Check topsite against predefined list of valid search engines
+// https://searchfox.org/mozilla-central/rev/ca869724246f4230b272ed1c8b9944596e80d920/toolkit/components/search/nsSearchService.js#939
+async function checkHasSearchEngine(keyword) {
+ return (await Services.search.getAppProvidedEngines()).find(
+ e => e.aliases.includes(keyword) && !e.hidden
+ );
+}
+
+const EXPORTED_SYMBOLS = [
+ "checkHasSearchEngine",
+ "getSearchProvider",
+ "getSearchFormURL",
+ "SEARCH_SHORTCUTS",
+ "CUSTOM_SEARCH_SHORTCUTS",
+ "SEARCH_SHORTCUTS_EXPERIMENT",
+ "SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF",
+ "SEARCH_SHORTCUTS_HAVE_PINNED_PREF",
+];
diff --git a/browser/components/newtab/lib/SectionsManager.jsm b/browser/components/newtab/lib/SectionsManager.jsm
new file mode 100644
index 0000000000..81c052397e
--- /dev/null
+++ b/browser/components/newtab/lib/SectionsManager.jsm
@@ -0,0 +1,720 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { EventEmitter } = ChromeUtils.importESModule(
+ "resource://gre/modules/EventEmitter.sys.mjs"
+);
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { getDefaultOptions } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamStorage.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+/*
+ * Generators for built in sections, keyed by the pref name for their feed.
+ * Built in sections may depend on options stored as serialised JSON in the pref
+ * `${feed_pref_name}.options`.
+ */
+const BUILT_IN_SECTIONS = ({ newtab, pocketNewtab }) => ({
+ "feeds.section.topstories": options => ({
+ id: "topstories",
+ pref: {
+ titleString: {
+ id: "home-prefs-recommended-by-header",
+ values: { provider: options.provider_name },
+ },
+ descString: {
+ id: "home-prefs-recommended-by-description-new",
+ values: { provider: options.provider_name },
+ },
+ nestedPrefs: [
+ ...(options.show_spocs
+ ? [
+ {
+ name: "showSponsored",
+ titleString:
+ "home-prefs-recommended-by-option-sponsored-stories",
+ icon: "icon-info",
+ eventSource: "POCKET_SPOCS",
+ },
+ ]
+ : []),
+ ...(pocketNewtab.recentSavesEnabled
+ ? [
+ {
+ name: "showRecentSaves",
+ titleString: "home-prefs-recommended-by-option-recent-saves",
+ icon: "icon-info",
+ eventSource: "POCKET_RECENT_SAVES",
+ },
+ ]
+ : []),
+ ],
+ learnMore: {
+ link: {
+ href: "https://getpocket.com/firefox/new_tab_learn_more",
+ id: "home-prefs-recommended-by-learn-more",
+ },
+ },
+ },
+ shouldHidePref: options.hidden,
+ eventSource: "TOP_STORIES",
+ icon: options.provider_icon,
+ title: {
+ id: "newtab-section-header-pocket",
+ values: { provider: options.provider_name },
+ },
+ learnMore: {
+ link: {
+ href: "https://getpocket.com/firefox/new_tab_learn_more",
+ message: { id: "newtab-pocket-learn-more" },
+ },
+ },
+ compactCards: false,
+ rowsPref: "section.topstories.rows",
+ maxRows: 4,
+ availableLinkMenuOptions: [
+ "CheckBookmarkOrArchive",
+ "CheckSavedToPocket",
+ "Separator",
+ "OpenInNewWindow",
+ "OpenInPrivateWindow",
+ "Separator",
+ "BlockUrl",
+ ],
+ emptyState: {
+ message: {
+ id: "newtab-empty-section-topstories",
+ values: { provider: options.provider_name },
+ },
+ icon: "check",
+ },
+ shouldSendImpressionStats: true,
+ dedupeFrom: ["highlights"],
+ }),
+ "feeds.section.highlights": options => ({
+ id: "highlights",
+ pref: {
+ titleString: {
+ id: "home-prefs-recent-activity-header",
+ },
+ descString: {
+ id: "home-prefs-recent-activity-description",
+ },
+ nestedPrefs: [
+ {
+ name: "section.highlights.includeVisited",
+ titleString: "home-prefs-highlights-option-visited-pages",
+ },
+ {
+ name: "section.highlights.includeBookmarks",
+ titleString: "home-prefs-highlights-options-bookmarks",
+ },
+ {
+ name: "section.highlights.includeDownloads",
+ titleString: "home-prefs-highlights-option-most-recent-download",
+ },
+ {
+ name: "section.highlights.includePocket",
+ titleString: "home-prefs-highlights-option-saved-to-pocket",
+ hidden: !Services.prefs.getBoolPref(
+ "extensions.pocket.enabled",
+ true
+ ),
+ },
+ ],
+ },
+ shouldHidePref: false,
+ eventSource: "HIGHLIGHTS",
+ icon: "chrome://global/skin/icons/highlights.svg",
+ title: {
+ id: "newtab-section-header-recent-activity",
+ },
+ compactCards: true,
+ rowsPref: "section.highlights.rows",
+ maxRows: 4,
+ emptyState: {
+ message: { id: "newtab-empty-section-highlights" },
+ icon: "chrome://global/skin/icons/highlights.svg",
+ },
+ shouldSendImpressionStats: false,
+ }),
+});
+
+const SectionsManager = {
+ ACTIONS_TO_PROXY: ["WEBEXT_CLICK", "WEBEXT_DISMISS"],
+ CONTEXT_MENU_PREFS: { CheckSavedToPocket: "extensions.pocket.enabled" },
+ CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES: {
+ history: [
+ "CheckBookmark",
+ "CheckSavedToPocket",
+ "Separator",
+ "OpenInNewWindow",
+ "OpenInPrivateWindow",
+ "Separator",
+ "BlockUrl",
+ "DeleteUrl",
+ ],
+ bookmark: [
+ "CheckBookmark",
+ "CheckSavedToPocket",
+ "Separator",
+ "OpenInNewWindow",
+ "OpenInPrivateWindow",
+ "Separator",
+ "BlockUrl",
+ "DeleteUrl",
+ ],
+ pocket: [
+ "ArchiveFromPocket",
+ "CheckSavedToPocket",
+ "Separator",
+ "OpenInNewWindow",
+ "OpenInPrivateWindow",
+ "Separator",
+ "BlockUrl",
+ ],
+ download: [
+ "OpenFile",
+ "ShowFile",
+ "Separator",
+ "GoToDownloadPage",
+ "CopyDownloadLink",
+ "Separator",
+ "RemoveDownload",
+ "BlockUrl",
+ ],
+ },
+ initialized: false,
+ sections: new Map(),
+ async init(prefs = {}, storage) {
+ this._storage = storage;
+ const featureConfig = {
+ newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {},
+ pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {},
+ };
+
+ for (const feedPrefName of Object.keys(BUILT_IN_SECTIONS(featureConfig))) {
+ const optionsPrefName = `${feedPrefName}.options`;
+ await this.addBuiltInSection(feedPrefName, prefs[optionsPrefName]);
+
+ this._dedupeConfiguration = [];
+ this.sections.forEach(section => {
+ if (section.dedupeFrom) {
+ this._dedupeConfiguration.push({
+ id: section.id,
+ dedupeFrom: section.dedupeFrom,
+ });
+ }
+ });
+ }
+
+ Object.keys(this.CONTEXT_MENU_PREFS).forEach(k =>
+ Services.prefs.addObserver(this.CONTEXT_MENU_PREFS[k], this)
+ );
+
+ this.initialized = true;
+ this.emit(this.INIT);
+ },
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "nsPref:changed":
+ for (const pref of Object.keys(this.CONTEXT_MENU_PREFS)) {
+ if (data === this.CONTEXT_MENU_PREFS[pref]) {
+ this.updateSections();
+ }
+ }
+ break;
+ }
+ },
+ updateSectionPrefs(id, collapsed) {
+ const section = this.sections.get(id);
+ if (!section) {
+ return;
+ }
+
+ const updatedSection = Object.assign({}, section, {
+ pref: Object.assign({}, section.pref, collapsed),
+ });
+ this.updateSection(id, updatedSection, true);
+ },
+ async addBuiltInSection(feedPrefName, optionsPrefValue = "{}") {
+ let options;
+ let storedPrefs;
+ const featureConfig = {
+ newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {},
+ pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {},
+ };
+ try {
+ options = JSON.parse(optionsPrefValue);
+ } catch (e) {
+ options = {};
+ console.error(`Problem parsing options pref for ${feedPrefName}`);
+ }
+ try {
+ storedPrefs = (await this._storage.get(feedPrefName)) || {};
+ } catch (e) {
+ storedPrefs = {};
+ console.error(`Problem getting stored prefs for ${feedPrefName}`);
+ }
+ const defaultSection = BUILT_IN_SECTIONS(featureConfig)[feedPrefName](
+ options
+ );
+ const section = Object.assign({}, defaultSection, {
+ pref: Object.assign(
+ {},
+ defaultSection.pref,
+ getDefaultOptions(storedPrefs)
+ ),
+ });
+ section.pref.feed = feedPrefName;
+ this.addSection(section.id, Object.assign(section, { options }));
+ },
+ addSection(id, options) {
+ this.updateLinkMenuOptions(options, id);
+ this.sections.set(id, options);
+ this.emit(this.ADD_SECTION, id, options);
+ },
+ removeSection(id) {
+ this.emit(this.REMOVE_SECTION, id);
+ this.sections.delete(id);
+ },
+ enableSection(id, isStartup = false) {
+ this.updateSection(id, { enabled: true }, true, isStartup);
+ this.emit(this.ENABLE_SECTION, id);
+ },
+ disableSection(id) {
+ this.updateSection(
+ id,
+ { enabled: false, rows: [], initialized: false },
+ true
+ );
+ this.emit(this.DISABLE_SECTION, id);
+ },
+ updateSections() {
+ this.sections.forEach((section, id) =>
+ this.updateSection(id, section, true)
+ );
+ },
+ updateSection(id, options, shouldBroadcast, isStartup = false) {
+ this.updateLinkMenuOptions(options, id);
+ if (this.sections.has(id)) {
+ const optionsWithDedupe = Object.assign({}, options, {
+ dedupeConfigurations: this._dedupeConfiguration,
+ });
+ this.sections.set(id, Object.assign(this.sections.get(id), options));
+ this.emit(
+ this.UPDATE_SECTION,
+ id,
+ optionsWithDedupe,
+ shouldBroadcast,
+ isStartup
+ );
+ }
+ },
+
+ /**
+ * Save metadata to places db and add a visit for that URL.
+ */
+ updateBookmarkMetadata({ url }) {
+ this.sections.forEach((section, id) => {
+ if (id === "highlights") {
+ // Skip Highlights cards, we already have that metadata.
+ return;
+ }
+ if (section.rows) {
+ section.rows.forEach(card => {
+ if (
+ card.url === url &&
+ card.description &&
+ card.title &&
+ card.image
+ ) {
+ lazy.PlacesUtils.history.update({
+ url: card.url,
+ title: card.title,
+ description: card.description,
+ previewImageURL: card.image,
+ });
+ // Highlights query skips bookmarks with no visits.
+ lazy.PlacesUtils.history.insert({
+ url,
+ title: card.title,
+ visits: [{}],
+ });
+ }
+ });
+ }
+ });
+ },
+
+ /**
+ * Sets the section's context menu options. These are all available context menu
+ * options minus the ones that are tied to a pref (see CONTEXT_MENU_PREFS) set
+ * to false.
+ *
+ * @param options section options
+ * @param id section ID
+ */
+ updateLinkMenuOptions(options, id) {
+ if (options.availableLinkMenuOptions) {
+ options.contextMenuOptions = options.availableLinkMenuOptions.filter(
+ o =>
+ !this.CONTEXT_MENU_PREFS[o] ||
+ Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o])
+ );
+ }
+
+ // Once we have rows, we can give each card it's own context menu based on it's type.
+ // We only want to do this for highlights because those have different data types.
+ // All other sections (built by the web extension API) will have the same context menu per section
+ if (options.rows && id === "highlights") {
+ this._addCardTypeLinkMenuOptions(options.rows);
+ }
+ },
+
+ /**
+ * Sets each card in highlights' context menu options based on the card's type.
+ * (See types.js for a list of types)
+ *
+ * @param rows section rows containing a type for each card
+ */
+ _addCardTypeLinkMenuOptions(rows) {
+ for (let card of rows) {
+ if (!this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[card.type]) {
+ console.error(
+ `No context menu for highlight type ${card.type} is configured`
+ );
+ } else {
+ card.contextMenuOptions = this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[
+ card.type
+ ];
+
+ // Remove any options that shouldn't be there based on CONTEXT_MENU_PREFS.
+ // For example: If the Pocket extension is disabled, we should remove the CheckSavedToPocket option
+ // for each card that has it
+ card.contextMenuOptions = card.contextMenuOptions.filter(
+ o =>
+ !this.CONTEXT_MENU_PREFS[o] ||
+ Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o])
+ );
+ }
+ }
+ },
+
+ /**
+ * Update a specific section card by its url. This allows an action to be
+ * broadcast to all existing pages to update a specific card without having to
+ * also force-update the rest of the section's cards and state on those pages.
+ *
+ * @param id The id of the section with the card to be updated
+ * @param url The url of the card to update
+ * @param options The options to update for the card
+ * @param shouldBroadcast Whether or not to broadcast the update
+ * @param isStartup If this update is during startup.
+ */
+ updateSectionCard(id, url, options, shouldBroadcast, isStartup = false) {
+ if (this.sections.has(id)) {
+ const card = this.sections.get(id).rows.find(elem => elem.url === url);
+ if (card) {
+ Object.assign(card, options);
+ }
+ this.emit(
+ this.UPDATE_SECTION_CARD,
+ id,
+ url,
+ options,
+ shouldBroadcast,
+ isStartup
+ );
+ }
+ },
+ removeSectionCard(sectionId, url) {
+ if (!this.sections.has(sectionId)) {
+ return;
+ }
+ const rows = this.sections
+ .get(sectionId)
+ .rows.filter(row => row.url !== url);
+ this.updateSection(sectionId, { rows }, true);
+ },
+ onceInitialized(callback) {
+ if (this.initialized) {
+ callback();
+ } else {
+ this.once(this.INIT, callback);
+ }
+ },
+ uninit() {
+ Object.keys(this.CONTEXT_MENU_PREFS).forEach(k =>
+ Services.prefs.removeObserver(this.CONTEXT_MENU_PREFS[k], this)
+ );
+ SectionsManager.initialized = false;
+ },
+};
+
+for (const action of [
+ "ACTION_DISPATCHED",
+ "ADD_SECTION",
+ "REMOVE_SECTION",
+ "ENABLE_SECTION",
+ "DISABLE_SECTION",
+ "UPDATE_SECTION",
+ "UPDATE_SECTION_CARD",
+ "INIT",
+ "UNINIT",
+]) {
+ SectionsManager[action] = action;
+}
+
+EventEmitter.decorate(SectionsManager);
+
+class SectionsFeed {
+ constructor() {
+ this.init = this.init.bind(this);
+ this.onAddSection = this.onAddSection.bind(this);
+ this.onRemoveSection = this.onRemoveSection.bind(this);
+ this.onUpdateSection = this.onUpdateSection.bind(this);
+ this.onUpdateSectionCard = this.onUpdateSectionCard.bind(this);
+ }
+
+ init() {
+ SectionsManager.on(SectionsManager.ADD_SECTION, this.onAddSection);
+ SectionsManager.on(SectionsManager.REMOVE_SECTION, this.onRemoveSection);
+ SectionsManager.on(SectionsManager.UPDATE_SECTION, this.onUpdateSection);
+ SectionsManager.on(
+ SectionsManager.UPDATE_SECTION_CARD,
+ this.onUpdateSectionCard
+ );
+ // Catch any sections that have already been added
+ SectionsManager.sections.forEach((section, id) =>
+ this.onAddSection(
+ SectionsManager.ADD_SECTION,
+ id,
+ section,
+ true /* isStartup */
+ )
+ );
+ }
+
+ uninit() {
+ SectionsManager.uninit();
+ SectionsManager.emit(SectionsManager.UNINIT);
+ SectionsManager.off(SectionsManager.ADD_SECTION, this.onAddSection);
+ SectionsManager.off(SectionsManager.REMOVE_SECTION, this.onRemoveSection);
+ SectionsManager.off(SectionsManager.UPDATE_SECTION, this.onUpdateSection);
+ SectionsManager.off(
+ SectionsManager.UPDATE_SECTION_CARD,
+ this.onUpdateSectionCard
+ );
+ }
+
+ onAddSection(event, id, options, isStartup = false) {
+ if (options) {
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.SECTION_REGISTER,
+ data: Object.assign({ id }, options),
+ meta: {
+ isStartup,
+ },
+ })
+ );
+
+ // Make sure the section is in sectionOrder pref. Otherwise, prepend it.
+ const orderedSections = this.orderedSectionIds;
+ if (!orderedSections.includes(id)) {
+ orderedSections.unshift(id);
+ this.store.dispatch(
+ ac.SetPref("sectionOrder", orderedSections.join(","))
+ );
+ }
+ }
+ }
+
+ onRemoveSection(event, id) {
+ this.store.dispatch(
+ ac.BroadcastToContent({ type: at.SECTION_DEREGISTER, data: id })
+ );
+ }
+
+ onUpdateSection(
+ event,
+ id,
+ options,
+ shouldBroadcast = false,
+ isStartup = false
+ ) {
+ if (options) {
+ const action = {
+ type: at.SECTION_UPDATE,
+ data: Object.assign(options, { id }),
+ meta: {
+ isStartup,
+ },
+ };
+ this.store.dispatch(
+ shouldBroadcast
+ ? ac.BroadcastToContent(action)
+ : ac.AlsoToPreloaded(action)
+ );
+ }
+ }
+
+ onUpdateSectionCard(
+ event,
+ id,
+ url,
+ options,
+ shouldBroadcast = false,
+ isStartup = false
+ ) {
+ if (options) {
+ const action = {
+ type: at.SECTION_UPDATE_CARD,
+ data: { id, url, options },
+ meta: {
+ isStartup,
+ },
+ };
+ this.store.dispatch(
+ shouldBroadcast
+ ? ac.BroadcastToContent(action)
+ : ac.AlsoToPreloaded(action)
+ );
+ }
+ }
+
+ get orderedSectionIds() {
+ return this.store.getState().Prefs.values.sectionOrder.split(",");
+ }
+
+ get enabledSectionIds() {
+ let sections = this.store
+ .getState()
+ .Sections.filter(section => section.enabled)
+ .map(s => s.id);
+ // Top Sites is a special case. Append if the feed is enabled.
+ if (this.store.getState().Prefs.values["feeds.topsites"]) {
+ sections.push("topsites");
+ }
+ return sections;
+ }
+
+ moveSection(id, direction) {
+ const orderedSections = this.orderedSectionIds;
+ const enabledSections = this.enabledSectionIds;
+ let index = orderedSections.indexOf(id);
+ orderedSections.splice(index, 1);
+ if (direction > 0) {
+ // "Move Down"
+ while (index < orderedSections.length) {
+ // If the section at the index is enabled/visible, insert moved section after.
+ // Otherwise, move on to the next spot and check it.
+ if (enabledSections.includes(orderedSections[index++])) {
+ break;
+ }
+ }
+ } else {
+ // "Move Up"
+ while (index > 0) {
+ // If the section at the previous index is enabled/visible, insert moved section there.
+ // Otherwise, move on to the previous spot and check it.
+ index--;
+ if (enabledSections.includes(orderedSections[index])) {
+ break;
+ }
+ }
+ }
+
+ orderedSections.splice(index, 0, id);
+ this.store.dispatch(ac.SetPref("sectionOrder", orderedSections.join(",")));
+ }
+
+ async onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ SectionsManager.onceInitialized(this.init);
+ break;
+ // Wait for pref values, as some sections have options stored in prefs
+ case at.PREFS_INITIAL_VALUES:
+ SectionsManager.init(
+ action.data,
+ this.store.dbStorage.getDbTable("sectionPrefs")
+ );
+ break;
+ case at.PREF_CHANGED: {
+ if (action.data) {
+ const matched = action.data.name.match(
+ /^(feeds.section.(\S+)).options$/i
+ );
+ if (matched) {
+ await SectionsManager.addBuiltInSection(
+ matched[1],
+ action.data.value
+ );
+ this.store.dispatch({
+ type: at.SECTION_OPTIONS_CHANGED,
+ data: matched[2],
+ });
+ }
+ }
+ break;
+ }
+ case at.UPDATE_SECTION_PREFS:
+ SectionsManager.updateSectionPrefs(action.data.id, action.data.value);
+ break;
+ case at.PLACES_BOOKMARK_ADDED:
+ SectionsManager.updateBookmarkMetadata(action.data);
+ break;
+ case at.WEBEXT_DISMISS:
+ if (action.data) {
+ SectionsManager.removeSectionCard(
+ action.data.source,
+ action.data.url
+ );
+ }
+ break;
+ case at.SECTION_DISABLE:
+ SectionsManager.disableSection(action.data);
+ break;
+ case at.SECTION_ENABLE:
+ SectionsManager.enableSection(action.data);
+ break;
+ case at.SECTION_MOVE:
+ this.moveSection(action.data.id, action.data.direction);
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ if (
+ SectionsManager.ACTIONS_TO_PROXY.includes(action.type) &&
+ SectionsManager.sections.size > 0
+ ) {
+ SectionsManager.emit(
+ SectionsManager.ACTION_DISPATCHED,
+ action.type,
+ action.data
+ );
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["SectionsFeed", "SectionsManager"];
diff --git a/browser/components/newtab/lib/ShortURL.jsm b/browser/components/newtab/lib/ShortURL.jsm
new file mode 100644
index 0000000000..19416904f0
--- /dev/null
+++ b/browser/components/newtab/lib/ShortURL.jsm
@@ -0,0 +1,83 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "IDNService",
+ "@mozilla.org/network/idn-service;1",
+ "nsIIDNService"
+);
+
+/**
+ * Properly convert internationalized domain names.
+ * @param {string} host Domain hostname.
+ * @returns {string} Hostname suitable to be displayed.
+ */
+function handleIDNHost(hostname) {
+ try {
+ return lazy.IDNService.convertToDisplayIDN(hostname, {});
+ } catch (e) {
+ // If something goes wrong (e.g. host is an IP address) just fail back
+ // to the full domain.
+ return hostname;
+ }
+}
+
+/**
+ * Get the effective top level domain of a host.
+ * @param {string} host The host to be analyzed.
+ * @return {str} The suffix or empty string if there's no suffix.
+ */
+function getETLD(host) {
+ try {
+ return Services.eTLD.getPublicSuffixFromHost(host);
+ } catch (err) {
+ return "";
+ }
+}
+
+/**
+ * shortURL - Creates a short version of a link's url, used for display purposes
+ * e.g. {url: http://www.foosite.com} => "foosite"
+ *
+ * @param {obj} link A link object
+ * {str} link.url (required)- The url of the link
+ * @return {str} A short url
+ */
+function shortURL({ url }) {
+ if (!url) {
+ return "";
+ }
+
+ // Make sure we have a valid / parseable url
+ let parsed;
+ try {
+ parsed = new URL(url);
+ } catch (ex) {
+ // Not entirely sure what we have, but just give it back
+ return url;
+ }
+
+ // Clean up the url (lowercase hostname via URL and remove www.)
+ const hostname = parsed.hostname.replace(/^www\./i, "");
+
+ // Remove the eTLD (e.g., com, net) and the preceding period from the hostname
+ const eTLD = getETLD(hostname);
+ const eTLDExtra = eTLD.length ? -(eTLD.length + 1) : Infinity;
+
+ // Ideally get the short eTLD-less host but fall back to longer url parts
+ return (
+ handleIDNHost(hostname.slice(0, eTLDExtra) || hostname) ||
+ parsed.pathname ||
+ parsed.href
+ );
+}
+
+const EXPORTED_SYMBOLS = ["shortURL", "getETLD"];
diff --git a/browser/components/newtab/lib/SiteClassifier.jsm b/browser/components/newtab/lib/SiteClassifier.jsm
new file mode 100644
index 0000000000..9527745bef
--- /dev/null
+++ b/browser/components/newtab/lib/SiteClassifier.jsm
@@ -0,0 +1,99 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { RemoteSettings } = ChromeUtils.import(
+ "resource://services-settings/remote-settings.js"
+);
+
+// Returns whether the passed in params match the criteria.
+// To match, they must contain all the params specified in criteria and the values
+// must match if a value is provided in criteria.
+function _hasParams(criteria, params) {
+ for (let param of criteria) {
+ const val = params.get(param.key);
+ if (
+ val === null ||
+ (param.value && param.value !== val) ||
+ (param.prefix && !val.startsWith(param.prefix))
+ ) {
+ return false;
+ }
+ }
+ return true;
+}
+
+/**
+ * classifySite
+ * Classifies a given URL into a category based on classification data from RemoteSettings.
+ * The data from remote settings can match a category by one of the following:
+ * - match the exact URL
+ * - match the hostname or second level domain (sld)
+ * - match query parameter(s), and optionally their values or prefixes
+ * - match both (hostname or sld) and query parameter(s)
+ *
+ * The data looks like:
+ * [{
+ * "type": "hostname-and-params-match",
+ * "criteria": [
+ * {
+ * "url": "https://matchurl.com",
+ * "hostname": "matchhostname.com",
+ * "sld": "secondleveldomain",
+ * "params": [
+ * {
+ * "key": "matchparam",
+ * "value": "matchvalue",
+ * "prefix": "matchpPrefix",
+ * },
+ * ],
+ * },
+ * ],
+ * "weight": 300,
+ * },...]
+ */
+async function classifySite(url, RS = RemoteSettings) {
+ let category = "other";
+ let parsedURL;
+
+ // Try to parse the url.
+ for (let _url of [url, `https://${url}`]) {
+ try {
+ parsedURL = new URL(_url);
+ break;
+ } catch (e) {}
+ }
+
+ if (parsedURL) {
+ // If we parsed successfully, find a match.
+ const hostname = parsedURL.hostname.replace(/^www\./i, "");
+ const params = parsedURL.searchParams;
+ // NOTE: there will be an initial/default local copy of the data in m-c.
+ // Therefore, this should never return an empty list [].
+ const siteTypes = await RS("sites-classification").get();
+ const sortedSiteTypes = siteTypes.sort(
+ (x, y) => (y.weight || 0) - (x.weight || 0)
+ );
+ for (let type of sortedSiteTypes) {
+ for (let criteria of type.criteria) {
+ if (criteria.url && criteria.url !== url) {
+ continue;
+ }
+ if (criteria.hostname && criteria.hostname !== hostname) {
+ continue;
+ }
+ if (criteria.sld && criteria.sld !== hostname.split(".")[0]) {
+ continue;
+ }
+ if (criteria.params && !_hasParams(criteria.params, params)) {
+ continue;
+ }
+ return type.type;
+ }
+ }
+ }
+ return category;
+}
+
+const EXPORTED_SYMBOLS = ["classifySite"];
diff --git a/browser/components/newtab/lib/SnippetsTestMessageProvider.jsm b/browser/components/newtab/lib/SnippetsTestMessageProvider.jsm
new file mode 100644
index 0000000000..ac4fe4892e
--- /dev/null
+++ b/browser/components/newtab/lib/SnippetsTestMessageProvider.jsm
@@ -0,0 +1,715 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const TEST_ICON = "chrome://branding/content/icon64.png";
+const TEST_ICON_16 = "chrome://branding/content/icon16.png";
+const TEST_ICON_BW =
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAQAAAC1+jfqAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQfjBQ8QDifrKGc/AAABf0lEQVQoz4WRO08UUQCFvztzd1AgG9jRgGwkhEoMIYGSygYt+A00tpZGY0jYxAJKEwkNjX9AK2xACx4dhFiQQCiMMRr2kYXdnQcz7L0z91qAMVac6hTfSU7OgVsk/prtyfSNfRb7ge2cd7dmVucP/wM2lwqVqoyICahRx9Nz71+8AnAAvlTct+dSYDBYcgJ+Fj68XFu/AfamnIoWFoHFYrAUuYMSn55/fAIOxIs1t4MhQpNxRYsUD0ld7r8DCfZph4QecrqkhCREgMLSeISQkAy0UBgE0CYgIkeRA9HdsCQhpEGCxichpItHigEcPH4XJLRbTf8STY0iiiuu60Ifxexx04F0N+aCgJCAhPQmD/cp/RC5A79WvUyhUHSIidAIoESv9VfAhW9n8+XqTCoyMsz1cviMMrGz9BrjAuboYHZajyXCInEocI8yvccbC+0muABanR4/tONjQz3DzgNKtj9sfv66XD9B/3tT9g/akb7h0bJwzxqqmlRHLr4rLPwBlYWoYj77l2AAAAAldEVYdGRhdGU6Y3JlYXRlADIwMTktMDUtMTVUMTY6MTQ6MzkrMDA6MDD5/4XBAAAAJXRFWHRkYXRlOm1vZGlmeQAyMDE5LTA1LTE1VDE2OjE0OjM5KzAwOjAwiKI9fQAAAABJRU5ErkJggg==";
+
+const MESSAGES = () => [
+ {
+ template: "simple_snippet",
+ template_version: "1.1.2",
+ content: {
+ text: "This is for <link0>preferences</link0> and <link1>about</link1>",
+ icon:
+ "https://snippets.cdn.mozilla.net/media/icons/1a8bb10e-8166-4e14-9e41-c1f85a41bcd2.png",
+ button_label: "Button Label",
+ section_title_icon:
+ "https://snippets.cdn.mozilla.net/media/icons/5878847e-a1fb-4204-aad9-09f6cf7f99ee.png",
+ section_title_text: "Messages from Firefox",
+ section_title_url:
+ "https://support.mozilla.org/kb/snippets-firefox-faq?utm_source=desktop-snippet&utm_medium=snippet&utm_campaign=&utm_term=&utm_content=",
+ tall: false,
+ block_button_text: "Remove this",
+ do_not_autoblock: true,
+ links: {
+ link0: {
+ action: "OPEN_PREFERENCES_PAGE",
+ entrypoint_value: "snippet",
+ args: "sync",
+ },
+ link1: {
+ action: "OPEN_ABOUT_PAGE",
+ args: "about",
+ entrypoint_name: "entryPoint",
+ entrypoint_value: "snippet",
+ },
+ },
+ button_action: "OPEN_PREFERENCES_PAGE",
+ button_entrypoint_value: "snippet",
+ },
+ id: "preview-13516_button_preferences",
+ },
+ {
+ template: "simple_snippet",
+ template_version: "1.1.2",
+ content: {
+ text: "This is for <link0>preferences</link0> and <link1>about</link1>",
+ icon:
+ "https://snippets.cdn.mozilla.net/media/icons/1a8bb10e-8166-4e14-9e41-c1f85a41bcd2.png",
+ button_label: "Button Label",
+ section_title_icon:
+ "https://snippets.cdn.mozilla.net/media/icons/5878847e-a1fb-4204-aad9-09f6cf7f99ee.png",
+ section_title_text: "Messages from Firefox",
+ section_title_url:
+ "https://support.mozilla.org/kb/snippets-firefox-faq?utm_source=desktop-snippet&utm_medium=snippet&utm_campaign=&utm_term=&utm_content=",
+ tall: false,
+ block_button_text: "Remove this",
+ do_not_autoblock: true,
+ links: {
+ link0: {
+ action: "OPEN_PREFERENCES_PAGE",
+ entrypoint_value: "snippet",
+ },
+ link1: {
+ action: "OPEN_ABOUT_PAGE",
+ args: "about",
+ entrypoint_name: "entryPoint",
+ entrypoint_value: "snippet",
+ },
+ },
+ button_action: "OPEN_ABOUT_PAGE",
+ button_action_args: "logins",
+ button_entrypoint_name: "entryPoint",
+ button_entrypoint_value: "snippet",
+ },
+ id: "preview-13517_button_about",
+ },
+ {
+ id: "SIMPLE_TEST_1",
+ template: "simple_snippet",
+ campaign: "test_campaign_blocking",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ title: "Firefox Account!",
+ title_icon: TEST_ICON_16,
+ title_icon_dark_theme: TEST_ICON_BW,
+ text:
+ "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_1_NO_DARK_THEME",
+ template: "simple_snippet",
+ campaign: "test_campaign_blocking",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: "",
+ title: "Firefox Account!",
+ title_icon: TEST_ICON_16,
+ title_icon_dark_theme: "",
+ text:
+ "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_1_SAME_CAMPAIGN",
+ template: "simple_snippet",
+ campaign: "test_campaign_blocking",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ text:
+ "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_TALL",
+ template: "simple_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ text:
+ "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ button_label: "Get one now!",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ block_button_text: "Block",
+ tall: true,
+ },
+ },
+ {
+ id: "SIMPLE_TEST_BUTTON_URL_1",
+ template: "simple_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Get one now!",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ text:
+ "Sync it, link it, take it with you. All this and more with a Firefox Account.",
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_BUTTON_ACTION_1",
+ template: "simple_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Open about:config",
+ button_action: "OPEN_ABOUT_PAGE",
+ button_action_args: "config",
+ text: "Testing the OPEN_ABOUT_PAGE action",
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_WITH_TITLE_TEST_1",
+ template: "simple_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ title: "Ready to sync?",
+ text: "Get connected with a <syncLink>Firefox account</syncLink>.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "NEWSLETTER_TEST_DEFAULTS",
+ template: "newsletter_snippet",
+ content: {
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_title: "Be a part of a movement.",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: TEST_ICON_BW,
+ scene1_text:
+ "Internet shutdowns, hackers, harassment &ndash; the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.",
+ scene1_button_label: "Continue",
+ scene1_button_color: "#712b00",
+ scene1_button_background_color: "#ff9400",
+ scene2_title: "Let's do this!",
+ locale: "en-CA",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_text:
+ "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.",
+ scene2_privacy_html:
+ "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.",
+ scene2_newsletter: "mozilla-foundation",
+ success_text: "Check your inbox for the confirmation!",
+ error_text: "Error!",
+ retry_button_label: "Try again?",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "NEWSLETTER_TEST_1",
+ template: "newsletter_snippet",
+ content: {
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_title: "Be a part of a movement.",
+ scene1_title_icon: "",
+ scene1_text:
+ "Internet shutdowns, hackers, harassment &ndash; the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.",
+ scene1_button_label: "Continue",
+ scene1_button_color: "#712b00",
+ scene1_button_background_color: "#ff9400",
+ scene2_title: "Let's do this!",
+ locale: "en-CA",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_text:
+ "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.",
+ scene2_privacy_html:
+ "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.",
+ scene2_button_label: "Sign Me up",
+ scene2_email_placeholder_text: "Your email here",
+ scene2_newsletter: "mozilla-foundation",
+ success_text: "Check your inbox for the confirmation!",
+ error_text: "Error!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "NEWSLETTER_TEST_SCENE1_SECTION_TITLE_ICON",
+ template: "newsletter_snippet",
+ content: {
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_title: "Be a part of a movement.",
+ scene1_title_icon: "",
+ scene1_text:
+ "Internet shutdowns, hackers, harassment &ndash; the health of the internet is on the line. Sign up and Mozilla will keep you updated on how you can help.",
+ scene1_button_label: "Continue",
+ scene1_button_color: "#712b00",
+ scene1_button_background_color: "#ff9400",
+ scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg",
+ scene1_section_title_text:
+ "All the Firefox news that's fit to Firefox print!",
+ scene2_title: "Let's do this!",
+ locale: "en-CA",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_text:
+ "Sign up for the Mozilla newsletter and we will keep you updated on how you can help.",
+ scene2_privacy_html:
+ "I'm okay with Mozilla handling my info as explained in this <privacyLink>Privacy Notice</privacyLink>.",
+ scene2_button_label: "Sign Me up",
+ scene2_email_placeholder_text: "Your email here",
+ scene2_newsletter: "mozilla-foundation",
+ success_text: "Check your inbox for the confirmation!",
+ error_text: "Error!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "FXA_SNIPPET_TEST_1",
+ template: "fxa_signup_snippet",
+ content: {
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_button_label: "Get connected with sync!",
+ scene1_button_color: "#712b00",
+ scene1_button_background_color: "#ff9400",
+
+ scene1_text:
+ "Connect to Firefox by securely syncing passwords, bookmarks, and open tabs.",
+ scene1_title: "Browser better.",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: TEST_ICON_BW,
+
+ scene2_text:
+ "Connect to your Firefox account to securely sync passwords, bookmarks, and open tabs.",
+ scene2_title: "Title 123",
+ scene2_email_placeholder_text: "Your email",
+ scene2_button_label: "Continue",
+ scene2_dismiss_button_text: "Dismiss",
+ },
+ },
+ {
+ id: "FXA_SNIPPET_TEST_TITLE_ICON",
+ template: "fxa_signup_snippet",
+ content: {
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_button_label: "Get connected with sync!",
+ scene1_button_color: "#712b00",
+ scene1_button_background_color: "#ff9400",
+
+ scene1_text:
+ "Connect to Firefox by securely syncing passwords, bookmarks, and open tabs.",
+ scene1_title: "Browser better.",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: TEST_ICON_BW,
+
+ scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg",
+ scene1_section_title_text: "Firefox Accounts: Receivable benefits",
+
+ scene2_text:
+ "Connect to your Firefox account to securely sync passwords, bookmarks, and open tabs.",
+ scene2_title: "Title 123",
+ scene2_email_placeholder_text: "Your email",
+ scene2_button_label: "Continue",
+ scene2_dismiss_button_text: "Dismiss",
+ },
+ },
+ {
+ id: "SNIPPETS_SEND_TO_DEVICE_TEST",
+ template: "send_to_device_snippet",
+ content: {
+ include_sms: true,
+ locale: "en-CA",
+ country: "us",
+ message_id_sms: "ff-mobilesn-download",
+ message_id_email: "download-firefox-mobile",
+
+ scene1_button_background_color: "#6200a4",
+ scene1_button_color: "#FFFFFF",
+ scene1_button_label: "Install now",
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_text: "Browse without compromise with Firefox Mobile.",
+ scene1_title: "Full-featured. Customizable. Lightning fast",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: TEST_ICON_BW,
+
+ scene2_button_label: "Send",
+ scene2_disclaimer_html:
+ "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_icon: TEST_ICON,
+ scene2_icon_dark_theme: TEST_ICON_BW,
+ scene2_input_placeholder: "Your email address or phone number",
+ scene2_text:
+ "Send Firefox to your phone and take a powerful independent browser with you.",
+ scene2_title: "Let's do this!",
+
+ error_text: "Oops, there was a problem.",
+ success_title: "Your download link was sent.",
+ success_text: "Check your device for the email message!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "SNIPPETS_SCENE2_SEND_TO_DEVICE_TEST",
+ template: "send_to_device_scene2_snippet",
+ content: {
+ include_sms: true,
+ locale: "en-CA",
+ country: "us",
+ message_id_sms: "ff-mobilesn-download",
+ message_id_email: "download-firefox-mobile",
+ scene2_icon: TEST_ICON,
+ section_title_icon:
+ "https://snippets.cdn.mozilla.net/media/icons/094b0707-ab65-4b2e-99a1-a84122b6ab26.png",
+ section_title_text: "Messages from Firefox",
+ section_title_url: "https://support.mozilla.org/kb",
+ scene2_button_label: "Send",
+ scene2_disclaimer_html:
+ "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.",
+ scene2_input_placeholder: "Your email address or phone number",
+ scene2_text:
+ "Send Firefox to your phone and take a powerful independent browser with you.",
+ error_text: "Oops, there was a problem.",
+ success_title: "Your download link was sent.",
+ success_text: "Check your device for the email message!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "SNIPPETS_SEND_TO_DEVICE_TEST_NO_DARK_THEME",
+ template: "send_to_device_snippet",
+ content: {
+ include_sms: true,
+ locale: "en-CA",
+ country: "us",
+ message_id_sms: "ff-mobilesn-download",
+ message_id_email: "download-firefox-mobile",
+
+ scene1_button_background_color: "#6200a4",
+ scene1_button_color: "#FFFFFF",
+ scene1_button_label: "Install now",
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: "",
+ scene1_text: "Browse without compromise with Firefox Mobile.",
+ scene1_title: "Full-featured. Customizable. Lightning fast",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: "",
+
+ scene2_button_label: "Send",
+ scene2_disclaimer_html:
+ "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_icon: TEST_ICON,
+ scene2_icon_dark_theme: "",
+ scene2_input_placeholder: "Your email address or phone number",
+ scene2_text:
+ "Send Firefox to your phone and take a powerful independent browser with you.",
+ scene2_title: "Let's do this!",
+
+ error_text: "Oops, there was a problem.",
+ success_title: "Your download link was sent.",
+ success_text: "Check your device for the email message!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "SNIPPETS_SEND_TO_DEVICE_TEST_SECTION_TITLE_ICON",
+ template: "send_to_device_snippet",
+ content: {
+ include_sms: true,
+ locale: "en-CA",
+ country: "us",
+ message_id_sms: "ff-mobilesn-download",
+ message_id_email: "download-firefox-mobile",
+
+ scene1_button_background_color: "#6200a4",
+ scene1_button_color: "#FFFFFF",
+ scene1_button_label: "Install now",
+ scene1_icon: TEST_ICON,
+ scene1_icon_dark_theme: TEST_ICON_BW,
+ scene1_text: "Browse without compromise with Firefox Mobile.",
+ scene1_title: "Full-featured. Customizable. Lightning fast",
+ scene1_title_icon: TEST_ICON_16,
+ scene1_title_icon_dark_theme: TEST_ICON_BW,
+ scene1_section_title_icon: "chrome://global/skin/icons/pocket.svg",
+ scene1_section_title_text: "Send Firefox to your mobile device!",
+
+ scene2_button_label: "Send",
+ scene2_disclaimer_html:
+ "The intended recipient of the email must have consented. <privacyLink>Learn more</privacyLink>.",
+ scene2_dismiss_button_text: "Dismiss",
+ scene2_icon: TEST_ICON,
+ scene2_icon_dark_theme: TEST_ICON_BW,
+ scene2_input_placeholder: "Your email address or phone number",
+ scene2_text:
+ "Send Firefox to your phone and take a powerful independent browser with you.",
+ scene2_title: "Let's do this!",
+
+ error_text: "Oops, there was a problem.",
+ success_title: "Your download link was sent.",
+ success_text: "Check your device for the email message!",
+ links: {
+ privacyLink: {
+ url:
+ "https://www.mozilla.org/privacy/websites/?sample_rate=0.001&snippet_name=7894",
+ },
+ },
+ },
+ },
+ {
+ id: "EOY_TEST_1",
+ template: "eoy_snippet",
+ content: {
+ highlight_color: "#f05",
+ background_color: "#ddd",
+ text_color: "yellow",
+ selected_button: "donation_amount_first",
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Donate",
+ monthly_checkbox_label_text: "Make my donation monthly",
+ currency_code: "usd",
+ donation_amount_first: 50,
+ donation_amount_second: 25,
+ donation_amount_third: 10,
+ donation_amount_fourth: 5,
+ donation_form_url:
+ "https://donate.mozilla.org/pl/?utm_source=desktop-snippet&amp;utm_medium=snippet&amp;utm_campaign=donate&amp;utm_term=7556",
+ text:
+ "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?",
+ },
+ },
+ {
+ id: "EOY_BOLD_TEST_1",
+ template: "eoy_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ selected_button: "donation_amount_second",
+ button_label: "Donate",
+ monthly_checkbox_label_text: "Make my donation monthly",
+ currency_code: "usd",
+ donation_amount_first: 50,
+ donation_amount_second: 25,
+ donation_amount_third: 10,
+ donation_amount_fourth: 5,
+ donation_form_url: "https://donate.mozilla.org",
+ text:
+ "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?",
+ test: "bold",
+ },
+ },
+ {
+ id: "EOY_TAKEOVER_TEST_1",
+ template: "eoy_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Donate",
+ monthly_checkbox_label_text: "Make my donation monthly",
+ currency_code: "usd",
+ donation_amount_first: 50,
+ donation_amount_second: 25,
+ donation_amount_third: 10,
+ donation_amount_fourth: 5,
+ donation_form_url: "https://donate.mozilla.org",
+ text:
+ "Big corporations want to restrict how we access the web. Fake news is making it harder for us to find the truth. Online bullies are silencing inspired voices. The <em>not-for-profit Mozilla Foundation</em> fights for a healthy internet with programs like our Tech Policy Fellowships and Internet Health Report; <b>will you donate today</b>?",
+ test: "takeover",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_WITH_SECTION_HEADING",
+ template: "simple_snippet",
+ content: {
+ button_label: "Get one now!",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ title: "Firefox Account!",
+ text:
+ "<syncLink>Sync it, link it, take it with you</syncLink>. All this and more with a Firefox Account.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ section_title_icon: "chrome://global/skin/icons/pocket.svg",
+ section_title_text: "Messages from Mozilla",
+ },
+ },
+ {
+ id: "SIMPLE_TEST_WITH_SECTION_HEADING_AND_LINK",
+ template: "simple_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ title: "Firefox Account!",
+ text:
+ "Sync it, link it, take it with you. All this and more with a Firefox Account.",
+ block_button_text: "Block",
+ section_title_icon: "chrome://global/skin/icons/pocket.svg",
+ section_title_text: "Messages from Mozilla (click for info)",
+ section_title_url: "https://www.mozilla.org/about",
+ },
+ },
+ {
+ id: "SIMPLE_BELOW_SEARCH_TEST_1",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ text:
+ "Securely store passwords, bookmarks, and more with a Firefox Account. <syncLink>Sign up</syncLink>",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_BELOW_SEARCH_TEST_2",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ text:
+ "<syncLink>Connect your Firefox Account to Sync</syncLink> your protected passwords, open tabs and bookmarks, and they'll always be available to you - on all of your devices.",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SIMPLE_BELOW_SEARCH_TEST_TITLE",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ title: "See if you've been part of an online data breach.",
+ text:
+ "Securely store passwords, bookmarks, and more with a Firefox Account. <syncLink>Sign up</syncLink>",
+ links: {
+ syncLink: { url: "https://www.mozilla.org/en-US/firefox/accounts" },
+ },
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SPECIAL_SNIPPET_BUTTON_1",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Find Out Now",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ title: "See if you've been part of an online data breach.",
+ text: "Firefox Monitor tells you what hackers already know about you.",
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SPECIAL_SNIPPET_LONG_CONTENT",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Find Out Now",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ title: "See if you've been part of an online data breach.",
+ text:
+ "Firefox Monitor tells you what hackers already know about you. Here's some extra text to make the content really long.",
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SPECIAL_SNIPPET_NO_TITLE",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ icon_dark_theme: TEST_ICON_BW,
+ button_label: "Find Out Now",
+ button_url: "https://www.mozilla.org/en-US/firefox/accounts",
+ text: "Firefox Monitor tells you what hackers already know about you.",
+ block_button_text: "Block",
+ },
+ },
+ {
+ id: "SPECIAL_SNIPPET_MONITOR",
+ template: "simple_below_search_snippet",
+ content: {
+ icon: TEST_ICON,
+ title: "See if you've been part of an online data breach.",
+ text: "Firefox Monitor tells you what hackers already know about you.",
+ button_label: "Get monitor",
+ button_action: "ENABLE_FIREFOX_MONITOR",
+ button_action_args: {
+ url:
+ "https://monitor.firefox.com/oauth/init?utm_source=snippets&utm_campaign=monitor-snippet-test&form_type=email&entrypoint=newtab",
+ flowRequestParams: {
+ entrypoint: "snippets",
+ utm_term: "monitor",
+ form_type: "email",
+ },
+ },
+ block_button_text: "Block",
+ },
+ },
+];
+
+const SnippetsTestMessageProvider = {
+ getMessages() {
+ return Promise.resolve(
+ MESSAGES()
+ // Ensures we never actually show test except when triggered by debug tools
+ .map(message => ({
+ ...message,
+ targeting: `providerCohorts.snippets_local_testing == "SHOW_TEST"`,
+ }))
+ );
+ },
+};
+
+const EXPORTED_SYMBOLS = ["SnippetsTestMessageProvider"];
diff --git a/browser/components/newtab/lib/Spotlight.jsm b/browser/components/newtab/lib/Spotlight.jsm
new file mode 100644
index 0000000000..b194929c64
--- /dev/null
+++ b/browser/components/newtab/lib/Spotlight.jsm
@@ -0,0 +1,117 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ AboutWelcomeTelemetry:
+ "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm",
+ RemoteImages: "resource://activity-stream/lib/RemoteImages.jsm",
+ SpecialMessageActions:
+ "resource://messaging-system/lib/SpecialMessageActions.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(
+ lazy,
+ "AWTelemetry",
+ () => new lazy.AboutWelcomeTelemetry()
+);
+
+const Spotlight = {
+ sendUserEventTelemetry(event, message, dispatch) {
+ const message_id =
+ message.template === "multistage" ? message.content.id : message.id;
+ const ping = {
+ message_id,
+ event,
+ };
+ dispatch({
+ type: "SPOTLIGHT_TELEMETRY",
+ data: { action: "spotlight_user_event", ...ping },
+ });
+ },
+
+ defaultDispatch(message) {
+ if (message.type === "SPOTLIGHT_TELEMETRY") {
+ const { message_id, event } = message.data;
+ lazy.AWTelemetry.sendTelemetry({ message_id, event });
+ }
+ },
+
+ /**
+ * Shows spotlight tab or window modal specific to the given browser
+ * @param browser The browser for spotlight display
+ * @param message Message containing content to show
+ * @param dispatchCFRAction A function to dispatch resulting actions
+ * @return boolean value capturing if spotlight was displayed
+ */
+ async showSpotlightDialog(browser, message, dispatch = this.defaultDispatch) {
+ const win = browser.ownerGlobal;
+ if (win.gDialogBox.isOpen) {
+ return false;
+ }
+ const spotlight_url = "chrome://browser/content/spotlight.html";
+
+ const dispatchCFRAction =
+ // This also blocks CFR impressions, which is fine for current use cases.
+ message.content?.metrics === "block" ? () => {} : dispatch;
+ let params = { primaryBtn: false, secondaryBtn: false };
+
+ // There are two events named `IMPRESSION` the first one refers to telemetry
+ // while the other refers to ASRouter impressions used for the frequency cap
+ this.sendUserEventTelemetry("IMPRESSION", message, dispatchCFRAction);
+ dispatchCFRAction({ type: "IMPRESSION", data: message });
+
+ const unload = await lazy.RemoteImages.patchMessage(message.content.logo);
+
+ if (message.content?.modal === "tab") {
+ let { closedPromise } = win.gBrowser.getTabDialogBox(browser).open(
+ spotlight_url,
+ {
+ features: "resizable=no",
+ allowDuplicateDialogs: false,
+ },
+ [message.content, params]
+ );
+ await closedPromise;
+ } else {
+ await win.gDialogBox.open(spotlight_url, [message.content, params]);
+ }
+
+ if (unload) {
+ unload();
+ }
+
+ // If dismissed report telemetry and exit
+ if (!params.secondaryBtn && !params.primaryBtn) {
+ this.sendUserEventTelemetry("DISMISS", message, dispatchCFRAction);
+ return true;
+ }
+
+ if (params.secondaryBtn) {
+ this.sendUserEventTelemetry("DISMISS", message, dispatchCFRAction);
+ lazy.SpecialMessageActions.handleAction(
+ message.content.body.secondary.action,
+ browser
+ );
+ }
+
+ if (params.primaryBtn) {
+ this.sendUserEventTelemetry("CLICK", message, dispatchCFRAction);
+ lazy.SpecialMessageActions.handleAction(
+ message.content.body.primary.action,
+ browser
+ );
+ }
+
+ return true;
+ },
+};
+
+const EXPORTED_SYMBOLS = ["Spotlight"];
diff --git a/browser/components/newtab/lib/Store.jsm b/browser/components/newtab/lib/Store.jsm
new file mode 100644
index 0000000000..12574e196a
--- /dev/null
+++ b/browser/components/newtab/lib/Store.jsm
@@ -0,0 +1,190 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { ActivityStreamMessageChannel } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamMessageChannel.jsm"
+);
+const { ActivityStreamStorage } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamStorage.jsm"
+);
+const { Prefs } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamPrefs.jsm"
+);
+const { reducers } = ChromeUtils.import(
+ "resource://activity-stream/common/Reducers.jsm"
+);
+const { redux } = ChromeUtils.import(
+ "resource://activity-stream/vendor/Redux.jsm"
+);
+
+/**
+ * Store - This has a similar structure to a redux store, but includes some extra
+ * functionality to allow for routing of actions between the Main processes
+ * and child processes via a ActivityStreamMessageChannel.
+ * It also accepts an array of "Feeds" on inititalization, which
+ * can listen for any action that is dispatched through the store.
+ */
+class Store {
+ /**
+ * constructor - The redux store and message manager are created here,
+ * but no listeners are added until "init" is called.
+ */
+ constructor() {
+ this._middleware = this._middleware.bind(this);
+ // Bind each redux method so we can call it directly from the Store. E.g.,
+ // store.dispatch() will call store._store.dispatch();
+ for (const method of ["dispatch", "getState", "subscribe"]) {
+ this[method] = (...args) => this._store[method](...args);
+ }
+ this.feeds = new Map();
+ this._prefs = new Prefs();
+ this._messageChannel = new ActivityStreamMessageChannel({
+ dispatch: this.dispatch,
+ });
+ this._store = redux.createStore(
+ redux.combineReducers(reducers),
+ redux.applyMiddleware(this._middleware, this._messageChannel.middleware)
+ );
+ this.storage = null;
+ }
+
+ /**
+ * _middleware - This is redux middleware consumed by redux.createStore.
+ * it calls each feed's .onAction method, if one
+ * is defined.
+ */
+ _middleware() {
+ return next => action => {
+ next(action);
+ for (const store of this.feeds.values()) {
+ if (store.onAction) {
+ store.onAction(action);
+ }
+ }
+ };
+ }
+
+ /**
+ * initFeed - Initializes a feed by calling its constructor function
+ *
+ * @param {string} feedName The name of a feed, as defined in the object
+ * passed to Store.init
+ * @param {Action} initAction An optional action to initialize the feed
+ */
+ initFeed(feedName, initAction) {
+ const feed = this._feedFactories.get(feedName)();
+ feed.store = this;
+ this.feeds.set(feedName, feed);
+ if (initAction && feed.onAction) {
+ feed.onAction(initAction);
+ }
+ }
+
+ /**
+ * uninitFeed - Removes a feed and calls its uninit function if defined
+ *
+ * @param {string} feedName The name of a feed, as defined in the object
+ * passed to Store.init
+ * @param {Action} uninitAction An optional action to uninitialize the feed
+ */
+ uninitFeed(feedName, uninitAction) {
+ const feed = this.feeds.get(feedName);
+ if (!feed) {
+ return;
+ }
+ if (uninitAction && feed.onAction) {
+ feed.onAction(uninitAction);
+ }
+ this.feeds.delete(feedName);
+ }
+
+ /**
+ * onPrefChanged - Listener for handling feed changes.
+ */
+ onPrefChanged(name, value) {
+ if (this._feedFactories.has(name)) {
+ if (value) {
+ this.initFeed(name, this._initAction);
+ } else {
+ this.uninitFeed(name, this._uninitAction);
+ }
+ }
+ }
+
+ /**
+ * init - Initializes the ActivityStreamMessageChannel channel, and adds feeds.
+ *
+ * Note that it intentionally initializes the TelemetryFeed first so that the
+ * addon is able to report the init errors from other feeds.
+ *
+ * @param {Map} feedFactories A Map of feeds with the name of the pref for
+ * the feed as the key and a function that
+ * constructs an instance of the feed.
+ * @param {Action} initAction An optional action that will be dispatched
+ * to feeds when they're created.
+ * @param {Action} uninitAction An optional action for when feeds uninit.
+ */
+ async init(feedFactories, initAction, uninitAction) {
+ this._feedFactories = feedFactories;
+ this._initAction = initAction;
+ this._uninitAction = uninitAction;
+
+ const telemetryKey = "feeds.telemetry";
+ if (feedFactories.has(telemetryKey) && this._prefs.get(telemetryKey)) {
+ this.initFeed(telemetryKey);
+ }
+
+ await this._initIndexedDB(telemetryKey);
+
+ for (const pref of feedFactories.keys()) {
+ if (pref !== telemetryKey && this._prefs.get(pref)) {
+ this.initFeed(pref);
+ }
+ }
+
+ this._prefs.observeBranch(this);
+ this._messageChannel.createChannel();
+
+ // Dispatch an initial action after all enabled feeds are ready
+ if (initAction) {
+ this.dispatch(initAction);
+ }
+
+ // Dispatch NEW_TAB_INIT/NEW_TAB_LOAD events after INIT event.
+ this._messageChannel.simulateMessagesForExistingTabs();
+ }
+
+ async _initIndexedDB(telemetryKey) {
+ this.dbStorage = new ActivityStreamStorage({
+ storeNames: ["sectionPrefs", "snippets"],
+ });
+ // Accessing the db causes the object stores to be created / migrated.
+ // This needs to happen before other instances try to access the db, which
+ // would update only a subset of the stores to the latest version.
+ try {
+ await this.dbStorage.db; // eslint-disable-line no-unused-expressions
+ } catch (e) {
+ this.dbStorage.telemetry = null;
+ }
+ }
+
+ /**
+ * uninit - Uninitalizes each feed, clears them, and destroys the message
+ * manager channel.
+ *
+ * @return {type} description
+ */
+ uninit() {
+ if (this._uninitAction) {
+ this.dispatch(this._uninitAction);
+ }
+ this._prefs.ignoreBranch(this);
+ this.feeds.clear();
+ this._feedFactories = null;
+ this._messageChannel.destroyChannel();
+ }
+}
+
+const EXPORTED_SYMBOLS = ["Store"];
diff --git a/browser/components/newtab/lib/SystemTickFeed.jsm b/browser/components/newtab/lib/SystemTickFeed.jsm
new file mode 100644
index 0000000000..ddf4762d0e
--- /dev/null
+++ b/browser/components/newtab/lib/SystemTickFeed.jsm
@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ clearInterval: "resource://gre/modules/Timer.sys.mjs",
+ setInterval: "resource://gre/modules/Timer.sys.mjs",
+});
+
+// Frequency at which SYSTEM_TICK events are fired
+const SYSTEM_TICK_INTERVAL = 5 * 60 * 1000;
+
+class SystemTickFeed {
+ init() {
+ this.intervalId = lazy.setInterval(
+ () => this.store.dispatch({ type: at.SYSTEM_TICK }),
+ SYSTEM_TICK_INTERVAL
+ );
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ break;
+ case at.UNINIT:
+ lazy.clearInterval(this.intervalId);
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = ["SystemTickFeed", "SYSTEM_TICK_INTERVAL"];
diff --git a/browser/components/newtab/lib/TelemetryFeed.jsm b/browser/components/newtab/lib/TelemetryFeed.jsm
new file mode 100644
index 0000000000..314272ecda
--- /dev/null
+++ b/browser/components/newtab/lib/TelemetryFeed.jsm
@@ -0,0 +1,1313 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/ActorConstants.sys.mjs"
+);
+
+const { actionTypes: at, actionUtils: au } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { Prefs } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamPrefs.jsm"
+);
+const { classifySite } = ChromeUtils.import(
+ "resource://activity-stream/lib/SiteClassifier.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "AboutNewTab",
+ "resource:///modules/AboutNewTab.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PingCentre",
+ "resource:///modules/PingCentre.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "pktApi",
+ "chrome://pocket/content/pktApi.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "UTEventReporting",
+ "resource://activity-stream/lib/UTEventReporting.jsm"
+);
+ChromeUtils.defineESModuleGetters(lazy, {
+ ClientID: "resource://gre/modules/ClientID.sys.mjs",
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+ TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs",
+ TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs",
+ UpdateUtils: "resource://gre/modules/UpdateUtils.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "HomePage",
+ "resource:///modules/HomePage.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "ExtensionSettingsStore",
+ "resource://gre/modules/ExtensionSettingsStore.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ ExperimentAPI: "resource://nimbus/ExperimentAPI.jsm",
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+const ACTIVITY_STREAM_ID = "activity-stream";
+const DOMWINDOW_OPENED_TOPIC = "domwindowopened";
+const DOMWINDOW_UNLOAD_TOPIC = "unload";
+const TAB_PINNED_EVENT = "TabPinned";
+
+// This is a mapping table between the user preferences and its encoding code
+const USER_PREFS_ENCODING = {
+ showSearch: 1 << 0,
+ "feeds.topsites": 1 << 1,
+ "feeds.section.topstories": 1 << 2,
+ "feeds.section.highlights": 1 << 3,
+ "feeds.snippets": 1 << 4,
+ showSponsored: 1 << 5,
+ "asrouter.userprefs.cfr.addons": 1 << 6,
+ "asrouter.userprefs.cfr.features": 1 << 7,
+ showSponsoredTopSites: 1 << 8,
+};
+
+const PREF_IMPRESSION_ID = "impressionId";
+const TELEMETRY_PREF = "telemetry";
+const EVENTS_TELEMETRY_PREF = "telemetry.ut.events";
+const STRUCTURED_INGESTION_ENDPOINT_PREF =
+ "telemetry.structuredIngestion.endpoint";
+// List of namespaces for the structured ingestion system.
+// They are defined in https://github.com/mozilla-services/mozilla-pipeline-schemas
+const STRUCTURED_INGESTION_NAMESPACE_AS = "activity-stream";
+const STRUCTURED_INGESTION_NAMESPACE_MS = "messaging-system";
+const STRUCTURED_INGESTION_NAMESPACE_CS = "contextual-services";
+
+// Used as the missing value for timestamps in the session ping
+const TIMESTAMP_MISSING_VALUE = -1;
+
+// Page filter for onboarding telemetry, any value other than these will
+// be set as "other"
+const ONBOARDING_ALLOWED_PAGE_VALUES = [
+ "about:welcome",
+ "about:home",
+ "about:newtab",
+];
+
+XPCOMUtils.defineLazyGetter(
+ lazy,
+ "browserSessionId",
+ () => lazy.TelemetrySession.getMetadata("").sessionId
+);
+
+// The scalar category for TopSites of Contextual Services
+const SCALAR_CATEGORY_TOPSITES = "contextual.services.topsites";
+// `contextId` is a unique identifier used by Contextual Services
+const CONTEXT_ID_PREF = "browser.contextual-services.contextId";
+XPCOMUtils.defineLazyGetter(lazy, "contextId", () => {
+ let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null);
+ if (!_contextId) {
+ _contextId = String(Services.uuid.generateUUID());
+ Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId);
+ }
+ return _contextId;
+});
+
+class TelemetryFeed {
+ constructor() {
+ this.sessions = new Map();
+ this._prefs = new Prefs();
+ this._impressionId = this.getOrCreateImpressionId();
+ this._aboutHomeSeen = false;
+ this._classifySite = classifySite;
+ this._addWindowListeners = this._addWindowListeners.bind(this);
+ this._browserOpenNewtabStart = null;
+ this.handleEvent = this.handleEvent.bind(this);
+ }
+
+ get telemetryEnabled() {
+ return this._prefs.get(TELEMETRY_PREF);
+ }
+
+ get eventTelemetryEnabled() {
+ return this._prefs.get(EVENTS_TELEMETRY_PREF);
+ }
+
+ get structuredIngestionEndpointBase() {
+ return this._prefs.get(STRUCTURED_INGESTION_ENDPOINT_PREF);
+ }
+
+ get telemetryClientId() {
+ Object.defineProperty(this, "telemetryClientId", {
+ value: lazy.ClientID.getClientID(),
+ });
+ return this.telemetryClientId;
+ }
+
+ get processStartTs() {
+ let startupInfo = Services.startup.getStartupInfo();
+ let processStartTs = startupInfo.process.getTime();
+
+ Object.defineProperty(this, "processStartTs", {
+ value: processStartTs,
+ });
+ return this.processStartTs;
+ }
+
+ init() {
+ this._beginObservingNewtabPingPrefs();
+ Services.obs.addObserver(
+ this.browserOpenNewtabStart,
+ "browser-open-newtab-start"
+ );
+ // Add pin tab event listeners on future windows
+ Services.obs.addObserver(this._addWindowListeners, DOMWINDOW_OPENED_TOPIC);
+ // Listen for pin tab events on all open windows
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ this._addWindowListeners(win);
+ }
+ // Set two scalars for the "deletion-request" ping (See bug 1602064 and 1729474)
+ Services.telemetry.scalarSet(
+ "deletion.request.impression_id",
+ this._impressionId
+ );
+ Services.telemetry.scalarSet("deletion.request.context_id", lazy.contextId);
+ Glean.newtab.locale.set(Services.locale.appLocaleAsBCP47);
+ }
+
+ handleEvent(event) {
+ switch (event.type) {
+ case TAB_PINNED_EVENT:
+ this.countPinnedTab(event.target);
+ break;
+ case DOMWINDOW_UNLOAD_TOPIC:
+ this._removeWindowListeners(event.target);
+ break;
+ }
+ }
+
+ _removeWindowListeners(win) {
+ win.removeEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent);
+ win.removeEventListener(TAB_PINNED_EVENT, this.handleEvent);
+ }
+
+ _addWindowListeners(win) {
+ win.addEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent);
+ win.addEventListener(TAB_PINNED_EVENT, this.handleEvent);
+ }
+
+ countPinnedTab(target, source = "TAB_CONTEXT_MENU") {
+ const win = target.ownerGlobal;
+ if (lazy.PrivateBrowsingUtils.isWindowPrivate(win)) {
+ return;
+ }
+ const event = Object.assign(this.createPing(), {
+ action: "activity_stream_user_event",
+ event: TAB_PINNED_EVENT.toUpperCase(),
+ value: { total_pinned_tabs: this.countTotalPinnedTabs() },
+ source,
+ // These fields are required but not relevant for this ping
+ page: "n/a",
+ session_id: "n/a",
+ });
+ this.sendEvent(event);
+ }
+
+ countTotalPinnedTabs() {
+ let pinnedTabs = 0;
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ if (win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win)) {
+ continue;
+ }
+ for (let tab of win.gBrowser.tabs) {
+ pinnedTabs += tab.pinned ? 1 : 0;
+ }
+ }
+
+ return pinnedTabs;
+ }
+
+ getOrCreateImpressionId() {
+ let impressionId = this._prefs.get(PREF_IMPRESSION_ID);
+ if (!impressionId) {
+ impressionId = String(Services.uuid.generateUUID());
+ this._prefs.set(PREF_IMPRESSION_ID, impressionId);
+ }
+ return impressionId;
+ }
+
+ browserOpenNewtabStart() {
+ let now = Cu.now();
+ this._browserOpenNewtabStart = Math.round(this.processStartTs + now);
+
+ ChromeUtils.addProfilerMarker(
+ "UserTiming",
+ now,
+ "browser-open-newtab-start"
+ );
+ }
+
+ setLoadTriggerInfo(port) {
+ // XXX note that there is a race condition here; we're assuming that no
+ // other tab will be interleaving calls to browserOpenNewtabStart and
+ // when at.NEW_TAB_INIT gets triggered by RemotePages and calls this
+ // method. For manually created windows, it's hard to imagine us hitting
+ // this race condition.
+ //
+ // However, for session restore, where multiple windows with multiple tabs
+ // might be restored much closer together in time, it's somewhat less hard,
+ // though it should still be pretty rare.
+ //
+ // The fix to this would be making all of the load-trigger notifications
+ // return some data with their notifications, and somehow propagate that
+ // data through closures into the tab itself so that we could match them
+ //
+ // As of this writing (very early days of system add-on perf telemetry),
+ // the hypothesis is that hitting this race should be so rare that makes
+ // more sense to live with the slight data inaccuracy that it would
+ // introduce, rather than doing the correct but complicated thing. It may
+ // well be worth reexamining this hypothesis after we have more experience
+ // with the data.
+
+ let data_to_save;
+ try {
+ if (!this._browserOpenNewtabStart) {
+ throw new Error("No browser-open-newtab-start recorded.");
+ }
+ data_to_save = {
+ load_trigger_ts: this._browserOpenNewtabStart,
+ load_trigger_type: "menu_plus_or_keyboard",
+ };
+ } catch (e) {
+ // if no mark was returned, we have nothing to save
+ return;
+ }
+ this.saveSessionPerfData(port, data_to_save);
+ }
+
+ /**
+ * Lazily initialize PingCentre for Activity Stream to send pings
+ */
+ get pingCentre() {
+ Object.defineProperty(this, "pingCentre", {
+ value: new lazy.PingCentre({ topic: ACTIVITY_STREAM_ID }),
+ });
+ return this.pingCentre;
+ }
+
+ /**
+ * Lazily initialize UTEventReporting to send pings
+ */
+ get utEvents() {
+ Object.defineProperty(this, "utEvents", {
+ value: new lazy.UTEventReporting(),
+ });
+ return this.utEvents;
+ }
+
+ /**
+ * Get encoded user preferences, multiple prefs will be combined via bitwise OR operator
+ */
+ get userPreferences() {
+ let prefs = 0;
+
+ for (const pref of Object.keys(USER_PREFS_ENCODING)) {
+ if (this._prefs.get(pref)) {
+ prefs |= USER_PREFS_ENCODING[pref];
+ }
+ }
+ return prefs;
+ }
+
+ /**
+ * Check if it is in the CFR experiment cohort by querying against the
+ * experiment manager of Messaging System
+ *
+ * @return {bool}
+ */
+ get isInCFRCohort() {
+ const experimentData = lazy.ExperimentAPI.getExperimentMetaData({
+ featureId: "cfr",
+ });
+ if (experimentData && experimentData.slug) {
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * addSession - Start tracking a new session
+ *
+ * @param {string} id the portID of the open session
+ * @param {string} the URL being loaded for this session (optional)
+ * @return {obj} Session object
+ */
+ addSession(id, url) {
+ // XXX refactor to use setLoadTriggerInfo or saveSessionPerfData
+
+ // "unexpected" will be overwritten when appropriate
+ let load_trigger_type = "unexpected";
+ let load_trigger_ts;
+
+ if (!this._aboutHomeSeen && url === "about:home") {
+ this._aboutHomeSeen = true;
+
+ // XXX note that this will be incorrectly set in the following cases:
+ // session_restore following by clicking on the toolbar button,
+ // or someone who has changed their default home page preference to
+ // something else and later clicks the toolbar. It will also be
+ // incorrectly unset if someone changes their "Home Page" preference to
+ // about:newtab.
+ //
+ // That said, the ratio of these mistakes to correct cases should
+ // be very small, and these issues should follow away as we implement
+ // the remaining load_trigger_type values for about:home in issue 3556.
+ //
+ // XXX file a bug to implement remaining about:home cases so this
+ // problem will go away and link to it here.
+ load_trigger_type = "first_window_opened";
+
+ // The real perceived trigger of first_window_opened is the OS-level
+ // clicking of the icon. We express this by using the process start
+ // absolute timestamp.
+ load_trigger_ts = this.processStartTs;
+ }
+
+ const session = {
+ session_id: String(Services.uuid.generateUUID()),
+ // "unknown" will be overwritten when appropriate
+ page: url ? url : "unknown",
+ perf: {
+ load_trigger_type,
+ is_preloaded: false,
+ },
+ };
+
+ if (load_trigger_ts) {
+ session.perf.load_trigger_ts = load_trigger_ts;
+ }
+
+ this.sessions.set(id, session);
+ return session;
+ }
+
+ /**
+ * endSession - Stop tracking a session
+ *
+ * @param {string} portID the portID of the session that just closed
+ */
+ endSession(portID) {
+ const session = this.sessions.get(portID);
+
+ if (!session) {
+ // It's possible the tab was never visible – in which case, there was no user session.
+ return;
+ }
+
+ this.sendDiscoveryStreamLoadedContent(portID, session);
+ this.sendDiscoveryStreamImpressions(portID, session);
+
+ Glean.newtab.closed.record({ newtab_visit_id: session.session_id });
+ if (
+ this.telemetryEnabled &&
+ (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true)
+ ) {
+ GleanPings.newtab.submit("newtab_session_end");
+ }
+
+ if (session.perf.visibility_event_rcvd_ts) {
+ let absNow = this.processStartTs + Cu.now();
+ session.session_duration = Math.round(
+ absNow - session.perf.visibility_event_rcvd_ts
+ );
+
+ // Rounding all timestamps in perf to ease the data processing on the backend.
+ // NB: use `TIMESTAMP_MISSING_VALUE` if the value is missing.
+ session.perf.visibility_event_rcvd_ts = Math.round(
+ session.perf.visibility_event_rcvd_ts
+ );
+ session.perf.load_trigger_ts = Math.round(
+ session.perf.load_trigger_ts || TIMESTAMP_MISSING_VALUE
+ );
+ session.perf.topsites_first_painted_ts = Math.round(
+ session.perf.topsites_first_painted_ts || TIMESTAMP_MISSING_VALUE
+ );
+ } else {
+ // This session was never shown (i.e. the hidden preloaded newtab), there was no user session either.
+ this.sessions.delete(portID);
+ return;
+ }
+
+ let sessionEndEvent = this.createSessionEndEvent(session);
+ this.sendEvent(sessionEndEvent);
+ this.sendUTEvent(sessionEndEvent, this.utEvents.sendSessionEndEvent);
+ this.sessions.delete(portID);
+ }
+
+ /**
+ * Send impression pings for Discovery Stream for a given session.
+ *
+ * @note the impression reports are stored in session.impressionSets for different
+ * sources, and will be sent separately accordingly.
+ *
+ * @param {String} port The session port with which this is associated
+ * @param {Object} session The session object
+ */
+ sendDiscoveryStreamImpressions(port, session) {
+ const { impressionSets } = session;
+
+ if (!impressionSets) {
+ return;
+ }
+
+ Object.keys(impressionSets).forEach(source => {
+ const { tiles, window_inner_width, window_inner_height } = impressionSets[
+ source
+ ];
+ const payload = this.createImpressionStats(port, {
+ source,
+ tiles,
+ window_inner_width,
+ window_inner_height,
+ });
+ this.sendStructuredIngestionEvent(
+ payload,
+ STRUCTURED_INGESTION_NAMESPACE_AS,
+ "impression-stats",
+ "1"
+ );
+ });
+ }
+
+ /**
+ * Send loaded content pings for Discovery Stream for a given session.
+ *
+ * @note the loaded content reports are stored in session.loadedContentSets for different
+ * sources, and will be sent separately accordingly.
+ *
+ * @param {String} port The session port with which this is associated
+ * @param {Object} session The session object
+ */
+ sendDiscoveryStreamLoadedContent(port, session) {
+ const { loadedContentSets } = session;
+
+ if (!loadedContentSets) {
+ return;
+ }
+
+ Object.keys(loadedContentSets).forEach(source => {
+ const tiles = loadedContentSets[source];
+ const payload = this.createImpressionStats(port, {
+ source,
+ tiles,
+ loaded: tiles.length,
+ });
+ this.sendStructuredIngestionEvent(
+ payload,
+ STRUCTURED_INGESTION_NAMESPACE_AS,
+ "impression-stats",
+ "1"
+ );
+ });
+ }
+
+ /**
+ * handleNewTabInit - Handle NEW_TAB_INIT, which creates a new session and sets the a flag
+ * for session.perf based on whether or not this new tab is preloaded
+ *
+ * @param {obj} action the Action object
+ */
+ handleNewTabInit(action) {
+ const session = this.addSession(
+ au.getPortIdOfSender(action),
+ action.data.url
+ );
+ session.perf.is_preloaded =
+ action.data.browser.getAttribute("preloadedState") === "preloaded";
+ }
+
+ /**
+ * createPing - Create a ping with common properties
+ *
+ * @param {string} id The portID of the session, if a session is relevant (optional)
+ * @return {obj} A telemetry ping
+ */
+ createPing(portID) {
+ const ping = {
+ addon_version: Services.appinfo.appBuildID,
+ locale: Services.locale.appLocaleAsBCP47,
+ user_prefs: this.userPreferences,
+ };
+
+ // If the ping is part of a user session, add session-related info
+ if (portID) {
+ const session = this.sessions.get(portID) || this.addSession(portID);
+ Object.assign(ping, { session_id: session.session_id });
+
+ if (session.page) {
+ Object.assign(ping, { page: session.page });
+ }
+ }
+ return ping;
+ }
+
+ /**
+ * createImpressionStats - Create a ping for an impression stats
+ *
+ * @param {string} portID The portID of the open session
+ * @param {ob} data The data object to be included in the ping.
+ * @return {obj} A telemetry ping
+ */
+ createImpressionStats(portID, data) {
+ let ping = Object.assign(this.createPing(portID), data, {
+ impression_id: this._impressionId,
+ });
+ // Make sure `session_id` and `client_id` are not in the ping.
+ delete ping.session_id;
+ delete ping.client_id;
+ return ping;
+ }
+
+ createUserEvent(action) {
+ return Object.assign(
+ this.createPing(au.getPortIdOfSender(action)),
+ action.data,
+ { action: "activity_stream_user_event" }
+ );
+ }
+
+ createSessionEndEvent(session) {
+ return Object.assign(this.createPing(), {
+ session_id: session.session_id,
+ page: session.page,
+ session_duration: session.session_duration,
+ action: "activity_stream_session",
+ perf: session.perf,
+ profile_creation_date:
+ lazy.TelemetryEnvironment.currentEnvironment.profile.resetDate ||
+ lazy.TelemetryEnvironment.currentEnvironment.profile.creationDate,
+ });
+ }
+
+ /**
+ * Create a ping for AS router event. The client_id is set to "n/a" by default,
+ * different component can override this by its own telemetry collection policy.
+ */
+ async createASRouterEvent(action) {
+ let event = {
+ ...action.data,
+ addon_version: Services.appinfo.appBuildID,
+ locale: Services.locale.appLocaleAsBCP47,
+ };
+ const session = this.sessions.get(au.getPortIdOfSender(action));
+ if (event.event_context && typeof event.event_context === "object") {
+ event.event_context = JSON.stringify(event.event_context);
+ }
+ switch (event.action) {
+ case "cfr_user_event":
+ event = await this.applyCFRPolicy(event);
+ break;
+ case "snippets_local_testing_user_event":
+ case "snippets_user_event":
+ event = await this.applySnippetsPolicy(event);
+ break;
+ case "badge_user_event":
+ case "whats-new-panel_user_event":
+ event = await this.applyWhatsNewPolicy(event);
+ break;
+ case "infobar_user_event":
+ event = await this.applyInfoBarPolicy(event);
+ break;
+ case "spotlight_user_event":
+ event = await this.applySpotlightPolicy(event);
+ break;
+ case "toast_notification_user_event":
+ event = await this.applyToastNotificationPolicy(event);
+ break;
+ case "moments_user_event":
+ event = await this.applyMomentsPolicy(event);
+ break;
+ case "onboarding_user_event":
+ event = await this.applyOnboardingPolicy(event, session);
+ break;
+ case "asrouter_undesired_event":
+ event = this.applyUndesiredEventPolicy(event);
+ break;
+ default:
+ event = { ping: event };
+ break;
+ }
+ return event;
+ }
+
+ /**
+ * Per Bug 1484035, CFR metrics comply with following policies:
+ * 1). In release, it collects impression_id, and treats bucket_id as message_id
+ * 2). In prerelease, it collects client_id and message_id
+ * 3). In shield experiments conducted in release, it collects client_id and message_id
+ */
+ async applyCFRPolicy(ping) {
+ if (
+ lazy.UpdateUtils.getUpdateChannel(true) === "release" &&
+ !this.isInCFRCohort
+ ) {
+ ping.message_id = "n/a";
+ ping.impression_id = this._impressionId;
+ } else {
+ ping.client_id = await this.telemetryClientId;
+ }
+ delete ping.action;
+ return { ping, pingType: "cfr" };
+ }
+
+ /**
+ * Per Bug 1482134, all the metrics for What's New panel use client_id in
+ * all the release channels
+ */
+ async applyWhatsNewPolicy(ping) {
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ // Attach page info to `event_context` if there is a session associated with this ping
+ delete ping.action;
+ return { ping, pingType: "whats-new-panel" };
+ }
+
+ async applyInfoBarPolicy(ping) {
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ delete ping.action;
+ return { ping, pingType: "infobar" };
+ }
+
+ async applySpotlightPolicy(ping) {
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ delete ping.action;
+ return { ping, pingType: "spotlight" };
+ }
+
+ async applyToastNotificationPolicy(ping) {
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ delete ping.action;
+ return { ping, pingType: "toast_notification" };
+ }
+
+ /**
+ * Per Bug 1484035, Moments metrics comply with following policies:
+ * 1). In release, it collects impression_id, and treats bucket_id as message_id
+ * 2). In prerelease, it collects client_id and message_id
+ * 3). In shield experiments conducted in release, it collects client_id and message_id
+ */
+ async applyMomentsPolicy(ping) {
+ if (
+ lazy.UpdateUtils.getUpdateChannel(true) === "release" &&
+ !this.isInCFRCohort
+ ) {
+ ping.message_id = "n/a";
+ ping.impression_id = this._impressionId;
+ } else {
+ ping.client_id = await this.telemetryClientId;
+ }
+ delete ping.action;
+ return { ping, pingType: "moments" };
+ }
+
+ /**
+ * Per Bug 1485069, all the metrics for Snippets in AS router use client_id in
+ * all the release channels
+ */
+ async applySnippetsPolicy(ping) {
+ ping.client_id = await this.telemetryClientId;
+ delete ping.action;
+ return { ping, pingType: "snippets" };
+ }
+
+ /**
+ * Per Bug 1482134, all the metrics for Onboarding in AS router use client_id in
+ * all the release channels
+ */
+ async applyOnboardingPolicy(ping, session) {
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ // Attach page info to `event_context` if there is a session associated with this ping
+ if (ping.action === "onboarding_user_event" && session && session.page) {
+ let event_context;
+
+ try {
+ event_context = ping.event_context
+ ? JSON.parse(ping.event_context)
+ : {};
+ } catch (e) {
+ // If `ping.event_context` is not a JSON serialized string, then we create a `value`
+ // key for it
+ event_context = { value: ping.event_context };
+ }
+
+ if (ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page)) {
+ event_context.page = session.page;
+ } else {
+ console.error(`Invalid 'page' for Onboarding event: ${session.page}`);
+ }
+ ping.event_context = JSON.stringify(event_context);
+ }
+ delete ping.action;
+ return { ping, pingType: "onboarding" };
+ }
+
+ applyUndesiredEventPolicy(ping) {
+ ping.impression_id = this._impressionId;
+ delete ping.action;
+ return { ping, pingType: "undesired-events" };
+ }
+
+ sendEvent(event_object) {
+ switch (event_object.action) {
+ case "activity_stream_user_event":
+ this.sendEventPing(event_object);
+ break;
+ case "activity_stream_session":
+ this.sendSessionPing(event_object);
+ break;
+ }
+ }
+
+ async sendEventPing(ping) {
+ delete ping.action;
+ ping.client_id = await this.telemetryClientId;
+ ping.browser_session_id = lazy.browserSessionId;
+ if (ping.value && typeof ping.value === "object") {
+ ping.value = JSON.stringify(ping.value);
+ }
+ this.sendStructuredIngestionEvent(
+ ping,
+ STRUCTURED_INGESTION_NAMESPACE_AS,
+ "events",
+ 1
+ );
+ }
+
+ async sendSessionPing(ping) {
+ delete ping.action;
+ ping.client_id = await this.telemetryClientId;
+ this.sendStructuredIngestionEvent(
+ ping,
+ STRUCTURED_INGESTION_NAMESPACE_AS,
+ "sessions",
+ 1
+ );
+ }
+
+ sendUTEvent(event_object, eventFunction) {
+ if (this.telemetryEnabled && this.eventTelemetryEnabled) {
+ eventFunction(event_object);
+ }
+ }
+
+ /**
+ * Generates an endpoint for Structured Ingestion telemetry pipeline. Note that
+ * Structured Ingestion requires a different endpoint for each ping. See more
+ * details about endpoint schema at:
+ * https://github.com/mozilla/gcp-ingestion/blob/master/docs/edge.md#postput-request
+ *
+ * @param {String} namespace Namespace of the ping, such as "activity-stream" or "messaging-system".
+ * @param {String} pingType Type of the ping, such as "impression-stats".
+ * @param {String} version Endpoint version for this ping type.
+ */
+ _generateStructuredIngestionEndpoint(namespace, pingType, version) {
+ const uuid = Services.uuid.generateUUID().toString();
+ // Structured Ingestion does not support the UUID generated by Services.uuid,
+ // because it contains leading and trailing braces. Need to trim them first.
+ const docID = uuid.slice(1, -1);
+ const extension = `${namespace}/${pingType}/${version}/${docID}`;
+ return `${this.structuredIngestionEndpointBase}/${extension}`;
+ }
+
+ sendStructuredIngestionEvent(eventObject, namespace, pingType, version) {
+ if (this.telemetryEnabled) {
+ this.pingCentre.sendStructuredIngestionPing(
+ eventObject,
+ this._generateStructuredIngestionEndpoint(namespace, pingType, version)
+ );
+ }
+ }
+
+ handleImpressionStats(action) {
+ const payload = this.createImpressionStats(
+ au.getPortIdOfSender(action),
+ action.data
+ );
+ this.sendStructuredIngestionEvent(
+ payload,
+ STRUCTURED_INGESTION_NAMESPACE_AS,
+ "impression-stats",
+ "1"
+ );
+ }
+
+ handleTopSitesImpressionStats(action) {
+ const { data } = action;
+ const { type, position, source, advertiser } = data;
+ let pingType;
+
+ const session = this.sessions.get(au.getPortIdOfSender(action));
+ if (type === "impression") {
+ pingType = "topsites-impression";
+ Services.telemetry.keyedScalarAdd(
+ `${SCALAR_CATEGORY_TOPSITES}.impression`,
+ `${source}_${position}`,
+ 1
+ );
+ if (session) {
+ Glean.topsites.impression.record({
+ newtab_visit_id: session.session_id,
+ is_sponsored: !!advertiser,
+ });
+ }
+ } else if (type === "click") {
+ pingType = "topsites-click";
+ Services.telemetry.keyedScalarAdd(
+ `${SCALAR_CATEGORY_TOPSITES}.click`,
+ `${source}_${position}`,
+ 1
+ );
+ if (session) {
+ Glean.topsites.click.record({
+ newtab_visit_id: session.session_id,
+ is_sponsored: !!advertiser,
+ });
+ }
+ } else {
+ console.error("Unknown ping type for TopSites impression");
+ return;
+ }
+
+ let payload = { ...data, context_id: lazy.contextId };
+ delete payload.type;
+ this.sendStructuredIngestionEvent(
+ payload,
+ STRUCTURED_INGESTION_NAMESPACE_CS,
+ pingType,
+ "1"
+ );
+ }
+
+ handleUserEvent(action) {
+ let userEvent = this.createUserEvent(action);
+ this.sendEvent(userEvent);
+ this.sendUTEvent(userEvent, this.utEvents.sendUserEvent);
+ }
+
+ handleDiscoveryStreamUserEvent(action) {
+ const pocket_logged_in_status = lazy.pktApi.isUserLoggedIn();
+ Glean.pocket.isSignedIn.set(pocket_logged_in_status);
+ this.handleUserEvent({
+ ...action,
+ data: {
+ ...(action.data || {}),
+ value: {
+ ...(action.data?.value || {}),
+ pocket_logged_in_status,
+ },
+ },
+ });
+ const session = this.sessions.get(au.getPortIdOfSender(action));
+ switch (action.data?.event) {
+ case "CLICK":
+ if (
+ action.data.source === "POPULAR_TOPICS" ||
+ action.data.value?.card_type === "topics_widget"
+ ) {
+ Glean.pocket.topicClick.record({
+ newtab_visit_id: session.session_id,
+ topic: action.data.value?.topic,
+ });
+ } else if (["spoc", "organic"].includes(action.data.value?.card_type)) {
+ Glean.pocket.click.record({
+ newtab_visit_id: session.session_id,
+ is_sponsored: action.data.value?.card_type === "spoc",
+ position: action.data.action_position,
+ });
+ }
+ break;
+ case "SAVE_TO_POCKET":
+ Glean.pocket.save.record({
+ newtab_visit_id: session.session_id,
+ is_sponsored: action.data.value?.card_type === "spoc",
+ position: action.data.action_position,
+ });
+ break;
+ }
+ }
+
+ async handleASRouterUserEvent(action) {
+ const { ping, pingType } = await this.createASRouterEvent(action);
+ if (!pingType) {
+ console.error("Unknown ping type for ASRouter telemetry");
+ return;
+ }
+ this.sendStructuredIngestionEvent(
+ ping,
+ STRUCTURED_INGESTION_NAMESPACE_MS,
+ pingType,
+ "1"
+ );
+ }
+
+ /**
+ * This function is used by ActivityStreamStorage to report errors
+ * trying to access IndexedDB.
+ */
+ SendASRouterUndesiredEvent(data) {
+ this.handleASRouterUserEvent({
+ data: { ...data, action: "asrouter_undesired_event" },
+ });
+ }
+
+ async sendPageTakeoverData() {
+ if (this.telemetryEnabled) {
+ const value = {};
+ let newtabAffected = false;
+ let homeAffected = false;
+ let newtabCategory = "disabled";
+ let homePageCategory = "disabled";
+
+ // Check whether or not about:home and about:newtab are set to a custom URL.
+ // If so, classify them.
+ if (Services.prefs.getBoolPref("browser.newtabpage.enabled")) {
+ newtabCategory = "enabled";
+ if (
+ lazy.AboutNewTab.newTabURLOverridden &&
+ !lazy.AboutNewTab.newTabURL.startsWith("moz-extension://")
+ ) {
+ value.newtab_url_category = await this._classifySite(
+ lazy.AboutNewTab.newTabURL
+ );
+ newtabAffected = true;
+ newtabCategory = value.newtab_url_category;
+ }
+ }
+ // Check if the newtab page setting is controlled by an extension.
+ await lazy.ExtensionSettingsStore.initialize();
+ const newtabExtensionInfo = lazy.ExtensionSettingsStore.getSetting(
+ "url_overrides",
+ "newTabURL"
+ );
+ if (newtabExtensionInfo && newtabExtensionInfo.id) {
+ value.newtab_extension_id = newtabExtensionInfo.id;
+ newtabAffected = true;
+ newtabCategory = "extension";
+ }
+
+ const homePageURL = lazy.HomePage.get();
+ if (
+ !["about:home", "about:blank"].includes(homePageURL) &&
+ !homePageURL.startsWith("moz-extension://")
+ ) {
+ value.home_url_category = await this._classifySite(homePageURL);
+ homeAffected = true;
+ homePageCategory = value.home_url_category;
+ }
+ const homeExtensionInfo = lazy.ExtensionSettingsStore.getSetting(
+ "prefs",
+ "homepage_override"
+ );
+ if (homeExtensionInfo && homeExtensionInfo.id) {
+ value.home_extension_id = homeExtensionInfo.id;
+ homeAffected = true;
+ homePageCategory = "extension";
+ }
+ if (!homeAffected && !lazy.HomePage.overridden) {
+ homePageCategory = "enabled";
+ }
+
+ let page;
+ if (newtabAffected && homeAffected) {
+ page = "both";
+ } else if (newtabAffected) {
+ page = "about:newtab";
+ } else if (homeAffected) {
+ page = "about:home";
+ }
+
+ if (page) {
+ const event = Object.assign(this.createPing(), {
+ action: "activity_stream_user_event",
+ event: "PAGE_TAKEOVER_DATA",
+ value,
+ page,
+ session_id: "n/a",
+ });
+ this.sendEvent(event);
+ }
+ Glean.newtab.newtabCategory.set(newtabCategory);
+ Glean.newtab.homepageCategory.set(homePageCategory);
+ if (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) {
+ GleanPings.newtab.submit("component_init");
+ }
+ }
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ this.sendPageTakeoverData();
+ break;
+ case at.NEW_TAB_INIT:
+ this.handleNewTabInit(action);
+ break;
+ case at.NEW_TAB_UNLOAD:
+ this.endSession(au.getPortIdOfSender(action));
+ break;
+ case at.SAVE_SESSION_PERF_DATA:
+ this.saveSessionPerfData(au.getPortIdOfSender(action), action.data);
+ break;
+ case at.TELEMETRY_IMPRESSION_STATS:
+ this.handleImpressionStats(action);
+ break;
+ case at.DISCOVERY_STREAM_IMPRESSION_STATS:
+ this.handleDiscoveryStreamImpressionStats(
+ au.getPortIdOfSender(action),
+ action.data
+ );
+ break;
+ case at.DISCOVERY_STREAM_LOADED_CONTENT:
+ this.handleDiscoveryStreamLoadedContent(
+ au.getPortIdOfSender(action),
+ action.data
+ );
+ break;
+ case at.DISCOVERY_STREAM_USER_EVENT:
+ this.handleDiscoveryStreamUserEvent(action);
+ break;
+ case at.TELEMETRY_USER_EVENT:
+ this.handleUserEvent(action);
+ break;
+ // The next few action types come from ASRouter, which doesn't use
+ // Actions from Actions.jsm, but uses these other custom strings.
+ case msg.TOOLBAR_BADGE_TELEMETRY:
+ // Intentional fall-through
+ case msg.TOOLBAR_PANEL_TELEMETRY:
+ // Intentional fall-through
+ case msg.MOMENTS_PAGE_TELEMETRY:
+ // Intentional fall-through
+ case msg.DOORHANGER_TELEMETRY:
+ // Intentional fall-through
+ case msg.INFOBAR_TELEMETRY:
+ // Intentional fall-through
+ case msg.SPOTLIGHT_TELEMETRY:
+ // Intentional fall-through
+ case msg.TOAST_NOTIFICATION_TELEMETRY:
+ // Intentional fall-through
+ case at.AS_ROUTER_TELEMETRY_USER_EVENT:
+ this.handleASRouterUserEvent(action);
+ break;
+ case at.TOP_SITES_IMPRESSION_STATS:
+ this.handleTopSitesImpressionStats(action);
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ }
+
+ /**
+ * Handle impression stats actions from Discovery Stream. The data will be
+ * stored into the session.impressionSets object for the given port, so that
+ * it is sent to the server when the session ends.
+ *
+ * @note session.impressionSets will be keyed on `source` of the `data`,
+ * all the data will be appended to an array for the same source.
+ *
+ * @param {String} port The session port with which this is associated
+ * @param {Object} data The impression data structured as {source: "SOURCE", tiles: [{id: 123}]}
+ *
+ */
+ handleDiscoveryStreamImpressionStats(port, data) {
+ let session = this.sessions.get(port);
+
+ if (!session) {
+ throw new Error("Session does not exist.");
+ }
+
+ const { window_inner_width, window_inner_height, source, tiles } = data;
+ const impressionSets = session.impressionSets || {};
+ const impressions = impressionSets[source] || {
+ tiles: [],
+ window_inner_width,
+ window_inner_height,
+ };
+ // The payload might contain other properties, we need `id`, `pos` and potentially `shim` here.
+ tiles.forEach(tile => {
+ impressions.tiles.push({
+ id: tile.id,
+ pos: tile.pos,
+ ...(tile.shim ? { shim: tile.shim } : {}),
+ });
+ Glean.pocket.impression.record({
+ newtab_visit_id: session.session_id,
+ is_sponsored: tile.type === "spoc",
+ position: tile.pos,
+ });
+ });
+ impressionSets[source] = impressions;
+ session.impressionSets = impressionSets;
+ }
+
+ /**
+ * Handle loaded content actions from Discovery Stream. The data will be
+ * stored into the session.loadedContentSets object for the given port, so that
+ * it is sent to the server when the session ends.
+ *
+ * @note session.loadedContentSets will be keyed on `source` of the `data`,
+ * all the data will be appended to an array for the same source.
+ *
+ * @param {String} port The session port with which this is associated
+ * @param {Object} data The loaded content structured as {source: "SOURCE", tiles: [{id: 123}]}
+ *
+ */
+ handleDiscoveryStreamLoadedContent(port, data) {
+ let session = this.sessions.get(port);
+
+ if (!session) {
+ throw new Error("Session does not exist.");
+ }
+
+ const loadedContentSets = session.loadedContentSets || {};
+ const loadedContents = loadedContentSets[data.source] || [];
+ // The payload might contain other properties, we need `id` and `pos` here.
+ data.tiles.forEach(tile =>
+ loadedContents.push({ id: tile.id, pos: tile.pos })
+ );
+ loadedContentSets[data.source] = loadedContents;
+ session.loadedContentSets = loadedContentSets;
+ }
+
+ /**
+ * Take all enumerable members of the data object and merge them into
+ * the session.perf object for the given port, so that it is sent to the
+ * server when the session ends. All members of the data object should
+ * be valid values of the perf object, as defined in pings.js and the
+ * data*.md documentation.
+ *
+ * @note Any existing keys with the same names already in the
+ * session perf object will be overwritten by values passed in here.
+ *
+ * @param {String} port The session with which this is associated
+ * @param {Object} data The perf data to be
+ */
+ saveSessionPerfData(port, data) {
+ // XXX should use try/catch and send a bad state indicator if this
+ // get blows up.
+ let session = this.sessions.get(port);
+
+ // XXX Partial workaround for #3118; avoids the worst incorrect associations
+ // of times with browsers, by associating the load trigger with the
+ // visibility event as the user is most likely associating the trigger to
+ // the tab just shown. This helps avoid associating with a preloaded
+ // browser as those don't get the event until shown. Better fix for more
+ // cases forthcoming.
+ //
+ // XXX the about:home check (and the corresponding test) should go away
+ // once the load_trigger stuff in addSession is refactored into
+ // setLoadTriggerInfo.
+ //
+ if (data.visibility_event_rcvd_ts && session.page !== "about:home") {
+ this.setLoadTriggerInfo(port);
+ }
+
+ let timestamp = data.topsites_first_painted_ts;
+
+ if (
+ timestamp &&
+ session.page === "about:home" &&
+ !lazy.HomePage.overridden &&
+ Services.prefs.getIntPref("browser.startup.page") === 1
+ ) {
+ lazy.AboutNewTab.maybeRecordTopsitesPainted(timestamp);
+ }
+
+ Object.assign(session.perf, data);
+
+ if (data.visibility_event_rcvd_ts && !session.newtabOpened) {
+ session.newtabOpened = true;
+ const source = ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page)
+ ? session.page
+ : "other";
+ Glean.newtab.opened.record({
+ newtab_visit_id: session.session_id,
+ source,
+ });
+ }
+ }
+
+ _beginObservingNewtabPingPrefs() {
+ const BRANCH = "browser.newtabpage.activity-stream.";
+ const NEWTAB_PING_PREFS = {
+ showSearch: Glean.newtabSearch.enabled,
+ "feeds.topsites": Glean.topsites.enabled,
+ showSponsoredTopSites: Glean.topsites.sponsoredEnabled,
+ "feeds.section.topstories": Glean.pocket.enabled,
+ showSponsored: Glean.pocket.sponsoredStoriesEnabled,
+ };
+ const setNewtabPrefMetrics = () => {
+ for (const [pref, metric] of Object.entries(NEWTAB_PING_PREFS)) {
+ metric.set(Services.prefs.getBoolPref(BRANCH + pref));
+ }
+ };
+ for (const pref of Object.keys(NEWTAB_PING_PREFS)) {
+ Services.prefs.addObserver(BRANCH + pref, setNewtabPrefMetrics);
+ }
+ setNewtabPrefMetrics();
+ Glean.pocket.isSignedIn.set(lazy.pktApi.isUserLoggedIn());
+ }
+
+ uninit() {
+ try {
+ Services.obs.removeObserver(
+ this.browserOpenNewtabStart,
+ "browser-open-newtab-start"
+ );
+ Services.obs.removeObserver(
+ this._addWindowListeners,
+ DOMWINDOW_OPENED_TOPIC
+ );
+ } catch (e) {
+ // Operation can fail when uninit is called before
+ // init has finished setting up the observer
+ }
+
+ // Only uninit if the getter has initialized it
+ if (Object.prototype.hasOwnProperty.call(this, "pingCentre")) {
+ this.pingCentre.uninit();
+ }
+ if (Object.prototype.hasOwnProperty.call(this, "utEvents")) {
+ this.utEvents.uninit();
+ }
+
+ // TODO: Send any unfinished sessions
+ }
+}
+
+const EXPORTED_SYMBOLS = [
+ "TelemetryFeed",
+ "USER_PREFS_ENCODING",
+ "PREF_IMPRESSION_ID",
+ "TELEMETRY_PREF",
+ "EVENTS_TELEMETRY_PREF",
+ "STRUCTURED_INGESTION_ENDPOINT_PREF",
+];
diff --git a/browser/components/newtab/lib/TippyTopProvider.jsm b/browser/components/newtab/lib/TippyTopProvider.jsm
new file mode 100644
index 0000000000..f0e50ad329
--- /dev/null
+++ b/browser/components/newtab/lib/TippyTopProvider.jsm
@@ -0,0 +1,62 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const TIPPYTOP_PATH = "chrome://activity-stream/content/data/content/tippytop/";
+const TIPPYTOP_JSON_PATH =
+ "chrome://activity-stream/content/data/content/tippytop/top_sites.json";
+
+/*
+ * Get a domain from a url optionally stripping subdomains.
+ */
+function getDomain(url, strip = "www.") {
+ let domain = "";
+ try {
+ domain = new URL(url).hostname;
+ } catch (ex) {}
+ if (strip === "*") {
+ try {
+ domain = Services.eTLD.getBaseDomainFromHost(domain);
+ } catch (ex) {}
+ } else if (domain.startsWith(strip)) {
+ domain = domain.slice(strip.length);
+ }
+ return domain;
+}
+
+class TippyTopProvider {
+ constructor() {
+ this._sitesByDomain = new Map();
+ this.initialized = false;
+ }
+
+ async init() {
+ // Load the Tippy Top sites from the json manifest.
+ try {
+ for (const site of await (
+ await fetch(TIPPYTOP_JSON_PATH, {
+ credentials: "omit",
+ })
+ ).json()) {
+ for (const domain of site.domains) {
+ this._sitesByDomain.set(domain, site);
+ }
+ }
+ this.initialized = true;
+ } catch (error) {
+ console.error("Failed to load tippy top manifest.");
+ }
+ }
+
+ processSite(site, strip) {
+ const tippyTop = this._sitesByDomain.get(getDomain(site.url, strip));
+ if (tippyTop) {
+ site.tippyTopIcon = TIPPYTOP_PATH + tippyTop.image_url;
+ site.smallFavicon = TIPPYTOP_PATH + tippyTop.favicon_url;
+ site.backgroundColor = tippyTop.background_color;
+ }
+ return site;
+ }
+}
+
+const EXPORTED_SYMBOLS = ["TippyTopProvider", "getDomain"];
diff --git a/browser/components/newtab/lib/ToastNotification.jsm b/browser/components/newtab/lib/ToastNotification.jsm
new file mode 100644
index 0000000000..4d6193d76f
--- /dev/null
+++ b/browser/components/newtab/lib/ToastNotification.jsm
@@ -0,0 +1,118 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ ExperimentAPI: "resource://nimbus/ExperimentAPI.jsm",
+ RemoteL10n: "resource://activity-stream/lib/RemoteL10n.jsm",
+});
+
+XPCOMUtils.defineLazyServiceGetters(lazy, {
+ AlertsService: ["@mozilla.org/alerts-service;1", "nsIAlertsService"],
+});
+
+const ToastNotification = {
+ // Allow testing to stub the alerts service.
+ get AlertsService() {
+ return lazy.AlertsService;
+ },
+
+ sendUserEventTelemetry(event, message, dispatch) {
+ const ping = {
+ message_id: message.id,
+ event,
+ };
+ dispatch({
+ type: "TOAST_NOTIFICATION_TELEMETRY",
+ data: { action: "toast_notification_user_event", ...ping },
+ });
+ },
+
+ /**
+ * Show a toast notification.
+ * @param message Message containing content to show.
+ * @param dispatch A function to dispatch resulting actions.
+ * @return boolean value capturing if toast notification was displayed.
+ */
+ async showToastNotification(message, dispatch) {
+ let { content } = message;
+ let title = await lazy.RemoteL10n.formatLocalizableText(content.title);
+ let body = await lazy.RemoteL10n.formatLocalizableText(content.body);
+
+ // The only link between background task message experiment and user
+ // re-engagement via the notification is the associated "tag". Said tag is
+ // usually controlled by the message content, but for message experiments,
+ // we want to avoid a missing tag and to ensure a deterministic tag for
+ // easier analysis, including across branches.
+ let { tag } = content;
+
+ let experimentMetadata =
+ lazy.ExperimentAPI.getExperimentMetaData({
+ featureId: "backgroundTaskMessage",
+ }) || {};
+
+ if (
+ experimentMetadata?.active &&
+ experimentMetadata?.slug &&
+ experimentMetadata?.branch?.slug
+ ) {
+ // Like `my-experiment:my-branch`.
+ tag = `${experimentMetadata?.slug}:${experimentMetadata?.branch?.slug}`;
+ }
+
+ // There are two events named `IMPRESSION` the first one refers to telemetry
+ // while the other refers to ASRouter impressions used for the frequency cap
+ this.sendUserEventTelemetry("IMPRESSION", message, dispatch);
+ dispatch({ type: "IMPRESSION", data: message });
+
+ let alert = Cc["@mozilla.org/alert-notification;1"].createInstance(
+ Ci.nsIAlertNotification
+ );
+ let systemPrincipal = Services.scriptSecurityManager.getSystemPrincipal();
+ alert.init(
+ tag,
+ content.image_url
+ ? Services.urlFormatter.formatURL(content.image_url)
+ : content.image_url,
+ title,
+ body,
+ true /* aTextClickable */,
+ content.data,
+ null /* aDir */,
+ null /* aLang */,
+ null /* aData */,
+ systemPrincipal,
+ null /* aInPrivateBrowsing */,
+ content.requireInteraction
+ );
+
+ if (content.actions) {
+ let actions = Cu.cloneInto(content.actions, {});
+ for (let action of actions) {
+ if (action.title) {
+ action.title = await lazy.RemoteL10n.formatLocalizableText(
+ action.title
+ );
+ }
+ }
+ alert.actions = actions;
+ }
+
+ if (content.launch_url) {
+ alert.launchURL = Services.urlFormatter.formatURL(content.launch_url);
+ }
+
+ this.AlertsService.showAlert(alert);
+
+ return true;
+ },
+};
+
+const EXPORTED_SYMBOLS = ["ToastNotification"];
diff --git a/browser/components/newtab/lib/ToolbarBadgeHub.jsm b/browser/components/newtab/lib/ToolbarBadgeHub.jsm
new file mode 100644
index 0000000000..f403ca9186
--- /dev/null
+++ b/browser/components/newtab/lib/ToolbarBadgeHub.jsm
@@ -0,0 +1,318 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+ clearTimeout: "resource://gre/modules/Timer.sys.mjs",
+ requestIdleCallback: "resource://gre/modules/Timer.sys.mjs",
+ setTimeout: "resource://gre/modules/Timer.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ EveryWindow: "resource:///modules/EveryWindow.jsm",
+ ToolbarPanelHub: "resource://activity-stream/lib/ToolbarPanelHub.jsm",
+});
+
+let notificationsByWindow = new WeakMap();
+
+class _ToolbarBadgeHub {
+ constructor() {
+ this.id = "toolbar-badge-hub";
+ this.state = {};
+ this.prefs = {
+ WHATSNEW_TOOLBAR_PANEL: "browser.messaging-system.whatsNewPanel.enabled",
+ };
+ this.removeAllNotifications = this.removeAllNotifications.bind(this);
+ this.removeToolbarNotification = this.removeToolbarNotification.bind(this);
+ this.addToolbarNotification = this.addToolbarNotification.bind(this);
+ this.registerBadgeToAllWindows = this.registerBadgeToAllWindows.bind(this);
+ this._sendPing = this._sendPing.bind(this);
+ this.sendUserEventTelemetry = this.sendUserEventTelemetry.bind(this);
+
+ this._handleMessageRequest = null;
+ this._addImpression = null;
+ this._blockMessageById = null;
+ this._sendTelemetry = null;
+ this._initialized = false;
+ }
+
+ async init(
+ waitForInitialized,
+ {
+ handleMessageRequest,
+ addImpression,
+ blockMessageById,
+ unblockMessageById,
+ sendTelemetry,
+ }
+ ) {
+ if (this._initialized) {
+ return;
+ }
+
+ this._initialized = true;
+ this._handleMessageRequest = handleMessageRequest;
+ this._blockMessageById = blockMessageById;
+ this._unblockMessageById = unblockMessageById;
+ this._addImpression = addImpression;
+ this._sendTelemetry = sendTelemetry;
+ // Need to wait for ASRouter to initialize before trying to fetch messages
+ await waitForInitialized;
+ this.messageRequest({
+ triggerId: "toolbarBadgeUpdate",
+ template: "toolbar_badge",
+ });
+ // Listen for pref changes that could trigger new badges
+ Services.prefs.addObserver(this.prefs.WHATSNEW_TOOLBAR_PANEL, this);
+ }
+
+ observe(aSubject, aTopic, aPrefName) {
+ switch (aPrefName) {
+ case this.prefs.WHATSNEW_TOOLBAR_PANEL:
+ this.messageRequest({
+ triggerId: "toolbarBadgeUpdate",
+ template: "toolbar_badge",
+ });
+ break;
+ }
+ }
+
+ maybeInsertFTL(win) {
+ win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl");
+ }
+
+ executeAction({ id, data, message_id }) {
+ switch (id) {
+ case "show-whatsnew-button":
+ lazy.ToolbarPanelHub.enableToolbarButton();
+ lazy.ToolbarPanelHub.enableAppmenuButton();
+ break;
+ }
+ }
+
+ _clearBadgeTimeout() {
+ if (this.state.showBadgeTimeoutId) {
+ lazy.clearTimeout(this.state.showBadgeTimeoutId);
+ }
+ }
+
+ removeAllNotifications(event) {
+ if (event) {
+ // ignore right clicks
+ if (
+ (event.type === "mousedown" || event.type === "click") &&
+ event.button !== 0
+ ) {
+ return;
+ }
+ // ignore keyboard access that is not one of the usual accessor keys
+ if (
+ event.type === "keypress" &&
+ event.key !== " " &&
+ event.key !== "Enter"
+ ) {
+ return;
+ }
+
+ event.target.removeEventListener(
+ "mousedown",
+ this.removeAllNotifications
+ );
+ event.target.removeEventListener("keypress", this.removeAllNotifications);
+ // If we have an event it means the user interacted with the badge
+ // we should send telemetry
+ if (this.state.notification) {
+ this.sendUserEventTelemetry("CLICK", this.state.notification);
+ }
+ }
+ // Will call uninit on every window
+ lazy.EveryWindow.unregisterCallback(this.id);
+ if (this.state.notification) {
+ this._blockMessageById(this.state.notification.id);
+ }
+ this._clearBadgeTimeout();
+ this.state = {};
+ }
+
+ removeToolbarNotification(toolbarButton) {
+ // Remove it from the element that displays the badge
+ toolbarButton
+ .querySelector(".toolbarbutton-badge")
+ .classList.remove("feature-callout");
+ toolbarButton.removeAttribute("badged");
+ // Remove id used for for aria-label badge description
+ const notificationDescription = toolbarButton.querySelector(
+ "#toolbarbutton-notification-description"
+ );
+ if (notificationDescription) {
+ notificationDescription.remove();
+ toolbarButton.removeAttribute("aria-labelledby");
+ toolbarButton.removeAttribute("aria-describedby");
+ }
+ }
+
+ addToolbarNotification(win, message) {
+ const document = win.browser.ownerDocument;
+ if (message.content.action) {
+ this.executeAction({ ...message.content.action, message_id: message.id });
+ }
+ let toolbarbutton = document.getElementById(message.content.target);
+ if (toolbarbutton) {
+ const badge = toolbarbutton.querySelector(".toolbarbutton-badge");
+ badge.classList.add("feature-callout");
+ toolbarbutton.setAttribute("badged", true);
+ // If we have additional aria-label information for the notification
+ // we add this content to the hidden `toolbarbutton-text` node.
+ // We then use `aria-labelledby` to link this description to the button
+ // that received the notification badge.
+ if (message.content.badgeDescription) {
+ // Insert strings as soon as we know we're showing them
+ this.maybeInsertFTL(win);
+ toolbarbutton.setAttribute(
+ "aria-labelledby",
+ `toolbarbutton-notification-description ${message.content.target}`
+ );
+ // Because tooltiptext is different to the label, it gets duplicated as
+ // the description. Setting `describedby` to the same value as
+ // `labelledby` will be detected by the a11y code and the description
+ // will be removed.
+ toolbarbutton.setAttribute(
+ "aria-describedby",
+ `toolbarbutton-notification-description ${message.content.target}`
+ );
+ const descriptionEl = document.createElement("span");
+ descriptionEl.setAttribute(
+ "id",
+ "toolbarbutton-notification-description"
+ );
+ descriptionEl.hidden = true;
+ document.l10n.setAttributes(
+ descriptionEl,
+ message.content.badgeDescription.string_id
+ );
+ toolbarbutton.appendChild(descriptionEl);
+ }
+ // `mousedown` event required because of the `onmousedown` defined on
+ // the button that prevents `click` events from firing
+ toolbarbutton.addEventListener("mousedown", this.removeAllNotifications);
+ // `keypress` event required for keyboard accessibility
+ toolbarbutton.addEventListener("keypress", this.removeAllNotifications);
+ this.state = { notification: { id: message.id } };
+
+ // Impression should be added when the badge becomes visible
+ this._addImpression(message);
+ // Send a telemetry ping when adding the notification badge
+ this.sendUserEventTelemetry("IMPRESSION", message);
+
+ return toolbarbutton;
+ }
+
+ return null;
+ }
+
+ registerBadgeToAllWindows(message) {
+ if (message.template === "update_action") {
+ this.executeAction({ ...message.content.action, message_id: message.id });
+ // No badge to set only an action to execute
+ return;
+ }
+
+ lazy.EveryWindow.registerCallback(
+ this.id,
+ win => {
+ if (notificationsByWindow.has(win)) {
+ // nothing to do
+ return;
+ }
+ const el = this.addToolbarNotification(win, message);
+ notificationsByWindow.set(win, el);
+ },
+ win => {
+ const el = notificationsByWindow.get(win);
+ if (el) {
+ this.removeToolbarNotification(el);
+ }
+ notificationsByWindow.delete(win);
+ }
+ );
+ }
+
+ registerBadgeNotificationListener(message, options = {}) {
+ // We need to clear any existing notifications and only show
+ // the one set by devtools
+ if (options.force) {
+ this.removeAllNotifications();
+ // When debugging immediately show the badge
+ this.registerBadgeToAllWindows(message);
+ return;
+ }
+
+ if (message.content.delay) {
+ this.state.showBadgeTimeoutId = lazy.setTimeout(() => {
+ lazy.requestIdleCallback(() => this.registerBadgeToAllWindows(message));
+ }, message.content.delay);
+ } else {
+ this.registerBadgeToAllWindows(message);
+ }
+ }
+
+ async messageRequest({ triggerId, template }) {
+ const telemetryObject = { triggerId };
+ TelemetryStopwatch.start("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ const message = await this._handleMessageRequest({
+ triggerId,
+ template,
+ });
+ TelemetryStopwatch.finish("MS_MESSAGE_REQUEST_TIME_MS", telemetryObject);
+ if (message) {
+ this.registerBadgeNotificationListener(message);
+ }
+ }
+
+ _sendPing(ping) {
+ this._sendTelemetry({
+ type: "TOOLBAR_BADGE_TELEMETRY",
+ data: { action: "badge_user_event", ...ping },
+ });
+ }
+
+ sendUserEventTelemetry(event, message) {
+ const win = Services.wm.getMostRecentWindow("navigator:browser");
+ // Only send pings for non private browsing windows
+ if (
+ win &&
+ !lazy.PrivateBrowsingUtils.isBrowserPrivate(
+ win.ownerGlobal.gBrowser.selectedBrowser
+ )
+ ) {
+ this._sendPing({
+ message_id: message.id,
+ event,
+ });
+ }
+ }
+
+ uninit() {
+ this._clearBadgeTimeout();
+ this.state = {};
+ this._initialized = false;
+ notificationsByWindow = new WeakMap();
+ Services.prefs.removeObserver(this.prefs.WHATSNEW_TOOLBAR_PANEL, this);
+ }
+}
+
+/**
+ * ToolbarBadgeHub - singleton instance of _ToolbarBadgeHub that can initiate
+ * message requests and render messages.
+ */
+const ToolbarBadgeHub = new _ToolbarBadgeHub();
+
+const EXPORTED_SYMBOLS = ["ToolbarBadgeHub", "_ToolbarBadgeHub"];
diff --git a/browser/components/newtab/lib/ToolbarPanelHub.jsm b/browser/components/newtab/lib/ToolbarPanelHub.jsm
new file mode 100644
index 0000000000..703e6a3c47
--- /dev/null
+++ b/browser/components/newtab/lib/ToolbarPanelHub.jsm
@@ -0,0 +1,612 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Preferences: "resource://gre/modules/Preferences.sys.mjs",
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ EveryWindow: "resource:///modules/EveryWindow.jsm",
+ SpecialMessageActions:
+ "resource://messaging-system/lib/SpecialMessageActions.jsm",
+ RemoteL10n: "resource://activity-stream/lib/RemoteL10n.jsm",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PanelMultiView",
+ "resource:///modules/PanelMultiView.jsm"
+);
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "TrackingDBService",
+ "@mozilla.org/tracking-db-service;1",
+ "nsITrackingDBService"
+);
+
+const idToTextMap = new Map([
+ [Ci.nsITrackingDBService.TRACKERS_ID, "trackerCount"],
+ [Ci.nsITrackingDBService.TRACKING_COOKIES_ID, "cookieCount"],
+ [Ci.nsITrackingDBService.CRYPTOMINERS_ID, "cryptominerCount"],
+ [Ci.nsITrackingDBService.FINGERPRINTERS_ID, "fingerprinterCount"],
+ [Ci.nsITrackingDBService.SOCIAL_ID, "socialCount"],
+]);
+
+const WHATSNEW_ENABLED_PREF = "browser.messaging-system.whatsNewPanel.enabled";
+const PROTECTIONS_PANEL_INFOMSG_PREF =
+ "browser.protections_panel.infoMessage.seen";
+
+const TOOLBAR_BUTTON_ID = "whats-new-menu-button";
+const APPMENU_BUTTON_ID = "appMenu-whatsnew-button";
+
+const BUTTON_STRING_ID = "cfr-whatsnew-button";
+const WHATS_NEW_PANEL_SELECTOR = "PanelUI-whatsNew-message-container";
+
+class _ToolbarPanelHub {
+ constructor() {
+ this.triggerId = "whatsNewPanelOpened";
+ this._showAppmenuButton = this._showAppmenuButton.bind(this);
+ this._hideAppmenuButton = this._hideAppmenuButton.bind(this);
+ this._showToolbarButton = this._showToolbarButton.bind(this);
+ this._hideToolbarButton = this._hideToolbarButton.bind(this);
+ this.insertProtectionPanelMessage = this.insertProtectionPanelMessage.bind(
+ this
+ );
+
+ this.state = {};
+ this._initialized = false;
+ }
+
+ async init(waitForInitialized, { getMessages, sendTelemetry }) {
+ if (this._initialized) {
+ return;
+ }
+
+ this._initialized = true;
+ this._getMessages = getMessages;
+ this._sendTelemetry = sendTelemetry;
+ // Wait for ASRouter messages to become available in order to know
+ // if we can show the What's New panel
+ await waitForInitialized;
+ // Enable the application menu button so that the user can access
+ // the panel outside of the toolbar button
+ await this.enableAppmenuButton();
+
+ this.state = {
+ protectionPanelMessageSeen: Services.prefs.getBoolPref(
+ PROTECTIONS_PANEL_INFOMSG_PREF,
+ false
+ ),
+ };
+ }
+
+ uninit() {
+ this._initialized = false;
+ lazy.EveryWindow.unregisterCallback(TOOLBAR_BUTTON_ID);
+ lazy.EveryWindow.unregisterCallback(APPMENU_BUTTON_ID);
+ }
+
+ get messages() {
+ return this._getMessages({
+ template: "whatsnew_panel_message",
+ triggerId: "whatsNewPanelOpened",
+ returnAll: true,
+ });
+ }
+
+ toggleWhatsNewPref(event) {
+ // Checkbox onclick handler gets called before the checkbox state gets toggled,
+ // so we have to call it with the opposite value.
+ let newValue = !event.target.checked;
+ lazy.Preferences.set(WHATSNEW_ENABLED_PREF, newValue);
+
+ this.sendUserEventTelemetry(
+ event.target.ownerGlobal,
+ "WNP_PREF_TOGGLE",
+ // Message id is not applicable in this case, the notification state
+ // is not related to a particular message
+ { id: "n/a" },
+ { value: { prefValue: newValue } }
+ );
+ }
+
+ maybeInsertFTL(win) {
+ win.MozXULElement.insertFTLIfNeeded("browser/newtab/asrouter.ftl");
+ win.MozXULElement.insertFTLIfNeeded("browser/branding/brandings.ftl");
+ win.MozXULElement.insertFTLIfNeeded("browser/branding/sync-brand.ftl");
+ }
+
+ maybeLoadCustomElement(win) {
+ if (!win.customElements.get("remote-text")) {
+ Services.scriptloader.loadSubScript(
+ "resource://activity-stream/data/custom-elements/paragraph.js",
+ win
+ );
+ }
+ }
+
+ // Turns on the Appmenu (hamburger menu) button for all open windows and future windows.
+ async enableAppmenuButton() {
+ if ((await this.messages).length) {
+ lazy.EveryWindow.registerCallback(
+ APPMENU_BUTTON_ID,
+ this._showAppmenuButton,
+ this._hideAppmenuButton
+ );
+ }
+ }
+
+ // Removes the button from the Appmenu.
+ // Only used in tests.
+ disableAppmenuButton() {
+ lazy.EveryWindow.unregisterCallback(APPMENU_BUTTON_ID);
+ }
+
+ // Turns on the Toolbar button for all open windows and future windows.
+ async enableToolbarButton() {
+ if ((await this.messages).length) {
+ lazy.EveryWindow.registerCallback(
+ TOOLBAR_BUTTON_ID,
+ this._showToolbarButton,
+ this._hideToolbarButton
+ );
+ }
+ }
+
+ // When the panel is hidden we want to run some cleanup
+ _onPanelHidden(win) {
+ const panelContainer = win.document.getElementById(
+ "customizationui-widget-panel"
+ );
+ // When the panel is hidden we want to remove any toolbar buttons that
+ // might have been added as an entry point to the panel
+ const removeToolbarButton = () => {
+ lazy.EveryWindow.unregisterCallback(TOOLBAR_BUTTON_ID);
+ };
+ if (!panelContainer) {
+ return;
+ }
+ panelContainer.addEventListener("popuphidden", removeToolbarButton, {
+ once: true,
+ });
+ }
+
+ // Newer messages first and use `order` field to decide between messages
+ // with the same timestamp
+ _sortWhatsNewMessages(m1, m2) {
+ // Sort by published_date in descending order.
+ if (m1.content.published_date === m2.content.published_date) {
+ // Ascending order
+ return m1.order - m2.order;
+ }
+ if (m1.content.published_date > m2.content.published_date) {
+ return -1;
+ }
+ return 1;
+ }
+
+ // Render what's new messages into the panel.
+ async renderMessages(win, doc, containerId, options = {}) {
+ // Set the checked status of the footer checkbox
+ let value = lazy.Preferences.get(WHATSNEW_ENABLED_PREF);
+ let checkbox = win.document.getElementById("panelMenu-toggleWhatsNew");
+
+ checkbox.checked = value;
+
+ this.maybeLoadCustomElement(win);
+ const messages =
+ (options.force && options.messages) ||
+ (await this.messages).sort(this._sortWhatsNewMessages);
+ const container = lazy.PanelMultiView.getViewNode(doc, containerId);
+
+ if (messages) {
+ // Targeting attribute state might have changed making new messages
+ // available and old messages invalid, we need to refresh
+ this.removeMessages(win, containerId);
+ let previousDate = 0;
+ // Get and store any variable part of the message content
+ this.state.contentArguments = await this._contentArguments();
+ for (let message of messages) {
+ container.appendChild(
+ this._createMessageElements(win, doc, message, previousDate)
+ );
+ previousDate = message.content.published_date;
+ }
+ }
+
+ this._onPanelHidden(win);
+
+ // Panel impressions are not associated with one particular message
+ // but with a set of messages. We concatenate message ids and send them
+ // back for every impression.
+ const eventId = {
+ id: messages
+ .map(({ id }) => id)
+ .sort()
+ .join(","),
+ };
+ // Check `mainview` attribute to determine if the panel is shown as a
+ // subview (inside the application menu) or as a toolbar dropdown.
+ // https://searchfox.org/mozilla-central/rev/07f7390618692fa4f2a674a96b9b677df3a13450/browser/components/customizableui/PanelMultiView.jsm#1268
+ const mainview = win.PanelUI.whatsNewPanel.hasAttribute("mainview");
+ this.sendUserEventTelemetry(win, "IMPRESSION", eventId, {
+ value: { view: mainview ? "toolbar_dropdown" : "application_menu" },
+ });
+ }
+
+ removeMessages(win, containerId) {
+ const doc = win.document;
+ const messageNodes = lazy.PanelMultiView.getViewNode(
+ doc,
+ containerId
+ ).querySelectorAll(".whatsNew-message");
+ for (const messageNode of messageNodes) {
+ messageNode.remove();
+ }
+ }
+
+ /**
+ * Dispatch the action defined in the message and user telemetry event.
+ */
+ _dispatchUserAction(win, message) {
+ let url;
+ try {
+ // Set platform specific path variables for SUMO articles
+ url = Services.urlFormatter.formatURL(message.content.cta_url);
+ } catch (e) {
+ console.error(e);
+ url = message.content.cta_url;
+ }
+ lazy.SpecialMessageActions.handleAction(
+ {
+ type: message.content.cta_type,
+ data: {
+ args: url,
+ where: message.content.cta_where || "tabshifted",
+ },
+ },
+ win.browser
+ );
+
+ this.sendUserEventTelemetry(win, "CLICK", message);
+ }
+
+ /**
+ * Attach event listener to dispatch message defined action.
+ */
+ _attachCommandListener(win, element, message) {
+ // Add event listener for `mouseup` not to overlap with the
+ // `mousedown` & `click` events dispatched from PanelMultiView.jsm
+ // https://searchfox.org/mozilla-central/rev/7531325c8660cfa61bf71725f83501028178cbb9/browser/components/customizableui/PanelMultiView.jsm#1830-1837
+ element.addEventListener("mouseup", () => {
+ this._dispatchUserAction(win, message);
+ });
+ element.addEventListener("keyup", e => {
+ if (e.key === "Enter" || e.key === " ") {
+ this._dispatchUserAction(win, message);
+ }
+ });
+ }
+
+ _createMessageElements(win, doc, message, previousDate) {
+ const { content } = message;
+ const messageEl = lazy.RemoteL10n.createElement(doc, "div");
+ messageEl.classList.add("whatsNew-message");
+
+ // Only render date if it is different from the one rendered before.
+ if (content.published_date !== previousDate) {
+ messageEl.appendChild(
+ lazy.RemoteL10n.createElement(doc, "p", {
+ classList: "whatsNew-message-date",
+ content: new Date(content.published_date).toLocaleDateString(
+ "default",
+ {
+ month: "long",
+ day: "numeric",
+ year: "numeric",
+ }
+ ),
+ })
+ );
+ }
+
+ const wrapperEl = lazy.RemoteL10n.createElement(doc, "div");
+ wrapperEl.doCommand = () => this._dispatchUserAction(win, message);
+ wrapperEl.classList.add("whatsNew-message-body");
+ messageEl.appendChild(wrapperEl);
+
+ if (content.icon_url) {
+ wrapperEl.classList.add("has-icon");
+ const iconEl = lazy.RemoteL10n.createElement(doc, "img");
+ iconEl.src = content.icon_url;
+ iconEl.classList.add("whatsNew-message-icon");
+ if (content.icon_alt && content.icon_alt.string_id) {
+ doc.l10n.setAttributes(iconEl, content.icon_alt.string_id);
+ } else {
+ iconEl.setAttribute("alt", content.icon_alt);
+ }
+ wrapperEl.appendChild(iconEl);
+ }
+
+ wrapperEl.appendChild(this._createMessageContent(win, doc, content));
+
+ if (content.link_text) {
+ const anchorEl = lazy.RemoteL10n.createElement(doc, "a", {
+ classList: "text-link",
+ content: content.link_text,
+ });
+ anchorEl.doCommand = () => this._dispatchUserAction(win, message);
+ wrapperEl.appendChild(anchorEl);
+ }
+
+ // Attach event listener on entire message container
+ this._attachCommandListener(win, messageEl, message);
+
+ return messageEl;
+ }
+
+ /**
+ * Return message title (optional subtitle) and body
+ */
+ _createMessageContent(win, doc, content) {
+ const wrapperEl = new win.DocumentFragment();
+
+ wrapperEl.appendChild(
+ lazy.RemoteL10n.createElement(doc, "h2", {
+ classList: "whatsNew-message-title",
+ content: content.title,
+ attributes: this.state.contentArguments,
+ })
+ );
+
+ wrapperEl.appendChild(
+ lazy.RemoteL10n.createElement(doc, "p", {
+ content: content.body,
+ classList: "whatsNew-message-content",
+ attributes: this.state.contentArguments,
+ })
+ );
+
+ return wrapperEl;
+ }
+
+ _createHeroElement(win, doc, message) {
+ this.maybeLoadCustomElement(win);
+
+ const messageEl = lazy.RemoteL10n.createElement(doc, "div");
+ messageEl.setAttribute("id", "protections-popup-message");
+ messageEl.classList.add("whatsNew-hero-message");
+ const wrapperEl = lazy.RemoteL10n.createElement(doc, "div");
+ wrapperEl.classList.add("whatsNew-message-body");
+ messageEl.appendChild(wrapperEl);
+
+ wrapperEl.appendChild(
+ lazy.RemoteL10n.createElement(doc, "h2", {
+ classList: "whatsNew-message-title",
+ content: message.content.title,
+ })
+ );
+ wrapperEl.appendChild(
+ lazy.RemoteL10n.createElement(doc, "p", {
+ classList: "protections-popup-content",
+ content: message.content.body,
+ })
+ );
+
+ if (message.content.link_text) {
+ let linkEl = lazy.RemoteL10n.createElement(doc, "a", {
+ classList: "text-link",
+ content: message.content.link_text,
+ });
+ linkEl.disabled = true;
+ wrapperEl.appendChild(linkEl);
+ this._attachCommandListener(win, linkEl, message);
+ } else {
+ this._attachCommandListener(win, wrapperEl, message);
+ }
+
+ return messageEl;
+ }
+
+ async _contentArguments() {
+ const { defaultEngine } = Services.search;
+ // Between now and 6 weeks ago
+ const dateTo = new Date();
+ const dateFrom = new Date(dateTo.getTime() - 42 * 24 * 60 * 60 * 1000);
+ const eventsByDate = await lazy.TrackingDBService.getEventsByDateRange(
+ dateFrom,
+ dateTo
+ );
+ // Make sure we set all types of possible values to 0 because they might
+ // be referenced by fluent strings
+ let totalEvents = { blockedCount: 0 };
+ for (let blockedType of idToTextMap.values()) {
+ totalEvents[blockedType] = 0;
+ }
+ // Count all events in the past 6 weeks. Returns an object with:
+ // `blockedCount` total number of blocked resources
+ // {tracker|cookie|social...} breakdown by event type as defined by `idToTextMap`
+ totalEvents = eventsByDate.reduce((acc, day) => {
+ const type = day.getResultByName("type");
+ const count = day.getResultByName("count");
+ acc[idToTextMap.get(type)] = (acc[idToTextMap.get(type)] || 0) + count;
+ acc.blockedCount += count;
+ return acc;
+ }, totalEvents);
+ return {
+ // Keys need to match variable names used in asrouter.ftl
+ // `earliestDate` will be either 6 weeks ago or when tracking recording
+ // started. Whichever is more recent.
+ earliestDate: Math.max(
+ new Date(await lazy.TrackingDBService.getEarliestRecordedDate()),
+ dateFrom
+ ),
+ ...totalEvents,
+ // Passing in `undefined` as string for the Fluent variable name
+ // in order to match and select the message that does not require
+ // the variable.
+ searchEngineName: defaultEngine ? defaultEngine.name : "undefined",
+ };
+ }
+
+ async _showAppmenuButton(win) {
+ this.maybeInsertFTL(win);
+ await this._showElement(
+ win.browser.ownerDocument,
+ APPMENU_BUTTON_ID,
+ BUTTON_STRING_ID
+ );
+ }
+
+ _hideAppmenuButton(win, windowClosed) {
+ // No need to do something if the window is going away
+ if (!windowClosed) {
+ this._hideElement(win.browser.ownerDocument, APPMENU_BUTTON_ID);
+ }
+ }
+
+ _showToolbarButton(win) {
+ const document = win.browser.ownerDocument;
+ this.maybeInsertFTL(win);
+ return this._showElement(document, TOOLBAR_BUTTON_ID, BUTTON_STRING_ID);
+ }
+
+ _hideToolbarButton(win) {
+ this._hideElement(win.browser.ownerDocument, TOOLBAR_BUTTON_ID);
+ }
+
+ _showElement(document, id, string_id) {
+ const el = lazy.PanelMultiView.getViewNode(document, id);
+ document.l10n.setAttributes(el, string_id);
+ el.hidden = false;
+ }
+
+ _hideElement(document, id) {
+ const el = lazy.PanelMultiView.getViewNode(document, id);
+ if (el) {
+ el.hidden = true;
+ }
+ }
+
+ _sendPing(ping) {
+ this._sendTelemetry({
+ type: "TOOLBAR_PANEL_TELEMETRY",
+ data: { action: "whats-new-panel_user_event", ...ping },
+ });
+ }
+
+ sendUserEventTelemetry(win, event, message, options = {}) {
+ // Only send pings for non private browsing windows
+ if (
+ win &&
+ !lazy.PrivateBrowsingUtils.isBrowserPrivate(
+ win.ownerGlobal.gBrowser.selectedBrowser
+ )
+ ) {
+ this._sendPing({
+ message_id: message.id,
+ event,
+ event_context: options.value,
+ });
+ }
+ }
+
+ /**
+ * Inserts a message into the Protections Panel. The message is visible once
+ * and afterwards set in a collapsed state. It can be shown again using the
+ * info button in the panel header.
+ */
+ async insertProtectionPanelMessage(event) {
+ const win = event.target.ownerGlobal;
+ this.maybeInsertFTL(win);
+
+ const doc = event.target.ownerDocument;
+ const container = doc.getElementById("messaging-system-message-container");
+ const infoButton = doc.getElementById("protections-popup-info-button");
+ const panelContainer = doc.getElementById("protections-popup");
+ const toggleMessage = () => {
+ const learnMoreLink = doc.querySelector(
+ "#messaging-system-message-container .text-link"
+ );
+ if (learnMoreLink) {
+ container.toggleAttribute("disabled");
+ infoButton.toggleAttribute("checked");
+ panelContainer.toggleAttribute("infoMessageShowing");
+ learnMoreLink.disabled = !learnMoreLink.disabled;
+ }
+ };
+ if (!container.childElementCount) {
+ const message = await this._getMessages({
+ template: "protections_panel",
+ triggerId: "protectionsPanelOpen",
+ });
+ if (message) {
+ const messageEl = this._createHeroElement(win, doc, message);
+ container.appendChild(messageEl);
+ infoButton.addEventListener("click", toggleMessage);
+ this.sendUserEventTelemetry(win, "IMPRESSION", message);
+ }
+ }
+ // Message is collapsed by default. If it was never shown before we want
+ // to expand it
+ if (
+ !this.state.protectionPanelMessageSeen &&
+ container.hasAttribute("disabled")
+ ) {
+ toggleMessage();
+ }
+ // Save state that we displayed the message
+ if (!this.state.protectionPanelMessageSeen) {
+ Services.prefs.setBoolPref(PROTECTIONS_PANEL_INFOMSG_PREF, true);
+ this.state.protectionPanelMessageSeen = true;
+ }
+ // Collapse the message after the panel is hidden so we don't get the
+ // animation when opening the panel
+ panelContainer.addEventListener(
+ "popuphidden",
+ () => {
+ if (
+ this.state.protectionPanelMessageSeen &&
+ !container.hasAttribute("disabled")
+ ) {
+ toggleMessage();
+ }
+ },
+ {
+ once: true,
+ }
+ );
+ }
+
+ /**
+ * @param {object} browser MessageChannel target argument as a response to a user action
+ * @param {object[]} messages Messages selected from devtools page
+ */
+ forceShowMessage(browser, messages) {
+ const win = browser.ownerGlobal;
+ const doc = browser.ownerDocument;
+ this.removeMessages(win, WHATS_NEW_PANEL_SELECTOR);
+ this.renderMessages(win, doc, WHATS_NEW_PANEL_SELECTOR, {
+ force: true,
+ messages: Array.isArray(messages) ? messages : [messages],
+ });
+ win.PanelUI.panel.addEventListener("popuphidden", event =>
+ this.removeMessages(event.target.ownerGlobal, WHATS_NEW_PANEL_SELECTOR)
+ );
+ }
+}
+
+/**
+ * ToolbarPanelHub - singleton instance of _ToolbarPanelHub that can initiate
+ * message requests and render messages.
+ */
+const ToolbarPanelHub = new _ToolbarPanelHub();
+
+const EXPORTED_SYMBOLS = ["ToolbarPanelHub", "_ToolbarPanelHub"];
diff --git a/browser/components/newtab/lib/TopSitesFeed.jsm b/browser/components/newtab/lib/TopSitesFeed.jsm
new file mode 100644
index 0000000000..768c6d8246
--- /dev/null
+++ b/browser/components/newtab/lib/TopSitesFeed.jsm
@@ -0,0 +1,1409 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+ "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+
+const { actionCreators: ac, actionTypes: at } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { TippyTopProvider } = ChromeUtils.import(
+ "resource://activity-stream/lib/TippyTopProvider.jsm"
+);
+const { insertPinned, TOP_SITES_MAX_SITES_PER_ROW } = ChromeUtils.import(
+ "resource://activity-stream/common/Reducers.jsm"
+);
+const { Dedupe } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Dedupe.sys.mjs"
+);
+const { shortURL } = ChromeUtils.import(
+ "resource://activity-stream/lib/ShortURL.jsm"
+);
+const { getDefaultOptions } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamStorage.jsm"
+);
+const {
+ CUSTOM_SEARCH_SHORTCUTS,
+ SEARCH_SHORTCUTS_EXPERIMENT,
+ SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF,
+ SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
+ checkHasSearchEngine,
+ getSearchProvider,
+ getSearchFormURL,
+} = ChromeUtils.import("resource://activity-stream/lib/SearchShortcuts.jsm");
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "FilterAdult",
+ "resource://activity-stream/lib/FilterAdult.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "LinksCache",
+ "resource://activity-stream/lib/LinksCache.jsm"
+);
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+ Region: "resource://gre/modules/Region.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "Screenshots",
+ "resource://activity-stream/lib/Screenshots.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "PageThumbs",
+ "resource://gre/modules/PageThumbs.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "RemoteSettings",
+ "resource://services-settings/remote-settings.js"
+);
+
+XPCOMUtils.defineLazyGetter(lazy, "log", () => {
+ const { Logger } = ChromeUtils.import(
+ "resource://messaging-system/lib/Logger.jsm"
+ );
+ return new Logger("TopSitesFeed");
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+});
+
+const DEFAULT_SITES_PREF = "default.sites";
+const SHOWN_ON_NEWTAB_PREF = "feeds.topsites";
+const DEFAULT_TOP_SITES = [];
+const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages)
+const MIN_FAVICON_SIZE = 96;
+const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"];
+const PINNED_FAVICON_PROPS_TO_MIGRATE = [
+ "favicon",
+ "faviconRef",
+ "faviconSize",
+];
+const SECTION_ID = "topsites";
+const ROWS_PREF = "topSitesRows";
+const SHOW_SPONSORED_PREF = "showSponsoredTopSites";
+const MAX_NUM_SPONSORED = 2;
+
+// Search experiment stuff
+const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile";
+const SEARCH_FILTERS = [
+ "google",
+ "search.yahoo",
+ "yahoo",
+ "bing",
+ "ask",
+ "duckduckgo",
+];
+
+const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting";
+const DEFAULT_SITES_OVERRIDE_PREF =
+ "browser.newtabpage.activity-stream.default.sites";
+const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment.";
+
+// Mozilla Tiles Service (Contile) prefs
+// Nimbus variable for the Contile integration. It falls back to the pref:
+// `browser.topsites.contile.enabled`.
+const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled";
+const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint";
+const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes
+const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors";
+
+function getShortURLForCurrentSearch() {
+ const url = shortURL({ url: Services.search.defaultEngine.searchForm });
+ return url;
+}
+
+class ContileIntegration {
+ constructor(topSitesFeed) {
+ this._topSitesFeed = topSitesFeed;
+ this._lastPeriodicUpdate = 0;
+ this._sites = [];
+ }
+
+ get sites() {
+ return this._sites;
+ }
+
+ periodicUpdate() {
+ let now = Date.now();
+ if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) {
+ this._lastPeriodicUpdate = now;
+ this.refresh();
+ }
+ }
+
+ async refresh() {
+ let updateDefaultSites = await this._fetchSites();
+ if (updateDefaultSites) {
+ this._topSitesFeed._readDefaults();
+ }
+ }
+
+ /**
+ * Filter the tiles whose sponsor is on the Top Sites sponsor blocklist.
+ *
+ * @param {array} tiles
+ * An array of the tile objects
+ */
+ _filterBlockedSponsors(tiles) {
+ const blocklist = JSON.parse(
+ Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
+ );
+ return tiles.filter(tile => !blocklist.includes(shortURL(tile)));
+ }
+
+ async _fetchSites() {
+ if (
+ !lazy.NimbusFeatures.newtab.getVariable(
+ NIMBUS_VARIABLE_CONTILE_ENABLED
+ ) ||
+ !this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF]
+ ) {
+ if (this._sites.length) {
+ this._sites = [];
+ return true;
+ }
+ return false;
+ }
+ try {
+ let url = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF);
+ const response = await fetch(url, { credentials: "omit" });
+ if (!response.ok) {
+ lazy.log.warn(
+ `Contile endpoint returned unexpected status: ${response.status}`
+ );
+ }
+
+ // Contile returns 204 indicating there is no content at the moment.
+ // If this happens, just return without signifying the change so that the
+ // existing tiles (`this._sites`) could retain. We might want to introduce
+ // other handling for this in the future.
+ if (response.status === 204) {
+ return false;
+ }
+ const body = await response.json();
+ if (body?.tiles && Array.isArray(body.tiles)) {
+ let { tiles } = body;
+ tiles = this._filterBlockedSponsors(tiles);
+ if (tiles.length > MAX_NUM_SPONSORED) {
+ lazy.log.warn(
+ `Contile provided more links than permitted. (${tiles.length} received, limit is ${MAX_NUM_SPONSORED})`
+ );
+ tiles.length = MAX_NUM_SPONSORED;
+ }
+ this._sites = tiles;
+ return true;
+ }
+ } catch (error) {
+ lazy.log.warn(
+ `Failed to fetch data from Contile server: ${error.message}`
+ );
+ }
+ return false;
+ }
+}
+
+class TopSitesFeed {
+ constructor() {
+ this._contile = new ContileIntegration(this);
+ this._tippyTopProvider = new TippyTopProvider();
+ XPCOMUtils.defineLazyGetter(
+ this,
+ "_currentSearchHostname",
+ getShortURLForCurrentSearch
+ );
+ this.dedupe = new Dedupe(this._dedupeKey);
+ this.frecentCache = new lazy.LinksCache(
+ lazy.NewTabUtils.activityStreamLinks,
+ "getTopSites",
+ CACHED_LINK_PROPS_TO_MIGRATE,
+ (oldOptions, newOptions) =>
+ // Refresh if no old options or requesting more items
+ !(oldOptions.numItems >= newOptions.numItems)
+ );
+ this.pinnedCache = new lazy.LinksCache(
+ lazy.NewTabUtils.pinnedLinks,
+ "links",
+ [...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE]
+ );
+ lazy.PageThumbs.addExpirationFilter(this);
+ this._nimbusChangeListener = this._nimbusChangeListener.bind(this);
+ }
+
+ _nimbusChangeListener(event, reason) {
+ // The Nimbus API current doesn't specify the changed variable(s) in the
+ // listener callback, so we have to refresh unconditionally on every change
+ // of the `newtab` feature. It should be a manageable overhead given the
+ // current update cadence (6 hours) of Nimbus.
+ //
+ // Skip the experiment and rollout loading reasons since this feature has
+ // `isEarlyStartup` enabled, the feature variables are already available
+ // before the experiment or rollout loads.
+ if (
+ !["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason)
+ ) {
+ this._contile.refresh();
+ }
+ }
+
+ init() {
+ // If the feed was previously disabled PREFS_INITIAL_VALUES was never received
+ this._readDefaults({ isStartup: true });
+ this._storage = this.store.dbStorage.getDbTable("sectionPrefs");
+ this._contile.refresh();
+ Services.obs.addObserver(this, "browser-search-engine-modified");
+ Services.obs.addObserver(this, "browser-region-updated");
+ Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
+ Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
+ Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
+ lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener);
+ }
+
+ uninit() {
+ lazy.PageThumbs.removeExpirationFilter(this);
+ Services.obs.removeObserver(this, "browser-search-engine-modified");
+ Services.obs.removeObserver(this, "browser-region-updated");
+ Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
+ Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
+ Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
+ lazy.NimbusFeatures.newtab.off(this._nimbusChangeListener);
+ }
+
+ observe(subj, topic, data) {
+ switch (topic) {
+ case "browser-search-engine-modified":
+ // We should update the current top sites if the search engine has been changed since
+ // the search engine that gets filtered out of top sites has changed.
+ // We also need to drop search shortcuts when their engine gets removed / hidden.
+ if (
+ data === "engine-default" &&
+ this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF]
+ ) {
+ delete this._currentSearchHostname;
+ this._currentSearchHostname = getShortURLForCurrentSearch();
+ }
+ this.refresh({ broadcast: true });
+ break;
+ case "browser-region-updated":
+ this._readDefaults();
+ break;
+ case "nsPref:changed":
+ if (
+ data === REMOTE_SETTING_DEFAULTS_PREF ||
+ data === DEFAULT_SITES_OVERRIDE_PREF ||
+ data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH)
+ ) {
+ this._readDefaults();
+ }
+ break;
+ }
+ }
+
+ _dedupeKey(site) {
+ return site && site.hostname;
+ }
+
+ /**
+ * _readDefaults - sets DEFAULT_TOP_SITES
+ */
+ async _readDefaults({ isStartup = false } = {}) {
+ this._useRemoteSetting = false;
+
+ if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) {
+ this.refreshDefaults(
+ this.store.getState().Prefs.values[DEFAULT_SITES_PREF],
+ { isStartup }
+ );
+ return;
+ }
+
+ // Try using default top sites from enterprise policies or tests. The pref
+ // is locked when set via enterprise policy. Tests have no default sites
+ // unless they set them via this pref.
+ if (
+ Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) ||
+ Cu.isInAutomation
+ ) {
+ let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, "");
+ this.refreshDefaults(sites, { isStartup });
+ return;
+ }
+
+ // Clear out the array of any previous defaults.
+ DEFAULT_TOP_SITES.length = 0;
+
+ // Read defaults from contile.
+ const contileEnabled = lazy.NimbusFeatures.newtab.getVariable(
+ NIMBUS_VARIABLE_CONTILE_ENABLED
+ );
+ let hasContileTiles = false;
+ if (contileEnabled) {
+ let sponsoredPosition = 1;
+ for (let site of this._contile.sites) {
+ let hostname = shortURL(site);
+ let link = {
+ isDefault: true,
+ url: site.url,
+ hostname,
+ sendAttributionRequest: false,
+ label: site.name,
+ show_sponsored_label: hostname !== "yandex",
+ sponsored_position: sponsoredPosition++,
+ sponsored_click_url: site.click_url,
+ sponsored_impression_url: site.impression_url,
+ sponsored_tile_id: site.id,
+ };
+ if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) {
+ // Only use the image from Contile if it's hi-res, otherwise, fallback
+ // to the built-in favicons.
+ link.favicon = site.image_url;
+ link.faviconSize = site.image_size;
+ }
+ DEFAULT_TOP_SITES.push(link);
+ }
+ hasContileTiles = sponsoredPosition > 1;
+ }
+
+ // Read defaults from remote settings.
+ this._useRemoteSetting = true;
+ let remoteSettingData = await this._getRemoteConfig();
+
+ const sponsoredBlocklist = JSON.parse(
+ Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
+ );
+
+ for (let siteData of remoteSettingData) {
+ let hostname = shortURL(siteData);
+ // Drop default sites when Contile already provided a sponsored one with
+ // the same host name.
+ if (
+ contileEnabled &&
+ DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1
+ ) {
+ continue;
+ }
+ // Also drop those sponsored sites that were blocked by the user before
+ // with the same hostname.
+ if (
+ siteData.sponsored_position &&
+ sponsoredBlocklist.includes(hostname)
+ ) {
+ continue;
+ }
+ let link = {
+ isDefault: true,
+ url: siteData.url,
+ hostname,
+ sendAttributionRequest: !!siteData.send_attribution_request,
+ };
+ if (siteData.url_urlbar_override) {
+ link.url_urlbar = siteData.url_urlbar_override;
+ }
+ if (siteData.title) {
+ link.label = siteData.title;
+ }
+ if (siteData.search_shortcut) {
+ link = await this.topSiteToSearchTopSite(link);
+ } else if (siteData.sponsored_position) {
+ if (contileEnabled && hasContileTiles) {
+ continue;
+ }
+ const {
+ sponsored_position,
+ sponsored_tile_id,
+ sponsored_impression_url,
+ sponsored_click_url,
+ } = siteData;
+ link = {
+ sponsored_position,
+ sponsored_tile_id,
+ sponsored_impression_url,
+ sponsored_click_url,
+ show_sponsored_label: link.hostname !== "yandex",
+ ...link,
+ };
+ }
+ DEFAULT_TOP_SITES.push(link);
+ }
+
+ this.refresh({ broadcast: true, isStartup });
+ }
+
+ refreshDefaults(sites, { isStartup = false } = {}) {
+ // Clear out the array of any previous defaults
+ DEFAULT_TOP_SITES.length = 0;
+
+ // Add default sites if any based on the pref
+ if (sites) {
+ for (const url of sites.split(",")) {
+ const site = {
+ isDefault: true,
+ url,
+ };
+ site.hostname = shortURL(site);
+ DEFAULT_TOP_SITES.push(site);
+ }
+ }
+
+ this.refresh({ broadcast: true, isStartup });
+ }
+
+ async _getRemoteConfig(firstTime = true) {
+ if (!this._remoteConfig) {
+ this._remoteConfig = await lazy.RemoteSettings("top-sites");
+ this._remoteConfig.on("sync", () => {
+ this._readDefaults();
+ });
+ }
+
+ let result = [];
+ let failed = false;
+ try {
+ result = await this._remoteConfig.get();
+ } catch (ex) {
+ console.error(ex);
+ failed = true;
+ }
+ if (!result.length) {
+ console.error("Received empty top sites configuration!");
+ failed = true;
+ }
+ // If we failed, or the result is empty, try loading from the local dump.
+ if (firstTime && failed) {
+ await this._remoteConfig.db.clear();
+ // Now call this again.
+ return this._getRemoteConfig(false);
+ }
+
+ // Sort sites based on the "order" attribute.
+ result.sort((a, b) => a.order - b.order);
+
+ result = result.filter(topsite => {
+ // Filter by region.
+ if (topsite.exclude_regions?.includes(lazy.Region.home)) {
+ return false;
+ }
+ if (
+ topsite.include_regions?.length &&
+ !topsite.include_regions.includes(lazy.Region.home)
+ ) {
+ return false;
+ }
+
+ // Filter by locale.
+ if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) {
+ return false;
+ }
+ if (
+ topsite.include_locales?.length &&
+ !topsite.include_locales.includes(Services.locale.appLocaleAsBCP47)
+ ) {
+ return false;
+ }
+
+ // Filter by experiment.
+ // Exclude this top site if any of the specified experiments are running.
+ if (
+ topsite.exclude_experiments?.some(experimentID =>
+ Services.prefs.getBoolPref(
+ DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
+ false
+ )
+ )
+ ) {
+ return false;
+ }
+ // Exclude this top site if none of the specified experiments are running.
+ if (
+ topsite.include_experiments?.length &&
+ topsite.include_experiments.every(
+ experimentID =>
+ !Services.prefs.getBoolPref(
+ DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
+ false
+ )
+ )
+ ) {
+ return false;
+ }
+
+ return true;
+ });
+
+ return result;
+ }
+
+ filterForThumbnailExpiration(callback) {
+ const { rows } = this.store.getState().TopSites;
+ callback(
+ rows.reduce((acc, site) => {
+ acc.push(site.url);
+ if (site.customScreenshotURL) {
+ acc.push(site.customScreenshotURL);
+ }
+ return acc;
+ }, [])
+ );
+ }
+
+ /**
+ * shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine?
+ *
+ * @param {string} hostname a top site hostname, such as "amazon" or "foo"
+ * @returns {bool}
+ */
+ shouldFilterSearchTile(hostname) {
+ if (
+ this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] &&
+ (SEARCH_FILTERS.includes(hostname) ||
+ hostname === this._currentSearchHostname)
+ ) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * _maybeInsertSearchShortcuts - if the search shortcuts experiment is running,
+ * insert search shortcuts if needed
+ * @param {Array} plainPinnedSites (from the pinnedSitesCache)
+ * @returns {Boolean} Did we insert any search shortcuts?
+ */
+ async _maybeInsertSearchShortcuts(plainPinnedSites) {
+ // Only insert shortcuts if the experiment is running
+ if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
+ // We don't want to insert shortcuts we've previously inserted
+ const prevInsertedShortcuts = this.store
+ .getState()
+ .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",")
+ .filter(s => s); // Filter out empty strings
+ const newInsertedShortcuts = [];
+
+ let shouldPin = this._useRemoteSetting
+ ? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname)
+ : this.store
+ .getState()
+ .Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(",");
+ shouldPin = shouldPin
+ .map(getSearchProvider)
+ .filter(s => s && s.shortURL !== this._currentSearchHostname);
+
+ // If we've previously inserted all search shortcuts return early
+ if (
+ shouldPin.every(shortcut =>
+ prevInsertedShortcuts.includes(shortcut.shortURL)
+ )
+ ) {
+ return false;
+ }
+
+ const numberOfSlots =
+ this.store.getState().Prefs.values[ROWS_PREF] *
+ TOP_SITES_MAX_SITES_PER_ROW;
+
+ // The plainPinnedSites array is populated with pinned sites at their
+ // respective indices, and null everywhere else, but is not always the
+ // right length
+ const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0);
+ const pinnedSites = [...plainPinnedSites].concat(
+ Array(emptySlots).fill(null)
+ );
+
+ const tryToInsertSearchShortcut = async shortcut => {
+ const nextAvailable = pinnedSites.indexOf(null);
+ // Only add a search shortcut if the site isn't already pinned, we
+ // haven't previously inserted it, there's space to pin it, and the
+ // search engine is available in Firefox
+ if (
+ !pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) &&
+ !prevInsertedShortcuts.includes(shortcut.shortURL) &&
+ nextAvailable > -1 &&
+ (await checkHasSearchEngine(shortcut.keyword))
+ ) {
+ const site = await this.topSiteToSearchTopSite({ url: shortcut.url });
+ this._pinSiteAt(site, nextAvailable);
+ pinnedSites[nextAvailable] = site;
+ newInsertedShortcuts.push(shortcut.shortURL);
+ }
+ };
+
+ for (let shortcut of shouldPin) {
+ await tryToInsertSearchShortcut(shortcut);
+ }
+
+ if (newInsertedShortcuts.length) {
+ this.store.dispatch(
+ ac.SetPref(
+ SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
+ prevInsertedShortcuts.concat(newInsertedShortcuts).join(",")
+ )
+ );
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ // eslint-disable-next-line max-statements
+ async getLinksWithDefaults(isStartup = false) {
+ const prefValues = this.store.getState().Prefs.values;
+ const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW;
+ const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT];
+ // We must wait for search services to initialize in order to access default
+ // search engine properties without triggering a synchronous initialization
+ await Services.search.init();
+
+ // Get all frecent sites from history.
+ let frecent = [];
+ const cache = await this.frecentCache.request({
+ // We need to overquery due to the top 5 alexa search + default search possibly being removed
+ numItems: numItems + SEARCH_FILTERS.length + 1,
+ topsiteFrecency: FRECENCY_THRESHOLD,
+ });
+ for (let link of cache) {
+ const hostname = shortURL(link);
+ if (!this.shouldFilterSearchTile(hostname)) {
+ frecent.push({
+ ...(searchShortcutsExperiment
+ ? await this.topSiteToSearchTopSite(link)
+ : link),
+ hostname,
+ });
+ }
+ }
+
+ // Get defaults.
+ let date = new Date();
+ let pad = number => number.toString().padStart(2, "0");
+ let yyyymmddhh =
+ String(date.getFullYear()) +
+ pad(date.getMonth() + 1) +
+ pad(date.getDate()) +
+ pad(date.getHours());
+ let notBlockedDefaultSites = [];
+ let sponsored = [];
+ for (let link of DEFAULT_TOP_SITES) {
+ // For sponsored Yandex links, default filtering is reversed: we only
+ // show them if Yandex is the default search engine.
+ if (link.sponsored_position && link.hostname === "yandex") {
+ if (link.hostname !== this._currentSearchHostname) {
+ continue;
+ }
+ } else if (this.shouldFilterSearchTile(link.hostname)) {
+ continue;
+ }
+ // Drop blocked default sites.
+ if (
+ lazy.NewTabUtils.blockedLinks.isBlocked({
+ url: link.url,
+ })
+ ) {
+ continue;
+ }
+ // Process %YYYYMMDDHH% tag in the URL.
+ let url_end;
+ let url_start;
+ if (this._useRemoteSetting) {
+ [url_start, url_end] = link.url.split("%YYYYMMDDHH%");
+ }
+ if (typeof url_end === "string") {
+ link = {
+ ...link,
+ // Save original URL without %YYYYMMDDHH% replaced so it can be
+ // blocked properly.
+ original_url: link.url,
+ url: url_start + yyyymmddhh + url_end,
+ };
+ if (link.url_urlbar) {
+ link.url_urlbar = link.url_urlbar.replace("%YYYYMMDDHH%", yyyymmddhh);
+ }
+ }
+ // If we've previously blocked a search shortcut, remove the default top site
+ // that matches the hostname
+ const searchProvider = getSearchProvider(shortURL(link));
+ if (
+ searchProvider &&
+ lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url })
+ ) {
+ continue;
+ }
+ if (link.sponsored_position) {
+ if (!prefValues[SHOW_SPONSORED_PREF]) {
+ continue;
+ }
+ sponsored[link.sponsored_position - 1] = link;
+
+ // Unpin search shortcut if present for the sponsored link to be shown
+ // instead.
+ this._unpinSearchShortcut(link.hostname);
+ } else {
+ notBlockedDefaultSites.push(
+ searchShortcutsExperiment
+ ? await this.topSiteToSearchTopSite(link)
+ : link
+ );
+ }
+ }
+
+ // Get pinned links augmented with desired properties
+ let plainPinned = await this.pinnedCache.request();
+
+ // Insert search shortcuts if we need to.
+ // _maybeInsertSearchShortcuts returns true if any search shortcuts are
+ // inserted, meaning we need to expire and refresh the pinnedCache
+ if (await this._maybeInsertSearchShortcuts(plainPinned)) {
+ this.pinnedCache.expire();
+ plainPinned = await this.pinnedCache.request();
+ }
+
+ const pinned = await Promise.all(
+ plainPinned.map(async link => {
+ if (!link) {
+ return link;
+ }
+
+ // Drop pinned search shortcuts when their engine has been removed / hidden.
+ if (link.searchTopSite) {
+ const searchProvider = getSearchProvider(shortURL(link));
+ if (
+ !searchProvider ||
+ !(await checkHasSearchEngine(searchProvider.keyword))
+ ) {
+ return null;
+ }
+ }
+
+ // Copy all properties from a frecent link and add more
+ const finder = other => other.url === link.url;
+
+ // Remove frecent link's screenshot if pinned link has a custom one
+ const frecentSite = frecent.find(finder);
+ if (frecentSite && link.customScreenshotURL) {
+ delete frecentSite.screenshot;
+ }
+ // If the link is a frecent site, do not copy over 'isDefault', else check
+ // if the site is a default site
+ const copy = Object.assign(
+ {},
+ frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) },
+ link,
+ { hostname: shortURL(link) },
+ { searchTopSite: !!link.searchTopSite }
+ );
+
+ // Add in favicons if we don't already have it
+ if (!copy.favicon) {
+ try {
+ lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI(
+ await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy])
+ );
+
+ for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) {
+ copy.__sharedCache.updateLink(prop, copy[prop]);
+ }
+ } catch (e) {
+ // Some issue with favicon, so just continue without one
+ }
+ }
+
+ return copy;
+ })
+ );
+
+ // Remove any duplicates from frecent and default sites
+ const [
+ ,
+ dedupedSponsored,
+ dedupedFrecent,
+ dedupedDefaults,
+ ] = this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites);
+ const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults];
+
+ // Remove adult sites if we need to
+ const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned);
+
+ // Insert the original pinned sites into the deduped frecent and defaults.
+ let withPinned = insertPinned(checkedAdult, pinned);
+ // Insert sponsored sites at their desired position.
+ dedupedSponsored.forEach(link => {
+ if (!link) {
+ return;
+ }
+ let index = link.sponsored_position - 1;
+ if (index > withPinned.length) {
+ withPinned[index] = link;
+ } else {
+ withPinned.splice(index, 0, link);
+ }
+ });
+ // Remove excess items after we inserted sponsored ones.
+ withPinned = withPinned.slice(0, numItems);
+
+ // Now, get a tippy top icon, a rich icon, or screenshot for every item
+ for (const link of withPinned) {
+ if (link) {
+ // If there is a custom screenshot this is the only image we display
+ if (link.customScreenshotURL) {
+ this._fetchScreenshot(link, link.customScreenshotURL, isStartup);
+ } else if (link.searchTopSite && !link.isDefault) {
+ await this._attachTippyTopIconForSearchShortcut(link, link.label);
+ } else {
+ this._fetchIcon(link, isStartup);
+ }
+
+ // Remove internal properties that might be updated after dispatch
+ delete link.__sharedCache;
+
+ // Indicate that these links should get a frecency bonus when clicked
+ link.typedBonus = true;
+ }
+ }
+
+ this._linksWithDefaults = withPinned;
+
+ return withPinned;
+ }
+
+ /**
+ * Attach TippyTop icon to the given search shortcut
+ *
+ * Note that it queries the search form URL from search service For Yandex,
+ * and uses it to choose the best icon for its shortcut variants.
+ *
+ * @param {Object} link A link object with a `url` property
+ * @param {string} keyword Search keyword
+ */
+ async _attachTippyTopIconForSearchShortcut(link, keyword) {
+ if (
+ ["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"].includes(keyword)
+ ) {
+ let site = { url: link.url };
+ site.url = (await getSearchFormURL(keyword)) || site.url;
+ this._tippyTopProvider.processSite(site);
+ link.tippyTopIcon = site.tippyTopIcon;
+ link.smallFavicon = site.smallFavicon;
+ link.backgroundColor = site.backgroundColor;
+ } else {
+ this._tippyTopProvider.processSite(link);
+ }
+ }
+
+ /**
+ * Refresh the top sites data for content.
+ * @param {bool} options.broadcast Should the update be broadcasted.
+ * @param {bool} options.isStartup Being called while TopSitesFeed is initting.
+ */
+ async refresh(options = {}) {
+ if (!this._startedUp && !options.isStartup) {
+ // Initial refresh still pending.
+ return;
+ }
+ this._startedUp = true;
+
+ if (!this._tippyTopProvider.initialized) {
+ await this._tippyTopProvider.init();
+ }
+
+ const links = await this.getLinksWithDefaults({
+ isStartup: options.isStartup,
+ });
+ const newAction = { type: at.TOP_SITES_UPDATED, data: { links } };
+ let storedPrefs;
+ try {
+ storedPrefs = (await this._storage.get(SECTION_ID)) || {};
+ } catch (e) {
+ storedPrefs = {};
+ console.error("Problem getting stored prefs for TopSites");
+ }
+ newAction.data.pref = getDefaultOptions(storedPrefs);
+
+ if (options.isStartup) {
+ newAction.meta = {
+ isStartup: true,
+ };
+ }
+
+ if (options.broadcast) {
+ // Broadcast an update to all open content pages
+ this.store.dispatch(ac.BroadcastToContent(newAction));
+ } else {
+ // Don't broadcast only update the state and update the preloaded tab.
+ this.store.dispatch(ac.AlsoToPreloaded(newAction));
+ }
+ }
+
+ async updateCustomSearchShortcuts(isStartup = false) {
+ if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
+ return;
+ }
+
+ if (!this._tippyTopProvider.initialized) {
+ await this._tippyTopProvider.init();
+ }
+
+ // Populate the state with available search shortcuts
+ let searchShortcuts = [];
+ for (const engine of await Services.search.getAppProvidedEngines()) {
+ const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s =>
+ engine.aliases.includes(s.keyword)
+ );
+ if (shortcut) {
+ let clone = { ...shortcut };
+ await this._attachTippyTopIconForSearchShortcut(clone, clone.keyword);
+ searchShortcuts.push(clone);
+ }
+ }
+
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.UPDATE_SEARCH_SHORTCUTS,
+ data: { searchShortcuts },
+ meta: {
+ isStartup,
+ },
+ })
+ );
+ }
+
+ async topSiteToSearchTopSite(site) {
+ const searchProvider = getSearchProvider(shortURL(site));
+ if (
+ !searchProvider ||
+ !(await checkHasSearchEngine(searchProvider.keyword))
+ ) {
+ return site;
+ }
+ return {
+ ...site,
+ searchTopSite: true,
+ label: searchProvider.keyword,
+ };
+ }
+
+ /**
+ * Get an image for the link preferring tippy top, rich favicon, screenshots.
+ */
+ async _fetchIcon(link, isStartup = false) {
+ // Nothing to do if we already have a rich icon from the page
+ if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) {
+ return;
+ }
+
+ // Nothing more to do if we can use a default tippy top icon
+ this._tippyTopProvider.processSite(link);
+ if (link.tippyTopIcon) {
+ return;
+ }
+
+ // Make a request for a better icon
+ this._requestRichIcon(link.url);
+
+ // Also request a screenshot if we don't have one yet
+ await this._fetchScreenshot(link, link.url, isStartup);
+ }
+
+ /**
+ * Fetch, cache and broadcast a screenshot for a specific topsite.
+ * @param link cached topsite object
+ * @param url where to fetch the image from
+ * @param isStartup Whether the screenshot is fetched while initting TopSitesFeed.
+ */
+ async _fetchScreenshot(link, url, isStartup = false) {
+ // We shouldn't bother caching screenshots if they won't be shown.
+ if (
+ link.screenshot ||
+ !this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF]
+ ) {
+ return;
+ }
+ await lazy.Screenshots.maybeCacheScreenshot(
+ link,
+ url,
+ "screenshot",
+ screenshot =>
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ data: { screenshot, url: link.url },
+ type: at.SCREENSHOT_UPDATED,
+ meta: {
+ isStartup,
+ },
+ })
+ )
+ );
+ }
+
+ /**
+ * Dispatch screenshot preview to target or notify if request failed.
+ * @param customScreenshotURL {string} The URL used to capture the screenshot
+ * @param target {string} Id of content process where to dispatch the result
+ */
+ async getScreenshotPreview(url, target) {
+ const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || "";
+ this.store.dispatch(
+ ac.OnlyToOneContent(
+ {
+ data: { url, preview },
+ type: at.PREVIEW_RESPONSE,
+ },
+ target
+ )
+ );
+ }
+
+ _requestRichIcon(url) {
+ this.store.dispatch({
+ type: at.RICH_ICON_MISSING,
+ data: { url },
+ });
+ }
+
+ updateSectionPrefs(collapsed) {
+ this.store.dispatch(
+ ac.BroadcastToContent({
+ type: at.TOP_SITES_PREFS_UPDATED,
+ data: { pref: collapsed },
+ })
+ );
+ }
+
+ /**
+ * Inform others that top sites data has been updated due to pinned changes.
+ */
+ _broadcastPinnedSitesUpdated() {
+ // Pinned data changed, so make sure we get latest
+ this.pinnedCache.expire();
+
+ // Refresh to update pinned sites with screenshots, trigger deduping, etc.
+ this.refresh({ broadcast: true });
+ }
+
+ /**
+ * Pin a site at a specific position saving only the desired keys.
+ * @param customScreenshotURL {string} User set URL of preview image for site
+ * @param label {string} User set string of custom site name
+ */
+ async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) {
+ const toPin = { url };
+ if (label) {
+ toPin.label = label;
+ }
+ if (customScreenshotURL) {
+ toPin.customScreenshotURL = customScreenshotURL;
+ }
+ if (searchTopSite) {
+ toPin.searchTopSite = searchTopSite;
+ }
+ lazy.NewTabUtils.pinnedLinks.pin(toPin, index);
+
+ await this._clearLinkCustomScreenshot({ customScreenshotURL, url });
+ }
+
+ async _clearLinkCustomScreenshot(site) {
+ // If screenshot url changed or was removed we need to update the cached link obj
+ if (site.customScreenshotURL !== undefined) {
+ const pinned = await this.pinnedCache.request();
+ const link = pinned.find(pin => pin && pin.url === site.url);
+ if (link && link.customScreenshotURL !== site.customScreenshotURL) {
+ link.__sharedCache.updateLink("screenshot", undefined);
+ }
+ }
+ }
+
+ /**
+ * Handle a pin action of a site to a position.
+ */
+ async pin(action) {
+ let { site, index } = action.data;
+ index = this._adjustPinIndexForSponsoredLinks(site, index);
+ // If valid index provided, pin at that position
+ if (index >= 0) {
+ await this._pinSiteAt(site, index);
+ this._broadcastPinnedSitesUpdated();
+ } else {
+ // Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option,
+ // then we want to make sure to unblock that link if it has previously been
+ // blocked. We know if the site has been added because the index will be -1.
+ if (index === -1) {
+ lazy.NewTabUtils.blockedLinks.unblock({ url: site.url });
+ this.frecentCache.expire();
+ }
+ this.insert(action);
+ }
+ }
+
+ /**
+ * Handle an unpin action of a site.
+ */
+ unpin(action) {
+ const { site } = action.data;
+ lazy.NewTabUtils.pinnedLinks.unpin(site);
+ this._broadcastPinnedSitesUpdated();
+ }
+
+ unpinAllSearchShortcuts() {
+ Services.prefs.clearUserPref(
+ `browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}`
+ );
+ for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
+ if (pinnedLink && pinnedLink.searchTopSite) {
+ lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
+ }
+ }
+ this.pinnedCache.expire();
+ }
+
+ _unpinSearchShortcut(vendor) {
+ for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
+ if (
+ pinnedLink &&
+ pinnedLink.searchTopSite &&
+ shortURL(pinnedLink) === vendor
+ ) {
+ lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
+ this.pinnedCache.expire();
+
+ const prevInsertedShortcuts = this.store
+ .getState()
+ .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",");
+ this.store.dispatch(
+ ac.SetPref(
+ SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
+ prevInsertedShortcuts.filter(s => s !== vendor).join(",")
+ )
+ );
+ break;
+ }
+ }
+ }
+
+ /**
+ * Reduces the given pinning index by the number of preceding sponsored
+ * sites, to accomodate for sponsored sites pushing pinned ones to the side,
+ * effectively increasing their index again.
+ */
+ _adjustPinIndexForSponsoredLinks(site, index) {
+ if (!this._linksWithDefaults) {
+ return index;
+ }
+ // Adjust insertion index for sponsored sites since their position is
+ // fixed.
+ let adjustedIndex = index;
+ for (let i = 0; i < index; i++) {
+ if (
+ this._linksWithDefaults[i]?.sponsored_position &&
+ this._linksWithDefaults[i]?.url !== site.url
+ ) {
+ adjustedIndex--;
+ }
+ }
+ return adjustedIndex;
+ }
+
+ /**
+ * Insert a site to pin at a position shifting over any other pinned sites.
+ */
+ _insertPin(site, originalIndex, draggedFromIndex) {
+ let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex);
+
+ // Don't insert any pins past the end of the visible top sites. Otherwise,
+ // we can end up with a bunch of pinned sites that can never be unpinned again
+ // from the UI.
+ const topSitesCount =
+ this.store.getState().Prefs.values[ROWS_PREF] *
+ TOP_SITES_MAX_SITES_PER_ROW;
+ if (index >= topSitesCount) {
+ return;
+ }
+
+ let pinned = lazy.NewTabUtils.pinnedLinks.links;
+ if (!pinned[index]) {
+ this._pinSiteAt(site, index);
+ } else {
+ pinned[draggedFromIndex] = null;
+ // Find the hole to shift the pinned site(s) towards. We shift towards the
+ // hole left by the site being dragged.
+ let holeIndex = index;
+ const indexStep = index > draggedFromIndex ? -1 : 1;
+ while (pinned[holeIndex]) {
+ holeIndex += indexStep;
+ }
+ if (holeIndex >= topSitesCount || holeIndex < 0) {
+ // There are no holes, so we will effectively unpin the last slot and shifting
+ // towards it. This only happens when adding a new top site to an already
+ // fully pinned grid.
+ holeIndex = topSitesCount - 1;
+ }
+
+ // Shift towards the hole.
+ const shiftingStep = holeIndex > index ? -1 : 1;
+ while (holeIndex !== index) {
+ const nextIndex = holeIndex + shiftingStep;
+ this._pinSiteAt(pinned[nextIndex], holeIndex);
+ holeIndex = nextIndex;
+ }
+ this._pinSiteAt(site, index);
+ }
+ }
+
+ /**
+ * Handle an insert (drop/add) action of a site.
+ */
+ async insert(action) {
+ let { index } = action.data;
+ // Treat invalid pin index values (e.g., -1, undefined) as insert in the first position
+ if (!(index > 0)) {
+ index = 0;
+ }
+
+ // Inserting a top site pins it in the specified slot, pushing over any link already
+ // pinned in the slot (unless it's the last slot, then it replaces).
+ this._insertPin(
+ action.data.site,
+ index,
+ action.data.draggedFromIndex !== undefined
+ ? action.data.draggedFromIndex
+ : this.store.getState().Prefs.values[ROWS_PREF] *
+ TOP_SITES_MAX_SITES_PER_ROW
+ );
+
+ await this._clearLinkCustomScreenshot(action.data.site);
+ this._broadcastPinnedSitesUpdated();
+ }
+
+ updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) {
+ // Unpin the deletedShortcuts.
+ deletedShortcuts.forEach(({ url }) => {
+ lazy.NewTabUtils.pinnedLinks.unpin({ url });
+ });
+
+ // Pin the addedShortcuts.
+ const numberOfSlots =
+ this.store.getState().Prefs.values[ROWS_PREF] *
+ TOP_SITES_MAX_SITES_PER_ROW;
+ addedShortcuts.forEach(shortcut => {
+ // Find first hole in pinnedLinks.
+ let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link);
+ if (
+ index < 0 &&
+ lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots
+ ) {
+ // pinnedLinks can have less slots than the total available.
+ index = lazy.NewTabUtils.pinnedLinks.links.length;
+ }
+ if (index >= 0) {
+ lazy.NewTabUtils.pinnedLinks.pin(shortcut, index);
+ } else {
+ // No slots available, we need to do an insert in first slot and push over other pinned links.
+ this._insertPin(shortcut, 0, numberOfSlots);
+ }
+ });
+
+ this._broadcastPinnedSitesUpdated();
+ }
+
+ onAction(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.init();
+ this.updateCustomSearchShortcuts(true /* isStartup */);
+ break;
+ case at.SYSTEM_TICK:
+ this.refresh({ broadcast: false });
+ this._contile.periodicUpdate();
+ break;
+ // All these actions mean we need new top sites
+ case at.PLACES_HISTORY_CLEARED:
+ case at.PLACES_LINKS_DELETED:
+ this.frecentCache.expire();
+ this.refresh({ broadcast: true });
+ break;
+ case at.PLACES_LINKS_CHANGED:
+ this.frecentCache.expire();
+ this.refresh({ broadcast: false });
+ break;
+ case at.PLACES_LINK_BLOCKED:
+ this.frecentCache.expire();
+ this.pinnedCache.expire();
+ this.refresh({ broadcast: true });
+ break;
+ case at.PREF_CHANGED:
+ switch (action.data.name) {
+ case DEFAULT_SITES_PREF:
+ if (!this._useRemoteSetting) {
+ this.refreshDefaults(action.data.value);
+ }
+ break;
+ case ROWS_PREF:
+ case FILTER_DEFAULT_SEARCH_PREF:
+ case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF:
+ this.refresh({ broadcast: true });
+ break;
+ case SHOW_SPONSORED_PREF:
+ if (
+ lazy.NimbusFeatures.newtab.getVariable(
+ NIMBUS_VARIABLE_CONTILE_ENABLED
+ )
+ ) {
+ this._contile.refresh();
+ } else {
+ this.refresh({ broadcast: true });
+ }
+ break;
+ case SEARCH_SHORTCUTS_EXPERIMENT:
+ if (action.data.value) {
+ this.updateCustomSearchShortcuts();
+ } else {
+ this.unpinAllSearchShortcuts();
+ }
+ this.refresh({ broadcast: true });
+ }
+ break;
+ case at.UPDATE_SECTION_PREFS:
+ if (action.data.id === SECTION_ID) {
+ this.updateSectionPrefs(action.data.value);
+ }
+ break;
+ case at.PREFS_INITIAL_VALUES:
+ if (!this._useRemoteSetting) {
+ this.refreshDefaults(action.data[DEFAULT_SITES_PREF]);
+ }
+ break;
+ case at.TOP_SITES_PIN:
+ this.pin(action);
+ break;
+ case at.TOP_SITES_UNPIN:
+ this.unpin(action);
+ break;
+ case at.TOP_SITES_INSERT:
+ this.insert(action);
+ break;
+ case at.PREVIEW_REQUEST:
+ this.getScreenshotPreview(action.data.url, action.meta.fromTarget);
+ break;
+ case at.UPDATE_PINNED_SEARCH_SHORTCUTS:
+ this.updatePinnedSearchShortcuts(action.data);
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = [
+ "TopSitesFeed",
+ "DEFAULT_TOP_SITES",
+ "ContileIntegration",
+];
diff --git a/browser/components/newtab/lib/TopStoriesFeed.jsm b/browser/components/newtab/lib/TopStoriesFeed.jsm
new file mode 100644
index 0000000000..639ced548d
--- /dev/null
+++ b/browser/components/newtab/lib/TopStoriesFeed.jsm
@@ -0,0 +1,751 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule(
+ "resource://activity-stream/common/Actions.sys.mjs"
+);
+const { Prefs } = ChromeUtils.import(
+ "resource://activity-stream/lib/ActivityStreamPrefs.jsm"
+);
+const { shortURL } = ChromeUtils.import(
+ "resource://activity-stream/lib/ShortURL.jsm"
+);
+const { SectionsManager } = ChromeUtils.import(
+ "resource://activity-stream/lib/SectionsManager.jsm"
+);
+const { PersistentCache } = ChromeUtils.import(
+ "resource://activity-stream/lib/PersistentCache.jsm"
+);
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
+});
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "pktApi",
+ "chrome://pocket/content/pktApi.jsm"
+);
+
+const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
+const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours
+const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours
+const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
+const SECTION_ID = "topstories";
+const IMPRESSION_SOURCE = "TOP_STORIES";
+const SPOC_IMPRESSION_TRACKING_PREF =
+ "feeds.section.topstories.spoc.impressions";
+const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled";
+const DISCOVERY_STREAM_PREF_ENABLED_PATH =
+ "browser.newtabpage.activity-stream.discoverystream.enabled";
+const REC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.rec.impressions";
+const PREF_USER_TOPSTORIES = "feeds.section.topstories";
+const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
+const DISCOVERY_STREAM_PREF = "discoverystream.config";
+
+class TopStoriesFeed {
+ constructor(ds) {
+ // Use discoverystream config pref default values for fast path and
+ // if needed lazy load activity stream top stories feed based on
+ // actual user preference when INIT and PREF_CHANGED is invoked
+ this.discoveryStreamEnabled =
+ ds &&
+ ds.value &&
+ JSON.parse(ds.value).enabled &&
+ Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false);
+ if (!this.discoveryStreamEnabled) {
+ this.initializeProperties();
+ }
+ }
+
+ initializeProperties() {
+ this.contentUpdateQueue = [];
+ this.spocCampaignMap = new Map();
+ this.cache = new PersistentCache(SECTION_ID, true);
+ this._prefs = new Prefs();
+ this.propertiesInitialized = true;
+ }
+
+ async onInit() {
+ SectionsManager.enableSection(SECTION_ID, true /* isStartup */);
+ if (this.discoveryStreamEnabled) {
+ return;
+ }
+
+ try {
+ const { options } = SectionsManager.sections.get(SECTION_ID);
+ const apiKey = this.getApiKeyFromPref(options.api_key_pref);
+ this.stories_endpoint = this.produceFinalEndpointUrl(
+ options.stories_endpoint,
+ apiKey
+ );
+ this.topics_endpoint = this.produceFinalEndpointUrl(
+ options.topics_endpoint,
+ apiKey
+ );
+ this.read_more_endpoint = options.read_more_endpoint;
+ this.stories_referrer = options.stories_referrer;
+ this.show_spocs = options.show_spocs;
+ this.storiesLastUpdated = 0;
+ this.topicsLastUpdated = 0;
+ this.storiesLoaded = false;
+ this.dispatchPocketCta(this._prefs.get("pocketCta"), false);
+
+ // Cache is used for new page loads, which shouldn't have changed data.
+ // If we have changed data, cache should be cleared,
+ // and last updated should be 0, and we can fetch.
+ let { stories, topics } = await this.loadCachedData();
+ if (this.storiesLastUpdated === 0) {
+ stories = await this.fetchStories();
+ }
+ if (this.topicsLastUpdated === 0) {
+ topics = await this.fetchTopics();
+ }
+ this.doContentUpdate({ stories, topics }, true);
+ this.storiesLoaded = true;
+
+ // This is filtered so an update function can return true to retry on the next run
+ this.contentUpdateQueue = this.contentUpdateQueue.filter(update =>
+ update()
+ );
+ } catch (e) {
+ console.error(`Problem initializing top stories feed: ${e.message}`);
+ }
+ }
+
+ init() {
+ SectionsManager.onceInitialized(this.onInit.bind(this));
+ }
+
+ async clearCache() {
+ await this.cache.set("stories", {});
+ await this.cache.set("topics", {});
+ await this.cache.set("spocs", {});
+ }
+
+ uninit() {
+ this.storiesLoaded = false;
+ SectionsManager.disableSection(SECTION_ID);
+ }
+
+ getPocketState(target) {
+ const action = {
+ type: at.POCKET_LOGGED_IN,
+ data: lazy.pktApi.isUserLoggedIn(),
+ };
+ this.store.dispatch(ac.OnlyToOneContent(action, target));
+ }
+
+ dispatchPocketCta(data, shouldBroadcast) {
+ const action = { type: at.POCKET_CTA, data: JSON.parse(data) };
+ this.store.dispatch(
+ shouldBroadcast
+ ? ac.BroadcastToContent(action)
+ : ac.AlsoToPreloaded(action)
+ );
+ }
+
+ /**
+ * doContentUpdate - Updates topics and stories in the topstories section.
+ *
+ * Sections have one update action for the whole section.
+ * Redux creates a state race condition if you call the same action,
+ * twice, concurrently. Because of this, doContentUpdate is
+ * one place to update both topics and stories in a single action.
+ *
+ * Section updates used old topics if none are available,
+ * but clear stories if none are available. Because of this, if no
+ * stories are passed, we instead use the existing stories in state.
+ *
+ * @param {Object} This is an object with potential new stories or topics.
+ * @param {Boolean} shouldBroadcast If we should update existing tabs or not. For first page
+ * loads or pref changes, we want to update existing tabs,
+ * for system tick or other updates we do not.
+ */
+ doContentUpdate({ stories, topics }, shouldBroadcast) {
+ let updateProps = {};
+ if (stories) {
+ updateProps.rows = stories;
+ } else {
+ const { Sections } = this.store.getState();
+ if (Sections && Sections.find) {
+ updateProps.rows = Sections.find(s => s.id === SECTION_ID).rows;
+ }
+ }
+ if (topics) {
+ Object.assign(updateProps, {
+ topics,
+ read_more_endpoint: this.read_more_endpoint,
+ });
+ }
+
+ // We should only be calling this once per init.
+ this.dispatchUpdateEvent(shouldBroadcast, updateProps);
+ }
+
+ async fetchStories() {
+ if (!this.stories_endpoint) {
+ return null;
+ }
+ try {
+ const response = await fetch(this.stories_endpoint, {
+ credentials: "omit",
+ });
+ if (!response.ok) {
+ throw new Error(
+ `Stories endpoint returned unexpected status: ${response.status}`
+ );
+ }
+
+ const body = await response.json();
+ this.updateSettings(body.settings);
+ this.stories = this.rotate(this.transform(body.recommendations));
+ this.cleanUpTopRecImpressionPref();
+
+ if (this.show_spocs && body.spocs) {
+ this.spocCampaignMap = new Map(
+ body.spocs.map(s => [s.id, `${s.campaign_id}`])
+ );
+ this.spocs = this.transform(body.spocs);
+ this.cleanUpCampaignImpressionPref();
+ }
+ this.storiesLastUpdated = Date.now();
+ body._timestamp = this.storiesLastUpdated;
+ this.cache.set("stories", body);
+ } catch (error) {
+ console.error(`Failed to fetch content: ${error.message}`);
+ }
+ return this.stories;
+ }
+
+ async loadCachedData() {
+ const data = await this.cache.get();
+ let stories = data.stories && data.stories.recommendations;
+ let topics = data.topics && data.topics.topics;
+
+ if (stories && !!stories.length && this.storiesLastUpdated === 0) {
+ this.updateSettings(data.stories.settings);
+ this.stories = this.rotate(this.transform(stories));
+ this.storiesLastUpdated = data.stories._timestamp;
+ if (data.stories.spocs && data.stories.spocs.length) {
+ this.spocCampaignMap = new Map(
+ data.stories.spocs.map(s => [s.id, `${s.campaign_id}`])
+ );
+ this.spocs = this.transform(data.stories.spocs);
+ this.cleanUpCampaignImpressionPref();
+ }
+ }
+ if (topics && !!topics.length && this.topicsLastUpdated === 0) {
+ this.topics = topics;
+ this.topicsLastUpdated = data.topics._timestamp;
+ }
+
+ return { topics: this.topics, stories: this.stories };
+ }
+
+ transform(items) {
+ if (!items) {
+ return [];
+ }
+
+ const calcResult = items
+ .filter(s => !lazy.NewTabUtils.blockedLinks.isBlocked({ url: s.url }))
+ .map(s => {
+ let mapped = {
+ guid: s.id,
+ hostname: s.domain || shortURL(Object.assign({}, s, { url: s.url })),
+ type:
+ Date.now() - s.published_timestamp * 1000 <= STORIES_NOW_THRESHOLD
+ ? "now"
+ : "trending",
+ context: s.context,
+ icon: s.icon,
+ title: s.title,
+ description: s.excerpt,
+ image: this.normalizeUrl(s.image_src),
+ referrer: this.stories_referrer,
+ url: s.url,
+ score: s.item_score || 1,
+ spoc_meta: this.show_spocs
+ ? { campaign_id: s.campaign_id, caps: s.caps }
+ : {},
+ };
+
+ // Very old cached spocs may not contain an `expiration_timestamp` property
+ if (s.expiration_timestamp) {
+ mapped.expiration_timestamp = s.expiration_timestamp;
+ }
+
+ return mapped;
+ })
+ .sort(this.compareScore);
+
+ return calcResult;
+ }
+
+ async fetchTopics() {
+ if (!this.topics_endpoint) {
+ return null;
+ }
+ try {
+ const response = await fetch(this.topics_endpoint, {
+ credentials: "omit",
+ });
+ if (!response.ok) {
+ throw new Error(
+ `Topics endpoint returned unexpected status: ${response.status}`
+ );
+ }
+ const body = await response.json();
+ const { topics } = body;
+ if (topics) {
+ this.topics = topics;
+ this.topicsLastUpdated = Date.now();
+ body._timestamp = this.topicsLastUpdated;
+ this.cache.set("topics", body);
+ }
+ } catch (error) {
+ console.error(`Failed to fetch topics: ${error.message}`);
+ }
+ return this.topics;
+ }
+
+ dispatchUpdateEvent(shouldBroadcast, data) {
+ SectionsManager.updateSection(SECTION_ID, data, shouldBroadcast);
+ }
+
+ compareScore(a, b) {
+ return b.score - a.score;
+ }
+
+ updateSettings(settings = {}) {
+ this.spocsPerNewTabs = settings.spocsPerNewTabs || 1; // Probability of a new tab getting a spoc [0,1]
+ this.recsExpireTime = settings.recsExpireTime;
+ }
+
+ // We rotate stories on the client so that
+ // active stories are at the front of the list, followed by stories that have expired
+ // impressions i.e. have been displayed for longer than recsExpireTime.
+ rotate(items) {
+ if (items.length <= 3) {
+ return items;
+ }
+
+ const maxImpressionAge = Math.max(
+ this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME,
+ DEFAULT_RECS_EXPIRE_TIME
+ );
+ const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
+ const expired = [];
+ const active = [];
+ for (const item of items) {
+ if (
+ impressions[item.guid] &&
+ Date.now() - impressions[item.guid] >= maxImpressionAge
+ ) {
+ expired.push(item);
+ } else {
+ active.push(item);
+ }
+ }
+ return active.concat(expired);
+ }
+
+ getApiKeyFromPref(apiKeyPref) {
+ if (!apiKeyPref) {
+ return apiKeyPref;
+ }
+
+ return (
+ this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref)
+ );
+ }
+
+ produceFinalEndpointUrl(url, apiKey) {
+ if (!url) {
+ return url;
+ }
+ if (url.includes("$apiKey") && !apiKey) {
+ throw new Error(`An API key was specified but none configured: ${url}`);
+ }
+ return url.replace("$apiKey", apiKey);
+ }
+
+ // Need to remove parenthesis from image URLs as React will otherwise
+ // fail to render them properly as part of the card template.
+ normalizeUrl(url) {
+ if (url) {
+ return url.replace(/\(/g, "%28").replace(/\)/g, "%29");
+ }
+ return url;
+ }
+
+ shouldShowSpocs() {
+ return this.show_spocs && this.store.getState().Prefs.values.showSponsored;
+ }
+
+ dispatchSpocDone(target) {
+ const action = { type: at.POCKET_WAITING_FOR_SPOC, data: false };
+ this.store.dispatch(ac.OnlyToOneContent(action, target));
+ }
+
+ filterSpocs() {
+ if (!this.shouldShowSpocs()) {
+ return [];
+ }
+
+ if (Math.random() > this.spocsPerNewTabs) {
+ return [];
+ }
+
+ if (!this.spocs || !this.spocs.length) {
+ // We have stories but no spocs so there's nothing to do and this update can be
+ // removed from the queue.
+ return [];
+ }
+
+ // Filter spocs based on frequency caps
+ const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
+ let spocs = this.spocs.filter(s =>
+ this.isBelowFrequencyCap(impressions, s)
+ );
+
+ // Filter out expired spocs based on `expiration_timestamp`
+ spocs = spocs.filter(spoc => {
+ // If cached data is so old it doesn't contain this property, assume the spoc is ok to show
+ if (!(`expiration_timestamp` in spoc)) {
+ return true;
+ }
+ // `expiration_timestamp` is the number of seconds elapsed since January 1, 1970 00:00:00 UTC
+ return spoc.expiration_timestamp * 1000 > Date.now();
+ });
+
+ return spocs;
+ }
+
+ maybeAddSpoc(target) {
+ const updateContent = () => {
+ let spocs = this.filterSpocs();
+
+ if (!spocs.length) {
+ this.dispatchSpocDone(target);
+ return false;
+ }
+
+ // Create a new array with a spoc inserted at index 2
+ const section = this.store
+ .getState()
+ .Sections.find(s => s.id === SECTION_ID);
+ let rows = section.rows.slice(0, this.stories.length);
+ rows.splice(2, 0, Object.assign(spocs[0], { pinned: true }));
+
+ // Send a content update to the target tab
+ const action = {
+ type: at.SECTION_UPDATE,
+ data: Object.assign({ rows }, { id: SECTION_ID }),
+ };
+ this.store.dispatch(ac.OnlyToOneContent(action, target));
+ this.dispatchSpocDone(target);
+ return false;
+ };
+
+ if (this.storiesLoaded) {
+ updateContent();
+ } else {
+ // Delay updating tab content until initial data has been fetched
+ this.contentUpdateQueue.push(updateContent);
+ }
+ }
+
+ // Frequency caps are based on campaigns, which may include multiple spocs.
+ // We currently support two types of frequency caps:
+ // - lifetime: Indicates how many times spocs from a campaign can be shown in total
+ // - period: Indicates how many times spocs from a campaign can be shown within a period
+ //
+ // So, for example, the feed configuration below defines that for campaign 1 no more
+ // than 5 spocs can be show in total, and no more than 2 per hour.
+ // "campaign_id": 1,
+ // "caps": {
+ // "lifetime": 5,
+ // "campaign": {
+ // "count": 2,
+ // "period": 3600
+ // }
+ // }
+ isBelowFrequencyCap(impressions, spoc) {
+ const campaignImpressions = impressions[spoc.spoc_meta.campaign_id];
+ if (!campaignImpressions) {
+ return true;
+ }
+
+ const lifeTimeCap = Math.min(
+ spoc.spoc_meta.caps && spoc.spoc_meta.caps.lifetime,
+ MAX_LIFETIME_CAP
+ );
+ const lifeTimeCapExceeded = campaignImpressions.length >= lifeTimeCap;
+ if (lifeTimeCapExceeded) {
+ return false;
+ }
+
+ const campaignCap =
+ (spoc.spoc_meta.caps && spoc.spoc_meta.caps.campaign) || {};
+ const campaignCapExceeded =
+ campaignImpressions.filter(
+ i => Date.now() - i < campaignCap.period * 1000
+ ).length >= campaignCap.count;
+ return !campaignCapExceeded;
+ }
+
+ // Clean up campaign impression pref by removing all campaigns that are no
+ // longer part of the response, and are therefore considered inactive.
+ cleanUpCampaignImpressionPref() {
+ const campaignIds = new Set(this.spocCampaignMap.values());
+ this.cleanUpImpressionPref(
+ id => !campaignIds.has(id),
+ SPOC_IMPRESSION_TRACKING_PREF
+ );
+ }
+
+ // Clean up rec impression pref by removing all stories that are no
+ // longer part of the response.
+ cleanUpTopRecImpressionPref() {
+ const activeStories = new Set(this.stories.map(s => `${s.guid}`));
+ this.cleanUpImpressionPref(
+ id => !activeStories.has(id),
+ REC_IMPRESSION_TRACKING_PREF
+ );
+ }
+
+ /**
+ * Cleans up the provided impression pref (spocs or recs).
+ *
+ * @param isExpired predicate (boolean-valued function) that returns whether or not
+ * the impression for the given key is expired.
+ * @param pref the impression pref to clean up.
+ */
+ cleanUpImpressionPref(isExpired, pref) {
+ const impressions = this.readImpressionsPref(pref);
+ let changed = false;
+
+ Object.keys(impressions).forEach(id => {
+ if (isExpired(id)) {
+ changed = true;
+ delete impressions[id];
+ }
+ });
+
+ if (changed) {
+ this.writeImpressionsPref(pref, impressions);
+ }
+ }
+
+ // Sets a pref mapping campaign IDs to timestamp arrays.
+ // The timestamps represent impressions which are used to calculate frequency caps.
+ recordCampaignImpression(campaignId) {
+ let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
+
+ const timeStamps = impressions[campaignId] || [];
+ timeStamps.push(Date.now());
+ impressions = Object.assign(impressions, { [campaignId]: timeStamps });
+
+ this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions);
+ }
+
+ // Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression).
+ // We use these timestamps to guarantee a story doesn't stay on top for longer than
+ // configured in the feed settings (settings.recsExpireTime).
+ recordTopRecImpressions(topItems) {
+ let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
+ let changed = false;
+
+ topItems.forEach(t => {
+ if (!impressions[t]) {
+ changed = true;
+ impressions = Object.assign(impressions, { [t]: Date.now() });
+ }
+ });
+
+ if (changed) {
+ this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions);
+ }
+ }
+
+ readImpressionsPref(pref) {
+ const prefVal = this._prefs.get(pref);
+ return prefVal ? JSON.parse(prefVal) : {};
+ }
+
+ writeImpressionsPref(pref, impressions) {
+ this._prefs.set(pref, JSON.stringify(impressions));
+ }
+
+ async removeSpocs() {
+ // Quick hack so that SPOCS are removed from all open and preloaded tabs when
+ // they are disabled. The longer term fix should probably be to remove them
+ // in the Reducer.
+ await this.clearCache();
+ this.uninit();
+ this.init();
+ }
+
+ lazyLoadTopStories(options = {}) {
+ let { dsPref, userPref } = options;
+ if (!dsPref) {
+ dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF];
+ }
+ if (!userPref) {
+ userPref = this.store.getState().Prefs.values[PREF_USER_TOPSTORIES];
+ }
+
+ try {
+ this.discoveryStreamEnabled =
+ JSON.parse(dsPref).enabled &&
+ this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED];
+ } catch (e) {
+ // Load activity stream top stories if fail to determine discovery stream state
+ this.discoveryStreamEnabled = false;
+ }
+
+ // Return without invoking initialization if top stories are loaded, or preffed off.
+ if (this.storiesLoaded || !userPref) {
+ return;
+ }
+
+ if (!this.discoveryStreamEnabled && !this.propertiesInitialized) {
+ this.initializeProperties();
+ }
+ this.init();
+ }
+
+ handleDisabled(action) {
+ switch (action.type) {
+ case at.INIT:
+ this.lazyLoadTopStories();
+ break;
+ case at.PREF_CHANGED:
+ if (action.data.name === DISCOVERY_STREAM_PREF) {
+ this.lazyLoadTopStories({ dsPref: action.data.value });
+ }
+ if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) {
+ this.lazyLoadTopStories();
+ }
+ if (action.data.name === PREF_USER_TOPSTORIES) {
+ if (action.data.value) {
+ // init topstories if value if true.
+ this.lazyLoadTopStories({ userPref: action.data.value });
+ } else {
+ this.uninit();
+ }
+ }
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ }
+ }
+
+ async onAction(action) {
+ if (this.discoveryStreamEnabled) {
+ this.handleDisabled(action);
+ return;
+ }
+ switch (action.type) {
+ // Check discoverystream pref and load activity stream top stories only if needed
+ case at.INIT:
+ this.lazyLoadTopStories();
+ break;
+ case at.SYSTEM_TICK:
+ let stories;
+ let topics;
+ if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) {
+ stories = await this.fetchStories();
+ }
+ if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) {
+ topics = await this.fetchTopics();
+ }
+ this.doContentUpdate({ stories, topics }, false);
+ break;
+ case at.UNINIT:
+ this.uninit();
+ break;
+ case at.NEW_TAB_REHYDRATED:
+ this.getPocketState(action.meta.fromTarget);
+ this.maybeAddSpoc(action.meta.fromTarget);
+ break;
+ case at.SECTION_OPTIONS_CHANGED:
+ if (action.data === SECTION_ID) {
+ await this.clearCache();
+ this.uninit();
+ this.init();
+ }
+ break;
+ case at.PLACES_LINK_BLOCKED:
+ if (this.spocs) {
+ this.spocs = this.spocs.filter(s => s.url !== action.data.url);
+ }
+ break;
+ case at.TELEMETRY_IMPRESSION_STATS: {
+ // We want to make sure we only track impressions from Top Stories,
+ // otherwise unexpected things that are not properly handled can happen.
+ // Example: Impressions from spocs on Discovery Stream can cause the
+ // Top Stories impressions pref to continuously grow, see bug #1523408
+ if (action.data.source === IMPRESSION_SOURCE) {
+ const payload = action.data;
+ const viewImpression = !(
+ "click" in payload ||
+ "block" in payload ||
+ "pocket" in payload
+ );
+ if (payload.tiles && viewImpression) {
+ if (this.shouldShowSpocs()) {
+ payload.tiles.forEach(t => {
+ if (this.spocCampaignMap.has(t.id)) {
+ this.recordCampaignImpression(this.spocCampaignMap.get(t.id));
+ }
+ });
+ }
+ const topRecs = payload.tiles
+ .filter(t => !this.spocCampaignMap.has(t.id))
+ .map(t => t.id);
+ this.recordTopRecImpressions(topRecs);
+ }
+ }
+ break;
+ }
+ case at.PREF_CHANGED:
+ if (action.data.name === DISCOVERY_STREAM_PREF) {
+ this.lazyLoadTopStories({ dsPref: action.data.value });
+ }
+ if (action.data.name === PREF_USER_TOPSTORIES) {
+ if (action.data.value) {
+ // init topstories if value if true.
+ this.lazyLoadTopStories({ userPref: action.data.value });
+ } else {
+ this.uninit();
+ }
+ }
+ // Check if spocs was disabled. Remove them if they were.
+ if (action.data.name === "showSponsored" && !action.data.value) {
+ await this.removeSpocs();
+ }
+ if (action.data.name === "pocketCta") {
+ this.dispatchPocketCta(action.data.value, true);
+ }
+ break;
+ }
+ }
+}
+
+const EXPORTED_SYMBOLS = [
+ "TopStoriesFeed",
+ "STORIES_UPDATE_TIME",
+ "TOPICS_UPDATE_TIME",
+ "SECTION_ID",
+ "SPOC_IMPRESSION_TRACKING_PREF",
+ "REC_IMPRESSION_TRACKING_PREF",
+ "DEFAULT_RECS_EXPIRE_TIME",
+];
diff --git a/browser/components/newtab/lib/UTEventReporting.jsm b/browser/components/newtab/lib/UTEventReporting.jsm
new file mode 100644
index 0000000000..612fb96938
--- /dev/null
+++ b/browser/components/newtab/lib/UTEventReporting.jsm
@@ -0,0 +1,66 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * Note: the schema can be found in
+ * https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml
+ */
+const EXTRAS_FIELD_NAMES = [
+ "addon_version",
+ "session_id",
+ "page",
+ "user_prefs",
+ "action_position",
+];
+
+class UTEventReporting {
+ constructor() {
+ Services.telemetry.setEventRecordingEnabled("activity_stream", true);
+ this.sendUserEvent = this.sendUserEvent.bind(this);
+ this.sendSessionEndEvent = this.sendSessionEndEvent.bind(this);
+ }
+
+ _createExtras(data) {
+ // Make a copy of the given data and delete/modify it as needed.
+ let utExtras = Object.assign({}, data);
+ for (let field of Object.keys(utExtras)) {
+ if (EXTRAS_FIELD_NAMES.includes(field)) {
+ utExtras[field] = String(utExtras[field]);
+ continue;
+ }
+ delete utExtras[field];
+ }
+ return utExtras;
+ }
+
+ sendUserEvent(data) {
+ let mainFields = ["event", "source"];
+ let eventFields = mainFields.map(field => String(data[field]) || null);
+
+ Services.telemetry.recordEvent(
+ "activity_stream",
+ "event",
+ ...eventFields,
+ this._createExtras(data)
+ );
+ }
+
+ sendSessionEndEvent(data) {
+ Services.telemetry.recordEvent(
+ "activity_stream",
+ "end",
+ "session",
+ String(data.session_duration),
+ this._createExtras(data)
+ );
+ }
+
+ uninit() {
+ Services.telemetry.setEventRecordingEnabled("activity_stream", false);
+ }
+}
+
+const EXPORTED_SYMBOLS = ["UTEventReporting"];
diff --git a/browser/components/newtab/lib/cache-worker.js b/browser/components/newtab/lib/cache-worker.js
new file mode 100644
index 0000000000..0996ecde45
--- /dev/null
+++ b/browser/components/newtab/lib/cache-worker.js
@@ -0,0 +1,205 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env mozilla/chrome-worker */
+
+/* global ReactDOMServer, NewtabRenderUtils */
+
+const PAGE_TEMPLATE_RESOURCE_PATH =
+ "resource://activity-stream/data/content/abouthomecache/page.html.template";
+const SCRIPT_TEMPLATE_RESOURCE_PATH =
+ "resource://activity-stream/data/content/abouthomecache/script.js.template";
+
+// If we don't stub these functions out, React throws warnings in the console
+// upon being loaded.
+let window = self;
+window.requestAnimationFrame = () => {};
+window.cancelAnimationFrame = () => {};
+window.ASRouterMessage = () => {
+ return Promise.resolve();
+};
+window.ASRouterAddParentListener = () => {};
+window.ASRouterRemoveParentListener = () => {};
+
+/* import-globals-from /toolkit/components/workerloader/require.js */
+importScripts("resource://gre/modules/workers/require.js");
+
+{
+ let oldChromeUtils = ChromeUtils;
+
+ // ChromeUtils is defined inside of a Worker, but we don't want the
+ // activity-stream.bundle.js to detect it when loading, since that results
+ // in it attempting to import JSMs on load, which is not allowed in
+ // a Worker. So we temporarily clear ChromeUtils so that activity-stream.bundle.js
+ // thinks its being loaded in content scope.
+ //
+ // eslint-disable-next-line no-global-assign
+ ChromeUtils = undefined;
+
+ /* import-globals-from ../vendor/react.js */
+ /* import-globals-from ../vendor/react-dom.js */
+ /* import-globals-from ../vendor/react-dom-server.js */
+ /* import-globals-from ../vendor/redux.js */
+ /* import-globals-from ../vendor/react-transition-group.js */
+ /* import-globals-from ../vendor/prop-types.js */
+ /* import-globals-from ../vendor/react-redux.js */
+ /* import-globals-from ../data/content/activity-stream.bundle.js */
+ importScripts(
+ "resource://activity-stream/vendor/react.js",
+ "resource://activity-stream/vendor/react-dom.js",
+ "resource://activity-stream/vendor/react-dom-server.js",
+ "resource://activity-stream/vendor/redux.js",
+ "resource://activity-stream/vendor/react-transition-group.js",
+ "resource://activity-stream/vendor/prop-types.js",
+ "resource://activity-stream/vendor/react-redux.js",
+ "resource://activity-stream/data/content/activity-stream.bundle.js"
+ );
+
+ // eslint-disable-next-line no-global-assign
+ ChromeUtils = oldChromeUtils;
+}
+
+let PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js");
+
+let Agent = {
+ _templates: null,
+
+ /**
+ * Synchronously loads the template files off of the file
+ * system, and returns them as an object. If the Worker has loaded
+ * these templates before, a cached copy of the templates is returned
+ * instead.
+ *
+ * @return Object
+ * An object with the following properties:
+ *
+ * pageTemplate (String):
+ * The template for the document markup.
+ *
+ * scriptTempate (String):
+ * The template for the script.
+ */
+ getOrCreateTemplates() {
+ if (this._templates) {
+ return this._templates;
+ }
+
+ const templateResources = new Map([
+ ["pageTemplate", PAGE_TEMPLATE_RESOURCE_PATH],
+ ["scriptTemplate", SCRIPT_TEMPLATE_RESOURCE_PATH],
+ ]);
+
+ this._templates = {};
+
+ for (let [name, path] of templateResources) {
+ const xhr = new XMLHttpRequest();
+ // Using a synchronous XHR in a worker is fine.
+ xhr.open("GET", path, false);
+ xhr.responseType = "text";
+ xhr.send(null);
+ this._templates[name] = xhr.responseText;
+ }
+
+ return this._templates;
+ },
+
+ /**
+ * Constructs the cached about:home document using ReactDOMServer. This will
+ * be called when "construct" messages are sent to this PromiseWorker.
+ *
+ * @param state (Object)
+ * The most recent Activity Stream Redux state.
+ * @return Object
+ * An object with the following properties:
+ *
+ * page (String):
+ * The generated markup for the document.
+ *
+ * script (String):
+ * The generated script for the document.
+ */
+ construct(state) {
+ // If anything in this function throws an exception, PromiseWorker
+ // runs the risk of leaving the Promise associated with this method
+ // forever unresolved. This is particularly bad when this method is
+ // called via AsyncShutdown, since the forever unresolved Promise can
+ // result in a AsyncShutdown timeout crash.
+ //
+ // To help ensure that no matter what, the Promise resolves with something,
+ // we wrap the whole operation in a try/catch.
+ try {
+ return this._construct(state);
+ } catch (e) {
+ console.error("about:home startup cache construction failed:", e);
+ return { page: null, script: null };
+ }
+ },
+
+ /**
+ * Internal method that actually does the work of constructing the cached
+ * about:home document using ReactDOMServer. This should be called from
+ * `construct` only.
+ *
+ * @param state (Object)
+ * The most recent Activity Stream Redux state.
+ * @return Object
+ * An object with the following properties:
+ *
+ * page (String):
+ * The generated markup for the document.
+ *
+ * script (String):
+ * The generated script for the document.
+ */
+ _construct(state) {
+ state.App.isForStartupCache = true;
+
+ // ReactDOMServer.renderToString expects a Redux store to pull
+ // the state from, so we mock out a minimal store implementation.
+ let fakeStore = {
+ getState() {
+ return state;
+ },
+ dispatch() {},
+ };
+
+ let markup = ReactDOMServer.renderToString(
+ NewtabRenderUtils.NewTab({
+ store: fakeStore,
+ isFirstrun: false,
+ })
+ );
+
+ let { pageTemplate, scriptTemplate } = this.getOrCreateTemplates();
+ let cacheTime = new Date().toUTCString();
+ let page = pageTemplate
+ .replace("{{ MARKUP }}", markup)
+ .replace("{{ CACHE_TIME }}", cacheTime);
+ let script = scriptTemplate.replace(
+ "{{ STATE }}",
+ JSON.stringify(state, null, "\t")
+ );
+
+ return { page, script };
+ },
+};
+
+// This boilerplate connects the PromiseWorker to the Agent so
+// that messages from the main thread map to methods on the
+// Agent.
+let worker = new PromiseWorker.AbstractWorker();
+worker.dispatch = function(method, args = []) {
+ return Agent[method](...args);
+};
+worker.postMessage = function(result, ...transfers) {
+ self.postMessage(result, ...transfers);
+};
+worker.close = function() {
+ self.close();
+};
+
+self.addEventListener("message", msg => worker.handleMessage(msg));
+self.addEventListener("unhandledrejection", function(error) {
+ throw error.reason;
+});