diff options
Diffstat (limited to 'browser/components/newtab/lib')
37 files changed, 16514 insertions, 0 deletions
diff --git a/browser/components/newtab/lib/AboutPreferences.sys.mjs b/browser/components/newtab/lib/AboutPreferences.sys.mjs new file mode 100644 index 0000000000..33f7ecdaeb --- /dev/null +++ b/browser/components/newtab/lib/AboutPreferences.sys.mjs @@ -0,0 +1,298 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionTypes as at, + actionCreators as ac, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +const HTML_NS = "http://www.w3.org/1999/xhtml"; +export const PREFERENCES_LOADED_EVENT = "home-pane-loaded"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", +}); + +// These "section" objects are formatted in a way to be similar to the ones from +// SectionsManager to construct the preferences view. +const PREFS_BEFORE_SECTIONS = () => [ + { + id: "search", + pref: { + feed: "showSearch", + titleString: "home-prefs-search-header", + }, + icon: "chrome://global/skin/icons/search-glass.svg", + }, + { + id: "topsites", + pref: { + feed: "feeds.topsites", + titleString: "home-prefs-shortcuts-header", + descString: "home-prefs-shortcuts-description", + get nestedPrefs() { + return Services.prefs.getBoolPref("browser.topsites.useRemoteSetting") + ? [ + { + name: "showSponsoredTopSites", + titleString: "home-prefs-shortcuts-by-option-sponsored", + eventSource: "SPONSORED_TOP_SITES", + }, + ] + : []; + }, + }, + icon: "chrome://browser/skin/topsites.svg", + maxRows: 4, + rowsPref: "topSitesRows", + eventSource: "TOP_SITES", + }, +]; + +export class AboutPreferences { + init() { + Services.obs.addObserver(this, PREFERENCES_LOADED_EVENT); + } + + uninit() { + Services.obs.removeObserver(this, PREFERENCES_LOADED_EVENT); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.uninit(); + break; + case at.SETTINGS_OPEN: + action._target.browser.ownerGlobal.openPreferences("paneHome"); + break; + // This is used to open the web extension settings page for an extension + case at.OPEN_WEBEXT_SETTINGS: + action._target.browser.ownerGlobal.BrowserOpenAddonsMgr( + `addons://detail/${encodeURIComponent(action.data)}` + ); + break; + } + } + + handleDiscoverySettings(sections) { + // Deep copy object to not modify original Sections state in store + let sectionsCopy = JSON.parse(JSON.stringify(sections)); + sectionsCopy.forEach(obj => { + if (obj.id === "topstories") { + obj.rowsPref = ""; + } + }); + return sectionsCopy; + } + + setupUserEvent(element, eventSource) { + element.addEventListener("command", e => { + const { checked } = e.target; + if (typeof checked === "boolean") { + this.store.dispatch( + ac.UserEvent({ + event: "PREF_CHANGED", + source: eventSource, + value: { status: checked, menu_source: "ABOUT_PREFERENCES" }, + }) + ); + } + }); + } + + observe(window) { + const discoveryStreamConfig = this.store.getState().DiscoveryStream.config; + let sections = this.store.getState().Sections; + + if (discoveryStreamConfig.enabled) { + sections = this.handleDiscoverySettings(sections); + } + + const featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + + this.renderPreferences(window, [ + ...PREFS_BEFORE_SECTIONS(featureConfig), + ...sections, + ]); + } + + /** + * Render preferences to an about:preferences content window with the provided + * preferences structure. + */ + renderPreferences({ document, Preferences, gHomePane }, prefStructure) { + // Helper to create a new element and append it + const createAppend = (tag, parent, options) => + parent.appendChild(document.createXULElement(tag, options)); + + // Helper to get fluentIDs sometimes encase in an object + const getString = message => + typeof message !== "object" ? message : message.id; + + // Helper to link a UI element to a preference for updating + const linkPref = (element, name, type) => { + const fullPref = `browser.newtabpage.activity-stream.${name}`; + element.setAttribute("preference", fullPref); + Preferences.add({ id: fullPref, type }); + + // Prevent changing the UI if the preference can't be changed + element.disabled = Preferences.get(fullPref).locked; + }; + + // Insert a new group immediately after the homepage one + const homeGroup = document.getElementById("homepageGroup"); + const contentsGroup = homeGroup.insertAdjacentElement( + "afterend", + homeGroup.cloneNode() + ); + contentsGroup.id = "homeContentsGroup"; + contentsGroup.setAttribute("data-subcategory", "contents"); + const homeHeader = createAppend("label", contentsGroup).appendChild( + document.createElementNS(HTML_NS, "h2") + ); + document.l10n.setAttributes(homeHeader, "home-prefs-content-header2"); + + const homeDescription = createAppend("description", contentsGroup); + homeDescription.classList.add("description-deemphasized"); + + document.l10n.setAttributes( + homeDescription, + "home-prefs-content-description2" + ); + + // Add preferences for each section + prefStructure.forEach(sectionData => { + const { + id, + pref: prefData, + icon = "webextension", + maxRows, + rowsPref, + shouldHidePref, + eventSource, + } = sectionData; + const { + feed: name, + titleString = {}, + descString, + nestedPrefs = [], + } = prefData || {}; + + // Don't show any sections that we don't want to expose in preferences UI + if (shouldHidePref) { + return; + } + + // Use full icon spec for certain protocols or fall back to packaged icon + const iconUrl = !icon.search(/^(chrome|moz-extension|resource):/) + ? icon + : `chrome://activity-stream/content/data/content/assets/glyph-${icon}-16.svg`; + + // Add the main preference for turning on/off a section + const sectionVbox = createAppend("vbox", contentsGroup); + sectionVbox.setAttribute("data-subcategory", id); + const checkbox = createAppend("checkbox", sectionVbox); + checkbox.classList.add("section-checkbox"); + checkbox.setAttribute("src", iconUrl); + // Setup a user event if we have an event source for this pref. + if (eventSource) { + this.setupUserEvent(checkbox, eventSource); + } + document.l10n.setAttributes( + checkbox, + getString(titleString), + titleString.values + ); + + linkPref(checkbox, name, "bool"); + + // Specially add a link for stories + if (id === "topstories") { + const sponsoredHbox = createAppend("hbox", sectionVbox); + sponsoredHbox.setAttribute("align", "center"); + sponsoredHbox.appendChild(checkbox); + checkbox.classList.add("tail-with-learn-more"); + + const link = createAppend("label", sponsoredHbox, { is: "text-link" }); + link.classList.add("learn-sponsored"); + link.setAttribute("href", sectionData.pref.learnMore.link.href); + document.l10n.setAttributes(link, sectionData.pref.learnMore.link.id); + } + + // Add more details for the section (e.g., description, more prefs) + const detailVbox = createAppend("vbox", sectionVbox); + detailVbox.classList.add("indent"); + if (descString) { + const description = createAppend("description", detailVbox); + description.classList.add("indent", "text-deemphasized"); + document.l10n.setAttributes( + description, + getString(descString), + descString.values + ); + + // Add a rows dropdown if we have a pref to control and a maximum + if (rowsPref && maxRows) { + const detailHbox = createAppend("hbox", detailVbox); + detailHbox.setAttribute("align", "center"); + description.setAttribute("flex", 1); + detailHbox.appendChild(description); + + // Add box so the search tooltip is positioned correctly + const tooltipBox = createAppend("hbox", detailHbox); + + // Add appropriate number of localized entries to the dropdown + const menulist = createAppend("menulist", tooltipBox); + menulist.setAttribute("crop", "none"); + const menupopup = createAppend("menupopup", menulist); + for (let num = 1; num <= maxRows; num++) { + const item = createAppend("menuitem", menupopup); + document.l10n.setAttributes( + item, + "home-prefs-sections-rows-option", + { num } + ); + item.setAttribute("value", num); + } + linkPref(menulist, rowsPref, "int"); + } + } + + const subChecks = []; + const fullName = `browser.newtabpage.activity-stream.${sectionData.pref.feed}`; + const pref = Preferences.get(fullName); + + // Add a checkbox pref for any nested preferences + nestedPrefs.forEach(nested => { + const subcheck = createAppend("checkbox", detailVbox); + // Setup a user event if we have an event source for this pref. + if (nested.eventSource) { + this.setupUserEvent(subcheck, nested.eventSource); + } + subcheck.classList.add("indent"); + document.l10n.setAttributes(subcheck, nested.titleString); + linkPref(subcheck, nested.name, "bool"); + subChecks.push(subcheck); + subcheck.disabled = !pref._value; + subcheck.hidden = nested.hidden; + }); + + // Disable any nested checkboxes if the parent pref is not enabled. + pref.on("change", () => { + subChecks.forEach(subcheck => { + subcheck.disabled = !pref._value; + }); + }); + }); + + // Update the visibility of the Restore Defaults btn based on checked prefs + gHomePane.toggleRestoreDefaultsBtn(); + } +} diff --git a/browser/components/newtab/lib/ActivityStream.sys.mjs b/browser/components/newtab/lib/ActivityStream.sys.mjs new file mode 100644 index 0000000000..f2287fe45e --- /dev/null +++ b/browser/components/newtab/lib/ActivityStream.sys.mjs @@ -0,0 +1,700 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// AppConstants, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AboutPreferences: "resource://activity-stream/lib/AboutPreferences.sys.mjs", + DEFAULT_SITES: "resource://activity-stream/lib/DefaultSites.sys.mjs", + DefaultPrefs: "resource://activity-stream/lib/ActivityStreamPrefs.sys.mjs", + DiscoveryStreamFeed: + "resource://activity-stream/lib/DiscoveryStreamFeed.sys.mjs", + FaviconFeed: "resource://activity-stream/lib/FaviconFeed.sys.mjs", + HighlightsFeed: "resource://activity-stream/lib/HighlightsFeed.sys.mjs", + NewTabInit: "resource://activity-stream/lib/NewTabInit.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PrefsFeed: "resource://activity-stream/lib/PrefsFeed.sys.mjs", + PlacesFeed: "resource://activity-stream/lib/PlacesFeed.sys.mjs", + RecommendationProvider: + "resource://activity-stream/lib/RecommendationProvider.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + SectionsFeed: "resource://activity-stream/lib/SectionsManager.sys.mjs", + Store: "resource://activity-stream/lib/Store.sys.mjs", + SystemTickFeed: "resource://activity-stream/lib/SystemTickFeed.sys.mjs", + TelemetryFeed: "resource://activity-stream/lib/TelemetryFeed.sys.mjs", + TopSitesFeed: "resource://activity-stream/lib/TopSitesFeed.sys.mjs", + TopStoriesFeed: "resource://activity-stream/lib/TopStoriesFeed.sys.mjs", +}); + +// NB: Eagerly load modules that will be loaded/constructed/initialized in the +// common case to avoid the overhead of wrapping and detecting lazy loading. +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +const REGION_BASIC_CONFIG = + "browser.newtabpage.activity-stream.discoverystream.region-basic-config"; + +// Determine if spocs should be shown for a geo/locale +function showSpocs({ geo }) { + const spocsGeoString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionSpocsConfig") || ""; + const spocsGeo = spocsGeoString.split(",").map(s => s.trim()); + return spocsGeo.includes(geo); +} + +// Configure default Activity Stream prefs with a plain `value` or a `getValue` +// that computes a value. A `value_local_dev` is used for development defaults. +export const PREFS_CONFIG = new Map([ + [ + "default.sites", + { + title: + "Comma-separated list of default top sites to fill in behind visited sites", + getValue: ({ geo }) => + lazy.DEFAULT_SITES.get(lazy.DEFAULT_SITES.has(geo) ? geo : ""), + }, + ], + [ + "feeds.section.topstories.options", + { + title: "Configuration options for top stories feed", + // This is a dynamic pref as it depends on the feed being shown or not + getValue: args => + JSON.stringify({ + api_key_pref: "extensions.pocket.oAuthConsumerKey", + // Use the opposite value as what default value the feed would have used + hidden: !PREFS_CONFIG.get("feeds.system.topstories").getValue(args), + provider_icon: "chrome://global/skin/icons/pocket.svg", + provider_name: "Pocket", + read_more_endpoint: + "https://getpocket.com/explore/trending?src=fx_new_tab", + stories_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=${ + args.locale + }&feed_variant=${ + showSpocs(args) ? "default_spocs_on" : "default_spocs_off" + }`, + stories_referrer: "https://getpocket.com/recommendations", + topics_endpoint: `https://getpocket.cdn.mozilla.net/v3/firefox/trending-topics?version=2&consumer_key=$apiKey&locale_lang=${args.locale}`, + show_spocs: showSpocs(args), + }), + }, + ], + [ + "feeds.topsites", + { + title: "Displays Top Sites on the New Tab Page", + value: true, + }, + ], + [ + "hideTopSitesTitle", + { + title: + "Hide the top sites section's title, including the section and collapse icons", + value: false, + }, + ], + [ + "showSponsored", + { + title: "User pref for sponsored Pocket content", + value: true, + }, + ], + [ + "system.showSponsored", + { + title: "System pref for sponsored Pocket content", + // This pref is dynamic as the sponsored content depends on the region + getValue: showSpocs, + }, + ], + [ + "showSponsoredTopSites", + { + title: "Show sponsored top sites", + value: true, + }, + ], + [ + "pocketCta", + { + title: "Pocket cta and button for logged out users.", + value: JSON.stringify({ + cta_button: "", + cta_text: "", + cta_url: "", + use_cta: false, + }), + }, + ], + [ + "showSearch", + { + title: "Show the Search bar", + value: true, + }, + ], + [ + "topSitesRows", + { + title: "Number of rows of Top Sites to display", + value: 1, + }, + ], + [ + "telemetry", + { + title: "Enable system error and usage data collection", + value: true, + value_local_dev: false, + }, + ], + [ + "telemetry.ut.events", + { + title: "Enable Unified Telemetry event data collection", + value: AppConstants.EARLY_BETA_OR_EARLIER, + value_local_dev: false, + }, + ], + [ + "telemetry.structuredIngestion.endpoint", + { + title: "Structured Ingestion telemetry server endpoint", + value: "https://incoming.telemetry.mozilla.org/submit", + }, + ], + [ + "section.highlights.includeVisited", + { + title: + "Boolean flag that decides whether or not to show visited pages in highlights.", + value: true, + }, + ], + [ + "section.highlights.includeBookmarks", + { + title: + "Boolean flag that decides whether or not to show bookmarks in highlights.", + value: true, + }, + ], + [ + "section.highlights.includePocket", + { + title: + "Boolean flag that decides whether or not to show saved Pocket stories in highlights.", + value: true, + }, + ], + [ + "section.highlights.includeDownloads", + { + title: + "Boolean flag that decides whether or not to show saved recent Downloads in highlights.", + value: true, + }, + ], + [ + "section.highlights.rows", + { + title: "Number of rows of Highlights to display", + value: 1, + }, + ], + [ + "section.topstories.rows", + { + title: "Number of rows of Top Stories to display", + value: 1, + }, + ], + [ + "sectionOrder", + { + title: "The rendering order for the sections", + value: "topsites,topstories,highlights", + }, + ], + [ + "improvesearch.noDefaultSearchTile", + { + title: "Remove tiles that are the same as the default search", + value: true, + }, + ], + [ + "improvesearch.topSiteSearchShortcuts.searchEngines", + { + title: + "An ordered, comma-delimited list of search shortcuts that we should try and pin", + // This pref is dynamic as the shortcuts vary depending on the region + getValue: ({ geo }) => { + if (!geo) { + return ""; + } + const searchShortcuts = []; + if (geo === "CN") { + searchShortcuts.push("baidu"); + } else if (["BY", "KZ", "RU", "TR"].includes(geo)) { + searchShortcuts.push("yandex"); + } else { + searchShortcuts.push("google"); + } + if (["DE", "FR", "GB", "IT", "JP", "US"].includes(geo)) { + searchShortcuts.push("amazon"); + } + return searchShortcuts.join(","); + }, + }, + ], + [ + "improvesearch.topSiteSearchShortcuts.havePinned", + { + title: + "A comma-delimited list of search shortcuts that have previously been pinned", + value: "", + }, + ], + [ + "asrouter.devtoolsEnabled", + { + title: "Are the asrouter devtools enabled?", + value: false, + }, + ], + [ + "asrouter.providers.onboarding", + { + title: "Configuration for onboarding provider", + value: JSON.stringify({ + id: "onboarding", + type: "local", + localProvider: "OnboardingMessageProvider", + enabled: true, + // Block specific messages from this local provider + exclude: [], + }), + }, + ], + // See browser/app/profile/firefox.js for other ASR preferences. They must be defined there to enable roll-outs. + [ + "discoverystream.flight.blocks", + { + title: "Track flight blocks", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "discoverystream.config", + { + title: "Configuration for the new pocket new tab", + getValue: ({ geo, locale }) => { + return JSON.stringify({ + api_key_pref: "extensions.pocket.oAuthConsumerKey", + collapsible: true, + enabled: true, + }); + }, + }, + ], + [ + "discoverystream.endpoints", + { + title: + "Endpoint prefixes (comma-separated) that are allowed to be requested", + value: + "https://getpocket.cdn.mozilla.net/,https://firefox-api-proxy.cdn.mozilla.net/,https://spocs.getpocket.com/", + }, + ], + [ + "discoverystream.isCollectionDismissible", + { + title: "Allows Pocket story collections to be dismissed", + value: false, + }, + ], + [ + "discoverystream.onboardingExperience.dismissed", + { + title: "Allows the user to dismiss the new Pocket onboarding experience", + skipBroadcast: true, + alsoToPreloaded: true, + value: false, + }, + ], + [ + "discoverystream.region-basic-layout", + { + title: "Decision to use basic layout based on region.", + getValue: ({ geo }) => { + const preffedRegionsString = + Services.prefs.getStringPref(REGION_BASIC_CONFIG) || ""; + // If no regions are set to basic, + // we don't need to bother checking against the region. + // We are also not concerned if geo is not set, + // because stories are going to be empty until we have geo. + if (!preffedRegionsString) { + return false; + } + const preffedRegions = preffedRegionsString + .split(",") + .map(s => s.trim()); + + return preffedRegions.includes(geo); + }, + }, + ], + [ + "discoverystream.spoc.impressions", + { + title: "Track spoc impressions", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "discoverystream.endpointSpocsClear", + { + title: + "Endpoint for when a user opts-out of sponsored content to delete the user's data from the ad server.", + value: "https://spocs.getpocket.com/user", + }, + ], + [ + "discoverystream.rec.impressions", + { + title: "Track rec impressions", + skipBroadcast: true, + value: "{}", + }, + ], + [ + "showRecentSaves", + { + title: "Control whether a user wants recent saves visible on Newtab", + value: true, + }, + ], +]); + +// Array of each feed's FEEDS_CONFIG factory and values to add to PREFS_CONFIG +const FEEDS_DATA = [ + { + name: "aboutpreferences", + factory: () => new lazy.AboutPreferences(), + title: "about:preferences rendering", + value: true, + }, + { + name: "newtabinit", + factory: () => new lazy.NewTabInit(), + title: "Sends a copy of the state to each new tab that is opened", + value: true, + }, + { + name: "places", + factory: () => new lazy.PlacesFeed(), + title: "Listens for and relays various Places-related events", + value: true, + }, + { + name: "prefs", + factory: () => new lazy.PrefsFeed(PREFS_CONFIG), + title: "Preferences", + value: true, + }, + { + name: "sections", + factory: () => new lazy.SectionsFeed(), + title: "Manages sections", + value: true, + }, + { + name: "section.highlights", + factory: () => new lazy.HighlightsFeed(), + title: "Fetches content recommendations from places db", + value: false, + }, + { + name: "system.topstories", + factory: () => + new lazy.TopStoriesFeed(PREFS_CONFIG.get("discoverystream.config")), + title: + "System pref that fetches content recommendations from a configurable content provider", + // Dynamically determine if Pocket should be shown for a geo / locale + getValue: ({ geo, locale }) => { + // If we don't have geo, we don't want to flash the screen with stories while geo loads. + // Best to display nothing until geo is ready. + if (!geo) { + return false; + } + const preffedRegionsBlockString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionStoriesBlock") || + ""; + const preffedRegionsString = + lazy.NimbusFeatures.pocketNewtab.getVariable("regionStoriesConfig") || + ""; + const preffedLocaleListString = + lazy.NimbusFeatures.pocketNewtab.getVariable("localeListConfig") || ""; + const preffedBlockRegions = preffedRegionsBlockString + .split(",") + .map(s => s.trim()); + const preffedRegions = preffedRegionsString.split(",").map(s => s.trim()); + const preffedLocales = preffedLocaleListString + .split(",") + .map(s => s.trim()); + const locales = { + US: ["en-CA", "en-GB", "en-US"], + CA: ["en-CA", "en-GB", "en-US"], + GB: ["en-CA", "en-GB", "en-US"], + AU: ["en-CA", "en-GB", "en-US"], + NZ: ["en-CA", "en-GB", "en-US"], + IN: ["en-CA", "en-GB", "en-US"], + IE: ["en-CA", "en-GB", "en-US"], + ZA: ["en-CA", "en-GB", "en-US"], + CH: ["de"], + BE: ["de"], + DE: ["de"], + AT: ["de"], + IT: ["it"], + FR: ["fr"], + ES: ["es-ES"], + PL: ["pl"], + JP: ["ja", "ja-JP-mac"], + }[geo]; + + const regionBlocked = preffedBlockRegions.includes(geo); + const localeEnabled = locale && preffedLocales.includes(locale); + const regionEnabled = + preffedRegions.includes(geo) && !!locales && locales.includes(locale); + return !regionBlocked && (localeEnabled || regionEnabled); + }, + }, + { + name: "systemtick", + factory: () => new lazy.SystemTickFeed(), + title: "Produces system tick events to periodically check for data expiry", + value: true, + }, + { + name: "telemetry", + factory: () => new lazy.TelemetryFeed(), + title: "Relays telemetry-related actions to PingCentre", + value: true, + }, + { + name: "favicon", + factory: () => new lazy.FaviconFeed(), + title: "Fetches tippy top manifests from remote service", + value: true, + }, + { + name: "system.topsites", + factory: () => new lazy.TopSitesFeed(), + title: "Queries places and gets metadata for Top Sites section", + value: true, + }, + { + name: "recommendationprovider", + factory: () => new lazy.RecommendationProvider(), + title: "Handles setup and interaction for the personality provider", + value: true, + }, + { + name: "discoverystreamfeed", + factory: () => new lazy.DiscoveryStreamFeed(), + title: "Handles new pocket ui for the new tab page", + value: true, + }, +]; + +const FEEDS_CONFIG = new Map(); +for (const config of FEEDS_DATA) { + const pref = `feeds.${config.name}`; + FEEDS_CONFIG.set(pref, config.factory); + PREFS_CONFIG.set(pref, config); +} + +export class ActivityStream { + /** + * constructor - Initializes an instance of ActivityStream + */ + constructor() { + this.initialized = false; + this.store = new lazy.Store(); + this.feeds = FEEDS_CONFIG; + this._defaultPrefs = new lazy.DefaultPrefs(PREFS_CONFIG); + } + + init() { + try { + this._updateDynamicPrefs(); + this._defaultPrefs.init(); + Services.obs.addObserver(this, "intl:app-locales-changed"); + + // Look for outdated user pref values that might have been accidentally + // persisted when restoring the original pref value at the end of an + // experiment across versions with a different default value. + const DS_CONFIG = + "browser.newtabpage.activity-stream.discoverystream.config"; + if ( + Services.prefs.prefHasUserValue(DS_CONFIG) && + [ + // Firefox 66 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.com/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + // Firefox 67 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","enabled":false,"show_spocs":true,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + // Firefox 68 + `{"api_key_pref":"extensions.pocket.oAuthConsumerKey","collapsible":true,"enabled":false,"show_spocs":true,"hardcoded_layout":true,"personalized":false,"layout_endpoint":"https://getpocket.cdn.mozilla.net/v3/newtab/layout?version=1&consumer_key=$apiKey&layout_variant=basic"}`, + ].includes(Services.prefs.getStringPref(DS_CONFIG)) + ) { + Services.prefs.clearUserPref(DS_CONFIG); + } + + // Hook up the store and let all feeds and pages initialize + this.store.init( + this.feeds, + ac.BroadcastToContent({ + type: at.INIT, + data: { + locale: this.locale, + }, + meta: { + isStartup: true, + }, + }), + { type: at.UNINIT } + ); + + this.initialized = true; + } catch (e) { + // TelemetryFeed could be unavailable if the telemetry is disabled, or + // the telemetry feed is not yet initialized. + const telemetryFeed = this.store.feeds.get("feeds.telemetry"); + if (telemetryFeed) { + telemetryFeed.handleUndesiredEvent({ + data: { event: "ADDON_INIT_FAILED" }, + }); + } + throw e; + } + } + + /** + * Check if an old pref has a custom value to migrate. Clears the pref so that + * it's the default after migrating (to avoid future need to migrate). + * + * @param oldPrefName {string} Pref to check and migrate + * @param cbIfNotDefault {function} Callback that gets the current pref value + */ + _migratePref(oldPrefName, cbIfNotDefault) { + // Nothing to do if the user doesn't have a custom value + if (!Services.prefs.prefHasUserValue(oldPrefName)) { + return; + } + + // Figure out what kind of pref getter to use + let prefGetter; + switch (Services.prefs.getPrefType(oldPrefName)) { + case Services.prefs.PREF_BOOL: + prefGetter = "getBoolPref"; + break; + case Services.prefs.PREF_INT: + prefGetter = "getIntPref"; + break; + case Services.prefs.PREF_STRING: + prefGetter = "getStringPref"; + break; + } + + // Give the callback the current value then clear the pref + cbIfNotDefault(Services.prefs[prefGetter](oldPrefName)); + Services.prefs.clearUserPref(oldPrefName); + } + + uninit() { + if (this.geo === "") { + Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC); + } + + Services.obs.removeObserver(this, "intl:app-locales-changed"); + + this.store.uninit(); + this.initialized = false; + } + + _updateDynamicPrefs() { + // Save the geo pref if we have it + if (lazy.Region.home) { + this.geo = lazy.Region.home; + } else if (this.geo !== "") { + // Watch for geo changes and use a dummy value for now + Services.obs.addObserver(this, lazy.Region.REGION_TOPIC); + this.geo = ""; + } + + this.locale = Services.locale.appLocaleAsBCP47; + + // Update the pref config of those with dynamic values + for (const pref of PREFS_CONFIG.keys()) { + // Only need to process dynamic prefs + const prefConfig = PREFS_CONFIG.get(pref); + if (!prefConfig.getValue) { + continue; + } + + // Have the dynamic pref just reuse using existing default, e.g., those + // set via Autoconfig or policy + try { + const existingDefault = this._defaultPrefs.get(pref); + if (existingDefault !== undefined && prefConfig.value === undefined) { + prefConfig.getValue = () => existingDefault; + } + } catch (ex) { + // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing + // default branch to believe there's a type) but no actual default value + } + + // Compute the dynamic value (potentially generic based on dummy geo) + const newValue = prefConfig.getValue({ + geo: this.geo, + locale: this.locale, + }); + + // If there's an existing value and it has changed, that means we need to + // overwrite the default with the new value. + if (prefConfig.value !== undefined && prefConfig.value !== newValue) { + this._defaultPrefs.set(pref, newValue); + } + + prefConfig.value = newValue; + } + } + + observe(subject, topic, data) { + switch (topic) { + case "intl:app-locales-changed": + case lazy.Region.REGION_TOPIC: + this._updateDynamicPrefs(); + break; + } + } +} diff --git a/browser/components/newtab/lib/ActivityStreamMessageChannel.sys.mjs b/browser/components/newtab/lib/ActivityStreamMessageChannel.sys.mjs new file mode 100644 index 0000000000..de9d2cb800 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamMessageChannel.sys.mjs @@ -0,0 +1,333 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AboutHomeStartupCache: "resource:///modules/BrowserGlue.sys.mjs", + AboutNewTabParent: "resource:///actors/AboutNewTabParent.sys.mjs", +}); + +import { + actionCreators as ac, + actionTypes as at, + actionUtils as au, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +const ABOUT_NEW_TAB_URL = "about:newtab"; + +export const DEFAULT_OPTIONS = { + dispatch(action) { + throw new Error( + `\nMessageChannel: Received action ${action.type}, but no dispatcher was defined.\n` + ); + }, + pageURL: ABOUT_NEW_TAB_URL, + outgoingMessageName: "ActivityStream:MainToContent", + incomingMessageName: "ActivityStream:ContentToMain", +}; + +export class ActivityStreamMessageChannel { + /** + * ActivityStreamMessageChannel - This module connects a Redux store to the new tab page actor. + * You should use the BroadcastToContent, AlsoToOneContent, and AlsoToMain action creators + * in common/Actions.sys.mjs to help you create actions that will be automatically routed + * to the correct location. + * + * @param {object} options + * @param {function} options.dispatch The dispatch method from a Redux store + * @param {string} options.pageURL The URL to which the channel is attached, such as about:newtab. + * @param {string} options.outgoingMessageName The name of the message sent to child processes + * @param {string} options.incomingMessageName The name of the message received from child processes + * @return {ActivityStreamMessageChannel} + */ + constructor(options = {}) { + Object.assign(this, DEFAULT_OPTIONS, options); + + this.middleware = this.middleware.bind(this); + this.onMessage = this.onMessage.bind(this); + this.onNewTabLoad = this.onNewTabLoad.bind(this); + this.onNewTabUnload = this.onNewTabUnload.bind(this); + this.onNewTabInit = this.onNewTabInit.bind(this); + } + + /** + * Get an iterator over the loaded tab objects. + */ + get loadedTabs() { + // In the test, AboutNewTabParent is not defined. + return lazy.AboutNewTabParent?.loadedTabs || new Map(); + } + + /** + * middleware - Redux middleware that looks for AlsoToOneContent and BroadcastToContent type + * actions, and sends them out. + * + * @param {object} store A redux store + * @return {function} Redux middleware + */ + middleware(store) { + return next => action => { + const skipMain = action.meta && action.meta.skipMain; + if (au.isSendToOneContent(action)) { + this.send(action); + } else if (au.isBroadcastToContent(action)) { + this.broadcast(action); + } else if (au.isSendToPreloaded(action)) { + this.sendToPreloaded(action); + } + + if (!skipMain) { + next(action); + } + }; + } + + /** + * onActionFromContent - Handler for actions from a content processes + * + * @param {object} action A Redux action + * @param {string} targetId The portID of the port that sent the message + */ + onActionFromContent(action, targetId) { + this.dispatch(ac.AlsoToMain(action, this.validatePortID(targetId))); + } + + /** + * broadcast - Sends an action to all ports + * + * @param {object} action A Redux action + */ + broadcast(action) { + // We're trying to update all tabs, so signal the AboutHomeStartupCache + // that its likely time to refresh the cache. + lazy.AboutHomeStartupCache.onPreloadedNewTabMessage(); + + for (let { actor } of this.loadedTabs.values()) { + try { + actor.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The target page is closed/closing by the user or test, so just ignore. + } + } + } + + /** + * send - Sends an action to a specific port + * + * @param {obj} action A redux action; it should contain a portID in the meta.toTarget property + */ + send(action) { + const targetId = action.meta && action.meta.toTarget; + const target = this.getTargetById(targetId); + try { + target.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The target page is closed/closing by the user or test, so just ignore. + } + } + + /** + * A valid portID is a combination of process id and a port number. + * It is generated in AboutNewTabChild.sys.mjs. + */ + validatePortID(id) { + if (typeof id !== "string" || !id.includes(":")) { + console.error("Invalid portID"); + } + + return id; + } + + /** + * getTargetById - Retrieve the message target by portID, if it exists + * + * @param {string} id A portID + * @return {obj|null} The message target, if it exists. + */ + getTargetById(id) { + this.validatePortID(id); + + for (let { portID, actor } of this.loadedTabs.values()) { + if (portID === id) { + return actor; + } + } + return null; + } + + /** + * sendToPreloaded - Sends an action to each preloaded browser, if any + * + * @param {obj} action A redux action + */ + sendToPreloaded(action) { + // We're trying to update the preloaded about:newtab, so signal + // the AboutHomeStartupCache that its likely time to refresh + // the cache. + lazy.AboutHomeStartupCache.onPreloadedNewTabMessage(); + + const preloadedActors = this.getPreloadedActors(); + if (preloadedActors && action.data) { + for (let preloadedActor of preloadedActors) { + try { + preloadedActor.sendAsyncMessage(this.outgoingMessageName, action); + } catch (e) { + // The preloaded page is no longer available, so just ignore. + } + } + } + } + + /** + * getPreloadedActors - Retrieve the preloaded actors + * + * @return {Array|null} An array of actors belonging to the preloaded browsers, or null + * if there aren't any preloaded browsers + */ + getPreloadedActors() { + let preloadedActors = []; + for (let { actor, browser } of this.loadedTabs.values()) { + if (this.isPreloadedBrowser(browser)) { + preloadedActors.push(actor); + } + } + return preloadedActors.length ? preloadedActors : null; + } + + /** + * isPreloadedBrowser - Returns true if the passed browser has been preloaded + * for faster rendering of new tabs. + * + * @param {<browser>} A <browser> to check. + * @return {bool} True if the browser is preloaded. + * if there aren't any preloaded browsers + */ + isPreloadedBrowser(browser) { + return browser.getAttribute("preloadedState") === "preloaded"; + } + + simulateMessagesForExistingTabs() { + // Some pages might have already loaded, so we won't get the usual message + for (const loadedTab of this.loadedTabs.values()) { + let simulatedDetails = { + actor: loadedTab.actor, + browser: loadedTab.browser, + browsingContext: loadedTab.browsingContext, + portID: loadedTab.portID, + url: loadedTab.url, + simulated: true, + }; + + this.onActionFromContent( + { + type: at.NEW_TAB_INIT, + data: simulatedDetails, + }, + loadedTab.portID + ); + + if (loadedTab.loaded) { + this.tabLoaded(simulatedDetails); + } + } + + // It's possible that those existing tabs had sent some messages up + // to us before the feeds / ActivityStreamMessageChannel was ready. + // + // AboutNewTabParent takes care of queueing those for us, so + // now that we're ready, we can flush these queued messages. + lazy.AboutNewTabParent.flushQueuedMessagesFromContent(); + } + + /** + * onNewTabInit - Handler for special RemotePage:Init message fired + * on initialization. + * + * @param {obj} msg The messsage from a page that was just initialized + * @param {obj} tabDetails details about a loaded tab + * + * tabDetails contains: + * actor, browser, browsingContext, portID, url + */ + onNewTabInit(msg, tabDetails) { + this.onActionFromContent( + { + type: at.NEW_TAB_INIT, + data: tabDetails, + }, + msg.data.portID + ); + } + + /** + * onNewTabLoad - Handler for special RemotePage:Load message fired on page load. + * + * @param {obj} msg The messsage from a page that was just loaded + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onNewTabLoad(msg, tabDetails) { + this.tabLoaded(tabDetails); + } + + tabLoaded(tabDetails) { + tabDetails.loaded = true; + + let { browser } = tabDetails; + if ( + this.isPreloadedBrowser(browser) && + browser.ownerGlobal.windowState !== browser.ownerGlobal.STATE_MINIMIZED && + !browser.ownerGlobal.isFullyOccluded + ) { + // As a perceived performance optimization, if this loaded Activity Stream + // happens to be a preloaded browser in a window that is not minimized or + // occluded, have it render its layers to the compositor now to increase + // the odds that by the time we switch to the tab, the layers are already + // ready to present to the user. + browser.renderLayers = true; + } + + this.onActionFromContent({ type: at.NEW_TAB_LOAD }, tabDetails.portID); + } + + /** + * onNewTabUnloadLoad - Handler for special RemotePage:Unload message fired + * on page unload. + * + * @param {obj} msg The messsage from a page that was just unloaded + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onNewTabUnload(msg, tabDetails) { + this.onActionFromContent({ type: at.NEW_TAB_UNLOAD }, tabDetails.portID); + } + + /** + * onMessage - Handles custom messages from content. It expects all messages to + * be formatted as Redux actions, and dispatches them to this.store + * + * @param {obj} msg A custom message from content + * @param {obj} msg.action A Redux action (e.g. {type: "HELLO_WORLD"}) + * @param {obj} msg.target A message target + * @param {obj} tabDetails details about a loaded tab, similar to onNewTabInit + */ + onMessage(msg, tabDetails) { + if (!msg.data || !msg.data.type) { + console.error( + new Error( + `Received an improperly formatted message from ${tabDetails.portID}` + ) + ); + return; + } + let action = {}; + Object.assign(action, msg.data); + // target is used to access a browser reference that came from the content + // and should only be used in feeds (not reducers) + action._target = { + browser: tabDetails.browser, + }; + + this.onActionFromContent(action, tabDetails.portID); + } +} diff --git a/browser/components/newtab/lib/ActivityStreamPrefs.sys.mjs b/browser/components/newtab/lib/ActivityStreamPrefs.sys.mjs new file mode 100644 index 0000000000..192ff30288 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamPrefs.sys.mjs @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// AppConstants, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +// eslint-disable-next-line mozilla/use-static-import +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); + +const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream."; + +export class Prefs extends Preferences { + /** + * Prefs - A wrapper around Preferences that always sets the branch to + * ACTIVITY_STREAM_PREF_BRANCH + */ + constructor(branch = ACTIVITY_STREAM_PREF_BRANCH) { + super({ branch }); + this._branchObservers = new Map(); + } + + ignoreBranch(listener) { + const observer = this._branchObservers.get(listener); + this._prefBranch.removeObserver("", observer); + this._branchObservers.delete(listener); + } + + observeBranch(listener) { + const observer = (subject, topic, pref) => { + listener.onPrefChanged(pref, this.get(pref)); + }; + this._prefBranch.addObserver("", observer); + this._branchObservers.set(listener, observer); + } +} + +export class DefaultPrefs extends Preferences { + /** + * DefaultPrefs - A helper for setting and resetting default prefs for the add-on + * + * @param {Map} config A Map with {string} key of the pref name and {object} + * value with the following pref properties: + * {string} .title (optional) A description of the pref + * {bool|string|number} .value The default value for the pref + * @param {string} branch (optional) The pref branch (defaults to ACTIVITY_STREAM_PREF_BRANCH) + */ + constructor(config, branch = ACTIVITY_STREAM_PREF_BRANCH) { + super({ + branch, + defaultBranch: true, + }); + this._config = config; + } + + /** + * init - Set default prefs for all prefs in the config + */ + init() { + // Local developer builds (with the default mozconfig) aren't OFFICIAL + const IS_UNOFFICIAL_BUILD = !AppConstants.MOZILLA_OFFICIAL; + + for (const pref of this._config.keys()) { + try { + // Avoid replacing existing valid default pref values, e.g., those set + // via Autoconfig or policy + if (this.get(pref) !== undefined) { + continue; + } + } catch (ex) { + // We get NS_ERROR_UNEXPECTED for prefs that have a user value (causing + // default branch to believe there's a type) but no actual default value + } + + const prefConfig = this._config.get(pref); + let value; + if (IS_UNOFFICIAL_BUILD && "value_local_dev" in prefConfig) { + value = prefConfig.value_local_dev; + } else { + value = prefConfig.value; + } + + try { + this.set(pref, value); + } catch (ex) { + // Potentially the user somehow set an unexpected value type, so we fail + // to set a default of our expected type + } + } + } +} diff --git a/browser/components/newtab/lib/ActivityStreamStorage.sys.mjs b/browser/components/newtab/lib/ActivityStreamStorage.sys.mjs new file mode 100644 index 0000000000..1e128ec3f2 --- /dev/null +++ b/browser/components/newtab/lib/ActivityStreamStorage.sys.mjs @@ -0,0 +1,119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + IndexedDB: "resource://gre/modules/IndexedDB.sys.mjs", +}); + +export class ActivityStreamStorage { + /** + * @param storeNames Array of strings used to create all the required stores + */ + constructor({ storeNames, telemetry }) { + if (!storeNames) { + throw new Error("storeNames required"); + } + + this.dbName = "ActivityStream"; + this.dbVersion = 3; + this.storeNames = storeNames; + this.telemetry = telemetry; + } + + get db() { + return this._db || (this._db = this.createOrOpenDb()); + } + + /** + * Public method that binds the store required by the consumer and exposes + * the private db getters and setters. + * + * @param storeName String name of desired store + */ + getDbTable(storeName) { + if (this.storeNames.includes(storeName)) { + return { + get: this._get.bind(this, storeName), + getAll: this._getAll.bind(this, storeName), + set: this._set.bind(this, storeName), + }; + } + + throw new Error(`Store name ${storeName} does not exist.`); + } + + async _getStore(storeName) { + return (await this.db).objectStore(storeName, "readwrite"); + } + + _get(storeName, key) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).get(key) + ); + } + + _getAll(storeName) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).getAll() + ); + } + + _set(storeName, key, value) { + return this._requestWrapper(async () => + (await this._getStore(storeName)).put(value, key) + ); + } + + _openDatabase() { + return lazy.IndexedDB.open(this.dbName, { version: this.dbVersion }, db => { + // If provided with array of objectStore names we need to create all the + // individual stores + this.storeNames.forEach(store => { + if (!db.objectStoreNames.contains(store)) { + this._requestWrapper(() => db.createObjectStore(store)); + } + }); + }); + } + + /** + * createOrOpenDb - Open a db (with this.dbName) if it exists. + * If it does not exist, create it. + * If an error occurs, deleted the db and attempt to + * re-create it. + * @returns Promise that resolves with a db instance + */ + async createOrOpenDb() { + try { + const db = await this._openDatabase(); + return db; + } catch (e) { + if (this.telemetry) { + this.telemetry.handleUndesiredEvent({ event: "INDEXEDDB_OPEN_FAILED" }); + } + await lazy.IndexedDB.deleteDatabase(this.dbName); + return this._openDatabase(); + } + } + + async _requestWrapper(request) { + let result = null; + try { + result = await request(); + } catch (e) { + if (this.telemetry) { + this.telemetry.handleUndesiredEvent({ event: "TRANSACTION_FAILED" }); + } + throw e; + } + + return result; + } +} + +export function getDefaultOptions(options) { + return { collapsed: !!options.collapsed }; +} diff --git a/browser/components/newtab/lib/DefaultSites.sys.mjs b/browser/components/newtab/lib/DefaultSites.sys.mjs new file mode 100644 index 0000000000..ea49cccc03 --- /dev/null +++ b/browser/components/newtab/lib/DefaultSites.sys.mjs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const DEFAULT_SITES_MAP = new Map([ + // This first item is the global list fallback for any unexpected geos + [ + "", + "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.reddit.com/,https://www.amazon.com/,https://twitter.com/", + ], + [ + "US", + "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://twitter.com/", + ], + [ + "CA", + "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.wikipedia.org/,https://www.amazon.ca/,https://twitter.com/", + ], + [ + "DE", + "https://www.youtube.com/,https://www.facebook.com/,https://www.amazon.de/,https://www.ebay.de/,https://www.wikipedia.org/,https://www.reddit.com/", + ], + [ + "PL", + "https://www.youtube.com/,https://www.facebook.com/,https://allegro.pl/,https://www.wikipedia.org/,https://www.olx.pl/,https://www.wykop.pl/", + ], + [ + "RU", + "https://vk.com/,https://www.youtube.com/,https://ok.ru/,https://www.avito.ru/,https://www.aliexpress.com/,https://www.wikipedia.org/", + ], + [ + "GB", + "https://www.youtube.com/,https://www.facebook.com/,https://www.reddit.com/,https://www.amazon.co.uk/,https://www.bbc.co.uk/,https://www.ebay.co.uk/", + ], + [ + "FR", + "https://www.youtube.com/,https://www.facebook.com/,https://www.wikipedia.org/,https://www.amazon.fr/,https://www.leboncoin.fr/,https://twitter.com/", + ], + [ + "CN", + "https://www.baidu.com/,https://www.zhihu.com/,https://www.ifeng.com/,https://weibo.com/,https://www.ctrip.com/,https://www.iqiyi.com/", + ], +]); + +// Immutable for export. +export const DEFAULT_SITES = Object.freeze(DEFAULT_SITES_MAP); diff --git a/browser/components/newtab/lib/DiscoveryStreamFeed.sys.mjs b/browser/components/newtab/lib/DiscoveryStreamFeed.sys.mjs new file mode 100644 index 0000000000..257036b9da --- /dev/null +++ b/browser/components/newtab/lib/DiscoveryStreamFeed.sys.mjs @@ -0,0 +1,2265 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", + PersistentCache: "resource://activity-stream/lib/PersistentCache.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", +}); + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// setTimeout / clearTimeout, and overrides importESModule +// to be a no-op (which can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { setTimeout, clearTimeout } = ChromeUtils.importESModule( + "resource://gre/modules/Timer.sys.mjs" +); +import { + actionTypes as at, + actionCreators as ac, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +const CACHE_KEY = "discovery_stream"; +const STARTUP_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week +const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour +const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server +const FETCH_TIMEOUT = 45 * 1000; +const SPOCS_URL = "https://spocs.getpocket.com/spocs"; +const FEED_URL = + "https://getpocket.cdn.mozilla.net/v3/firefox/global-recs?version=3&consumer_key=$apiKey&locale_lang=$locale®ion=$region&count=30"; +const PREF_CONFIG = "discoverystream.config"; +const PREF_ENDPOINTS = "discoverystream.endpoints"; +const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId"; +const PREF_ENABLED = "discoverystream.enabled"; +const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout"; +const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint"; +const PREF_SPOCS_ENDPOINT_QUERY = "discoverystream.spocs-endpoint-query"; +const PREF_REGION_BASIC_LAYOUT = "discoverystream.region-basic-layout"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories"; +const PREF_USER_TOPSITES = "feeds.topsites"; +const PREF_SYSTEM_TOPSITES = "feeds.system.topsites"; +const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear"; +const PREF_SHOW_SPONSORED = "showSponsored"; +const PREF_SYSTEM_SHOW_SPONSORED = "system.showSponsored"; +const PREF_SHOW_SPONSORED_TOPSITES = "showSponsoredTopSites"; +// Nimbus variable to enable the SOV feature for sponsored tiles. +const NIMBUS_VARIABLE_CONTILE_SOV_ENABLED = "topSitesContileSovEnabled"; +const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions"; +const PREF_FLIGHT_BLOCKS = "discoverystream.flight.blocks"; +const PREF_REC_IMPRESSIONS = "discoverystream.rec.impressions"; +const PREF_COLLECTIONS_ENABLED = + "discoverystream.sponsored-collections.enabled"; +const PREF_POCKET_BUTTON = "extensions.pocket.enabled"; +const PREF_COLLECTION_DISMISSIBLE = "discoverystream.isCollectionDismissible"; + +let getHardcodedLayout; + +export class DiscoveryStreamFeed { + constructor() { + // Internal state for checking if we've intialized all our data + this.loaded = false; + + // Persistent cache for remote endpoint data. + this.cache = new lazy.PersistentCache(CACHE_KEY, true); + this.locale = Services.locale.appLocaleAsBCP47; + this._impressionId = this.getOrCreateImpressionId(); + // Internal in-memory cache for parsing json prefs. + this._prefCache = {}; + } + + getOrCreateImpressionId() { + let impressionId = Services.prefs.getCharPref(PREF_IMPRESSION_ID, ""); + if (!impressionId) { + impressionId = String(Services.uuid.generateUUID()); + Services.prefs.setCharPref(PREF_IMPRESSION_ID, impressionId); + } + return impressionId; + } + + get config() { + if (this._prefCache.config) { + return this._prefCache.config; + } + try { + this._prefCache.config = JSON.parse( + this.store.getState().Prefs.values[PREF_CONFIG] + ); + } catch (e) { + // istanbul ignore next + this._prefCache.config = {}; + // istanbul ignore next + console.error( + `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config.`, + e + ); + } + this._prefCache.config.enabled = + this._prefCache.config.enabled && + this.store.getState().Prefs.values[PREF_ENABLED]; + + return this._prefCache.config; + } + + resetConfigDefauts() { + this.store.dispatch({ + type: at.CLEAR_PREF, + data: { + name: PREF_CONFIG, + }, + }); + } + + get region() { + return lazy.Region.home; + } + + get isBff() { + if (this._isBff === undefined) { + const pocketConfig = + this.store.getState().Prefs.values?.pocketConfig || {}; + + const preffedRegionBffConfigString = pocketConfig.regionBffConfig || ""; + const preffedRegionBffConfig = preffedRegionBffConfigString + .split(",") + .map(s => s.trim()); + const regionBff = preffedRegionBffConfig.includes(this.region); + this._isBff = regionBff; + } + + return this._isBff; + } + + get showSpocs() { + // High level overall sponsored check, if one of these is true, + // we know we need some sort of spoc control setup. + return this.showSponsoredStories || this.showSponsoredTopsites; + } + + get showSponsoredStories() { + // Combine user-set sponsored opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SHOW_SPONSORED] && + this.store.getState().Prefs.values[PREF_SYSTEM_SHOW_SPONSORED] + ); + } + + get showSponsoredTopsites() { + const placements = this.getPlacements(); + // Combine user-set sponsored opt-out with placement data + return !!( + this.store.getState().Prefs.values[PREF_SHOW_SPONSORED_TOPSITES] && + placements.find(placement => placement.name === "sponsored-topsites") + ); + } + + get showStories() { + // Combine user-set stories opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] && + this.store.getState().Prefs.values[PREF_USER_TOPSTORIES] + ); + } + + get showTopsites() { + // Combine user-set topsites opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SYSTEM_TOPSITES] && + this.store.getState().Prefs.values[PREF_USER_TOPSITES] + ); + } + + get personalized() { + return this.recommendationProvider.personalized; + } + + get recommendationProvider() { + if (this._recommendationProvider) { + return this._recommendationProvider; + } + this._recommendationProvider = this.store.feeds.get( + "feeds.recommendationprovider" + ); + return this._recommendationProvider; + } + + setupConfig(isStartup = false) { + // Send the initial state of the pref on our reducer + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_SETUP, + data: this.config, + meta: { + isStartup, + }, + }) + ); + } + + setupPrefs(isStartup = false) { + const pocketNewtabExperiment = lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "pocketNewtab", + }); + + const pocketNewtabRollout = lazy.ExperimentAPI.getRolloutMetaData({ + featureId: "pocketNewtab", + }); + + // We want to know if the user is in an experiment or rollout, + // but we prioritize experiments over rollouts. + const experimentMetaData = pocketNewtabExperiment || pocketNewtabRollout; + + let utmSource = "pocket-newtab"; + let utmCampaign = experimentMetaData?.slug; + let utmContent = experimentMetaData?.branch?.slug; + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_EXPERIMENT_DATA, + data: { + utmSource, + utmCampaign, + utmContent, + }, + meta: { + isStartup, + }, + }) + ); + + const pocketButtonEnabled = Services.prefs.getBoolPref(PREF_POCKET_BUTTON); + + const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {}; + const { region } = this.store.getState().Prefs.values; + + this.setupSpocsCacheUpdateTime(); + const saveToPocketCardRegions = nimbusConfig.saveToPocketCardRegions + ?.split(",") + .map(s => s.trim()); + const saveToPocketCard = + pocketButtonEnabled && + (nimbusConfig.saveToPocketCard || + saveToPocketCardRegions?.includes(region)); + + const hideDescriptionsRegions = nimbusConfig.hideDescriptionsRegions + ?.split(",") + .map(s => s.trim()); + const hideDescriptions = + nimbusConfig.hideDescriptions || + hideDescriptionsRegions?.includes(region); + + // We don't BroadcastToContent for this, as the changes may + // shift around elements on an open newtab the user is currently reading. + // So instead we AlsoToPreloaded so the next tab is updated. + // This is because setupPrefs is called by the system and not a user interaction. + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_PREFS_SETUP, + data: { + recentSavesEnabled: nimbusConfig.recentSavesEnabled, + pocketButtonEnabled, + saveToPocketCard, + hideDescriptions, + compactImages: nimbusConfig.compactImages, + imageGradient: nimbusConfig.imageGradient, + newSponsoredLabel: nimbusConfig.newSponsoredLabel, + titleLines: nimbusConfig.titleLines, + descLines: nimbusConfig.descLines, + readTime: nimbusConfig.readTime, + }, + meta: { + isStartup, + }, + }) + ); + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_COLLECTION_DISMISSIBLE_TOGGLE, + data: { + value: + this.store.getState().Prefs.values[PREF_COLLECTION_DISMISSIBLE], + }, + meta: { + isStartup, + }, + }) + ); + } + + async setupPocketState(target) { + let dispatch = action => + this.store.dispatch(ac.OnlyToOneContent(action, target)); + const isUserLoggedIn = lazy.pktApi.isUserLoggedIn(); + dispatch({ + type: at.DISCOVERY_STREAM_POCKET_STATE_SET, + data: { + isUserLoggedIn, + }, + }); + + // If we're not logged in, don't bother fetching recent saves, we're done. + if (isUserLoggedIn) { + let recentSaves = await lazy.pktApi.getRecentSavesCache(); + if (recentSaves) { + // We have cache, so we can use those. + dispatch({ + type: at.DISCOVERY_STREAM_RECENT_SAVES, + data: { + recentSaves, + }, + }); + } else { + // We don't have cache, so fetch fresh stories. + lazy.pktApi.getRecentSaves({ + success(data) { + dispatch({ + type: at.DISCOVERY_STREAM_RECENT_SAVES, + data: { + recentSaves: data, + }, + }); + }, + error(error) {}, + }); + } + } + } + + uninitPrefs() { + // Reset in-memory cache + this._prefCache = {}; + } + + async fetchFromEndpoint(rawEndpoint, options = {}) { + if (!rawEndpoint) { + console.error("Tried to fetch endpoint but none was configured."); + return null; + } + + const apiKeyPref = this.config.api_key_pref; + const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); + + const endpoint = rawEndpoint + .replace("$apiKey", apiKey) + .replace("$locale", this.locale) + .replace("$region", this.region); + + try { + // Make sure the requested endpoint is allowed + const allowed = this.store + .getState() + .Prefs.values[PREF_ENDPOINTS].split(","); + if (!allowed.some(prefix => endpoint.startsWith(prefix))) { + throw new Error(`Not one of allowed prefixes (${allowed})`); + } + + const controller = new AbortController(); + const { signal } = controller; + + const fetchPromise = fetch(endpoint, { + ...options, + credentials: "omit", + signal, + }); + // istanbul ignore next + const timeoutId = setTimeout(() => { + controller.abort(); + }, FETCH_TIMEOUT); + + const response = await fetchPromise; + if (!response.ok) { + throw new Error(`Unexpected status (${response.status})`); + } + clearTimeout(timeoutId); + + return response.json(); + } catch (error) { + console.error(`Failed to fetch ${endpoint}:`, error.message); + } + return null; + } + + get spocsCacheUpdateTime() { + if (this._spocsCacheUpdateTime) { + return this._spocsCacheUpdateTime; + } + this.setupSpocsCacheUpdateTime(); + return this._spocsCacheUpdateTime; + } + + setupSpocsCacheUpdateTime() { + const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {}; + const { spocsCacheTimeout } = nimbusConfig; + const MAX_TIMEOUT = 30; + const MIN_TIMEOUT = 5; + // We do a bit of min max checking the the configured value is between + // 5 and 30 minutes, to protect against unreasonable values. + if ( + spocsCacheTimeout && + spocsCacheTimeout <= MAX_TIMEOUT && + spocsCacheTimeout >= MIN_TIMEOUT + ) { + // This value is in minutes, but we want ms. + this._spocsCacheUpdateTime = spocsCacheTimeout * 60 * 1000; + } else { + // The const is already in ms. + this._spocsCacheUpdateTime = SPOCS_FEEDS_UPDATE_TIME; + } + } + + /** + * Returns true if data in the cache for a particular key has expired or is missing. + * @param {object} cachedData data returned from cache.get() + * @param {string} key a cache key + * @param {string?} url for "feed" only, the URL of the feed. + * @param {boolean} is this check done at initial browser load + */ + isExpired({ cachedData, key, url, isStartup }) { + const { spocs, feeds } = cachedData; + const updateTimePerComponent = { + spocs: this.spocsCacheUpdateTime, + feed: COMPONENT_FEEDS_UPDATE_TIME, + }; + const EXPIRATION_TIME = isStartup + ? STARTUP_CACHE_EXPIRE_TIME + : updateTimePerComponent[key]; + switch (key) { + case "spocs": + return !spocs || !(Date.now() - spocs.lastUpdated < EXPIRATION_TIME); + case "feed": + return ( + !feeds || + !feeds[url] || + !(Date.now() - feeds[url].lastUpdated < EXPIRATION_TIME) + ); + default: + // istanbul ignore next + throw new Error(`${key} is not a valid key`); + } + } + + async _checkExpirationPerComponent() { + const cachedData = (await this.cache.get()) || {}; + const { feeds } = cachedData; + return { + spocs: this.showSpocs && this.isExpired({ cachedData, key: "spocs" }), + feeds: + this.showStories && + (!feeds || + Object.keys(feeds).some(url => + this.isExpired({ cachedData, key: "feed", url }) + )), + }; + } + + /** + * Returns true if any data for the cached endpoints has expired or is missing. + */ + async checkIfAnyCacheExpired() { + const expirationPerComponent = await this._checkExpirationPerComponent(); + return expirationPerComponent.spocs || expirationPerComponent.feeds; + } + + updatePlacements(sendUpdate, layout, isStartup = false) { + const placements = []; + const placementsMap = {}; + for (const row of layout.filter(r => r.components && r.components.length)) { + for (const component of row.components.filter( + c => c.placement && c.spocs + )) { + // If we find a valid placement, we set it to this value. + let placement; + + // We need to check to see if this placement is on or not. + // If this placement has a prefs array, check against that. + if (component.spocs.prefs) { + // Check every pref in the array to see if this placement is turned on. + if ( + component.spocs.prefs.length && + component.spocs.prefs.every( + p => this.store.getState().Prefs.values[p] + ) + ) { + // This placement is on. + placement = component.placement; + } + } else if (this.showSponsoredStories) { + // If we do not have a prefs array, use old check. + // This is because Pocket spocs uses an old non pref method. + placement = component.placement; + } + + // Validate this placement and check for dupes. + if (placement?.name && !placementsMap[placement.name]) { + placementsMap[placement.name] = placement; + placements.push(placement); + } + } + } + + // Update placements data. + // Even if we have no placements, we still want to update it to clear it. + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_PLACEMENTS, + data: { placements }, + meta: { + isStartup, + }, + }); + } + + /** + * Adds a query string to a URL. + * A query can be any string literal accepted by https://developer.mozilla.org/docs/Web/API/URLSearchParams + * Examples: "?foo=1&bar=2", "&foo=1&bar=2", "foo=1&bar=2", "?bar=2" or "bar=2" + */ + addEndpointQuery(url, query) { + if (!query) { + return url; + } + + const urlObject = new URL(url); + const params = new URLSearchParams(query); + + for (let [key, val] of params.entries()) { + urlObject.searchParams.append(key, val); + } + + return urlObject.toString(); + } + + parseGridPositions(csvPositions) { + let gridPositions; + + // Only accept parseable non-negative integers + try { + gridPositions = csvPositions.map(index => { + let parsedInt = parseInt(index, 10); + + if (!isNaN(parsedInt) && parsedInt >= 0) { + return parsedInt; + } + + throw new Error("Bad input"); + }); + } catch (e) { + // Catch spoc positions that are not numbers or negative, and do nothing. + // We have hard coded backup positions. + gridPositions = undefined; + } + + return gridPositions; + } + + generateFeedUrl(isBff) { + if (isBff) { + return `https://${lazy.NimbusFeatures.saveToPocket.getVariable( + "bffApi" + )}/desktop/v1/recommendations?locale=$locale®ion=$region&count=30`; + } + return FEED_URL; + } + + loadLayout(sendUpdate, isStartup) { + let layoutData = {}; + let url = ""; + + const isBasicLayout = + this.config.hardcoded_basic_layout || + this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT] || + this.store.getState().Prefs.values[PREF_REGION_BASIC_LAYOUT]; + + const sponsoredCollectionsEnabled = + this.store.getState().Prefs.values[PREF_COLLECTIONS_ENABLED]; + + const pocketConfig = this.store.getState().Prefs.values?.pocketConfig || {}; + const onboardingExperience = + this.isBff && pocketConfig.onboardingExperience; + const { spocTopsitesPlacementEnabled } = pocketConfig; + + let items = isBasicLayout ? 3 : 21; + if (pocketConfig.fourCardLayout || pocketConfig.hybridLayout) { + items = isBasicLayout ? 4 : 24; + } + + const ctaButtonSponsors = pocketConfig.ctaButtonSponsors + ?.split(",") + .map(s => s.trim().toLowerCase()); + let ctaButtonVariant = ""; + // We specifically against hard coded values, instead of applying whatever is in the pref. + // This is to ensure random class names from a user modified pref doesn't make it into the class list. + if ( + pocketConfig.ctaButtonVariant === "variant-a" || + pocketConfig.ctaButtonVariant === "variant-b" + ) { + ctaButtonVariant = pocketConfig.ctaButtonVariant; + } + + const prepConfArr = arr => { + return arr + ?.split(",") + .filter(item => item) + .map(item => parseInt(item, 10)); + }; + + const spocAdTypes = prepConfArr(pocketConfig.spocAdTypes); + const spocZoneIds = prepConfArr(pocketConfig.spocZoneIds); + const spocTopsitesAdTypes = prepConfArr(pocketConfig.spocTopsitesAdTypes); + const spocTopsitesZoneIds = prepConfArr(pocketConfig.spocTopsitesZoneIds); + const { spocSiteId } = pocketConfig; + let spocPlacementData; + let spocTopsitesPlacementData; + let spocsUrl; + + if (spocAdTypes?.length && spocZoneIds?.length) { + spocPlacementData = { + ad_types: spocAdTypes, + zone_ids: spocZoneIds, + }; + } + + if (spocTopsitesAdTypes?.length && spocTopsitesZoneIds?.length) { + spocTopsitesPlacementData = { + ad_types: spocTopsitesAdTypes, + zone_ids: spocTopsitesZoneIds, + }; + } + + if (spocSiteId) { + const newUrl = new URL(SPOCS_URL); + newUrl.searchParams.set("site", spocSiteId); + spocsUrl = newUrl.href; + } + + let feedUrl = this.generateFeedUrl(this.isBff); + + // Set layout config. + // Changing values in this layout in memory object is unnecessary. + layoutData = getHardcodedLayout({ + spocsUrl, + feedUrl, + items, + sponsoredCollectionsEnabled, + spocPlacementData, + spocTopsitesPlacementEnabled, + spocTopsitesPlacementData, + spocPositions: this.parseGridPositions( + pocketConfig.spocPositions?.split(`,`) + ), + spocTopsitesPositions: this.parseGridPositions( + pocketConfig.spocTopsitesPositions?.split(`,`) + ), + widgetPositions: this.parseGridPositions( + pocketConfig.widgetPositions?.split(`,`) + ), + widgetData: [ + ...(this.locale.startsWith("en-") ? [{ type: "TopicsWidget" }] : []), + ], + hybridLayout: pocketConfig.hybridLayout, + hideCardBackground: pocketConfig.hideCardBackground, + fourCardLayout: pocketConfig.fourCardLayout, + newFooterSection: pocketConfig.newFooterSection, + compactGrid: pocketConfig.compactGrid, + // For now essentialReadsHeader and editorsPicksHeader are English only. + essentialReadsHeader: + this.locale.startsWith("en-") && pocketConfig.essentialReadsHeader, + editorsPicksHeader: + this.locale.startsWith("en-") && pocketConfig.editorsPicksHeader, + onboardingExperience, + // For now button variants are for experimentation and English only. + ctaButtonSponsors: this.locale.startsWith("en-") ? ctaButtonSponsors : [], + ctaButtonVariant: this.locale.startsWith("en-") ? ctaButtonVariant : "", + }); + + sendUpdate({ + type: at.DISCOVERY_STREAM_LAYOUT_UPDATE, + data: layoutData, + meta: { + isStartup, + }, + }); + + if (layoutData.spocs) { + url = + this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] || + this.config.spocs_endpoint || + layoutData.spocs.url; + + const spocsEndpointQuery = + this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT_QUERY]; + + // For QA, testing, or debugging purposes, there may be a query string to add. + url = this.addEndpointQuery(url, spocsEndpointQuery); + + if ( + url && + url !== this.store.getState().DiscoveryStream.spocs.spocs_endpoint + ) { + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT, + data: { + url, + }, + meta: { + isStartup, + }, + }); + this.updatePlacements(sendUpdate, layoutData.layout, isStartup); + } + } + } + + /** + * buildFeedPromise - Adds the promise result to newFeeds and + * pushes a promise to newsFeedsPromises. + * @param {Object} Has both newFeedsPromises (Array) and newFeeds (Object) + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Function} We return a function so we can contain + * the scope for isStartup and the promises object. + * Combines feed results and promises for each component with a feed. + */ + buildFeedPromise( + { newFeedsPromises, newFeeds }, + isStartup = false, + sendUpdate + ) { + return component => { + const { url } = component.feed; + + if (!newFeeds[url]) { + // We initially stub this out so we don't fetch dupes, + // we then fill in with the proper object inside the promise. + newFeeds[url] = {}; + const feedPromise = this.getComponentFeed(url, isStartup); + + feedPromise + .then(feed => { + // If we stored the result of filter in feed cache as it happened, + // I think we could reduce doing this for cache fetches. + // Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606277 + newFeeds[url] = this.filterRecommendations(feed); + sendUpdate({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed: newFeeds[url], + url, + }, + meta: { + isStartup, + }, + }); + }) + .catch( + /* istanbul ignore next */ error => { + console.error( + `Error trying to load component feed ${url}:`, + error + ); + } + ); + newFeedsPromises.push(feedPromise); + } + }; + } + + filterRecommendations(feed) { + if ( + feed && + feed.data && + feed.data.recommendations && + feed.data.recommendations.length + ) { + const { data: recommendations } = this.filterBlocked( + feed.data.recommendations + ); + return { + ...feed, + data: { + ...feed.data, + recommendations, + }, + }; + } + return feed; + } + + /** + * reduceFeedComponents - Filters out components with no feeds, and combines + * all feeds on this component with the feeds from other components. + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Function} We return a function so we can contain the scope for isStartup. + * Reduces feeds into promises and feed data. + */ + reduceFeedComponents(isStartup, sendUpdate) { + return (accumulator, row) => { + row.components + .filter(component => component && component.feed) + .forEach(this.buildFeedPromise(accumulator, isStartup, sendUpdate)); + return accumulator; + }; + } + + /** + * buildFeedPromises - Filters out rows with no components, + * and gets us a promise for each unique feed. + * @param {Object} layout This is the Discovery Stream layout object. + * @param {Boolean} isStartup We have different cache handling for startup. + * @returns {Object} An object with newFeedsPromises (Array) and newFeeds (Object), + * we can Promise.all newFeedsPromises to get completed data in newFeeds. + */ + buildFeedPromises(layout, isStartup, sendUpdate) { + const initialData = { + newFeedsPromises: [], + newFeeds: {}, + }; + return layout + .filter(row => row && row.components) + .reduce(this.reduceFeedComponents(isStartup, sendUpdate), initialData); + } + + async loadComponentFeeds(sendUpdate, isStartup = false) { + const { DiscoveryStream } = this.store.getState(); + + if (!DiscoveryStream || !DiscoveryStream.layout) { + return; + } + + // Reset the flag that indicates whether or not at least one API request + // was issued to fetch the component feed in `getComponentFeed()`. + this.componentFeedFetched = false; + const { newFeedsPromises, newFeeds } = this.buildFeedPromises( + DiscoveryStream.layout, + isStartup, + sendUpdate + ); + + // Each promise has a catch already built in, so no need to catch here. + await Promise.all(newFeedsPromises); + + if (this.componentFeedFetched) { + this.cleanUpTopRecImpressionPref(newFeeds); + } + await this.cache.set("feeds", newFeeds); + sendUpdate({ + type: at.DISCOVERY_STREAM_FEEDS_UPDATE, + meta: { + isStartup, + }, + }); + } + + getPlacements() { + const { placements } = this.store.getState().DiscoveryStream.spocs; + return placements; + } + + // I wonder, can this be better as a reducer? + // See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717 + placementsForEach(callback) { + this.getPlacements().forEach(callback); + } + + // Bug 1567271 introduced meta data on a list of spocs. + // This involved moving the spocs array into an items prop. + // However, old data could still be returned, and cached data might also be old. + // For ths reason, we want to ensure if we don't find an items array, + // we use the previous array placement, and then stub out title and context to empty strings. + // We need to do this *after* both fresh fetches and cached data to reduce repetition. + normalizeSpocsItems(spocs) { + const items = spocs.items || spocs; + const title = spocs.title || ""; + const context = spocs.context || ""; + const sponsor = spocs.sponsor || ""; + // We do not stub sponsored_by_override with an empty string. It is an override, and an empty string + // explicitly means to override the client to display an empty string. + // An empty string is not an no op in this case. Undefined is the proper no op here. + const { sponsored_by_override } = spocs; + // Undefined is fine here. It's optional and only used by collections. + // If we leave it out, you get a collection that cannot be dismissed. + const { flight_id } = spocs; + return { + items, + title, + context, + sponsor, + sponsored_by_override, + ...(flight_id ? { flight_id } : {}), + }; + } + + updateSponsoredCollectionsPref(collectionEnabled = false) { + const currentState = + this.store.getState().Prefs.values[PREF_COLLECTIONS_ENABLED]; + + // If the current state does not match the new state, update the pref. + if (currentState !== collectionEnabled) { + this.store.dispatch( + ac.SetPref(PREF_COLLECTIONS_ENABLED, collectionEnabled) + ); + } + } + + async loadSpocs(sendUpdate, isStartup) { + const cachedData = (await this.cache.get()) || {}; + let spocsState = cachedData.spocs; + let placements = this.getPlacements(); + + if ( + this.showSpocs && + placements?.length && + this.isExpired({ cachedData, key: "spocs", isStartup }) + ) { + // We optimistically set this to true, because if SOV is not ready, we fetch them. + let useTopsitesPlacement = true; + + // If SOV is turned off or not available, we optimistically fetch sponsored topsites. + if ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_SOV_ENABLED + ) + ) { + let { positions, ready } = this.store.getState().TopSites.sov; + if (ready) { + // We don't need to await here, because we don't need it now. + this.cache.set("sov", positions); + } else { + // If SOV is not available, and there is a SOV cache, use it. + positions = cachedData.sov; + } + + if (positions?.length) { + // If SOV is ready and turned on, we can check if we need moz-sales position. + useTopsitesPlacement = positions.some( + allocation => allocation.assignedPartner === "moz-sales" + ); + } + } + + // We can filter out the topsite placement from the fetch. + if (!useTopsitesPlacement) { + placements = placements.filter( + placement => placement.name !== "sponsored-topsites" + ); + } + + if (placements?.length) { + const endpoint = + this.store.getState().DiscoveryStream.spocs.spocs_endpoint; + + const headers = new Headers(); + headers.append("content-type", "application/json"); + + const apiKeyPref = this.config.api_key_pref; + const apiKey = Services.prefs.getCharPref(apiKeyPref, ""); + + const spocsResponse = await this.fetchFromEndpoint(endpoint, { + method: "POST", + headers, + body: JSON.stringify({ + pocket_id: this._impressionId, + version: 2, + consumer_key: apiKey, + ...(placements.length ? { placements } : {}), + }), + }); + + if (spocsResponse) { + spocsState = { + lastUpdated: Date.now(), + spocs: { + ...spocsResponse, + }, + }; + + if (spocsResponse.settings && spocsResponse.settings.feature_flags) { + this.store.dispatch( + ac.OnlyToMain({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_OVERRIDE, + data: { + override: !spocsResponse.settings.feature_flags.spoc_v2, + }, + }) + ); + this.updateSponsoredCollectionsPref( + spocsResponse.settings.feature_flags.collections + ); + } + + const spocsResultPromises = this.getPlacements().map( + async placement => { + const freshSpocs = spocsState.spocs[placement.name]; + + if (!freshSpocs) { + return; + } + + // spocs can be returns as an array, or an object with an items array. + // We want to normalize this so all our spocs have an items array. + // There can also be some meta data for title and context. + // This is mostly because of backwards compat. + const { + items: normalizedSpocsItems, + title, + context, + sponsor, + sponsored_by_override, + } = this.normalizeSpocsItems(freshSpocs); + + if (!normalizedSpocsItems || !normalizedSpocsItems.length) { + // In the case of old data, we still want to ensure we normalize the data structure, + // even if it's empty. We expect the empty data to be an object with items array, + // and not just an empty array. + spocsState.spocs = { + ...spocsState.spocs, + [placement.name]: { + title, + context, + items: [], + }, + }; + return; + } + + // Migrate flight_id + const { data: migratedSpocs } = + this.migrateFlightId(normalizedSpocsItems); + + const { data: capResult } = this.frequencyCapSpocs(migratedSpocs); + + const { data: blockedResults } = this.filterBlocked(capResult); + + const { data: scoredResults, personalized } = + await this.scoreItems(blockedResults, "spocs"); + + spocsState.spocs = { + ...spocsState.spocs, + [placement.name]: { + title, + context, + sponsor, + sponsored_by_override, + personalized, + items: scoredResults, + }, + }; + } + ); + await Promise.all(spocsResultPromises); + + this.cleanUpFlightImpressionPref(spocsState.spocs); + } else { + console.error("No response for spocs_endpoint prop"); + } + } + } + + // Use good data if we have it, otherwise nothing. + // We can have no data if spocs set to off. + // We can have no data if request fails and there is no good cache. + // We want to send an update spocs or not, so client can render something. + spocsState = + spocsState && spocsState.spocs + ? spocsState + : { + lastUpdated: Date.now(), + spocs: {}, + }; + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.spocs, + }); + + sendUpdate({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.spocs, + }, + meta: { + isStartup, + }, + }); + } + + async clearSpocs() { + const endpoint = + this.store.getState().Prefs.values[PREF_SPOCS_CLEAR_ENDPOINT]; + if (!endpoint) { + return; + } + const headers = new Headers(); + headers.append("content-type", "application/json"); + + await this.fetchFromEndpoint(endpoint, { + method: "DELETE", + headers, + body: JSON.stringify({ + pocket_id: this._impressionId, + }), + }); + } + + observe(subject, topic, data) { + switch (topic) { + case "nsPref:changed": + // If the Pocket button was turned on or off, we need to update the cards + // because cards show menu options for the Pocket button that need to be removed. + if (data === PREF_POCKET_BUTTON) { + this.configReset(); + } + break; + } + } + + /* + * This function is used to sort any type of story, both spocs and recs. + * This uses hierarchical sorting, first sorting by priority, then by score within a priority. + * This function could be sorting an array of spocs or an array of recs. + * A rec would have priority undefined, and a spoc would probably have a priority set. + * Priority is sorted ascending, so low numbers are the highest priority. + * Score is sorted descending, so high numbers are the highest score. + * Undefined priority values are considered the lowest priority. + * A negative priority is considered the same as undefined, lowest priority. + * A negative priority is unlikely and not currently supported or expected. + * A negative score is a possible use case. + */ + sortItem(a, b) { + // If the priorities are the same, sort based on score. + // If both item priorities are undefined, + // we can safely sort via score. + if (a.priority === b.priority) { + return b.score - a.score; + } else if (!a.priority || a.priority <= 0) { + // If priority is undefined or an unexpected value, + // consider it lowest priority. + return 1; + } else if (!b.priority || b.priority <= 0) { + // Also consider this case lowest priority. + return -1; + } + // Our primary sort for items with priority. + return a.priority - b.priority; + } + + async scoreItems(items, type) { + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + const personalizedByType = + type === "feed" ? recsPersonalized : spocsPersonalized; + // If this is initialized, we are ready to go. + const personalized = this.store.getState().Personalization.initialized; + + const data = ( + await Promise.all( + items.map(item => this.scoreItem(item, personalizedByType)) + ) + ) + // Sort by highest scores. + .sort(this.sortItem); + + return { data, personalized }; + } + + async scoreItem(item, personalizedByType) { + item.score = item.item_score; + if (item.score !== 0 && !item.score) { + item.score = 1; + } + if (this.personalized && personalizedByType) { + await this.recommendationProvider.calculateItemRelevanceScore(item); + } + return item; + } + + filterBlocked(data) { + if (data && data.length) { + let flights = this.readDataPref(PREF_FLIGHT_BLOCKS); + const filteredItems = data.filter(item => { + const blocked = + lazy.NewTabUtils.blockedLinks.isBlocked({ url: item.url }) || + flights[item.flight_id]; + return !blocked; + }); + return { data: filteredItems }; + } + return { data }; + } + + // For backwards compatibility, older spoc endpoint don't have flight_id, + // but instead had campaign_id we can use + // + // @param {Object} data An object that might have a SPOCS array. + // @returns {Object} An object with a property `data` as the result. + migrateFlightId(spocs) { + if (spocs && spocs.length) { + return { + data: spocs.map(s => { + return { + ...s, + ...(s.flight_id || s.campaign_id + ? { + flight_id: s.flight_id || s.campaign_id, + } + : {}), + ...(s.caps + ? { + caps: { + ...s.caps, + flight: s.caps.flight || s.caps.campaign, + }, + } + : {}), + }; + }), + }; + } + return { data: spocs }; + } + + // Filter spocs based on frequency caps + // + // @param {Object} data An object that might have a SPOCS array. + // @returns {Object} An object with a property `data` as the result, and a property + // `filterItems` as the frequency capped items. + frequencyCapSpocs(spocs) { + if (spocs && spocs.length) { + const impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); + const caps = []; + const result = spocs.filter(s => { + const isBelow = this.isBelowFrequencyCap(impressions, s); + if (!isBelow) { + caps.push(s); + } + return isBelow; + }); + // send caps to redux if any. + if (caps.length) { + this.store.dispatch({ + type: at.DISCOVERY_STREAM_SPOCS_CAPS, + data: caps, + }); + } + return { data: result, filtered: caps }; + } + return { data: spocs, filtered: [] }; + } + + // Frequency caps are based on flight, which may include multiple spocs. + // We currently support two types of frequency caps: + // - lifetime: Indicates how many times spocs from a flight can be shown in total + // - period: Indicates how many times spocs from a flight can be shown within a period + // + // So, for example, the feed configuration below defines that for flight 1 no more + // than 5 spocs can be shown in total, and no more than 2 per hour. + // "flight_id": 1, + // "caps": { + // "lifetime": 5, + // "flight": { + // "count": 2, + // "period": 3600 + // } + // } + isBelowFrequencyCap(impressions, spoc) { + const flightImpressions = impressions[spoc.flight_id]; + if (!flightImpressions) { + return true; + } + + const lifetime = spoc.caps && spoc.caps.lifetime; + + const lifeTimeCap = Math.min( + lifetime || MAX_LIFETIME_CAP, + MAX_LIFETIME_CAP + ); + const lifeTimeCapExceeded = flightImpressions.length >= lifeTimeCap; + if (lifeTimeCapExceeded) { + return false; + } + + const flightCap = spoc.caps && spoc.caps.flight; + if (flightCap) { + const flightCapExceeded = + flightImpressions.filter(i => Date.now() - i < flightCap.period * 1000) + .length >= flightCap.count; + return !flightCapExceeded; + } + return true; + } + + async retryFeed(feed) { + const { url } = feed; + const result = await this.getComponentFeed(url); + const newFeed = this.filterRecommendations(result); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed: newFeed, + url, + }, + }) + ); + } + + async getComponentFeed(feedUrl, isStartup) { + const cachedData = (await this.cache.get()) || {}; + const { feeds } = cachedData; + + let feed = feeds ? feeds[feedUrl] : null; + if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) { + let options = {}; + if (this.isBff) { + const headers = new Headers(); + const oAuthConsumerKey = lazy.NimbusFeatures.saveToPocket.getVariable( + "oAuthConsumerKeyBff" + ); + headers.append("consumer_key", oAuthConsumerKey); + options = { + method: "GET", + headers, + }; + } + + const feedResponse = await this.fetchFromEndpoint(feedUrl, options); + if (feedResponse) { + const { settings = {} } = feedResponse; + let { recommendations } = feedResponse; + if (this.isBff) { + recommendations = feedResponse.data.map(item => ({ + id: item.tileId, + url: item.url, + title: item.title, + excerpt: item.excerpt, + publisher: item.publisher, + time_to_read: item.timeToRead, + raw_image_src: item.imageUrl, + recommendation_id: item.recommendationId, + })); + } + const { data: scoredItems, personalized } = await this.scoreItems( + recommendations, + "feed" + ); + const { recsExpireTime } = settings; + const rotatedItems = this.rotate(scoredItems, recsExpireTime); + this.componentFeedFetched = true; + feed = { + lastUpdated: Date.now(), + personalized, + data: { + settings, + recommendations: rotatedItems, + status: "success", + }, + }; + } else { + console.error("No response for feed"); + } + } + + // If we have no feed at this point, both fetch and cache failed for some reason. + return ( + feed || { + data: { + status: "failed", + }, + } + ); + } + + /** + * Called at startup to update cached data in the background. + */ + async _maybeUpdateCachedData() { + const expirationPerComponent = await this._checkExpirationPerComponent(); + // Pass in `store.dispatch` to send the updates only to main + if (expirationPerComponent.spocs) { + await this.loadSpocs(this.store.dispatch); + } + if (expirationPerComponent.feeds) { + await this.loadComponentFeeds(this.store.dispatch); + } + } + + async scoreFeeds(feedsState) { + if (feedsState.data) { + const feeds = {}; + const feedsPromises = Object.keys(feedsState.data).map(url => { + let feed = feedsState.data[url]; + if (feed.personalized) { + // Feed was previously personalized then cached, we don't need to do this again. + return Promise.resolve(); + } + const feedPromise = this.scoreItems(feed.data.recommendations, "feed"); + feedPromise.then(({ data: scoredItems, personalized }) => { + const { recsExpireTime } = feed.data.settings; + const recommendations = this.rotate(scoredItems, recsExpireTime); + feed = { + ...feed, + personalized, + data: { + ...feed.data, + recommendations, + }, + }; + + feeds[url] = feed; + + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_FEED_UPDATE, + data: { + feed, + url, + }, + }) + ); + }); + return feedPromise; + }); + await Promise.all(feedsPromises); + await this.cache.set("feeds", feeds); + } + } + + async scoreSpocs(spocsState) { + const spocsResultPromises = this.getPlacements().map(async placement => { + const nextSpocs = spocsState.data[placement.name] || {}; + const { items } = nextSpocs; + + if (nextSpocs.personalized || !items || !items.length) { + return; + } + + const { data: scoreResult, personalized } = await this.scoreItems( + items, + "spocs" + ); + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...nextSpocs, + personalized, + items: scoreResult, + }, + }; + }); + await Promise.all(spocsResultPromises); + + // Update cache here so we don't need to re calculate scores on loads from cache. + // Related Bug 1606276 + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }, + }) + ); + } + + /** + * @typedef {Object} RefreshAll + * @property {boolean} updateOpenTabs - Sends updates to open tabs immediately if true, + * updates in background if false + * @property {boolean} isStartup - When the function is called at browser startup + * + * Refreshes component feeds, and spocs in order if caches have expired. + * @param {RefreshAll} options + */ + async refreshAll(options = {}) { + const { updateOpenTabs, isStartup } = options; + + const dispatch = updateOpenTabs + ? action => this.store.dispatch(ac.BroadcastToContent(action)) + : this.store.dispatch; + + this.loadLayout(dispatch, isStartup); + if (this.showStories || this.showTopsites) { + const promises = []; + // We could potentially have either or both sponsored topsites or stories. + // We only make one fetch, and control which to request when we fetch. + // So for now we only care if we need to make this request at all. + const spocsPromise = this.loadSpocs(dispatch, isStartup).catch(error => + console.error("Error trying to load spocs feeds:", error) + ); + promises.push(spocsPromise); + if (this.showStories) { + const storiesPromise = this.loadComponentFeeds( + dispatch, + isStartup + ).catch(error => + console.error("Error trying to load component feeds:", error) + ); + promises.push(storiesPromise); + } + await Promise.all(promises); + if (isStartup) { + // We don't pass isStartup in _maybeUpdateCachedData on purpose, + // because startup loads have a longer cache timer, + // and we want this to update in the background sooner. + await this._maybeUpdateCachedData(); + } + } + } + + // We have to rotate stories on the client so that + // active stories are at the front of the list, followed by stories that have expired + // impressions i.e. have been displayed for longer than recsExpireTime. + rotate(recommendations, recsExpireTime) { + const maxImpressionAge = Math.max( + recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, + DEFAULT_RECS_EXPIRE_TIME + ); + const impressions = this.readDataPref(PREF_REC_IMPRESSIONS); + const expired = []; + const active = []; + for (const item of recommendations) { + if ( + impressions[item.id] && + Date.now() - impressions[item.id] >= maxImpressionAge + ) { + expired.push(item); + } else { + active.push(item); + } + } + return active.concat(expired); + } + + enableStories() { + if (this.config.enabled) { + // If stories are being re enabled, ensure we have stories. + this.refreshAll({ updateOpenTabs: true }); + } + } + + async enable(options = {}) { + await this.refreshAll(options); + this.loaded = true; + } + + async reset() { + this.resetDataPrefs(); + await this.resetCache(); + this.resetState(); + } + + async resetCache() { + await this.resetAllCache(); + } + + async resetContentCache() { + await this.cache.set("feeds", {}); + await this.cache.set("spocs", {}); + await this.cache.set("sov", {}); + } + + async resetAllCache() { + await this.resetContentCache(); + // Reset in-memory caches. + this._isBff = undefined; + this._spocsCacheUpdateTime = undefined; + } + + resetDataPrefs() { + this.writeDataPref(PREF_SPOC_IMPRESSIONS, {}); + this.writeDataPref(PREF_REC_IMPRESSIONS, {}); + this.writeDataPref(PREF_FLIGHT_BLOCKS, {}); + } + + resetState() { + // Reset reducer + this.store.dispatch( + ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET }) + ); + this.setupPrefs(false /* isStartup */); + this.loaded = false; + } + + async onPrefChange() { + // We always want to clear the cache/state if the pref has changed + await this.reset(); + if (this.config.enabled) { + // Load data from all endpoints + await this.enable({ updateOpenTabs: true }); + } + } + + // This is a request to change the config from somewhere. + // Can be from a specific pref related to Discovery Stream, + // or can be a generic request from an external feed that + // something changed. + configReset() { + this._prefCache.config = null; + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_CHANGE, + data: this.config, + }) + ); + } + + recordFlightImpression(flightId) { + let impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS); + + const timeStamps = impressions[flightId] || []; + timeStamps.push(Date.now()); + impressions = { ...impressions, [flightId]: timeStamps }; + + this.writeDataPref(PREF_SPOC_IMPRESSIONS, impressions); + } + + recordTopRecImpressions(recId) { + let impressions = this.readDataPref(PREF_REC_IMPRESSIONS); + if (!impressions[recId]) { + impressions = { ...impressions, [recId]: Date.now() }; + this.writeDataPref(PREF_REC_IMPRESSIONS, impressions); + } + } + + recordBlockFlightId(flightId) { + const flights = this.readDataPref(PREF_FLIGHT_BLOCKS); + if (!flights[flightId]) { + flights[flightId] = 1; + this.writeDataPref(PREF_FLIGHT_BLOCKS, flights); + } + } + + cleanUpFlightImpressionPref(data) { + let flightIds = []; + this.placementsForEach(placement => { + const newSpocs = data[placement.name]; + if (!newSpocs) { + return; + } + + const items = newSpocs.items || []; + flightIds = [...flightIds, ...items.map(s => `${s.flight_id}`)]; + }); + if (flightIds && flightIds.length) { + this.cleanUpImpressionPref( + id => !flightIds.includes(id), + PREF_SPOC_IMPRESSIONS + ); + } + } + + // Clean up rec impression pref by removing all stories that are no + // longer part of the response. + cleanUpTopRecImpressionPref(newFeeds) { + // Need to build a single list of stories. + const activeStories = Object.keys(newFeeds) + .filter(currentValue => newFeeds[currentValue].data) + .reduce((accumulator, currentValue) => { + const { recommendations } = newFeeds[currentValue].data; + return accumulator.concat(recommendations.map(i => `${i.id}`)); + }, []); + this.cleanUpImpressionPref( + id => !activeStories.includes(id), + PREF_REC_IMPRESSIONS + ); + } + + writeDataPref(pref, impressions) { + this.store.dispatch(ac.SetPref(pref, JSON.stringify(impressions))); + } + + readDataPref(pref) { + const prefVal = this.store.getState().Prefs.values[pref]; + return prefVal ? JSON.parse(prefVal) : {}; + } + + cleanUpImpressionPref(isExpired, pref) { + const impressions = this.readDataPref(pref); + let changed = false; + + Object.keys(impressions).forEach(id => { + if (isExpired(id)) { + changed = true; + delete impressions[id]; + } + }); + + if (changed) { + this.writeDataPref(pref, impressions); + } + } + + onCollectionsChanged() { + // Update layout, and reload any off screen tabs. + // This does not change any existing open tabs. + // It also doesn't update any spoc or rec data, just the layout. + const dispatch = action => this.store.dispatch(ac.AlsoToPreloaded(action)); + this.loadLayout(dispatch, false); + } + + async onPrefChangedAction(action) { + switch (action.data.name) { + case PREF_CONFIG: + case PREF_ENABLED: + case PREF_HARDCODED_BASIC_LAYOUT: + case PREF_SPOCS_ENDPOINT: + case PREF_SPOCS_ENDPOINT_QUERY: + case PREF_SPOCS_CLEAR_ENDPOINT: + case PREF_ENDPOINTS: + // This is a config reset directly related to Discovery Stream pref. + this.configReset(); + break; + case PREF_COLLECTIONS_ENABLED: + this.onCollectionsChanged(); + break; + case PREF_USER_TOPSITES: + case PREF_SYSTEM_TOPSITES: + if ( + !( + this.showTopsites || + (this.showStories && this.showSponsoredStories) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + break; + case PREF_USER_TOPSTORIES: + case PREF_SYSTEM_TOPSTORIES: + if ( + !( + this.showStories || + (this.showTopsites && this.showSponsoredTopsites) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + if (action.data.value) { + this.enableStories(); + } + break; + // Check if spocs was disabled. Remove them if they were. + case PREF_SHOW_SPONSORED: + case PREF_SHOW_SPONSORED_TOPSITES: + const dispatch = update => + this.store.dispatch(ac.BroadcastToContent(update)); + // We refresh placements data because one of the spocs were turned off. + this.updatePlacements( + dispatch, + this.store.getState().DiscoveryStream.layout + ); + // Currently the order of this is important. + // We need to check this after updatePlacements is called, + // because some of the spoc logic depends on the result of placement updates. + if ( + !( + (this.showSponsoredStories || + (this.showTopSites && this.showSponsoredTopSites)) && + (this.showSponsoredTopsites || + (this.showStories && this.showSponsoredStories)) + ) + ) { + // Ensure we delete any remote data potentially related to spocs. + this.clearSpocs(); + } + // Placements have changed so consider spocs expired, and reload them. + await this.cache.set("spocs", {}); + await this.loadSpocs(dispatch); + break; + } + } + + async onAction(action) { + switch (action.type) { + case at.INIT: + // During the initialization of Firefox: + // 1. Set-up listeners and initialize the redux state for config; + this.setupConfig(true /* isStartup */); + this.setupPrefs(true /* isStartup */); + // 2. If config.enabled is true, start loading data. + if (this.config.enabled) { + await this.enable({ updateOpenTabs: true, isStartup: true }); + } + Services.prefs.addObserver(PREF_POCKET_BUTTON, this); + break; + case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK: + case at.SYSTEM_TICK: + // Only refresh if we loaded once in .enable() + if ( + this.config.enabled && + this.loaded && + (await this.checkIfAnyCacheExpired()) + ) { + await this.refreshAll({ updateOpenTabs: false }); + } + break; + case at.DISCOVERY_STREAM_DEV_SYNC_RS: + lazy.RemoteSettings.pollChanges(); + break; + case at.DISCOVERY_STREAM_DEV_EXPIRE_CACHE: + // Personalization scores update at a slower interval than content, so in order to debug, + // we want to be able to expire just content to trigger the earlier expire times. + await this.resetContentCache(); + break; + case at.DISCOVERY_STREAM_CONFIG_SET_VALUE: + // Use the original string pref to then set a value instead of + // this.config which has some modifications + this.store.dispatch( + ac.SetPref( + PREF_CONFIG, + JSON.stringify({ + ...JSON.parse(this.store.getState().Prefs.values[PREF_CONFIG]), + [action.data.name]: action.data.value, + }) + ) + ); + break; + case at.DISCOVERY_STREAM_POCKET_STATE_INIT: + this.setupPocketState(action.meta.fromTarget); + break; + case at.DISCOVERY_STREAM_PERSONALIZATION_UPDATED: + if (this.personalized) { + const { feeds, spocs } = this.store.getState().DiscoveryStream; + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + if (recsPersonalized && feeds.loaded) { + this.scoreFeeds(feeds); + } + if (spocsPersonalized && spocs.loaded) { + this.scoreSpocs(spocs); + } + } + break; + case at.DISCOVERY_STREAM_CONFIG_RESET: + // This is a generic config reset likely related to an external feed pref. + this.configReset(); + break; + case at.DISCOVERY_STREAM_CONFIG_RESET_DEFAULTS: + this.resetConfigDefauts(); + break; + case at.DISCOVERY_STREAM_RETRY_FEED: + this.retryFeed(action.data.feed); + break; + case at.DISCOVERY_STREAM_CONFIG_CHANGE: + // When the config pref changes, load or unload data as needed. + await this.onPrefChange(); + break; + case at.DISCOVERY_STREAM_IMPRESSION_STATS: + if ( + action.data.tiles && + action.data.tiles[0] && + action.data.tiles[0].id + ) { + this.recordTopRecImpressions(action.data.tiles[0].id); + } + break; + case at.DISCOVERY_STREAM_SPOC_IMPRESSION: + if (this.showSpocs) { + this.recordFlightImpression(action.data.flightId); + + // Apply frequency capping to SPOCs in the redux store, only update the + // store if the SPOCs are changed. + const spocsState = this.store.getState().DiscoveryStream.spocs; + + let frequencyCapped = []; + this.placementsForEach(placement => { + const spocs = spocsState.data[placement.name]; + if (!spocs || !spocs.items) { + return; + } + + const { data: capResult, filtered } = this.frequencyCapSpocs( + spocs.items + ); + frequencyCapped = [...frequencyCapped, ...filtered]; + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...spocs, + items: capResult, + }, + }; + }); + + if (frequencyCapped.length) { + // Update cache here so we don't need to re calculate frequency caps on loads from cache. + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_SPOCS_UPDATE, + data: { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }, + }) + ); + } + } + break; + // This is fired from the browser, it has no concept of spocs, flight or pocket. + // We match the blocked url with our available spoc urls to see if there is a match. + // I suspect we *could* instead do this in BLOCK_URL but I'm not sure. + case at.PLACES_LINK_BLOCKED: + if (this.showSpocs) { + let blockedItems = []; + const spocsState = this.store.getState().DiscoveryStream.spocs; + + this.placementsForEach(placement => { + const spocs = spocsState.data[placement.name]; + if (spocs && spocs.items && spocs.items.length) { + const blockedResults = []; + const blocks = spocs.items.filter(s => { + const blocked = s.url === action.data.url; + if (!blocked) { + blockedResults.push(s); + } + return blocked; + }); + + blockedItems = [...blockedItems, ...blocks]; + + spocsState.data = { + ...spocsState.data, + [placement.name]: { + ...spocs, + items: blockedResults, + }, + }; + } + }); + + if (blockedItems.length) { + // Update cache here so we don't need to re calculate blocks on loads from cache. + await this.cache.set("spocs", { + lastUpdated: spocsState.lastUpdated, + spocs: spocsState.data, + }); + + // If we're blocking a spoc, we want open tabs to have + // a slightly different treatment from future tabs. + // AlsoToPreloaded updates the source data and preloaded tabs with a new spoc. + // BroadcastToContent updates open tabs with a non spoc instead of a new spoc. + this.store.dispatch( + ac.AlsoToPreloaded({ + type: at.DISCOVERY_STREAM_LINK_BLOCKED, + data: action.data, + }) + ); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_SPOC_BLOCKED, + data: action.data, + }) + ); + break; + } + } + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_LINK_BLOCKED, + data: action.data, + }) + ); + break; + case at.UNINIT: + // When this feed is shutting down: + this.uninitPrefs(); + this._recommendationProvider = null; + Services.prefs.removeObserver(PREF_POCKET_BUTTON, this); + break; + case at.BLOCK_URL: { + // If we block a story that also has a flight_id + // we want to record that as blocked too. + // This is because a single flight might have slightly different urls. + action.data.forEach(site => { + const { flight_id } = site; + if (flight_id) { + this.recordBlockFlightId(flight_id); + } + }); + break; + } + case at.PREF_CHANGED: + await this.onPrefChangedAction(action); + if (action.data.name === "pocketConfig") { + await this.onPrefChange(); + this.setupPrefs(false /* isStartup */); + } + break; + } + } +} + +/* This function generates a hardcoded layout each call. + This is because modifying the original object would + persist across pref changes and system_tick updates. + + NOTE: There is some branching logic in the template. + `spocsUrl` Changing the url for spocs is used for adding a siteId query param. + `feedUrl` Where to fetch stories from. + `items` How many items to include in the primary card grid. + `spocPositions` Changes the position of spoc cards. + `spocTopsitesPositions` Changes the position of spoc topsites. + `spocPlacementData` Used to set the spoc content. + `spocTopsitesPlacementEnabled` Tuns on and off the sponsored topsites placement. + `spocTopsitesPlacementData` Used to set spoc content for topsites. + `sponsoredCollectionsEnabled` Tuns on and off the sponsored collection section. + `hybridLayout` Changes cards to smaller more compact cards only for specific breakpoints. + `hideCardBackground` Removes Pocket card background and borders. + `fourCardLayout` Enable four Pocket cards per row. + `newFooterSection` Changes the layout of the topics section. + `compactGrid` Reduce the number of pixels between the Pocket cards. + `essentialReadsHeader` Updates the Pocket section header and title to say "Today’s Essential Reads", moves the "Recommended by Pocket" header to the right side. + `editorsPicksHeader` Updates the Pocket section header and title to say "Editor’s Picks", if used with essentialReadsHeader, creates a second section 2 rows down for editorsPicks. + `onboardingExperience` Show new users some UI explaining Pocket above the Pocket section. + `ctaButtonSponsors` An array of sponsors we want to show a cta button on the card for. + `ctaButtonVariant` Sets the variant for the cta sponsor button. +*/ +getHardcodedLayout = ({ + spocsUrl = SPOCS_URL, + feedUrl = FEED_URL, + items = 21, + spocPositions = [1, 5, 7, 11, 18, 20], + spocTopsitesPositions = [1], + spocPlacementData = { ad_types: [3617], zone_ids: [217758, 217995] }, + spocTopsitesPlacementEnabled = false, + spocTopsitesPlacementData = { ad_types: [3120], zone_ids: [280143] }, + widgetPositions = [], + widgetData = [], + sponsoredCollectionsEnabled = false, + hybridLayout = false, + hideCardBackground = false, + fourCardLayout = false, + newFooterSection = false, + compactGrid = false, + essentialReadsHeader = false, + editorsPicksHeader = false, + onboardingExperience = false, + ctaButtonSponsors = [], + ctaButtonVariant = "", +}) => ({ + lastUpdate: Date.now(), + spocs: { + url: spocsUrl, + }, + layout: [ + { + width: 12, + components: [ + { + type: "TopSites", + header: { + title: { + id: "newtab-section-header-topsites", + }, + }, + ...(spocTopsitesPlacementEnabled && spocTopsitesPlacementData + ? { + placement: { + name: "sponsored-topsites", + ad_types: spocTopsitesPlacementData.ad_types, + zone_ids: spocTopsitesPlacementData.zone_ids, + }, + spocs: { + probability: 1, + prefs: [PREF_SHOW_SPONSORED_TOPSITES], + positions: spocTopsitesPositions.map(position => { + return { index: position }; + }), + }, + } + : {}), + properties: {}, + }, + ...(sponsoredCollectionsEnabled + ? [ + { + type: "CollectionCardGrid", + properties: { + items: 3, + }, + header: { + title: "", + }, + placement: { + name: "sponsored-collection", + ad_types: [3617], + zone_ids: [217759, 218031], + }, + spocs: { + probability: 1, + positions: [ + { + index: 0, + }, + { + index: 1, + }, + { + index: 2, + }, + ], + }, + }, + ] + : []), + { + type: "Message", + essentialReadsHeader, + editorsPicksHeader, + header: { + title: { + id: "newtab-section-header-stories", + }, + subtitle: "", + link_text: { + id: "newtab-pocket-learn-more", + }, + link_url: "https://getpocket.com/firefox/new_tab_learn_more", + icon: "chrome://global/skin/icons/pocket.svg", + }, + properties: {}, + styles: { + ".ds-message": "margin-bottom: -20px", + }, + }, + { + type: "CardGrid", + properties: { + items, + hybridLayout, + hideCardBackground, + fourCardLayout, + compactGrid, + essentialReadsHeader, + editorsPicksHeader, + onboardingExperience, + ctaButtonSponsors, + ctaButtonVariant, + }, + widgets: { + positions: widgetPositions.map(position => { + return { index: position }; + }), + data: widgetData, + }, + cta_variant: "link", + header: { + title: "", + }, + placement: { + name: "spocs", + ad_types: spocPlacementData.ad_types, + zone_ids: spocPlacementData.zone_ids, + }, + feed: { + embed_reference: null, + url: feedUrl, + }, + spocs: { + probability: 1, + positions: spocPositions.map(position => { + return { index: position }; + }), + }, + }, + { + type: "Navigation", + newFooterSection, + properties: { + alignment: "left-align", + links: [ + { + name: "Self improvement", + url: "https://getpocket.com/explore/self-improvement?utm_source=pocket-newtab", + }, + { + name: "Food", + url: "https://getpocket.com/explore/food?utm_source=pocket-newtab", + }, + { + name: "Entertainment", + url: "https://getpocket.com/explore/entertainment?utm_source=pocket-newtab", + }, + { + name: "Health & fitness", + url: "https://getpocket.com/explore/health?utm_source=pocket-newtab", + }, + { + name: "Science", + url: "https://getpocket.com/explore/science?utm_source=pocket-newtab", + }, + { + name: "More recommendations ›", + url: "https://getpocket.com/explore?utm_source=pocket-newtab", + }, + ], + extraLinks: [ + { + name: "Career", + url: "https://getpocket.com/explore/career?utm_source=pocket-newtab", + }, + { + name: "Technology", + url: "https://getpocket.com/explore/technology?utm_source=pocket-newtab", + }, + ], + privacyNoticeURL: { + url: "https://www.mozilla.org/privacy/firefox/#recommend-relevant-content", + title: { + id: "newtab-section-menu-privacy-notice", + }, + }, + }, + header: { + title: { + id: "newtab-pocket-read-more", + }, + }, + styles: { + ".ds-navigation": "margin-top: -10px;", + }, + }, + ...(newFooterSection + ? [ + { + type: "PrivacyLink", + properties: { + url: "https://www.mozilla.org/privacy/firefox/", + title: { + id: "newtab-section-menu-privacy-notice", + }, + }, + }, + ] + : []), + ], + }, + ], +}); diff --git a/browser/components/newtab/lib/DownloadsManager.sys.mjs b/browser/components/newtab/lib/DownloadsManager.sys.mjs new file mode 100644 index 0000000000..f095645d41 --- /dev/null +++ b/browser/components/newtab/lib/DownloadsManager.sys.mjs @@ -0,0 +1,188 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { actionTypes as at } from "resource://activity-stream/common/Actions.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DownloadsCommon: "resource:///modules/DownloadsCommon.sys.mjs", + DownloadsViewUI: "resource:///modules/DownloadsViewUI.sys.mjs", + FileUtils: "resource://gre/modules/FileUtils.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", +}); + +const DOWNLOAD_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for downloads changed events + +export class DownloadsManager { + constructor(store) { + this._downloadData = null; + this._store = null; + this._downloadItems = new Map(); + this._downloadTimer = null; + } + + setTimeout(callback, delay) { + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT); + return timer; + } + + formatDownload(download) { + let referrer = download.source.referrerInfo?.originalReferrer?.spec || null; + return { + hostname: new URL(download.source.url).hostname, + url: download.source.url, + path: download.target.path, + title: lazy.DownloadsViewUI.getDisplayName(download), + description: + lazy.DownloadsViewUI.getSizeWithUnits(download) || + lazy.DownloadsCommon.strings.sizeUnknown, + referrer, + date_added: download.endTime, + }; + } + + init(store) { + this._store = store; + this._downloadData = lazy.DownloadsCommon.getData( + null /* null for non-private downloads */, + true, + false, + true + ); + this._downloadData.addView(this); + } + + onDownloadAdded(download) { + if (!this._downloadItems.has(download.source.url)) { + this._downloadItems.set(download.source.url, download); + + // On startup, all existing downloads fire this notification, so debounce them + if (this._downloadTimer) { + this._downloadTimer.delay = DOWNLOAD_CHANGED_DELAY_TIME; + } else { + this._downloadTimer = this.setTimeout(() => { + this._downloadTimer = null; + this._store.dispatch({ type: at.DOWNLOAD_CHANGED }); + }, DOWNLOAD_CHANGED_DELAY_TIME); + } + } + } + + onDownloadRemoved(download) { + if (this._downloadItems.has(download.source.url)) { + this._downloadItems.delete(download.source.url); + this._store.dispatch({ type: at.DOWNLOAD_CHANGED }); + } + } + + async getDownloads( + threshold, + { + numItems = this._downloadItems.size, + onlySucceeded = false, + onlyExists = false, + } + ) { + if (!threshold) { + return []; + } + let results = []; + + // Only get downloads within the time threshold specified and sort by recency + const downloadThreshold = Date.now() - threshold; + let downloads = [...this._downloadItems.values()] + .filter(download => download.endTime > downloadThreshold) + .sort((download1, download2) => download1.endTime < download2.endTime); + + for (const download of downloads) { + // Ignore blocked links, but allow long (data:) uris to avoid high CPU + if ( + download.source.url.length < 10000 && + lazy.NewTabUtils.blockedLinks.isBlocked(download.source) + ) { + continue; + } + + // Only include downloads where the file still exists + if (onlyExists) { + // Refresh download to ensure the 'exists' attribute is up to date + await download.refresh(); + if (!download.target.exists) { + continue; + } + } + // Only include downloads that were completed successfully + if (onlySucceeded) { + if (!download.succeeded) { + continue; + } + } + const formattedDownloadForHighlights = this.formatDownload(download); + results.push(formattedDownloadForHighlights); + if (results.length === numItems) { + break; + } + } + return results; + } + + uninit() { + if (this._downloadData) { + this._downloadData.removeView(this); + this._downloadData = null; + } + if (this._downloadTimer) { + this._downloadTimer.cancel(); + this._downloadTimer = null; + } + } + + onAction(action) { + let doDownloadAction = callback => { + let download = this._downloadItems.get(action.data.url); + if (download) { + callback(download); + } + }; + + switch (action.type) { + case at.COPY_DOWNLOAD_LINK: + doDownloadAction(download => { + lazy.DownloadsCommon.copyDownloadLink(download); + }); + break; + case at.REMOVE_DOWNLOAD_FILE: + doDownloadAction(download => { + lazy.DownloadsCommon.deleteDownload(download).catch(console.error); + }); + break; + case at.SHOW_DOWNLOAD_FILE: + doDownloadAction(download => { + lazy.DownloadsCommon.showDownloadedFile( + new lazy.FileUtils.File(download.target.path) + ); + }); + break; + case at.OPEN_DOWNLOAD_FILE: + const win = action._target.browser.ownerGlobal; + const openWhere = + action.data.event && win.whereToOpenLink(action.data.event); + doDownloadAction(download => { + lazy.DownloadsCommon.openDownload(download, { + // Replace "current" or unknown value with "tab" as the default behavior + // for opening downloads when handled internally + openWhere: ["window", "tab", "tabshifted"].includes(openWhere) + ? openWhere + : "tab", + }); + }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} diff --git a/browser/components/newtab/lib/FaviconFeed.sys.mjs b/browser/components/newtab/lib/FaviconFeed.sys.mjs new file mode 100644 index 0000000000..a76566d3e8 --- /dev/null +++ b/browser/components/newtab/lib/FaviconFeed.sys.mjs @@ -0,0 +1,198 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { actionTypes as at } from "resource://activity-stream/common/Actions.sys.mjs"; +import { getDomain } from "resource://activity-stream/lib/TippyTopProvider.sys.mjs"; + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// RemoteSettings, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +const MIN_FAVICON_SIZE = 96; + +/** + * Get favicon info (uri and size) for a uri from Places. + * + * @param uri {nsIURI} Page to check for favicon data + * @returns A promise of an object (possibly null) containing the data + */ +function getFaviconInfo(uri) { + return new Promise(resolve => + lazy.PlacesUtils.favicons.getFaviconDataForPage( + uri, + // Package up the icon data in an object if we have it; otherwise null + (iconUri, faviconLength, favicon, mimeType, faviconSize) => + resolve(iconUri ? { iconUri, faviconSize } : null), + lazy.NewTabUtils.activityStreamProvider.THUMB_FAVICON_SIZE + ) + ); +} + +/** + * Fetches visit paths for a given URL from its most recent visit in Places. + * + * Note that this includes the URL itself as well as all the following + * permenent&temporary redirected URLs if any. + * + * @param {String} a URL string + * + * @returns {Array} Returns an array containing objects as + * {int} visit_id: ID of the visit in moz_historyvisits. + * {String} url: URL of the redirected URL. + */ +async function fetchVisitPaths(url) { + const query = ` + WITH RECURSIVE path(visit_id) + AS ( + SELECT v.id + FROM moz_places h + JOIN moz_historyvisits v + ON v.place_id = h.id + WHERE h.url_hash = hash(:url) AND h.url = :url + AND v.visit_date = h.last_visit_date + + UNION + + SELECT id + FROM moz_historyvisits + JOIN path + ON visit_id = from_visit + WHERE visit_type IN + (${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_PERMANENT}, + ${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_TEMPORARY}) + ) + SELECT visit_id, ( + SELECT ( + SELECT url + FROM moz_places + WHERE id = place_id) + FROM moz_historyvisits + WHERE id = visit_id) AS url + FROM path + `; + + const visits = + await lazy.NewTabUtils.activityStreamProvider.executePlacesQuery(query, { + columns: ["visit_id", "url"], + params: { url }, + }); + return visits; +} + +/** + * Fetch favicon for a url by following its redirects in Places. + * + * This can improve the rich icon coverage for Top Sites since Places only + * associates the favicon to the final url if the original one gets redirected. + * Note this is not an urgent request, hence it is dispatched to the main + * thread idle handler to avoid any possible performance impact. + */ +export async function fetchIconFromRedirects(url) { + const visitPaths = await fetchVisitPaths(url); + if (visitPaths.length > 1) { + const lastVisit = visitPaths.pop(); + const redirectedUri = Services.io.newURI(lastVisit.url); + const iconInfo = await getFaviconInfo(redirectedUri); + if (iconInfo && iconInfo.faviconSize >= MIN_FAVICON_SIZE) { + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(url), + iconInfo.iconUri, + false, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } + } +} + +export class FaviconFeed { + constructor() { + this._queryForRedirects = new Set(); + } + + /** + * fetchIcon attempts to fetch a rich icon for the given url from two sources. + * First, it looks up the tippy top feed, if it's still missing, then it queries + * the places for rich icon with its most recent visit in order to deal with + * the redirected visit. See Bug 1421428 for more details. + */ + async fetchIcon(url) { + // Avoid initializing and fetching icons if prefs are turned off + if (!this.shouldFetchIcons) { + return; + } + + const site = await this.getSite(getDomain(url)); + if (!site) { + if (!this._queryForRedirects.has(url)) { + this._queryForRedirects.add(url); + Services.tm.idleDispatchToMainThread(() => fetchIconFromRedirects(url)); + } + return; + } + + let iconUri = Services.io.newURI(site.image_url); + // The #tippytop is to be able to identify them for telemetry. + iconUri = iconUri.mutate().setRef("tippytop").finalize(); + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(url), + iconUri, + false, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + null, + Services.scriptSecurityManager.getSystemPrincipal() + ); + } + + /** + * Get the site tippy top data from Remote Settings. + */ + async getSite(domain) { + const sites = await this.tippyTop.get({ + filters: { domain }, + syncIfEmpty: false, + }); + return sites.length ? sites[0] : null; + } + + /** + * Get the tippy top collection from Remote Settings. + */ + get tippyTop() { + if (!this._tippyTop) { + this._tippyTop = RemoteSettings("tippytop"); + } + return this._tippyTop; + } + + /** + * Determine if we should be fetching and saving icons. + */ + get shouldFetchIcons() { + return Services.prefs.getBoolPref("browser.chrome.site_icons"); + } + + onAction(action) { + switch (action.type) { + case at.RICH_ICON_MISSING: + this.fetchIcon(action.data.url); + break; + } + } +} diff --git a/browser/components/newtab/lib/FilterAdult.sys.mjs b/browser/components/newtab/lib/FilterAdult.sys.mjs new file mode 100644 index 0000000000..a60ba3baa6 --- /dev/null +++ b/browser/components/newtab/lib/FilterAdult.sys.mjs @@ -0,0 +1,3040 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// XPCOMUtils, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "gFilterAdultEnabled", + "browser.newtabpage.activity-stream.filterAdult", + true +); + +// Keep a Set of adult base domains for lookup (initialized at end of file) +let gAdultSet; + +// Keep a hasher for repeated hashings +let gCryptoHash = null; + +/** + * Run some text through md5 and return the base64 result. + */ +function md5Hash(text) { + // Lazily create a reusable hasher + if (gCryptoHash === null) { + gCryptoHash = Cc["@mozilla.org/security/hash;1"].createInstance( + Ci.nsICryptoHash + ); + } + + gCryptoHash.init(gCryptoHash.MD5); + + // Convert the text to a byte array for hashing + gCryptoHash.update( + text.split("").map(c => c.charCodeAt(0)), + text.length + ); + + // Request the has result as ASCII base64 + return gCryptoHash.finish(true); +} + +export const FilterAdult = { + /** + * Filter out any link objects that have a url with an adult base domain. + * + * @param {string[]} links + * An array of links to test. + * @returns {string[]} + * A filtered array without adult links. + */ + filter(links) { + if (!lazy.gFilterAdultEnabled) { + return links; + } + + return links.filter(({ url }) => { + try { + const uri = Services.io.newURI(url); + return !gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri))); + } catch (ex) { + return true; + } + }); + }, + + /** + * Determine if the supplied url is an adult url or not. + * + * @param {string} url + * The url to test. + * @returns {boolean} + * True if it is an adult url. + */ + isAdultUrl(url) { + if (!lazy.gFilterAdultEnabled) { + return false; + } + try { + const uri = Services.io.newURI(url); + return gAdultSet.has(md5Hash(Services.eTLD.getBaseDomain(uri))); + } catch (ex) { + return false; + } + }, + + /** + * For tests, adds a domain to the adult list. + */ + addDomainToList(url) { + gAdultSet.add( + md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url))) + ); + }, + + /** + * For tests, removes a domain to the adult list. + */ + removeDomainFromList(url) { + gAdultSet.delete( + md5Hash(Services.eTLD.getBaseDomain(Services.io.newURI(url))) + ); + }, +}; + +// These are md5 hashes of base domains to be filtered out. Originally from: +// https://hg.mozilla.org/mozilla-central/log/default/browser/base/content/newtab/newTab.inadjacent.json +gAdultSet = new Set([ + "+/UCpAhZhz368iGioEO8aQ==", + "+1e7jvUo8f2/2l0TFrQqfA==", + "+1gcqAqaRZwCj5BGiZp3CA==", + "+25t/2lo0FUEtWYK8LdQZQ==", + "+8PiQt6O7pJI/nIvQpDaAg==", + "+CLf5witKkuOvPCulTlkqw==", + "+CvLiih/gf2ugXAF+LgWqw==", + "+DWs0vvFGt6d3mzdcsdsyA==", + "+H0Rglt/HnhZwdty2hsDHg==", + "+L1FDsr5VQtuYc2Is5QGjw==", + "+LJYVZl1iPrdMU3L5+nxZw==", + "+Mp+JIyO0XC5urvMyi3wvQ==", + "+NMUaQ7XPsAi0rk7tTT9wQ==", + "+NmjwjsPhGJh9bM10SFkLw==", + "+OERSmo7OQUUjudkccSMOA==", + "+OLntmlsMBBYPREPnS6iVw==", + "+OXdvbTxHtSoLg7bZMho4w==", + "+P5q4YD1Rr5SX26Xr+tzlw==", + "+PUVXkoTqHxJHO18z4KMfw==", + "+Pl0bSMBAdXpRIA+zE02JA==", + "+QosBAnSM2h4lsKuBlqEZw==", + "+S+WXgVDSU1oGmCzGwuT3g==", + "+SclwwY8R2RPrnX54Z+A6w==", + "+VfRcTBQ80KSeJRdg0cDfw==", + "+WpF8+poKmHPUBB4UYh/ig==", + "+YVxSyViJfrme/ENe1zA7A==", + "+YrqTEJlJCv0A2RHQ8tr1A==", + "+ZozWaPWw8ws1cE5DJACeg==", + "+aF4ilbjQbLpAuFXQEYMWQ==", + "+dBv88reDrjEz6a2xX3Hzw==", + "+dIEf5FBrHpkjmwUmGS6eg==", + "+edqJYGvcy1AH2mEjJtSIg==", + "+fcjH2kZKNj8quOytUk4nQ==", + "+gO0bg8LY+py2dLM1sM7Ag==", + "+gbitI/gpxebN/rK7qj8Fw==", + "+gpHnUj2GWocP74t5XWz4w==", + "+jVN/3ASc2O44sX6ab8/cg==", + "+mJLK+6qq8xFv7O/mbILTw==", + "+n0K7OB2ItzhySZ4rhUrMg==", + "+p8pofUlwn8vV6Rp6+sz9g==", + "+tuUmnRDRWVLA+1k0dcUvg==", + "+zBkeHF4P8vLzk1iO1Zn3Q==", + "//eHwmDOQRSrv+k9C/k3ZQ==", + "/2Chaw2M9DzsadFFkCu6WQ==", + "/2c4oNniwhL3z5IOngfggg==", + "/2jGyMekNu7U136K+2N3Jg==", + "/Bwpt5fllzDHq2Ul6v86fA==", + "/DJgKE9ouibewuZ2QEnk6w==", + "/DiUApY7cVp5W9o24rkgRA==", + "/FchS2nPezycB8Bcqc2dbg==", + "/FdZzSprPnNDPwbhV1C0Cg==", + "/FsJYFNe+7UvsSkiotNJEQ==", + "/G26n5Xoviqldr5sg/Jl3w==", + "/HU2+fBqfWTEuqINc0UZSA==", + "/IarsLzJB8bf0AupJJ+/Eg==", + "/KYZdUWrkfxSsIrp46xxow==", + "/MEOgAhwb7F0nBnV4tIRZA==", + "/MeHciFhvFzQsCIw39xIZA==", + "/Ph/6l/lFNVqxAje1+PgFA==", + "/SP6pOdYFzcAl2OL05z4uQ==", + "/TSsi/AwKHtP6kQaeReI3w==", + "/VnKh/NDv7y/bfO6CWsLaQ==", + "/XC/FmMIOdhMTPqmy4DfUA==", + "/XjB6c5fxFGcKVAQ4o+OMw==", + "/YuQw7oAF08KDptxJEBS9g==", + "/a+bLXOq02sa/s8h7PhUTg==", + "/a9O7kWeXa0le45ab3+nVw==", + "/c34NtdUZAHWIwGl3JM8Tw==", + "/cJ0Nn5YbXeUpOHMfWXNHQ==", + "/cdR1i5TuQvO+u3Ov3b0KQ==", + "/gi3UZmunVOIXhZSktZ8zQ==", + "/hFhjFGJx2wRfz6hyrIpvA==", + "/jDVt9dRIn+o4IQ1DPwbsg==", + "/jH6imhTPZ/tHI4gYz2+HA==", + "/kGxvyEokQsVz0xlKzCn2A==", + "/mFp3GFkGNLhx2CiDvJv4A==", + "/mrqas0eDX+sFUNJvCQY8g==", + "/n1RLTTVpygre1dl36PDwQ==", + "/ngbFuKIAVpdSwsA3VxvNw==", + "/p/aCTIhi1bU0/liuO/a2Q==", + "/u5W2Gab4GgCMIc4KTp2mg==", + "/wIZAye9h1TUiZmDW0ZmYA==", + "/wiA2ltAuWyBhIvQAYBTQw==", + "/y/jHHEpUu5TR+R2o96kXA==", + "/zFLRvi75UL8qvg+a6zqGg==", + "00TVKawojyqrJkC7YqT41Q==", + "022B0oiRMx8Xb4Af98mTvQ==", + "02im2RooJQ/9UfUrh5LO+A==", + "0G93AxGPVwmr66ZOleM90A==", + "0HN6MIGtkdzNPsrGs611xA==", + "0K4NBxqEa3RYpnrkrD/XjQ==", + "0L0FVcH5Dlj3oL8+e9Na7g==", + "0NrvBuyjcJ2q6yaHpz/FOA==", + "0ODJyWKJSfObo+FNdRQkkA==", + "0QB0OUW5x2JLHfrtmpZQ+w==", + "0QCQORCYfLuSbq94Sbt0bQ==", + "0QbH4oI8IjZ9BRcqRyvvDQ==", + "0QxPAqRF8inBuFEEzNmLjA==", + "0SkC/4PtnX1bMYgD6r6CLA==", + "0TxcYwG72dT7Tg+eG8pP1w==", + "0UeRwDID2RBIikInqFI7uw==", + "0VsaJHR0Ms8zegsCpAKoyg==", + "0Y6iiZjCwPDwD/CwJzfioQ==", + "0ZEC3hy411LkOhKblvTcqg==", + "0ZRGz+oj2infCAkuKKuHiQ==", + "0a4SafpDIe8V4FlFWYkMHw==", + "0b/xj6fd0x+aB8EB0LC4SA==", + "0bj069wXgEJbw7dpiPr8Tg==", + "0dIeIM5Zvm5nSVWLy94LWg==", + "0e8hM3E5tnABRyy29A8yFw==", + "0egBaMnAf0CQEXf1pCIKnA==", + "0fN+eHlbRS6mVZBbH/B9FQ==", + "0fnruVOCxEczscBuv4yL9A==", + "0fpe9E6m3eLp/5j5rLrz2Q==", + "0klouNfZRHFFpdHi4ZR2hA==", + "0nOg18ZJ/NicqVUz5Jr0Hg==", + "0ofMbUCA3/v5L8lHnX4S5w==", + "0p1jMr06OyBoXQuSLYN4aQ==", + "0p8YbEMxeb73HbAfvPLQRw==", + "0q+erphtrB+6HBnnYg7O6w==", + "0rTYcuVYdilO7zEfKrxY3A==", + "0rfG4gRugAwVP0i3AGVxxg==", + "0u+0WHr7WI6IlVBBgiRi6w==", + "0yJ7TQYzcp3DXVSvwavr+w==", + "1+A9FCGP3bZhk6gU3LQtNg==", + "1+XWdu4qCqLLVjqkKz3nmA==", + "1+qmrbC8c7MJ6pxmDMcKuA==", + "1/Hxu8M9N/oNwk8bCj4FNQ==", + "1/SGIab+NnizimUmNDC4wA==", + "1/ZheMsbojazxt31j/l3iA==", + "10OltdxPXOvfatJuwPVKbQ==", + "11FE2kknwYi2Qu0JUKMn3A==", + "11U5XEwfMI7avx014LfC8g==", + "16d+fhFlgayu3ttKVV/pbg==", + "16iT/jCcPDrJEfi2bE5F+Q==", + "18RKixTv12q3xoBLz6eKiA==", + "18ndtDM9UaNfBR1cr3SHdA==", + "19yQHaBemtlgo2QkU5M6jQ==", + "1AeReq55UQotRQVKJ66pmg==", + "1ApqwW7pE+XUB2Cs2M6y7g==", + "1B5gxGQSGzVKoNd5Ol4N7g==", + "1BjsijOzgHt/0i36ZGffoQ==", + "1C50kisi9nvyVJNfq2hOEQ==", + "1E3pMgAHOnHx3ALdNoHr8Q==", + "1EI9aa955ejNo1dJepcZJw==", + "1FSrgkUXgZot2CsmbAtkPw==", + "1Gpj4TPXhdPEI4zfQFsOCg==", + "1HDgfU7xU7LWO/BXsODZAQ==", + "1I+UVx3krrD4NhzO7dgfHQ==", + "1JI9bT92UzxI8txjhst9LQ==", + "1JRgSHnfAQFQtSkFTttkqQ==", + "1LPC0BzhJbepHTSAiZ3QTw==", + "1MIn73MLroxXirrb+vyg2Q==", + "1Oykse0jQVbuR3MvW5ot4A==", + "1Pmnur6TbZ9cmemvu0+dSA==", + "1PvTn90xwZJPoVfyT5/uIQ==", + "1QGhj9NONF2rC44UdO+Izw==", + "1RQZ2pWSxT+RKyhBigtSFg==", + "1Vtrv6QUAfiYQjlLTpNovg==", + "1WIi4I62GqkjDXOYqHWJfQ==", + "1Wc8jQlDSB4Dp32wkL2odw==", + "1X14kHeKwGmLeYqpe60XEA==", + "1YO9G8qAhLIu2rShvekedw==", + "1Ym0lyBJ9aFjhJb/GdUPvQ==", + "1b2uf+CdVjufqiVpUShvHw==", + "1buQEv2YlH/ljTgH0uJEtw==", + "1cj1Fpd3+UiBAOahEhsluA==", + "1d7RPHdZ9qzAbG3Vi9BdFA==", + "1dhq3ozNCx0o4dV1syLVDA==", + "1dsKN1nG6upj7kKTKuJWsQ==", + "1eCHcz4swFH+uRhiilOinQ==", + "1eRUCdIJe3YGD5jOMbkkOg==", + "1fztTtQWNMIMSAc5Hr6jMQ==", + "1gA65t5FiBTEgMELTQFUPQ==", + "1jBaRO8Bg5l6TH7qJ8EPiw==", + "1k8tL2xmGFVYMgKUcmDcEw==", + "1lCcQWGDePPYco4vYrA5vw==", + "1m1yD4L9A7Q1Ot+wCsrxJQ==", + "1mw6LfTiirFyfjejf8QNGA==", + "1nXByug2eKq0kR3H3VjnWQ==", + "1tpM0qgdo7JDFwvT0TD78g==", + "1vqRt79ukuvdJNyIlIag8Q==", + "1wBuHqS1ciup31WTfm3NPg==", + "1xWx5V3G9murZP7srljFmA==", + "1zDfWw5LdG20ClNP1HYxgw==", + "203EqmJI9Q4tWxTJaBdSzA==", + "23C4eh3yBb5n/RNZeTyJkA==", + "23d9B9Gz5kUOi1I//EYsSQ==", + "24H9q+E8pgCEdFS7JO5kzQ==", + "25w3ZRUzCvJwAVHYCIO5uw==", + "26+yXbqI+fmIZsYl4UhUzw==", + "26Wmdp6SkKN74W0/XPcnmA==", + "29EybnMEO95Ng4l/qK4NWQ==", + "2Ct+pLXrK6Ku1f4qehjurQ==", + "2D6yhuABiaFFoXz0Lh0C+w==", + "2DNbXVgesUa7PgYQ4zX5Lw==", + "2E41e0MgM3WhFx2oasIQeA==", + "2HHqeGRMfzf3RXwVybx+ZQ==", + "2Hc5oyl0AYRy2VzcDKy+VA==", + "2QQtKtBAm2AjJ5c0WQ6BQA==", + "2QS/6OBA1T01NlIbfkTYJg==", + "2RFaMPlSbVuoEqKXgkIa5A==", + "2SI4F7Vvde2yjzMLAwxOog==", + "2SwIiUwT4vRZPrg7+vZqDA==", + "2W6lz1Z7PhkvObEAg2XKJw==", + "2Wvk/kouEEOY0evUkQLhOQ==", + "2XrR2hjDEvx8MQpHk9dnjw==", + "2aDK0tGNgMLyxT+BQPDE8Q==", + "2aIx9UdMxxZWvrfeJ+DcTw==", + "2abfl3N46tznOpr+94VONQ==", + "2bsIpvnGcFhTCSrK9EW1FQ==", + "2hEzujfG3mR5uQJXbvOPTQ==", + "2j83jrPwPfYlpJJ2clEBYQ==", + "2ksediOVrh4asSBxKcudTg==", + "2melaInV0wnhBpiI3da6/A==", + "2nSTEYzLK77h5Rgyti+ULQ==", + "2os5s7j7Tl46ZmoZJH8FjA==", + "2rOkEVl90EPqfHOF5q2FYw==", + "2rhjiY0O0Lo36wTHjmlNyw==", + "2vm7g3rk1ACJOTCXkLB3zA==", + "2wesXiib76wM9sqRZ7JYwQ==", + "2ywo4t5PPSVUCWDwUlOVwQ==", + "3++dZXzZ6AFEz7hK+i5hww==", + "3+9nURtBK3FKn0J9DQDa3g==", + "3+zsjCi7TnJhti//YXK35w==", + "3/1puZTGSrD9qNKPGaUZww==", + "300hoYyMR/mk1mfWJxS8/w==", + "301utVPZ93AnPLYbsiJggw==", + "312g8iTB9oJgk/OqcgR7Cw==", + "342VOUOxoLHUqtHANt83Hw==", + "36XDmX6j542q+Oei1/x0gw==", + "37Nkh06O979nt7xzspOFyQ==", + "3AKEYQqpkfW7CZMFQZoxOw==", + "3AVYtcIv7A5mVbVnQMaCeA==", + "3BjLFon1Il0SsjxHE2A1LQ==", + "3CJbrUdW68E3Drhe4ahUnQ==", + "3EhLkC9NqD3A6ApV6idmgg==", + "3Ejtsqw3Iep/UQd0tXnSlg==", + "3FH4D31nKV13sC9RpRZFIg==", + "3Gg9N7vjAfQEYOtQKuF/Eg==", + "3HPOzIZxoaQAmWRy9OkoSg==", + "3JhnM6G4L06NHt31lR0zXA==", + "3L3KEBHhgDwH615w4OvgZA==", + "3Leu2Sc+YOntJFlrvhaXeg==", + "3P2aJxV8Trll2GH9ptElYA==", + "3RTtSaMp1TZegJo5gFtwwA==", + "3TbRZtFtsh9ez8hqZuTDeA==", + "3TjntNWtpG7VqBt3729L6Q==", + "3UBYBMejKInSbCHRoJJ7dg==", + "3UNJ37f+gnNyYk9yLFeoYA==", + "3WVBP9fyAiBPZAq3DpMwOQ==", + "3Wfj05vCLFAB9vII5AU9tw==", + "3WwITQML938W9+MUM56a3A==", + "3XyoREdvhmSbyvAbgw2y/A==", + "3Y4w0nETru3SiSVUMcWXqw==", + "3Y6/HqS1trYc9Dh778sefg==", + "3YXp1PmMldUjBz3hC6ItbA==", + "3djRJvkZk9O2bZeUTe+7xQ==", + "3go7bJ9WqH/PPUTjNP3q/Q==", + "3hVslsq98QCDIiO40JNOuA==", + "3iC21ByW/YVL+pSyppanWw==", + "3itfXtlLPRmPCSYaSvc39Q==", + "3j0kFUZ6g+yeeEljx+WXGg==", + "3jmCreW5ytSuGfmeLv7NfQ==", + "3jqsY8/xTWELmu/az3Daug==", + "3kREs/qaMX0AwFXN0LO5ow==", + "3ltw31yJuAl4VT6MieEXXw==", + "3nthUmLZ30HxQrzr2d7xFA==", + "3oMTbWf7Bv83KRlfjNWQZA==", + "3pi3aNVq1QNJmu1j0iyL0g==", + "3rbml1D0gfXnwOs5jRZ3gA==", + "3sNJJIx1NnjYcgJhjOLJOg==", + "3v09RHCPTLUztqapThYaHg==", + "3xw8+0/WU51Yz4TWIMK8mw==", + "3y5Xk65ShGvWFbQxcZaQAQ==", + "3yDD+xT8iRfUVdxcc7RxKw==", + "3yavzOJ1mM44pOSFLLszgA==", + "4+htiqjEz9oq0YcI/ErBVg==", + "40HzgVKYnqIb6NJhpSIF0A==", + "40gCrW4YWi+2lkqMSPKBPg==", + "41WEjhYUlG6jp2UPGj11eQ==", + "444F9T6Y7J67Y9sULG81qg==", + "46FCwqh+eMkf+czjhjworw==", + "46piyANQVvvLqcoMq5G8tQ==", + "49jZr/mEW6fvnyzskyN40w==", + "49z/15Nx9Og7dN9ebVqIzg==", + "4A+RHIw+aDzw0rSRYfbc7g==", + "4BkqgraeXY7yaI1FE07Evw==", + "4CfEP8TeMKX33ktwgifGgA==", + "4DIPP/yWRgRuFqVeqIyxMQ==", + "4FBBtWPvqJ3dv4w25tRHiQ==", + "4ID0PHTzIMZz2rQqDGBVfA==", + "4KJZPCE9NKTfzFxl76GWjg==", + "4LtQrahKXVtsbXrEzYU1zQ==", + "4LvQSicqsgxQFWauqlcEjw==", + "4NHQwbb3zWq2klqbT/pG6g==", + "4NP8EFFJyPcuQKnBSxzKgQ==", + "4PBaoeEwUj79njftnYYqLg==", + "4Qinl7cWmVeLJgah8bcNkw==", + "4SdHWowXgCpCDL28jEFpAw==", + "4TQkMnRsXBobbtnBmfPKnA==", + "4VR5LiXLew6Nyn91zH9L4w==", + "4WO6eT0Rh6sokb29zSJQnQ==", + "4WRdAjiUmOQg2MahsunjAg==", + "4WcFEswYU/HHQPw77DYnyA==", + "4XNUmgwxsqDYsNmPkgNQYQ==", + "4Xh/B3C16rrjbES+FM1W8g==", + "4ZFYKa7ZgvHyZLS6WpM8gA==", + "4aPU6053cfMLHgLwAZJRNg==", + "4ekt4m38G9m599xJCmhlug==", + "4erEA42TqGA9K4iFKkxMMA==", + "4ifNsmjYf1iOn2YpMfzihg==", + "4iiCq+HhC+hPMldNQMt0NA==", + "4itEKfbRCJvqlgKnyEdIOQ==", + "4jeOFKuKpCmMXUVJSh9y0g==", + "4kXlJNuT79XXf1HuuFOlHw==", + "4kj0S8XlmhHXoUP7dQItUw==", + "4mQVNv7FHj+/O6XFqWFt/Q==", + "4mig4AMLUw+T/ect9p4CfA==", + "4qMSNAxichi3ori/pR+o0w==", + "4rrSL6N0wyucuxeRELfAmw==", + "4u3eyKc+y3uRnkASrgBVUw==", + "4wnUAbPT3AHRJrPwTTEjyw==", + "4xojeUxTFmMLGm6jiMYh/Q==", + "4yEkKp2FYZ09mAhw2IcrrA==", + "4yVqq66iHYQjiTSxGgX2oA==", + "4yrFNgqWq17zVCyffULocA==", + "50jASqzGm4VyHJbFv8qVRA==", + "50xwiYvGQytEDyVgeeOnMg==", + "51yLpfEdvqXmtB6+q27/AQ==", + "520wTzrysiRi2Td92Zq0HQ==", + "53UccFNzMi9mKmdeD82vAw==", + "54XELlPm8gBvx8D5bN3aUg==", + "59ipbMH7cKBsF9bNf4PLeQ==", + "5CMadLqS2KWwwMCpzlDmLw==", + "5DDb7fFJQEb3XTc3YyOTjg==", + "5HovoyHtul8lXh+z8ywq9A==", + "5I/heFSQG/UpWGx0uhAqGQ==", + "5KOgetfZR+O2wHQSKt41BQ==", + "5LJqHFRyIwQKA4HbtqAYQQ==", + "5LuFDNKzMd2BzpWEIYO2Ww==", + "5M3dFrAOemzQ0MAbA8bI5w==", + "5N2oi2pB69NxeNt08yPLhw==", + "5NEP7Xt7ynj6xCzWzt21hQ==", + "5Nk2Z94DhlIdfG5HNgvBbQ==", + "5PfGtbH9fmVuNnq83xIIgQ==", + "5Q/Y2V0iSVTK8HE8JerEig==", + "5S5/asYfWjOwnzYpbK6JDw==", + "5SbwLDNT6sBOy6nONtUcTg==", + "5T39s5CtSrK5awMPUcEWJg==", + "5VO1inwXMvLDBQSOahT6rg==", + "5VY++KiWgo7jXSdFJsPN3A==", + "5Wcq+6hgnWsQZ/bojERpUw==", + "5Yrj6uevT8wHRyqqgnSfeg==", + "5dUry23poD+0wxZ3hH6WmA==", + "5eHStFN7wEmIE+uuRwIlPQ==", + "5eXpiczlRdmqMYSaodOUiQ==", + "5gGoDPTc/sOIDLngmlEq4A==", + "5jHgQF4SfO/zy9xy9t+9dw==", + "5jyuDp82Fux+B0+zlx8EXw==", + "5kvyy902llnYGQdn2Py04w==", + "5l6kDfjtZjkTZPJvNNOVFw==", + "5lfLJAk1L3QzGMML3fOuSw==", + "5m1ijXEW+4RTNGZsDA/rxQ==", + "5oD/aGqoakxaezq43x0Tvw==", + "5pje7qyz8BRsa8U4a4rmoA==", + "5pqqzC/YmRIMA9tMFPi7rg==", + "5r1ZsGkrzNQEpgt/gENibw==", + "5u2PdDcIY3RQgtchSGDCGg==", + "5ugVOraop5P5z5XLlYPJyQ==", + "5w/c9WkI/FA+4lOtdPxoww==", + "5w4FbRhWACP7k2WnNitiHg==", + "6+jhreeBLfw64tJ+Nhyipw==", + "600bwlyhcy754W1E6tuyYg==", + "600mjiWke4u0CDaSQKLOOg==", + "60suecbWRfexSh7C67RENA==", + "61V74uIjaSfZM8au1dxr1A==", + "62RHCbpGU8Hb+Ubn+SCTBg==", + "63OTPaKM0xCfJOy9EDto+Q==", + "64AA4jLHXc1Dp15aMaGVcA==", + "64QzHOYX0A9++FqRzZRHlQ==", + "64YsV2qeDxk2Q6WK/h7OqA==", + "65KhGKUBFQubRRIEdh9SwQ==", + "6706ncrH1OANFnaK6DUMqQ==", + "68jPYo3znYoU4uWI7FH3/g==", + "68nqDtXOuxF7DSw6muEZvg==", + "6ACvJNfryPSjGOK39ov8Qg==", + "6CjtF1S2Y6RCbhl7hMsD+g==", + "6G2bD3Y7qbGmfPqH9TqLFA==", + "6GXHGF62/+jZ7PfIBlMxZw==", + "6HGeEPyTAu9oiKhNVLjQnA==", + "6HnWgYNKohqhoa1tnjjU3A==", + "6M6QapJ5xtMXfiD3bMaiLA==", + "6NP81geiL14BeQW6TpLnUA==", + "6PzjncEw2wHZg7SP7SQk9w==", + "6QAtjOK9enNLRhcVa2iaTg==", + "6QUGE2S8oFYx4T4nW56cCw==", + "6W79FmpUN1ByNtv5IEXY4w==", + "6WhHPWlqEUqXC52rHGRHjA==", + "6XYqR2WvDzx4fWO7BIOTjA==", + "6Z9myGCF5ylWljgIYAmhqw==", + "6ZKmm7IW7IdWuVytLr68CQ==", + "6ZMs9vCzK9lsbS6eyzZlIA==", + "6b7ue29cBDsvmj1VSa5njw==", + "6c0iuya20Ys8BsvoI4iQaQ==", + "6cTETZ9iebhWl+4W5CB+YQ==", + "6dshA8knH5qqD+KmR/kdSQ==", + "6e8boFcyc8iF0/tHVje4eQ==", + "6erpZS36qZRXeZ9RN9L+kw==", + "6fWom3YoKvW6NIg6y9o9CQ==", + "6k2cuk0McTThSMW/QRHfjA==", + "6lVSzYUQ/r0ep4W2eCzFpg==", + "6leyDVmC5jglAa98NQ3+Hg==", + "6nwR+e9Qw0qp8qIwH9S/Mg==", + "6o5g9JfKLKQ2vBPqKs6kjg==", + "6rIWazDEWU5WPZHLkqznuQ==", + "6rqK8sjLPJUIp7ohkEwfZg==", + "6sBemZt4qY/TBwqk3YcLOQ==", + "6sNP0rzCCm3w976I2q2s/w==", + "6tfM6dx3R5TiVKaqYQjnCg==", + "6txm8z4/LGCH0cpaet/Hsg==", + "6uMF5i0b/xsk55DlPumT7A==", + "6uT7LZiWjLnnqnnSEW4e/Q==", + "6v3eTZtPYBfKFSjfOo2UaA==", + "6wkfN8hyKmKU6tG3YetCmw==", + "6z8CRivao3IMyV4p4gMh7g==", + "71w3aSvuh2mBLtdqJCN3wA==", + "734u4Y1R3u7UNUnD+wWUoA==", + "74FW/QYTzr/P1k6QwVHMcw==", + "778O1hdVKHLG2q9dycUS0Q==", + "78b8sDBp28zUlYPV5UTnYw==", + "79uTykH43voFC3XhHHUzKg==", + "7E6V6/zSjbtqraG7Umj+Jw==", + "7Ephy+mklG2Y3MFdqmXqlA==", + "7Eqzyb+Kep+dIahYJWNNxQ==", + "7GgNLBppgAKcgJCDSsRqOQ==", + "7J3FoFGuTIW36q0PZkgBiw==", + "7K8l6KoP0BH82/WMLntfrg==", + "7R5rFaXCxM3moIUtoCfM2g==", + "7Tauesu7bgs5lJmQROVFiQ==", + "7VHlLw20dWck+I8tCEZilA==", + "7W9aF7dxnL+E8lbS/F7brg==", + "7XRiYvytcwscemlxd9iXIQ==", + "7Y87wVJok20UfuwkGbXxLg==", + "7b0oo4+qphu6HRvJq6qkHQ==", + "7bM/pn4G7g7Zl6Xf1r62Lg==", + "7br49X11xc2GxQLSpZWjKQ==", + "7btpMFgeGkUsiTtsmNxGQA==", + "7cnUHeaPO8txZGGWHL9tKg==", + "7dz+W494zwU5sg63v5flCg==", + "7k5rBuh8FbTTI4TP87wBPQ==", + "7l0RMKbONGS/goW/M+gnMQ==", + "7mxU5fJl/c6dXss9H3vGcQ==", + "7nr3zyWL+HHtJhRrCPhYZA==", + "7p4NpnoNSQR7ISg+w+4yFg==", + "7pkUY2UzSbGnwLvyRrbxfA==", + "7sCJ4RxbxRqVnF4MBoKfuQ==", + "7w3b73nN/fIBvuLuGZDCYQ==", + "7w4PDRJxptG8HMe/ijL6cQ==", + "7wgT9WIiMVcrj48PVAMIgw==", + "7xDIG/80SnhgxAYPL9YJtg==", + "7xTKFcog69nTmMfr5qFUTA==", + "80C9TB9/XT1gGFfQDJxRoA==", + "80PCwYh4llIKAplcDvMj4g==", + "80UE+Ivby3nwplO/HA7cPw==", + "81ZH3SO0NrOO+xoR/Ngw1g==", + "81iQLU+YwxNwq4of6e9z7A==", + "81nkjWtpBhqhvOp6K8dcWg==", + "81pAhreEPxcKse+++h1qBg==", + "82hTTe1Nr4N2g7zwgGjxkw==", + "83ERX2XJV3ST4XwvN7YWCg==", + "83WGpQGWyt6mCV+emaomog==", + "83wtvSoSP9FVBsdWaiWfpA==", + "861mBNvjIkVgkBiocCUj/Q==", + "88PNi9+yn3Bp4/upgxtWGA==", + "88tB/HgUIUnqWXEX++b5Aw==", + "897ptlztTjr7yk+pk8MT0Q==", + "8AfCSZC0uasVON9Y/0P2Pw==", + "8B12CamjOGzJDnQ+RkUf4w==", + "8BLkvEkfnOizJq0OTCYGzw==", + "8CjmgWQSAAGcXX9kz3kssw==", + "8Cm19vJW8ivhFPy0oQXVNA==", + "8DtgIyYiNFqDc5qVrpFUng==", + "8GyPup4QAiolFJ9v80/Nkw==", + "8JVHFRwAd/SCLU0CRJYofg==", + "8LNNoHe6rEQyJ0ebl151Mw==", + "8M0kSvjn5KN8bjsMdUqKZQ==", + "8N3mhHt29FZDHn1P2WH1wQ==", + "8OFxXwnPmrogpNoueZlC4Q==", + "8QK7emHS6rAcAF5QQemW/A==", + "8RtLlzkGEiisy1v9Xo0sbw==", + "8VqeoQELbCs232+Mu+HblA==", + "8WU1vLKV1GhrL7oS9PpABg==", + "8ZBiwr842ZMKphlqmNngHw==", + "8ZFPMJJYVJHsfRpU4DigSg==", + "8ZqmPJDnQSOFXvNMRQYG2Q==", + "8c+lvG5sZNimvx9NKNH3ug==", + "8cXqZub6rjgJXmh1CYJBOg==", + "8dBIsHMEAk7aoArLZKDZtg==", + "8dUcSkd2qnX5lD9B+fUe+Q==", + "8dbyfox/isKLsnVjQNsEXg==", + "8fJLQeIHaTnJ8wGqUiKU6g==", + "8g08gjG/QtvAYer32xgNAg==", + "8hsfXqi4uiuL+bV1VrHqCw==", + "8iYdEleTXGM+Wc85/7vU9w==", + "8j9GVPiFdfIRm/+ho7hpoA==", + "8nOTDhFyZ8YUA4b6M5p84w==", + "8snljTGo/uICl9q0Hxy7/A==", + "8uP4HUnSodw88yoiWXOIcw==", + "8vLA9MOdmLTo3Qg+/2GzLA==", + "8vr+ERVrM99dp+IGnCWDGQ==", + "8ylI1AS3QJpAi3I/NLMYdg==", + "9+hjTVMQUsvVKs7Tmp52tg==", + "90dtIMq0ozJXezT2r79vMQ==", + "91+Yms6Oy/rP0rVjha5z9w==", + "91LQuW6bMSxl10J/UDX23A==", + "91SdBFJEZ65M+ixGaprY/A==", + "91VcAVv7YDzkC1XtluPigw==", + "91vfsZ7Lx9x5gqWTOdM4sg==", + "96ORaz1JRHY1Gk8H74+C2g==", + "99+SBN45LwKCPfrjUKRPmw==", + "9Bet5waJF5/ZvsYaHUVEjQ==", + "9DRHdyX8ECKHUoEsGuqR4Q==", + "9DtM1vls4rFTdrSnQ7uWXw==", + "9FdpxlIFu11qIPdO7WC5nw==", + "9Gkw+hvsR/tFY1cO89topg==", + "9J53kk+InE3CKa7cPyCXMw==", + "9JKIJrlQjhNSC46H3Cstcw==", + "9L6yLO93sRN70+3qq3ObfA==", + "9MDG0WeBPpjGJLEmUJgBWg==", + "9QFYrCXsGsInUb4SClS3cQ==", + "9RGIQ2qyevNbSSEF36xk/A==", + "9RXymE9kCkDvBzWGyMgIWA==", + "9SUOfKtfKmkGICJnvbIDMg==", + "9SgfpAY0UhNC6sYGus9GgQ==", + "9T7gB0ZkdWB0VpbKIXiujQ==", + "9TalxEyFgy6hFCM73hgb7Q==", + "9UhKmKtr4vMzXTEn74BEhg==", + "9W57pTzc572EvSURqwrRhw==", + "9Y1ZmfiHJd9vCiZ6KfO1xQ==", + "9aKH1u5+4lgYhhLztQ4KWA==", + "9ajIS45NTicqRANzRhDWFA==", + "9bAWYElyRN1oJ6eJwPtCtQ==", + "9cvHJmim9e0pOaoUEtiM6A==", + "9dbn0Kzwr9adCEfBJh78uQ==", + "9iB7+VwXRbi6HLkWyh9/kg==", + "9inw7xzbqAnZDKOl/MfCqA==", + "9jxA/t3TQx8dQ+FBsn/YCg==", + "9k17UqdR1HzlF7OBAjpREA==", + "9k1u/5TgPmXrsx3/NsYUhg==", + "9lLhHcrPWI4EsA4fHIIXuw==", + "9nMltdrrBmM5ESBY2FRjGA==", + "9oQ/SVNJ4Ye9lq8AaguGAQ==", + "9oUawSwUGOmb0sDn3XS6og==", + "9onh6QKp70glZk9cX3s34A==", + "9pdeedz1UZUlv8jPfPeZ1g==", + "9pk75mBzhmcdT+koHvgDlw==", + "9qWLbRLXWIBJUXYjYhY2pg==", + "9rL8nC/VbSqrvnUtH9WsxQ==", + "9reBKZ1Rp6xcdH1pFQacjw==", + "9s3ar9q32Y5A3tla5GW/2Q==", + "9sYLg75/hudZaBA3FrzKHw==", + "9tiibT8V9VwnPOErWGNT3w==", + "9vEgJVJLEfed6wJ7hBUGgQ==", + "9viAzLFGYYudBYFu7kFamg==", + "9vmJUS7WIVOlhMqwipAknQ==", + "9wUIeSgNN36SFxy8v2unVg==", + "9xIgKpZGqq0/OU6wM5ZSHw==", + "9xmtuClkFlpz/X5E9JBWBA==", + "A+DLpIlYyCb9DaarpLN76g==", + "A2ODff+ImIkreJtDPUVrlg==", + "A3dX2ShyL9+WOi6MNJBoYQ==", + "A6TLWhipfymkjPYq8kaoDQ==", + "AChOz8avRYsvxlbWcorQ3w==", + "AEpTVUQhIEJGlXJB6rS26A==", + "AFdelaqvxRj6T3YdLgCFyg==", + "AGd0rcLnQ0n+meYyJur1Pw==", + "AGoVLd0QPcXnTedT5T95JQ==", + "ALJWKUImVE40MbEooqsrng==", + "ALlGgVDO8So71ccX0D6u2g==", + "AMfL0rH+g8c0VqOUSgNzQw==", + "ARCWkHAnVgBOIkCDQ19ZuA==", + "ARKIvf4+zRF8eCvUITWPng==", + "ATmMzriwGLl+M3ppkfcZNA==", + "AUGmvZkpkKBry5bHZn4DJA==", + "AV/YJfdoDUdRcrXVwinhQg==", + "AVjwqrTBQH1VREuBlOyUOg==", + "AX1HxQKXD12Yv5HWi39aPQ==", + "AYxGETZs477n2sa1Ulu/RQ==", + "AZs3v4KJYxdi8T1gjVjI2Q==", + "AcKwfS8FRVqb72uSkDNY/Q==", + "AcbG0e6xN8pZfYAv7QJe1Q==", + "Af9j1naGtnZf0u1LyYmK1w==", + "AfVPdxD3FyfwwNrQnVNQ7A==", + "AgDJsaW0LkpGE65Kxk5+IA==", + "Ahpi9+nl13kPTdzL+jgqMw==", + "AiMtfedwGcddA+XYNc+21g==", + "AjHz9GkRTFPjrqBokCDzFw==", + "Ak3rlzEOds6ykivfg39xmw==", + "AkAes5oErTaJiGD2I4A1Pw==", + "AklOdt9/2//3ylUhWebHRw==", + "Al8+d/dlOA5BXsUc5GL8Tg==", + "Ao1Zc0h5AdSHtYt1caWZnQ==", + "AoN/pnK4KEUaGw4V9SFjpg==", + "ApiuEPWr8UjuRyJjsYZQBw==", + "AqHVaj3JcR44hnMzUPvVYg==", + "Ar1Eb/f/LtuIjXnnVPYQlA==", + "Ar9N1VYgE7riwmcrM3bA2Q==", + "AsAHrIkMgc3RRWnklY9lJw==", + "AvdeYb9XNOUFWiiz+XGfng==", + "AwPTZpC28NJQhf5fNiJuLA==", + "AxEjImKz4tMFieSo7m60Sg==", + "AyWlT+EGzIXc395zTlEU5Q==", + "B+TsxQZf0IiQrU8X9S4dsQ==", + "B0TaUQ6dKhPfSc5V/MjLEQ==", + "B1VVUbl8pU0Phyl1RYrmBg==", + "B6reUwMkQFaCHb9BYZExpw==", + "BA18GEAOOyVXO2yZt2U35w==", + "BAJ+/jbk2HyobezZyB9LiQ==", + "BB/R8oQOcoE4j63Hrh8ifg==", + "BB9PTlwKAWkExt3kKC/Wog==", + "BDNM1u/9mefjuW1YM2DuBg==", + "BDbfe/xa9Mz1lVD82ZYRGA==", + "BH+rkZWQjTp7au6vtll/CQ==", + "BL3buzSCV78rCXNEhUhuKQ==", + "BLJk9wA88z6e0IQNrWJIVw==", + "BLbTFLSb4mkxMaq4/B2khg==", + "BMOi5JmFUg5sCkbTTffXHw==", + "BMZB1FwvAuEqyrd0rZrEzw==", + "BPT4PQxeQcsZsUQl33VGmg==", + "BTiGLT6XdZIpFBc91IJY6g==", + "BV1moliPL15M14xkL+H1zw==", + "BW0A06zoQw7S+YMGaegT7g==", + "BXGlq54wIH6R3OdYfSSDRw==", + "BYpHADmEnzBsegdYTv8B5Q==", + "BYz52gYI/Z6AbYbjWefcEA==", + "BZTzHJGhzhs3mCXHDqMjnQ==", + "BaRwTrc5ulyKbW4+QqD0dw==", + "BhKO1s1O693Fjy1LItR/Jw==", + "BjfOelfc1IBgmUxMJFjlbQ==", + "BlCgDd7EYDIqnoAiKOXX6Q==", + "BophnnMszW5o+ywgb+3Qbw==", + "Bq82MoMcDjIo/exqd/6UoA==", + "BuDVDLl0OGdomEcr+73XhQ==", + "BuENxPg7JNrWXcCxBltOPg==", + "Bv4mNIC72KppYw/nHQxfpQ==", + "Bvk8NX4l6WktLcRDRKsK/A==", + "BwRA+tMtwEvth28IwpZx+w==", + "BxFP+4o6PSlGN78eSVT1pA==", + "BxsDnI8jXr4lBwDbyHaYXw==", + "Byhi4ymFqqH8uIeoMRvPug==", + "BzkNYH03gF/mQY71RwO3VA==", + "C+Ssp+v1r+00+qiTy2d7kA==", + "C4QEzQKGxyRi2rjwioHttA==", + "C65PZm8rZxJ6tTEb6d08Eg==", + "C7UaoIEXsVRxjeA0u99Qmw==", + "CBAGa5l95f3hVzNi6MPWeQ==", + "CCK+6Dr72G3WlNCzV7nmqw==", + "CDsanJz7e3r/eQe+ZYFeVQ==", + "CF1sAlhjDQY/KWOBnSSveA==", + "CHLHizLruvCrVi9chj9sXA==", + "CHsFJfsvZkPWDXkA6ZMsDQ==", + "CJoZn5wdTXbhrWO5LkiW0g==", + "CLPzjXKGGpJ0VrkSJp7wPQ==", + "CPDs+We/1wvsGdaiqxzeCQ==", + "CQ0PPwgdG3N6Ohfwx1C8xA==", + "CQpJFrpOvcQhsTXIlJli+Q==", + "CRiL6zpjfznhGXhCIbz8pQ==", + "CRmAj3JcasAb4iZ9ZbNIbw==", + "CT3ldhWpS1SEEmPtjejR/Q==", + "CT9g8mKsIN/VeHLSTFJcNQ==", + "CUCjG2UaEBmiYWQc6+AS1Q==", + "CUEueo8QXRxkfVdfNIk/gg==", + "CWBGcRFYwZ0va6115vV/oQ==", + "CX/N/lHckmAtHKysYtGdZA==", + "CXMKIdGvm60bgfsNc+Imvg==", + "CYJB3qy5GalPLAv1KGFEZA==", + "CZNoTy26VUQirvYxSPc/5A==", + "CZbd+UoTz0Qu1kkCS3k8Xg==", + "CazLJMJjQMeHhYLwXW7YNg==", + "Ci7sS7Yi1+IwAM3VMAB4ew==", + "CiiUeJ0LeWfm7+gmEmYXtg==", + "CkDIoAFLlIRXra78bxT/ZA==", + "CkZUmKBAGu0FLpgPDrybpw==", + "Cl1u5nGyXaoGyDmNdt38Bw==", + "CmBf5qchS1V3C2mS6Rl4bw==", + "CmVD6nh8b/04/6JV9SovlA==", + "CmkmWcMK4eqPBcRbdnQvhw==", + "CnIwpRVC2URVfoiymnsdYQ==", + "CoLvjQDQGldGDqRxfQo+WQ==", + "CrJDgdfzOea2M2hVedTrIg==", + "CsPkyTZADMnKcgSuNu1qxg==", + "CtDj/h2Q/lRey20G8dzSgA==", + "CuGIxWhRLN7AalafBZLCKQ==", + "Cv079ZF55RnbsDT27MOQIA==", + "Cz1G77hsDtAjpe0WzEgQog==", + "CzP13PM/mNpJcJg8JD3s6w==", + "CzSumIcYrZlxOUwUnLR2Zw==", + "CzWhuxwYbNB/Ffj/uSCtbw==", + "D09afzGpwCEH0EgZUSmIZA==", + "D0Qt9sRlMaPnOv1xaq+XUg==", + "D0W5F7gKMljoG5rlue1jrg==", + "D175i+2bZ7aWa4quSSkQpA==", + "D2JcY4zWwqaCKebLM8lPiQ==", + "D31ZticrjGWAO45l5hFh7A==", + "D5ibbo8UJMfFZ48RffuhgQ==", + "D5jaV+HtXkSpSxJPmaBDXg==", + "D66Suu3tWBD+eurBpPXfjA==", + "D7piVoB2NJlBxK5owyo4+g==", + "D7wN7b5u5PKkMaLJBP9Ksw==", + "DA+3fjr7mgpwf6BZcExj0w==", + "DB706G73NpBSRS8TKQOVZw==", + "DBKrdpCE0awppxST4o/zzg==", + "DCjgaGV5hgSVtFY5tcwkuA==", + "DCvI9byhw0wOFwF1uP6xIQ==", + "DDitrRSvovaiXe2nfAtp4g==", + "DEaZD/8aWV6+zkiLSVN/gA==", + "DG2Qe2DqPs5MkZPOqX363Q==", + "DJ+a37tCaGF5OgUhG+T0NA==", + "DJmrmNRKARzsTCKSMLmcNA==", + "DJoy1NSZZw87oxWGlNHhfg==", + "DJscTYNFPyPmTb57g/1w+Q==", + "DKApp/alXiaPSRNm3MfSuA==", + "DLzHkTjjuH6LpWHo2ITD0Q==", + "DMHmyn2U2n+UXxkqdvKpnA==", + "DO1/jfP/xBI9N0RJNqB2Rw==", + "DQJRsUwO1fOuGlkgJavcwQ==", + "DQQB/l55iPN9XcySieNX3A==", + "DQeib845UqBMEl96sqsaSg==", + "DQlZWBgdTCoYB1tJrNS5YQ==", + "DRiFNojs7wM8sfkWcmLnhQ==", + "DWKsPfKDAtfuwgmc2dKUNg==", + "DY0IolKTYlW+jbKLPAlYjQ==", + "DYWCPUq/hpjr6puBE7KBHg==", + "DbWQI3H2tcJsVJThszfHGA==", + "DdaT4JLC7U0EkF50LzIj9w==", + "DdiNGiOSoIZxrMrGNvqkXw==", + "DinJuuBX9OKsK5fUtcaTcQ==", + "DjHszpS8Dgocv3oQkW/VZQ==", + "DjeSrUoWW2QAZOAybeLGJg==", + "Dk0L/lQizPEb3Qud6VHb1Q==", + "DmxgZsQg+Qy1GP0fPkW3VA==", + "Dmyb+a7/QFsU4d2cVQsxDw==", + "DnF6TYSJxlc+cwdfevLYng==", + "Do3aqbRKtmlQI2fXtSZfxQ==", + "DoiItHSms0B9gYmunVbRkQ==", + "DqzWt1gfyu/e7RQl5zWnuQ==", + "Dt6hvhPJu94CJpiyJ5uUkg==", + "Dt8Q5ORzTmpPR2Wdk0k+Aw==", + "DuEKxykezAvyaFO2/5ZmKQ==", + "Dulw855DfgIwiK7hr3X8vg==", + "Duz/8Ebbd0w6oHwOs0Wnwg==", + "DwOTyyCoUfaSShHZx9u6xg==", + "DwP0MQf71VsqvAbAMtC3QQ==", + "DwrNdmU5VFFf3TwCCcptPA==", + "Dz90OhYEjpaJ/pxwg1Qxhg==", + "E+02smwQGBIxv42LIF2Y4Q==", + "E1CvxFbuu9AYW604mnpGTw==", + "E2LR1aZ3DcdCBuVT7BhReA==", + "E2v8Kk60qVpQ232YzjS2ow==", + "E3jMjAgXwvwR8PA53g4+PQ==", + "E4NtzxQruLcetC23zKVIng==", + "E4ojRDwGsIiyuxBuXHsKBA==", + "E8yMPK7W0SIGTK6gIqhxiQ==", + "E9IlDyULLdeaVUzN6eky8g==", + "E9ajQQMe02gyUiW3YLjO/A==", + "E9yeifEZtpqlD0N3pomnGw==", + "EATnlYm0p3h04cLAL95JgA==", + "EC0+iUdSZvmIEzipXgj7Gg==", + "EGLOaMe6Nvzs/cmb7pNpbg==", + "EJgedRYsZPc4cT9rlwaZhg==", + "EKU3OVlT4b/8j3MTBqpMNg==", + "ENFfP93LA257G6pXQkmIdg==", + "EUXQZwLgnDG+C8qxVoBNdw==", + "EXveRXjzsjh8zbbQY2pM9g==", + "EZVQGsXTZvht1qedRLF8bQ==", + "EbGG4X18upaiVQmPfwKytg==", + "EdvIAKdRAXj7e42mMlFOGQ==", + "Ee4A3lTMLQ7iDQ7b8QP8Qg==", + "EfXDc6h69aBPE6qsB+6+Ig==", + "Egs14xVbRWjfBBX7X5Z60g==", + "Ej7W3+67kCIng3yulXGpRQ==", + "ElTNyMR4Rg8ApKrPw88WPg==", + "Epm0d/DvXkOFeM4hoPCBrg==", + "EqMlrz1to7HG4GIFTPaehQ==", + "EqYq2aVOrdX5r7hBqUJP7g==", + "Err1mbWJud80JNsDEmXcYg==", + "EuGWtIbyKToOe6DN3NkVpQ==", + "Ev/xjTi7akYBI7IeZJ4Igw==", + "EvSB+rCggob2RBeXyDQRvQ==", + "Ex3x5HeDPhgO2S9jjCFy4g==", + "EyIsYQxgFa4huyo/Lomv7g==", + "EzjbinBHx3Wr08eXpH3HXA==", + "F50iXjRo1aSTr37GQQXuJA==", + "F58ktE4O0f7C9HdsXYm+lw==", + "F5FcNti7lUa9DyF2iEpBug==", + "F5bs0GGWBx9eBwcJJpXbqg==", + "F8l+Qd9TZgzV+r8G584lKA==", + "F8tEIT5EhcvLNRU5f0zlXQ==", + "FA+nK6mpFWdD0kLFcEdhxA==", + "FAXzjjIr8l1nsQFPpgxM/g==", + "FCLQocqxxhJeleARZ6kSPg==", + "FH5Z60RXXUiDk+dSZBxD3g==", + "FHvI0IVNvih8tC7JgzvCOw==", + "FI2WhaSMb3guFLe3e9il8Q==", + "FIOCTEbzb2+KMCnEdJ7jZw==", + "FL/j3GJBuXdAo54JYiWklQ==", + "FLvED9nB9FEl9LqPn7OOrA==", + "FN7oLGBQGHXXn5dLnr/ElA==", + "FNvQqYoe0s/SogpAB7Hr1Q==", + "FUQySDFodnRhr+NUsWt0KA==", + "FV/D5uSco+Iz8L+5t7E8SA==", + "FWphIPZMumqnXr1glnbK4w==", + "FXzaxi3nAXBc8WZfFElQeA==", + "FbxScyuRacAQkdQ034ShTA==", + "FcFcn4qmPse5mJCX5yNlsA==", + "FcKjlHKfQAGoovtpf+DxWQ==", + "Fd0c8f2eykUp9GYhqOcKoA==", + "Fd2fYFs8vtjws2kx1gf6Rw==", + "FeRovookFQIsXmHXUJhGOw==", + "FhthAO5IkMyW4dFwpFS7RA==", + "Fiy3hkcGZQjNKSQP9vRqyA==", + "FltEN+7NKvzt+XAktHpfHA==", + "FnVNxl5AFH1AieYru2ZG+A==", + "FoJZ61VrU8i084pAuoWhDQ==", + "FpWDTLTDmkUhH/Sgo+g1Gg==", + "FpgdsQ2OG+bVEy3AeuLXFQ==", + "FqWLkhWl0iiD/u2cp+XK9A==", + "FrTgaF5YZCNkyfR1kVzTLQ==", + "Ft2wXUokFdUf6d2Y/lwriw==", + "FtxpWdhEmC6MT61qQv4DGA==", + "FuWspiqu5g8Eeli5Az+BkA==", + "FxnbKnuDct4OWcnFMT/a5w==", + "Fz8EI+ZpYlbcttSHs5PfpA==", + "FzqIpOcTsckSNHExrl+9jg==", + "Fzuq+Wg7clo6DTujNrxsSA==", + "G+sGF13VXPH4Ih6XgFEXxg==", + "G/PA+kt0N+jXDVKjR/054A==", + "G0LChrb0OE5YFqsfTpIL1Q==", + "G0MlFNCbRjXk4ekcPO/chQ==", + "G2UponGde3/Z+9b2m9abpQ==", + "G37U8XTFyshfCs7qzFxATg==", + "G3PmmPGHaWHpPW30xQgm3Q==", + "G4qzBI1sFP2faN+tlRL/Bw==", + "G736AX070whraDxChqUrqw==", + "G7J/za99BFbAZH+Q+/B8WA==", + "G8LFBop8u6IIng+gQuVg3w==", + "GA8k6GQ20DGduVoC+gieRA==", + "GCYI9Dn1h3gOuueKc7pdKA==", + "GDMqfhPQN0PxfJPnK1Bb9A==", + "GF0lY77rx1NQzAsZpFtXIQ==", + "GF2yvI9UWf1WY7V7HXmKPA==", + "GFRJoPcXlkKSvJRuBOAYHQ==", + "GG8a3BlwGrYIwZH9j3cnPA==", + "GHEdXgGWOeOa6RuPMF0xXg==", + "GIHKW6plyLra0BmMOurFgA==", + "GKzs8mlnQQc58CyOBTlfIg==", + "GLDNTSwygNBmuFwCIm7HtA==", + "GLmWLXURlUOJ+PMjpWEXVA==", + "GLnS9wDCje7TOMvBX9jJVA==", + "GNak/LFeoHWlTdLW1iU4eg==", + "GNrMvNXQkW7PydlyJa+f1w==", + "GQJxu1SoMBH14KPV/G/KrQ==", + "GSWncBq4nwomZCBoxCULww==", + "GT6WUDXiheKAM7tPg3he9A==", + "GTNttXfMniNhrbhn92Aykg==", + "GUiinC3vgBjbQC2ybMrMNQ==", + "GW1Uaq622QamiiF24QUA0g==", + "GWwJ32SZqD5wldrXUdNTLA==", + "GdTanUprpE3X/YjJDPpkhQ==", + "Gdf4VEDLBrKJNQ8qzDsIyw==", + "GglPoW5fvr4JSM3Zv99oiA==", + "GhpJfRSWZigLg/azTssyVA==", + "Ghuj9hAyfehmYgebBktfgA==", + "GmC+0rNDMIR+YbUudoNUXw==", + "GnJKlRzmgKN9vWyGfMq3aA==", + "GncGQgmWpI/fZyb/6zaFCg==", + "GrSbnecYAC3j5gtoKntL0A==", + "Gt4/MMrLBErhbFjGbiNqQQ==", + "GzbeM7snhe+M+J7X+gAsQw==", + "H+NHjk/GJDh/GaNzMQSzjg==", + "H+yPRiooEh5J7lAJB4RZ7Q==", + "H0UMAUfHFQH92A2AXRCBKA==", + "H1NJEI+fvOQbI51kaNQQjQ==", + "H1y2iXVaQYwP0SakN6sa+Q==", + "H1zH9I8RwfEy5DGz3z+dHw==", + "H6HPFAcdHFbQUNrYnB74dA==", + "H6j2nPbBaxHecXruxiWYkA==", + "HBRzLacCVYfwUVGzrefZYg==", + "HCbHUfsTDl6+bxPjT57lrA==", + "HCu4ZMrcLMZbPXbTlWuvvQ==", + "HDxGhvdQwGh0aLRYEGFqnw==", + "HEcOaEd9zCoOVbEmroSvJg==", + "HEghmKg3GN60K7otpeNhaA==", + "HFCQEiZf7/SNc+oNSkkwlA==", + "HFHMGgfOeO0UPrray1G+Zw==", + "HGxe+5/kkh6R9GXzEOOFHA==", + "HHxn4iIQ7m0tF1rSd+BZBg==", + "HI4ZIE5s8ez8Rb+Mv39FxA==", + "HITIVoFoWNg04NExe13dNA==", + "HJYgUxFZ66fRT8Ka73RaUg==", + "HK0yf7F97bkf1VYCrEFoWA==", + "HK9xG03FjgCy8vSR+hx8+Q==", + "HLesnV3DL+FhWF3h6RXe8g==", + "HLxROy6fx/mLXFTDSX4eLA==", + "HMQarkPWOUDIg5+5ja2dBQ==", + "HMWOlMmzocOIiJ7yG1YaDQ==", + "HOi+vsGAae4vhr+lJ5ATnQ==", + "HPvYV94ufwiNHEImu4OYvQ==", + "HRF3WL/ue3/QlYyu7NUTrA==", + "HRWYX2XOdsOqYzCcqkwIyw==", + "HYylUirJRqLm+dkp39fSOQ==", + "HaHTsLzx7V3G1SFknXpGxA==", + "HaIRV9SNPRTPDOSX9sK/bg==", + "HaSc7MZphCMysTy2JbTJkw==", + "Hb+pdSavvJ9lUXkSVZW8Og==", + "HbT6W1Ssd3W7ApKzrmsbcg==", + "HbXv8InyZqFT7i3VrllBgg==", + "HdB7Se47cWjPgpJN0pZuiA==", + "HdXg64DBy5WcL5fRRiUVOg==", + "HeQbUuBM9sqfXFXRBDISSw==", + "HfvsiCQN/3mT0FabCU5ygQ==", + "HgIFX42oUdRPu7sKAXhNWg==", + "HhBHt5lQauNl7EZXpsDHJA==", + "HiAgt86AyznvbI2pnLalVQ==", + "HjlPM2FQWdILUXHalIhQ5w==", + "HjyxyL0db2hGDq2ZjwOOhg==", + "HkbdaMuDTPBDnt3wAn5RpQ==", + "Hm6MG6BXbAGURVJKWRM6ZA==", + "HnVfyqgJ+1xSsN4deTXcIA==", + "HoaBBw2aPCyhh0f5GxF+/Q==", + "Hs3vUOOs2TWQdQZHs+FaQQ==", + "Hst3yfyTB7yBUinvVzYROQ==", + "HtDXgMuF8PJ1haWk88S0Ew==", + "HuDuxs2KiGqmeyY1s1PjpQ==", + "HwLSUie8bzH+pOJT3XQFyg==", + "HxEU37uBMeiR5y8q/pM42g==", + "Hy1nqC40l5ItxumkIC2LAA==", + "I+wVQA+jpPTJ6xEsAlYucg==", + "I07W2eDQwe6DVsm1zHKM8A==", + "I5qDndyelK4Njv4YrX7S6w==", + "I9KNZC1tijiG1T72C4cVqQ==", + "IA1jmtfpYkz/E2wD0+27WA==", + "IADk81pIu8NIL/+9Fi94pA==", + "IAMInfSYb76GxDlAr1dsTg==", + "ICPdBCdONUqPwD5BXU5lrw==", + "IEz72W2/W8xBx5aCobUFOQ==", + "IHhyR6+5sZXTH+/NrghIPg==", + "IHyIeMad23fSDisblwyfpA==", + "IKgNa2oPaFVGYnOsL+GC5Q==", + "INNBBin5ePwTyhPIyndHHg==", + "IPLD9nT5EEYG9ioaSIYuuA==", + "ITYL3tDwddEdWSD6J6ULaA==", + "ITZ3P47ALS0JguFms6/cDA==", + "IUZ5aGpkJ9rLgSg6oAmMlw==", + "IUwVHH6+8/0c+nOrjclOWA==", + "IWZnTJ3Hb9qw9HAK/M9gTw==", + "IYIP2UBRyWetVfYLRsi1SQ==", + "IYIbEaErHoFBn8sTT9ICIQ==", + "IbN736G1Px5bsYqE5gW1JQ==", + "IdadoCPmSgHDHzn1zyf8Jw==", + "IdmcpJXyVDajzeiGZixhSA==", + "IhHyHbHGyQS+VawxteLP0w==", + "IhpXs1TK7itQ3uTzZPRP5Q==", + "IindlAnepkazs5DssBCPhA==", + "IjmLaf3stWDAwvjzNbJpQA==", + "Ily2MKoFI1zr5LxBy93EmQ==", + "Iqszlv4R49UevjGxIPMhIA==", + "IrDuBrVu1HWm0BthAHyOLQ==", + "Is3uxoSNqoIo5I15z6Z2UQ==", + "IshzWega6zr3979khNVFQQ==", + "It+K/RCYMOfNrDZxo7lbcA==", + "IwLbkL33z+LdTjaFYh93kg==", + "IwfeA6d0cT4nDTCCRhK+pA==", + "J/PNYu4y6ZMWFFXsAhaoow==", + "J/eAtAPswMELIj8K2ai+Xg==", + "J0NauydfKsACUUEpMhQg8A==", + "J1nYqJ7tIQK1+a/3sMXI/Q==", + "J2NFyb8cXEpZyxWDthYQiA==", + "J4MC9He6oqjOWsYQh9nl3Q==", + "J8v2f6hWFu8oLuwhOeoQjA==", + "JATLdpQm//SQnkyCfI5x7Q==", + "JBkbaBiorCtFq9M9lSUdMg==", + "JC8Q+8yOJ52NvtVeyHo68w==", + "JFFeXsFsMA59iNtZey7LAA==", + "JFHutgSe1/SlcYKIbNNYwQ==", + "JFi6N1PlrpKaYECOnI7GFg==", + "JGEy6VP3sz3LHiyT2UwNHQ==", + "JGeqHRQpf4No74aCs+YTfA==", + "JGx8sTyvr4bLREIhSqpFkw==", + "JHBjKpCgSgrNNACZW1W+1w==", + "JIC8R48jGVqro6wmG2KXIw==", + "JJJkp1TpuDx5wrua2Wml7g==", + "JJbzQ/trOeqQomsKXKwUpQ==", + "JKg64m6mU7C/CkTwVn4ASg==", + "JKmZqz9cUnj6eTsWnFaB0A==", + "JKphO0UYjFqcbPr6EeBuqg==", + "JLq/DrW2f26NaRwfpDXIEA==", + "JPxEncA4IkfBDvpjHsQzig==", + "JQf9UmutPh3tAnu7FDk3nA==", + "JSr/lqDej81xqUvd/O2s7w==", + "JSyhTcHLTfzHsPrxJyiVrA==", + "JSyq2MIuObPnEgEUDyALjQ==", + "JVSLiwurnCelNBiG2nflpQ==", + "JXCYeWjFqcdSf6QwB54G+A==", + "JYJvOZ4CHktLrYJyAbdOnA==", + "JZRjdJLgZ+S0ieWVDj8IJg==", + "Ja3ECL7ClwDrWMTdcSQ6Ug==", + "JaYQXntiyznQzrTlEeZMIw==", + "Jbxl8Nw1vlHO9rtu0q/Fpg==", + "Jcxjli2tcIAjCe+5LyvqdQ==", + "Je1UESovkBa9T6wS0hevLw==", + "JgXSPXDqaS1G9NqmJXZG0A==", + "JgxNrUlL8wutG04ogKFPvw==", + "JipruVZx4ban3Zo5nNM37g==", + "Jit0X0srSNFnn8Ymi1EY+g==", + "Jj4IrSVpqQnhFrzNvylSzA==", + "Jm862vBTCYbv/V4T1t46+Q==", + "JnE6BK0vpWIhNkaeaYNUzw==", + "JoATsk/aJH0UcDchFMksWA==", + "JquDByOmaQEpFb47ZJ4+JA==", + "JrKGKAKdjfAaYeQH8Y2ZRQ==", + "Js7g8Dr6XsnGURA4UNF0Ug==", + "Jt4Eg6MJn8O4Ph/K2LeSUA==", + "Ju4YwtPw+MKzpbC0wJsZow==", + "JvXTdChcE3AqMbFYTT3/wg==", + "JyIDGL1m/w+pQDOyyeYupA==", + "JyUJEnU6hJu8x2NCnGrYFw==", + "JzW+yhrjXW1ivKu3mUXPXg==", + "K1CGbMfhlhIuS0YHLG30PQ==", + "K1RL+tLjICBvMupe7QppIQ==", + "K1RgR6HR5uDEQgZ32TAFgA==", + "K2gk9zWGd0lJFRMQ1AjQ/Q==", + "K3NBEG8jJTJbSrYSOC3FKw==", + "K4VS+DDkTdBblG93l2eNkA==", + "K4yZNVoqHjXNhrZzz2gTew==", + "K5lhaAIZkGeP5rH2ebSJFw==", + "K8PVQhEJCEH1ghwOdztjRw==", + "K9A87aMlJC8XB9LuFM913g==", + "KCJJfgLe00+tjSfP6EBcUg==", + "KGI/cXVz6v6CfL8H6akcUQ==", + "KI7tQFYW38zYHOzkKp9/lQ==", + "KO2XVYyNZadcQv8aCNn5JA==", + "KOm8PTa+ICgDrgK9QxCJZw==", + "KOmdvm+wJuZ/nT/o1+xOuw==", + "KPh6TwYpspne4KZA6NyMbw==", + "KQw25X4LnQ9is+qdqfxo0w==", + "KR401XBdgCrtVDSaXqPEiA==", + "KSorNz/PLR/YYkxaj1fuqw==", + "KSumhnbKxMXQDkZIpDSWmQ==", + "KTjwL+qswa+Bid8xLdjMTg==", + "KXuFON8tMBizNkCC48ICLA==", + "KXvdjZ3rRKn60djPTCENGA==", + "KYuUNrkTvjUWQovw9dNakA==", + "Kh/J1NpDBGoyDU+Mrnnxkg==", + "KhUT2buOXavGCpcDOcbOYg==", + "KhrIIHfqXl9zGE9aGrkRVg==", + "Kj1QI+s9261S3lTtPKd9eg==", + "KjfL7YyVqmCJGBGDFdJ0gw==", + "KjnL3x+56r3M2pDj1pPihA==", + "KkXlgPJPen6HLxbNn5llBw==", + "KkwQL0DeUM3nPFfHb2ej+A==", + "KlY5TGg0pR/57TVX+ik1KQ==", + "KmcGEE0pacQ/HDUgjlt7Pg==", + "KodYHHN62zESrXUye7M01g==", + "Koiog/hpN7ew5kgJbty34A==", + "Kt6BTG1zdeBZ3nlVk+BZKQ==", + "KuNY8qAJBce+yUIluW8AYw==", + "KujFdhhgB9q4oJfjYMSsLg==", + "KyLQxi5UP+qOiyZl0PoHNQ==", + "KzWdWPP2gH0DoMYV4ndJRg==", + "Kzs+/IZJO8v4uIv9mlyJ2Q==", + "L+N/6geuokiLPPSDXM9Qkg==", + "L2D7G0btrwxl9V4dP3XM5Q==", + "L2IeUnATZHqOPcrnW2APbA==", + "L2RofFWDO0fVgSz4D2mtdw==", + "L3Jt5dHQpWQk74IAuDOL8g==", + "L4+C6I7ausPl6JbIbmozAg==", + "LATQEY7f47i77M6p11wjWA==", + "LCj4hI520tA685Sscq6uLw==", + "LCvz/h9hbouXCmdWDPGWqg==", + "LDuBcL5r3PUuzKKZ9x6Kfw==", + "LEVYAE54618FrlXkDN01Kw==", + "LFcpCtnSnsCPD2gT/RA+Zg==", + "LGwcvetzQ3QqKjNh5vA8vw==", + "LHQETSI5zsejvDaPpsO29g==", + "LJeLdqmriyAQp+QjZGFkdQ==", + "LJtRcR70ug6UHiuqbT6NGw==", + "LKyOFgUKKGUU/PxpFYMILw==", + "LMCZqd3UoF/kHHwzTdj7Tw==", + "LMEtzh0+J27+4zORfcjITw==", + "LPYFDbTEp5nGtG6uO8epSw==", + "LQttmX92SI94+hDNVd8Gtw==", + "LSN9GmT6LUHlCAMFqpuPIA==", + "LUWxfy4lfgB5wUrqCOUisw==", + "LWWfRqgtph1XrpxF4N64TA==", + "LWd0+N3M94n81qd346LfJQ==", + "LZAKplVoNjeQgfaHqkyEJA==", + "La0gzdbDyXUq6YAXeKPuJA==", + "LawT9ZygiVtBk0XJ+KkQgQ==", + "LbPp1oL0t3K2BAlIN+l8DA==", + "LblwOqNiciHmt2NXjd89tg==", + "LcF0OqPWrcpHby8RwXz1Yg==", + "LcoJBEPTlSsQwfuoKQUxEw==", + "LhqRc9oewY4XaaXTcnXIHQ==", + "Lo1xTCEWSxVuIGEbBEkVxA==", + "LoUv/f2lcWpjftzpdivMww==", + "LpoayYsTO8WLFLCSh2kf2w==", + "Lqel4GdU0ZkfoJVXI5WC/Q==", + "LqgzKxbI6WTMz0AMIDJR5w==", + "LsmsPokAwWNCuC74MaqFCQ==", + "Lt/pVD4TFRoiikmgAxEWEw==", + "Lu02ic/E94s42A14m7NGCA==", + "LyPXOoOPMieqINtX8C9Zag==", + "LyYPOZKm8bBegMr5NTSBfg==", + "M/cQja3uIk1im9++brbBOA==", + "M0ESOGwJ4WZ4Ons1ljP0bQ==", + "M20iX2sUfw5SXaZLZYlTaA==", + "M2JMnViESVHTZaru6LDM6w==", + "M2suCoFHJ5fh9oKEpUG3xA==", + "M55eersiJuN9v61r8DoAjQ==", + "M98hjSxCwvZ27aBaJTGozQ==", + "M9oqlPb63e0kZE0zWOm+JQ==", + "MArbGuIAGnw4+fw6mZIxaw==", + "MBjMU/17AXBK0tqyARZP5w==", + "MFeXfNZy6Q9wBfZmPQy3xg==", + "MI+HSMRh8KTW+Afiaxd/Fw==", + "MJ1FuK8PXcmnBAG9meU84A==", + "MK7AqlJIGqK2+K5mCvMXRQ==", + "ML7ipnY/g8mA1PUIju1j8Q==", + "MLHt6Ak288G0RGhCVaOeqA==", + "MLlVniZ08FHAS5xe+ZKRaA==", + "MMaegl2Md9s/wOx5o9564w==", + "MN94B0r5CNAF9sl3Kccdbw==", + "MOrAbuJTyGKPC6MgYJlx5Q==", + "MQYM3BT77i35LG9HcqxY2Q==", + "MQvAr+OOfnYnr/Il/2Ubkg==", + "MUkRa/PjeWMhbCTq43g6Aw==", + "MVoxyIA+emaulH8Oks8Weg==", + "MWcV03ULc0vSt/pFPYPvFA==", + "MbI04HlTGCoc/6WDejwtaQ==", + "MdvhC1cuXqni/0mtQlSOCw==", + "MeKXnEfxeuQu9t3r/qWvcw==", + "MfkyURTBfkNZwB+wZKjP4g==", + "Mj87ajJ/yR41XwAbFzJbcA==", + "Ml3mi1lGS1IspHp3dYYClg==", + "MlKWxeEh8404vXenBLq4bw==", + "MlOOZOwcRGIkifaktEq0aQ==", + "MnStiFQAr3QlaRZ02SYGaQ==", + "Mofqu40zMRrlcGRLS42eBw==", + "MpAwWMt7bcs4eL7hCSLudQ==", + "MqqDg9Iyt4k3vYVW5F+LDw==", + "Mr5mCtC53+wwmwujOU/fWw==", + "MrbEUlTagbesBNg0OemHpw==", + "MrxR3cJaDHp0t3jQNThEyg==", + "MsCloSmTFoBpm7XWYb+ueQ==", + "Muf2Eafcf9G3U2ZvQ9OgtQ==", + "MvMbvZNKbXFe2XdN+HtnpQ==", + "N+K1ibXAOyMWdfYctNDSZQ==", + "N/HgDydvaXuJvTCBhG/KtA==", + "N2KovXW14hN/6+iWa1Yv3g==", + "N2X7KWekNN+fMmwyXgKD5w==", + "N3YDSkBUqSmrmNvZZx4a1Q==", + "N4/mQFyhDpPzmihjFJJn6w==", + "N65PqIWiQeS082D6qpfrAg==", + "N7fHwb397tuQHtBz1P80ZQ==", + "N8dXCawxSBX40fgRRSDqlQ==", + "N9nD7BGEM7LDwWIMDB+rEQ==", + "NBmB/cQfS+ipERd7j9+oVg==", + "ND2hYtAIQGMxBF7o7+u7nQ==", + "ND9l4JWcncRaSLATsq0LVw==", + "NDZWIhhixq7NT8baJUR4VQ==", + "NGApiVkDSwzO45GT57GDQw==", + "NKGY0ANVZ0gnUtzVx1pKSw==", + "NKRzJndo2uXNiNppVnqy1g==", + "NMbAjbnuK7EkVeY3CQI5VA==", + "NN/ymVQNa17JOTGr6ki3eQ==", + "NOmu8oZc6CcKLu+Wfz2YOQ==", + "NQVQfN3nIg9ipHiFh4BvfQ==", + "NRyFx6jqO/oo9ojvbYzsAg==", + "NSrzwNlB0bde3ph8k6ZQcQ==", + "NZtcY8fIpSKPso/KA6ZfzA==", + "Nc5kiwXCAyjpzt43G5RF1A==", + "NdULoUDGhIolzw1PyYKV0A==", + "NdVyHoTbBhX6Umz/9vbi0g==", + "Ndx5LDiVyyTz/Fh3oBTgvA==", + "Nf9fbRHm844KZ2sqUjNgkA==", + "NfxVYc3RNWZwzh2RmfXpiA==", + "Ng5v/B9Z10TTfsDFQ/XrXQ==", + "NhZbSq0CjDNOAIvBHBM9zA==", + "NiQ/m4DZXUbpca9aZdzWAw==", + "NiawWuMBDo0Q3P2xK/vnLQ==", + "NjeDgQ1nzH1XGRnLNqCmSg==", + "NmQrsmb8PVP05qnSulPe5Q==", + "NmWmDxwK5FpKlZbo0Rt8RA==", + "NoX8lkY+kd2GPuGjp+s0tQ==", + "NquRbPn8fFQhBrUCQeRRoQ==", + "Nr4zGo5VUrjXbI8Lr4YVWQ==", + "Nsd+DfRX6L54xs+iWeMjCQ==", + "NtwqUO3SKZE/9MXLbTJo/g==", + "NuBYjwlxadAH+vLWYRZ3bg==", + "NvkR0inSzAdetpI4SOXGhw==", + "NvurnIHin4O+wNP7MnrZ1w==", + "NxSdT2+MUkQN49pyNO2bJw==", + "NyF+4VRog7etp90B9FuEjA==", + "O/EizzJSuFY8MpusBRn7Tg==", + "O1ckWUwuhD44MswpaD6/rw==", + "O209ftgvu0vSr0UZywRFXA==", + "O538ibsrI4gkE5tfwjxjmg==", + "O5N2yd+QQggPBinQ+zIhtQ==", + "O7JiE0bbp583G6ZWRGBcfw==", + "O839JUrR+JS30/nOp428QA==", + "OChiB4BzcRE8Qxilu6TgJg==", + "OEJ40VmMDYzc2ESEMontRA==", + "OERGn45uzfDfglzFFn6JAg==", + "OFLn4wun6lq484I7f6yEwg==", + "OGpsXRHlaN8BvZftxh1e7A==", + "OHJBT2SEv5b5NxBpiAf7oQ==", + "OIwtfdq37eQ0qoXuB2j7Hw==", + "OMO4pqzfcbQ11YO4nkTXfg==", + "OONAvFS/kmH7+vPhAGTNSg==", + "OOS6wQCJsXH8CsWEidB35A==", + "OVHqwV8oQMC5KSMzd5VemA==", + "OaNpzwshdHUZMphQXa6i8w==", + "Oc3BqTF3ZBW3xE0QsnFn/A==", + "OlpA9HsF8MBh7b45WZSSlg==", + "OlwHO6Sg2zIwsCOCRu0HiQ==", + "Omi2ZB9kdR1HrVP2nueQkA==", + "Omr+zPWVucPCSfkgOzLmSQ==", + "OnmvXbyT2BYsSDJYZhLScA==", + "OpC/sL320wl5anx6AVEL+A==", + "OpL+vHwPasW30s2E1TYgpA==", + "OrqJKjRndcZ8OjE3cSQv7g==", + "Otz/PgYOEZ1CQDW54FWJIQ==", + "OwArFF1hpdBupCkanpwT+Q==", + "OwIGvTh8FPFqa4ijNkguAw==", + "Owg8qCpjZa+PmbhZew6/sw==", + "OzFRv+PzPqTNmOnvZGoo5g==", + "OzH7jTcyeM7RPVFtBdakpQ==", + "OzMR5D2LriC5yrVd5hchnA==", + "P0Pc8owrqt6spdf7FgBFSw==", + "P14k+fyz0TG9yIPdojp52w==", + "P3y5MoXrkRTSLhCdLlnc4A==", + "P430CeF2MDkuq11YdjvV8A==", + "P5WPQc5NOaK7WQiRtFabkw==", + "P5fucOJhtcRIoElFJS4ffg==", + "P5wS+xB8srW4a5KDp/JVkA==", + "P7eMlOz9YUcJO+pJy0Kpkw==", + "P8lUiLFoL100c9YSQWYqDA==", + "PAlx9+U+yQCAc5Fi0BOG0w==", + "PBULPuFXb6V3Di713n3Gug==", + "PCOGl7GIqbizAKj/sZmlwQ==", + "PD+yHtJxZJ2XEvjIPIJHsQ==", + "PF0lpolQQXlpc3qTLMBk8w==", + "PHwJ5ZAqqftZ4ypr8H1qiQ==", + "PKtXc4x4DEjM45dnmPWzyg==", + "PMCWKgog/G+GFZcIruSONw==", + "PMvG4NqJP76kMRAup6TSZA==", + "PPa7BDMpRdxJdBxkuWCxKA==", + "PTAm/jGkie7OlgVOvPKpaA==", + "PTW+fhZq/ErxHqpM0DZwHQ==", + "PXC6ZpdMH0ATis/jGW12iA==", + "PaROi5U16Tk35p0EKX5JpA==", + "ParhxI6RtLETBSwB0vwChQ==", + "PbDVq2Iw1eeM8c2o/XYdTA==", + "PbnxuVerGwHyshkumqAARg==", + "Pc+u0MAzp4lndTz4m6oQ5w==", + "PcdBtV8pfKU0YbDpsjPgwg==", + "PcoVtZrS1x1Q+6nfm4f80w==", + "PdBgXFq5mBqNxgCiqaRnkw==", + "PeJS+mXnAA6jQ0WxybRQ8w==", + "PfkWkSbAxIt1Iso0znW0+Q==", + "PggVPQL5YKqSU/1asihcrg==", + "PibGJQNw7VHPTgqeCzGUGA==", + "Po0lhBfiMaXhl+vYh1D8gA==", + "PolhKCedOsplEcaX4hQ0YQ==", + "Pp1ZMxJ8yajdbfKM4HAQxA==", + "PqLCd/pwc+q5GkL6MB0jTg==", + "Pt3i49uweYVgWze3OjkjJA==", + "Pu9pEf+Tek3J+3jmQNqrKw==", + "Pv9FWQEDLKnG/9K9EIz4Gw==", + "PwvPBc+4L73xK22S9kTrdA==", + "PxReytUUn/BbxYTFMu1r2Q==", + "PybPZhJErbRTuAafrrkb3g==", + "Q0TJZxpn3jk67L7N+YDaNA==", + "Q1pdQadt12anX1QRmU2Y/A==", + "Q3TpCE+wnmH/1h/EPWsBtQ==", + "Q4bfQslDSqU64MOQbBQEUw==", + "Q6vGRQiNwoyz7bDETGvi5g==", + "Q7Df6zGwvb4rC+EtIKfaSw==", + "Q7teXmTHAC5qBy+t7ugf0w==", + "Q8RVI/kRbKuXa8HAQD7zUA==", + "QAz7FA+jpz9GgLvwdoNTEQ==", + "QCpzCTReHxGm5lcLsgwPCA==", + "QGYFMpkv37CS2wmyp42ppg==", + "QH36wzyIhh6I56Vnx79hRA==", + "QH3lAwOYBAJ0Fd5pULAZqw==", + "QIKjir/ppRyS63BwUcHWmw==", + "QJEbr3+42P9yiAfrekKdRQ==", + "QTz21WkhpPjfK8YoBrpo+w==", + "QV0OG5bpjrjku4AzDvp9yw==", + "QVwuN66yPajcjiRnVk/V8g==", + "QWURrsEgxbJ8MWcaRmOWqw==", + "Qc+XYy2qyWJ5VVwd2PExbw==", + "Qf7JFJJuuacSzl6djUT2EQ==", + "Qg1ubGl+orphvT990e5ZPA==", + "QiozlNcQCbqXtwItWExqJQ==", + "QmSBVvdk0tqH9RAicXq2zA==", + "QmcURiMzmVeUNaYPSOtTTg==", + "QoUC9nyK1BAzoUVnBLV2zw==", + "QoqHzpHDHTwQD5UF30NruQ==", + "QozQL0DTtr+PXNKifv6l6g==", + "Qrh7OEHjp80IW+YzQwzlJg==", + "QsquNcCZL9wv7oZFqm64vQ==", + "QtD35QhE8sAccPrDnhtQmQ==", + "Qv6wWP4PpycDGxe7EZNSCw==", + "QvYZxsLdu+3nV/WhY1DsYg==", + "Qx6rVv9Xj8CBjqikWI9KFA==", + "QyyiJ5I/OZC50o89fa5EmQ==", + "R+beucURp/H5jLs4kW6wmg==", + "R/y6+JJP8rzz1KITJ4qWBw==", + "R1TCCfgltnXBvt5AiUnCtQ==", + "R2OOV18CV/YpWL1xzr/VQg==", + "R2Use39If2C0FVBP7KDerA==", + "R36O31Pj8jn0AWSuqI7X2Q==", + "R3ijnutzvK6IKV3AKHQZSA==", + "R5oOM58zdbVxFSDQnNWqeA==", + "R6Me6sSGP5xpNI8R0xGOWw==", + "R6cO8GzYfOGTIi773jtkXw==", + "R81DX/5a7DYKkS4CU+TL+w==", + "R8FxgXWKBpEVbnl41+tWEw==", + "R8ULpSNu9FcCwXZM0QedSg==", + "R906Kxp2VFVR3VD+o6Vxcw==", + "R97chlspND/sE9/HMScXjQ==", + "RAAw14BA1ws5Wu/rU7oegw==", + "RAECgYZmcF4WxcFcZ4A0Ww==", + "RBMv0IxXEO3o7MnV47Bzow==", + "RClzwwKh51rbB4ekl99EZA==", + "RDgGGxTtcPvRg/5KRRlz4w==", + "REnDNe9mGfqVGZt+GdsmjQ==", + "RHKCMAqrPjvUYt13BVcmvw==", + "RHToSGASrwEmvzjX6VPvNQ==", + "RIVYGO2smx9rmRoDVYMPXw==", + "RIZYDgXqsIdTf9o2Tp/S7g==", + "RJJqFMeiCZHdsqs72J17MQ==", + "RKVDdE1AkILTFndYWi9wFg==", + "RM5CpIiB94Sqxi462G7caA==", + "RNK9G1hfuz3ETY/RmA9+aA==", + "RNdyt6ZRGvwYG5Ws3QTuEA==", + "ROSt+NlEoiPFtpRqKtDUrQ==", + "RQOlmzHwQKFpafKPJj0D8w==", + "RQywrOLZEKw9+kG6qTzr3g==", + "RUmhye56tQu9xXs4SRJpOQ==", + "RVD3Ij6sRwwxTUDAxwELtA==", + "RWI0HfpP7643OSEZR8kxzw==", + "RYkDwwng6eeffPHxt8iD9A==", + "RZTpYKxOAH9JgF1QFGN+hw==", + "RfSwpO/ywQx4lfgeYlBr2w==", + "RgtwfY5pTolKrUGT+6Pp6g==", + "RhcqXY4OsZlVVF7ZlkTeRw==", + "RiahBXX2JbPzt8baPiP/8g==", + "RkQK9S1ezo+dFYHQP57qrw==", + "RlNPyhgYOIn28R4vKCVtYA==", + "RnOXOygwJFqrD+DlM3R5Ew==", + "RnxOYPSQdHS6fw4KkDJtrA==", + "RppDe/WGt1Ed6Vqg1+cCkQ==", + "RqYpA5AY7mKPaSxoQfI1CA==", + "RrE3B3X/SJi3CqCUlTYwaw==", + "Rrq0ak9YexLqqbSD4SSXlw==", + "Rs8deApkoosIJSfX7NXtAA==", + "RuLeQHP1wHsxhdmYMcgtrQ==", + "RvXWAFwM+mUAPW1MjPBaHA==", + "Rvchz/xjcY9uKiDAkRBMmA==", + "Rww3qkF3kWSd+AaMT0kfdw==", + "RxmdoO8ak8y/HzMSIm+yBQ==", + "Ry3zgZ6KHrpNyb7+Tt2Pkw==", + "RzeH+G3gvuK1z+nJGYqARQ==", + "S+b37XhKRm8cDwRb1gSsKQ==", + "S2MAIYeDQeJ1pl9vhtYtUg==", + "S3VQa6DH+BdlSrxT/g6B5g==", + "S47hklz3Ow+n5aY6+qsCoA==", + "S4RvORcJ3m6WhnAgV4YfYA==", + "S4rFuiKLFKZ+cL7ldiTwpg==", + "S7Vjy/gOWp0HozPP1RUOZw==", + "S8jlvuYuankCnvIvMVMzmg==", + "S9L29U2P5K8wNW+sWbiH7w==", + "SCO9nQncEcyVXGCtx30Jdg==", + "SChDh/Np1HyTPWfICfE1uA==", + "SDi5+FoP9bMyKYp+vVv1XA==", + "SEGu+cSbeeeZg4xWwsSErQ==", + "SEIZhyguLoyH7So0p1KY0A==", + "SESKbGF35rjO64gktmLTWA==", + "SElc2+YVi3afE1eG1MI7dQ==", + "SFn78uklZfMtKoz2N0xDaQ==", + "SIuKH/Qediq0TyvqUF93HQ==", + "SM7E98MyViSSS9G0Pwzwyw==", + "SNPYH4r/J9vpciGN2ybP5Q==", + "SOdpdrk2ayeyv0xWdNuy9g==", + "SPGpjEJrpflv1hF0qsFlPw==", + "SPHU6ES1WVm0Mu2LB+YjrA==", + "SSKhl2L3Mvy93DcZulADtA==", + "SUAwMWLMml8uGqagz5oqhQ==", + "SVFbcjXbV7HRg+7jUrzpwg==", + "SVLHWPCCH7GPVCF7QApPbw==", + "SVuEYfQ9FGyVMo1672n0Yg==", + "SbMjjI8/P8B9a9H2G0wHEQ==", + "Scto+9TWxj1eZgvNKo+a9A==", + "SfwnYZCKP1iUJyU1yq4eKg==", + "SiSlasZ+6U2IZYogqr2UPg==", + "Slu3z535ijcs5kzDnR7kfA==", + "SmRWEzqddY9ucGAP5jXjAg==", + "Sr9c0ReRpkDYGAiqSy683g==", + "Srl4HivgHMxMOUHyM3jvNw==", + "StDtLMlCI75g4XC59mESEQ==", + "StoXC7TBzyRViPzytAlzyQ==", + "StpQm/cQF8cT0LFzKUhC5w==", + "SusSOsWNoAerAIMBVWHtfA==", + "Swjn3YkWgj0uxbZ1Idtk+A==", + "SzCGM8ypE58FLaR1+1ccxQ==", + "Szko0IPE7RX2+mfsWczrMg==", + "T/6gSz2HwWJDFIVrmcm8Ug==", + "T1pMWdoNDpIsHF8nKuOn2A==", + "T6LA+daQqRI38iDKZTdg1A==", + "T7waQc3PvTFr0yWGKmFQdQ==", + "T9WoUJNwp8h4Yydixbx6nA==", + "TA9WjiLAFgJubLN4StPwLw==", + "TAD0Lk95CD86vbwrcRogaQ==", + "TBQpcKq2huNC5OmI2wzRQw==", + "TDrq23VUdzEU/8L5i8jRJQ==", + "TGB+FIzzKnouLh5bAiVOQg==", + "THfzE2G2NVKKfO+A2TjeFw==", + "THs1r8ZEPChSGrrhrNTlsA==", + "TI90EuS/bHq/CAlX32UFXg==", + "TIKadc6FAaRWSQUg5OATgg==", + "TIWSM78m0RprwgPGK/e0JA==", + "TLJbasOoVO435E5NE5JDcA==", + "TNyvLixb03aP2f8cDozzfA==", + "TSGL3iQYUgVg/O9SBKP9EA==", + "TSPFvkgw6uLsJh66Ou0H9w==", + "TVlHoi8J7sOZ2Ti7Dm92cQ==", + "TXab/hqNGWaSK+fXAoB2bg==", + "TYlnrwgyeZoRgOpBYneRAg==", + "TZ3ATPOFjNqFGSKY3vP2Hw==", + "TZT86wXfzFffjt0f95UF5w==", + "TafM7nTE5d+tBpRCsb8TjQ==", + "TahqPgS7kEg+y6Df0HBASw==", + "TcFinyBrUoAEcLzWdFymow==", + "TcGhAJHRr7eMwGeFgpFBhg==", + "TcyyXrSsQsnz0gJ36w4Dxw==", + "TeBGJCqSqbzvljIh9viAqA==", + "TfHvdbl2M4deg65QKBTPng==", + "TfNHjSTV8w6Pg6+FaGlxvA==", + "TgWe70YalDPyyUz6n88ujg==", + "Tk5MAqd1gyHpkYi8ErlbWg==", + "TlJizlASbPtShZhkPww4UA==", + "Tm4zk2Lmg8w4ITMI31NfTA==", + "Tmx0suRHzlUK4FdBivwOwA==", + "Tp52d1NndiC9w3crFqFm9g==", + "TrLmfgwaNATh24eSrOT+pw==", + "TrWS+reCJ0vbrDNT5HDR9w==", + "Tu6w6DtX2RJJ3Ym3o3QAWw==", + "TuaG3wRdM9BWKAxh2UmAsg==", + "Tud+AMyuFkWYYZ73yoJGpQ==", + "Tug3eh+28ttyf+U7jfpg5w==", + "U+bB5NjFIuQr/Y5UpXHwxA==", + "U+oTpcjhc0E+6UjP11OE/Q==", + "U0KmEI6e5zJkaI4YJyA5Ew==", + "U49SfOBeqQV9wzsNkboi8Q==", + "U6VQghxOXsydh3Naa5Nz4A==", + "U9kE50Wq5/EHO03c5hE4Ug==", + "UAqf4owQ+EmrE45hBcUMEw==", + "UEMwF4kwgIGxGT4jrBhMPQ==", + "UHpge5Bldt9oPGo2oxnYvQ==", + "UIXytIHyVODxlrg+eQoARA==", + "UK+R+hAoVeZ4xvsoZjdWpw==", + "UNRlg6+CYVOt68NwgufGNA==", + "UNdKik7Vy23LjjPzEdzNsg==", + "UNt7CNMtltJWq8giDciGyA==", + "UP7NXAE0uxHRXUAWPhto0w==", + "UP9mmAKzeQqGhod7NCqzhg==", + "UPYR575ASaBSZIR3aX1IgQ==", + "UPzS4LR3p/h0u69+7YemrQ==", + "UQTQk5rrs6lEb1a+nkLwfg==", + "USCvrMEm/Wqeu9oX6FrgcQ==", + "USq1iF90eUv41QBebs3bhw==", + "UTmTgvl+vGiCDQpLXyVgOg==", + "UVEZPoH9cysC+17MKHFraw==", + "UXUNYEOffgW3AdBs7zTMFA==", + "UZoibx+y1YJy/uRSa9Oa2w==", + "Ua6aO6HwM+rY4sPR19CNFA==", + "UbABE6ECnjB+9YvblE9CYw==", + "UbSFw5jtyLk5MealqJw++A==", + "Ugt8HVC/aUzyWpiHd0gCOQ==", + "UgvtdE2eBZBUCAJG/6c0og==", + "Uh1mvZNGehK1AaI4a1auKQ==", + "Uje3Ild84sN41JEg3PEHDg==", + "UjmDFO7uzjl4RZDPeMeNyg==", + "Um1ftRBycvb+363a90Osog==", + "Umd+5fTcxa3mzRFDL9Z8Ww==", + "Uo+FIhw1mfjF6/M8cE1c/Q==", + "Uo1ebgsOxc3eDRds1ah3ag==", + "UreSZCIdDgloih8KLeX7gg==", + "UtLYUlQJ02oKcjNR3l+ktg==", + "Uudn69Kcv2CGz2FbfJSSEA==", + "UvC1WADanMrhT+gPp/yVqA==", + "Uw6Iw+TP9ZdZGm2b/DAmkg==", + "UwqBVd4Wfias4ElOjk2BzQ==", + "Uy4QI8D2y1bq/HDNItCtAw==", + "UymZUnEEQWVnLDdRemv+Tw==", + "UzPPFSXgeV7KW4CN5GIQXA==", + "V+QzdKh5gxTPp2yPC9ZNEg==", + "V/xG5QFyx1pihimKmAo8ZA==", + "V1fvtnJ0L3sluj9nI5KzRw==", + "V2P75JFB4Se9h7TCUMfeNA==", + "V5HEaY3v9agOhsbYOAZgJA==", + "V5HKdaTHjA8IzvHNd9C51g==", + "V6CRKrKezPwsRdbm0DJ2Yg==", + "V6zyoX6MERIybGhhULnZiw==", + "V7eji28JSg3vTi30BCS7gw==", + "V8m51xgUgywRoV6BGKUrgg==", + "V8q+xz4ljszLZMrOMOngug==", + "V9G1we3DOIQGKXjjPqIppQ==", + "V9vkAanK+Pkc4FGAokJsTA==", + "VAg/aU5nl72O+cdNuPRO4g==", + "VCL3xfPVCL5RjihQM59fgg==", + "VE4sLM5bKlLdk85sslxiLQ==", + "VGRCSrgGTkBNb8sve0fYnQ==", + "VH70dN82yPCRctmAHMfCig==", + "VI8pgqBZeGWNaxkuqQVe7g==", + "VIC7inSiqzM6v9VqtXDyCw==", + "VIkS30v268x+M1GCcq/A8A==", + "VJt2kPVBLEBpGpgvuv1oUw==", + "VK95g27ws2C6J2h/7rC2qA==", + "VOB+9Bcfu8aHKGdNO0iMRw==", + "VOvrzqiZ1EHw+ZzzTWtpsw==", + "VPa7DG6v7KnzMvtJPb88LQ==", + "VPqyIomYm7HbK5biVDvlpw==", + "VQIpquUqmeyt/q6OgxzduQ==", + "VRnx+kd6VdxChwsfbo1oeQ==", + "VUDsc9RMS1fSM43c+Jo9dQ==", + "VWNDBOtjiiI4uVNntOlu/A==", + "VWb8U4jF/Ic0+wpoXi/y/g==", + "VWy9lB5t4fNCp4O/4n8S4w==", + "VX+cVXV8p9i5EBTMoiQOQQ==", + "VXu4ARjq7DS2IR/gT24Pfw==", + "VZX1FnyC8NS2k3W+RGQm4g==", + "VaJc9vtYlqJbRPGb5Tf0ow==", + "VbCoGr8apEcN7xfdaVwVXw==", + "VbHoWmtiiPdABvkbt+3XKQ==", + "Vg2E5qEDfC+QxZTZDCu9yQ==", + "VhYGC8KYe5Up+UJ2OTLKUw==", + "Vik8tGNxO0xfdV0pFmmFDw==", + "ViweSJuNWbx5Lc49ETEs/A==", + "VjclDY8HN4fSpB263jsEiQ==", + "VllbOAjeW3Dpbj5lp2OSmA==", + "VoPth5hDHhkQcrQTxHXbuw==", + "VpmBstwR7qPVqPgKYQTA3g==", + "VsXEBIaMkVftkxt1kIh7TA==", + "Vu0E+IJXBnc25x4n41kQig==", + "VzQ1NwNv9btxUzxwVqvHQg==", + "VznvTPAAwAev+yhl9oZT0w==", + "W+M4BcYNmjj7xAximDGWsA==", + "W/0s1x3Qm+wN8DhROk6FrQ==", + "W/5ThNLu43uT1O+fg0Fzwg==", + "W04GeDh+Tk/I1S85KlozRA==", + "W2x0SBzSIsTRgyWUCOZ/lg==", + "W4CfeVp9mXgk04flryL7iA==", + "W4utAK3ws0zjiba/3i91YA==", + "W5now3RWSzzMDAxsHSl++Q==", + "W8bATujVUT80v2XGJTKXDg==", + "W8y32OLHihfeV0XFw7LmOg==", + "WADmxH7R6B4LR+W6HqQQ6A==", + "WBu0gJmmjVdVbjDmQOkU6w==", + "WGKFTWJac8uehn3N59yHJw==", + "WHutPin+uUEqtrA7L8878A==", + "WKehT4nGF2T7aKuzABDMlA==", + "WLsh3UF4WXdHwgnbKEwRlQ==", + "WLwpjgr9KzevuogoHZaVUw==", + "WN7lFJfw4lSnTCcbmt5nsg==", + "WNfDNaWUOqABQ6c6kR+eyw==", + "WQMffxULFKJ+bun6NrCURA==", + "WQznrwqvMhUlM3CzmbhAOQ==", + "WRjYdKdtnd1G9e/vFXCt0g==", + "WRoJMO0BCJyn5V6qnpUi4Q==", + "WTr3q/gDkmB4Zyj7Ly20+w==", + "WVhfn2yJZ43qCTu0TVWJwA==", + "WWN44lbUnEdHmxSfMCZc6w==", + "WY7mCUGvpXrC8gkBB46euw==", + "WbAdlac/PhYUq7J2+n5f+w==", + "Wd0dOs7eIMqW5wnILTQBtg==", + "WdCWezJU4JK43EOZ9YHVdg==", + "Wf2olJCYZRGTTZxZoBePuQ==", + "WjDqf1LyFyhdd8qkwWk+MA==", + "WkSJpxBa45XJRWWZFee7hw==", + "Wn+Vj4eiWx0WPUHr3nFbyA==", + "WnHK5ZQDR6Da5cGODXeo0A==", + "WrJMOuXSLKKzgmIDALkyNw==", + "WtT0QAERZSiIt2SFDiAizg==", + "WwraoO97OTalvavjUsqhxQ==", + "Wx9jh/teM0LJHrvTScssyQ==", + "WyCFB4+6lVtlzu3ExHAGbQ==", + "WzjvUJ4jZAEK7sBqw+m07A==", + "X/Gha4Ajjm/GStp/tv+Jvw==", + "X1PaCfEDScclLtOTiF5JUw==", + "X2Tawm2Cra6H7WtXi1Z4Qw==", + "X2YfnPXgF2VHVX95ZcBaxQ==", + "X4hrgqMIcApsjA9qOWBoCw==", + "X4kdXUuhcUqMSduqhfLpxA==", + "X4o0OkTz0ec70mzgwRfltA==", + "X6Ln4si8G5aKar52ZH/FEQ==", + "X6ulLp4noBgefQTsbuIbYQ==", + "X9QAaNjgiOeAWSphrGtyVw==", + "XA2hUgq3GVPpxtRYiqnclg==", + "XAq/C+XyR6m3uzzLlMWO5Q==", + "XEwOJG24eaEtAuBWtMxhwg==", + "XF/yncdoT4ruPeXCxEhl9Q==", + "XGAXhUFjORwKmAq9gGEcRg==", + "XHHEg/8KZioW/4/wgSEkbQ==", + "XHjrTLXkm/bBY/BewmJcCQ==", + "XJihma9zSRrXLC+T+VcFDA==", + "XLq/nWX8lQqjxsK9jlCqUg==", + "XOG1PYgqoG8gVLIbVLTQgg==", + "XSb71ae0v+yDxNF5HJXGbQ==", + "XTCcsVfEvqxnjc0K5PLcyw==", + "XV13yK0QypJXmgI+dj4KYw==", + "XV5MYe0Q7YMtoBD6/iMdSw==", + "XVVy3e6dTnO3HpgD6BtwQw==", + "XXFr0WUuGsH5nXPas7hR3Q==", + "Xconi1dtldH90Wou9swggw==", + "XddlSluOH6VkR7spFIFmdQ==", + "XdkxmYYooeDKzy7PXVigBQ==", + "XePy/hhnQwHXFeXUQQ55Vg==", + "XfBOCJwi2dezYzLe316ivw==", + "XfY+QUriCAA1+3QAsswdgg==", + "XgPHx2+ULpm14IOZU2lrDg==", + "XjjrIpsmATV/lyln4tPb+g==", + "Xo8ZjXOIoXlBjFCGdlPuZw==", + "XpGXh76RDgXC4qnTCsnNHA==", + "XqFSbgvgZn0CpaZoZiRauQ==", + "XqTK/2QuGWj50tGmiDxysA==", + "XqUO7ULEYhDOuT/I2J8BOA==", + "XqW7UBTobbV4lt1yfh0LZw==", + "XrFDomoH2qFjQ2jJ2yp9lA==", + "XsF7R12agx/KkRWl0TyXRA==", + "Xv0mNYedaBc57RrcbHr9OA==", + "XwKWd03sAz8MmvJEuN08xA==", + "Y1Nm3omeWX2MXaCjDDYnWQ==", + "Y1flEyZZAYxauMo4cmtJ1w==", + "Y26jxXvl79RcffH8O8b9Ew==", + "Y5KKN7t/v9JSxG/m1GMPSA==", + "Y5XR8Igvau/h+c1pRgKayg==", + "Y5iDQySR2c3MK7RPMCgSrw==", + "Y78dviyBS3Jq9zoRD5sZtQ==", + "Y7OofF9eUvp7qlpgdrzvkg==", + "Y7XpxIwsGK3Lm/7jX/rRmg==", + "Y7iDCWYrO1coopM3RZWIPg==", + "YA+zdEC+yEgFWRIgS1Eiqw==", + "YA0kMTJ82PYuLA4pkn4rfw==", + "YHM6NNHjmodv+G0mRLK7kw==", + "YK+q7uJObkQZvOwQ9hplMg==", + "YLz+HA6qIneP+4naavq44Q==", + "YNqIHCmBp/EbCgaPKJ7phw==", + "YPgMthbpcBN2CMkugV60hQ==", + "YVlRQHQglkbj3J2nHiP/Hw==", + "YXHQ3JI9+oca8pc/jMH6mA==", + "YZ39RIXpeLAhyMgmW2vfkQ==", + "YZt6HwCvdI5DRQqndA/hBQ==", + "YaUKOTyByjUvp1XaoLiW5Q==", + "YfbfE3WyYOW7083Y8sGfwQ==", + "YgVpC5d5V6K/BpOD663yQA==", + "YhLEPsi/TNyeUJw69SPYzQ==", + "Yig+Wh18VIqdsmwtwfoUQw==", + "Yjm5tSq1ejZn3aWqqysNvA==", + "YmaksRzoU+OwlpiEaBDYaQ==", + "YmjZJyNfHN5FaTL/HAm8ww==", + "YodhkayN5wsgPZEYN7/KNA==", + "YrEP9z2WPQ8l7TY1qWncDA==", + "YtZ8CYfnIpMd2FFA5fJ+1Q==", + "Yw4ztKv6yqxK9U1L0noFXg==", + "Yy2pPhITTmkEwoudXizHqQ==", + "YzTV0esAxBFVls3e0qRsnA==", + "Z+bsbVP91KrJvxrujBLrrQ==", + "Z0sjccxzKylgEiPCFBqPSA==", + "Z2MkqmpQXdlctCTCUDPyzw==", + "Z2rwGmVEMCY6nCfHO3qOzw==", + "Z5B+uOmPZbpbFWHpI9WhPw==", + "Z8T1b9RsUWf59D06MUrXCQ==", + "Z9bDWIgcq6XwMoU2ECDR5Q==", + "ZAQHWU6RMg4IadOxuaukyw==", + "ZCdad3AwhVArttapWFwT/Q==", + "ZH5Es/4lJ+D5KEkF1BVSGg==", + "ZIZx4MehWTVXPN9cVQBmyA==", + "ZItMIn1vhGqAlpDHclg0Ig==", + "ZJY+hujfd58mTKTdsmHoQQ==", + "ZJc7GV0Yb6MrXkpDVIuc8g==", + "ZKXxq9yr7NGBOHidht34uQ==", + "ZKeTDCboOgCptrjSfgu0xw==", + "ZKvox7BaQg4/p5jIX69Umw==", + "ZNrjP1fLdQpGykFXoLBNPw==", + "ZQ0ZnTsZKWxbRj7Tilh24Q==", + "ZQSDYgpsimK+lYGdXBWE/w==", + "ZRWyfXyXqAaOEjkzWl949Q==", + "ZRnR6i+5WKMRfs3BDRBCJg==", + "ZSmN8mmI9lDEHkJqBBg0Nw==", + "ZV8mEgJweIYk0/l0BFKetA==", + "ZVnErH1Si4u51QoT0OT7pA==", + "ZWXfE3uGU91WpPMGyknmqw==", + "ZXeMG5eqQpZO/SGKC4WQkA==", + "ZYW30FfgwHmW6nAbUGmwzA==", + "ZZImGypBWwYOAW43xDRWCQ==", + "ZaPsR9X77SNt7dLjMJUh8A==", + "ZbLVNTQSVZQWTNgC4ZGfQg==", + "ZcuIvc8fDI+2uF0I0uLiVA==", + "ZfRlID+pC1Rr4IY14jolMw==", + "ZgdpqFrVGiaHkh9o3rDszg==", + "ZgjifTVKmxOieco81gnccQ==", + "ZiJ/kJ9GneF3TIEm08lfvQ==", + "ZlBNHAiYsfaEEiPQ1z+rCA==", + "ZlOAnCLV1PkR0kb3E+Nfuw==", + "ZmVpw1TUVuT13Zw/MNI5hQ==", + "ZmblZauRqO5tGysY3/0kDw==", + "ZoNSxARrRiKZF5Wvpg7bew==", + "Zqd6+81TwYuiIgLrToFOTQ==", + "ZqjnqxZE/BjOUY0CMdVl0g==", + "ZqkmoGB0p5uT5J6XBGh7Tw==", + "ZrCezGLz38xKmzAom6yCTQ==", + "ZrCnZB/U/vcqEtI1cSvnww==", + "ZtWvgitOSRDWq7LAKYYd4Q==", + "ZtmnX24AwYAXHb2ZDC6MeQ==", + "ZuayB6IpbeITokKGVi9R5w==", + "ZvvxwDd0I6MsYd7aobjLUA==", + "ZyDh3vCQWzS5DI1zSasXWA==", + "ZybIEGf1Rn/26vlHmuMxhw==", + "ZydKlOpn2ySBW0G3uAqwuw==", + "ZygAjaN62XhW5smlLkks+Q==", + "Zyo0fzewcqXiKe2mAwKx5g==", + "ZyoaR1cMiKAsElmYZqKjLA==", + "Zz/5VMbw1TqwazReplvsEg==", + "ZzT5b0dYQXkQHTXySpWEaA==", + "ZzduJxTnXLD9EPKMn1LI4Q==", + "a/Y6IAVFv0ykRs9WD+ming==", + "a1aL8zQ+ie3YPogE3hyFFg==", + "a4EYNljinYTx9vb1VvUA6A==", + "a4rPqbDWiMivVzaRxvAj7g==", + "a5gZ5uuRrXEAjgaoh7PXAg==", + "a6IszND1m+6w+W+CvseC7g==", + "a6vem8n6WmRZAalDrHNP0g==", + "a7Pv1SOWYnkhIUC22dhdDA==", + "aD4QvtMlr8Lk/zZgZ6zIMg==", + "aEnHUfn7UE/Euh6jsMuZ7g==", + "aFJuE/s+Kbge4ppn+wulkA==", + "aIPde9CtyZrhbHLK740bfw==", + "aJFbBhYtMbTyMFBFIz/dTA==", + "aK9nybtiIBUvxgs1iQFgsw==", + "aLY2pCT0WfFO5EJyinLpPg==", + "aLh1XEUrfR9W82gzusKcOg==", + "aMa1yVA71/w6Uf1Szc9rMA==", + "aMmrAzoRWLOMPHhBuxczKg==", + "aN5x46Gw1VihRalwCt1CGg==", + "aOeJZUIZM9YWjIEokFPnzQ==", + "aRpdnrOyu5mWB1P5YMbvOA==", + "aRrcmH+Ud3mF1vEXcpEm4w==", + "aTWiWjyeSDVY/q8y9xc2zg==", + "aWZRql2IUPVe9hS3dxgVfQ==", + "aXqiibI6BpW3qilV6izHaQ==", + "aXrbsro7KLV8s4I4NMi4Eg==", + "aXs9qTEXLTkN956ch3pnOA==", + "aY6B28XdPnuYnbOy9uSP8A==", + "adJAjAFyR2ne1puEgRiH+g==", + "adT+OjEB2kqpeYi4kQ6FPg==", + "afMd/Hr3rYz/l7a3CfdDjg==", + "ahAbmGJZvUOXrcK6OydNGQ==", + "alJtvTAD7dH/zss/Ek1DMQ==", + "alqHQBz8V446EdzuVfeY5Q==", + "anyANMnNkUqr3JuPJz5Qzw==", + "apWEPWUvMC24Y+2vTSLXoA==", + "aqcOby9QyEbizPsgO3g0yw==", + "ash1r2J6B0PUxJe8P0otVQ==", + "asouSfUjJa8yfMG7BBe+fA==", + "auvG6kWMnhCMi7c7e9eHrw==", + "avFTp3rS6z5zxQUZQuaBHQ==", + "avZp5K7zJvRvJvpLSldNAw==", + "aw4CzX8pYbPVMuNrGCEcWg==", + "axEl7xXt/bwlvxKhI7hx4g==", + "ayBGGPEy++biljvGcwIjXA==", + "aySnrShOW4/xRSzl/dtSKQ==", + "ays5/F7JANIgPHN0vp2dqQ==", + "b06KGv5zDYsTxyTbQ9/eyA==", + "b0vZfEyuTja2JYMa20Rtbg==", + "b16O4LF7sVqB7aLU2f3F1A==", + "b3BQG9/9qDNC/bNSTBY/sQ==", + "b3q8kjHJPj9DWrz3yNgwjQ==", + "b4BoZmzVErvuynxirLxn0w==", + "b4aFwwcWMXsSdgS1AdFOXA==", + "b53qqLnrTBthRXmmnuXWvw==", + "b6rrRA0W247O+FfvDHbVCQ==", + "b85nxzs8xiHxaqezuDVWvg==", + "b8BZV1NfBdLi70ir4vYvZg==", + "bA2kaTpeXflTElTnQRp6GQ==", + "bBEndaOStXBpAK79FrgHaw==", + "bG+P+p34t/IJ1ubRiWg6IA==", + "bGGUhiG9SqJMHQWitXTcYQ==", + "bIk7Fa6SW7X18hfDjTKowg==", + "bJ1cZW7KsXmoLw0BcoppJg==", + "bJgsuw29cO2WozqsGZxl7w==", + "bK045TkBlz+/3+6n6Qwvrg==", + "bL2FuwsPT7a7oserJQnPcw==", + "bLEntCrCHFy9pg3T3gbBzg==", + "bLd38ZNkVeuhf0joEAxnBQ==", + "bLsStF0DDebpO+xulqGNtg==", + "bMWFvjM8eVezU1ZXKmdgqw==", + "bMb1ia0rElr2ZpZVhva0Jw==", + "bNDKcFu8T5Y6OoLSV+o/Sw==", + "bNq/hj0Cjt4lkLQeVxDVdQ==", + "bO55S58bqDiRWXSAIUGJKw==", + "bPRX2zl+K1S0iWAWUn1DZw==", + "bQ7J5mebp38rfP/fuqQOsg==", + "bQKkL+/KUCsAXlwwIH0N3w==", + "bTNRjJm+FfSQVfd56nNNqQ==", + "bUF0JIfS4uKd3JZj2xotLQ==", + "bUxQBaqKyvlSHcuRL9whjg==", + "bV9r7j2kNJpDCEM5E2339Q==", + "bWwtTFlhO3xEh/pdw0uWaQ==", + "bb/U8UynPHwczew/hxLQxw==", + "bbBsi6tXMVWyq3SDVTIXUg==", + "beSrliUu0BOadCWmx+yZyA==", + "bfUD03N2PRDT+MZ+WFVtow==", + "bhVbgJ4Do4v56D9mBuR/EA==", + "birqO8GOwGEI97zYaHyAuw==", + "bjLZ7ot/X/vWSVx4EYwMCg==", + "bkRdUHAksJZGzE1gugizYQ==", + "blygTgAHZJ3NzyAT33Bfww==", + "bs2QG8yYWxPzhtyMqO6u3A==", + "bsHIShcLS134C+dTxFQHyA==", + "bvbMJZMHScwjJALxEyGIyg==", + "bvyB6OEwhwCIfJ6KRhjnRw==", + "bz294kSG4egZnH2dJ8HwEg==", + "bzVeU2qM9zHuzf7cVIsSZw==", + "bzXXzQGZs8ustv0K4leklA==", + "c1wbFbN7AdUERO/xVPJlgw==", + "c3WVxyC5ZFtzGeQlH5Gw+w==", + "c5Tc7rTFXNJqYyc0ppW+Iw==", + "c5q/8n7Oeffv3B1snHM/lA==", + "c5ymZKqx/td1MiS2ERiz9A==", + "c6Yhwy/q3j7skXq52l36Ww==", + "cBBOQn7ZjxDku0CUrxq2ng==", + "cFFE2R4GztNoftYkqalqUQ==", + "cHSj5dpQ04h/WyefjABfmQ==", + "cHkOsVd80Rgwepeweq4S1g==", + "cLR0Ry4/N5swqga1R6QDMw==", + "cMo6l1EQESx1rIo+R4Vogg==", + "cNsC9bH30eM1EZS6IdEdtQ==", + "cSHSg9xJz/3F6kc+hKXkwg==", + "cT3PwwS6ALZA/na9NjtdzA==", + "cTvDd8okNUx0RCMer6O8sw==", + "cUyqCa7Oue934riyC17F8g==", + "cVhdRFuZaW/09CYPmtNv5g==", + "cWUg7AfqhiiEmBIu+ryImA==", + "cWdlhVZD7NWHUGte24tMjg==", + "cXpfd6Io6Glj2/QzrDMCvA==", + "ca+kx+kf7JuZ3pfYKDwFlg==", + "caepyBOAFu0MxbcXrGf6TA==", + "catI+QUNk3uJ+mUBY3bY8Q==", + "cbBXgB1WQ/i8Xul0bYY2fg==", + "ccK42Lm8Tsv73YMVZRwL6A==", + "cchuqe+CWCJpoakjHLvUfA==", + "ccmy4GVuX967KaQyycmO0w==", + "ccy3Ke2k4+evIw0agHlh3w==", + "cdWUm6uLNzR/knuj2x75eA==", + "cffrYrBX3UQhfX1TbAF+GQ==", + "cfh5VZFmIqJH/bKboDvtlA==", + "cgSEbLqqvDsNUyeA3ryJ6Q==", + "chwv4+xbEAa93PHg8q9zgQ==", + "ck86G8HsbXflyrK7MBntLg==", + "ckugAisBNX18eQz+EnEjjw==", + "cl4t9FXabQg7tbh1g7a0OA==", + "coGEgMVs2b314qrXMjNumQ==", + "cszpMdGbsbe6BygqMlnC9Q==", + "ctJYJegZhG42i+vnPFWAWw==", + "cu4ZluwohhfIYLkWp72pqA==", + "cuQslgfqD2VOMhAdnApHrA==", + "cvMJ714elj/HUh89a9lzOQ==", + "cvOg7N4DmTM+ok1NBLyBiQ==", + "cvZT1pvNbIL8TWg+SoTZdA==", + "cvrGmub2LoJ+FaM5HTPt9A==", + "cw1gBLtxH/m4H7dSM7yvFg==", + "cwBNvZc0u4bGABo88YUsVQ==", + "cxpZ4bloGv734LBf4NpVhA==", + "cxqHS4UbPolcYUwMMzgoOA==", + "czBWiYsQtNFrksWwoQxlOw==", + "d+ctfXU0j07rpRRzb5/HDA==", + "d/Wd3Ma1xYyoMByPQnA9Cw==", + "d0NBFiwGlQNclKObRtGVMQ==", + "d0VAZLbLcDUgLgIfT1GmVQ==", + "d0qvm3bl38rRCpYdWqolCQ==", + "d13Rj3NJdcat0K/kxlHLFw==", + "dAq8/1JSQf1f4QPLUitp0g==", + "dCDaYYrgASXPMGFRV0RCGg==", + "dChBe9QR29ObPFu/9PusLg==", + "dFSavcNwGd8OaLUdWq3sng==", + "dFetwmFw+D6bPMAZodUMZQ==", + "dG98w8MynOoX7aWmkvt+jg==", + "dGjcKAOGBd4gIjJq7fL+qQ==", + "dGrf9SWJ13+eWS6BtmKCNw==", + "dJHKDkfMFJeoULg7U4wwDQ==", + "dK2DU3t1ns+DWDwfBvH3SQ==", + "dL6n/JsK+Iq6UTbQuo/GOw==", + "dM9up4vKQV5LeX82j//1jQ==", + "dMRx4Mf6LrN64tiJuyWmDw==", + "dNTU+/2DdZyGGTdc+3KMhQ==", + "dNq2InSVDGnYXjkxPNPRxA==", + "dOS+mVCy3rFX9FvpkTxGXA==", + "dRFCIbVu0Y8XbjG5i+UFCQ==", + "dTMoNd6DDr1Tu8tuZWLudw==", + "dUx1REyXKiDFAABooqrKEA==", + "dVh/XMTUIx1nYN4q1iH1bA==", + "dXDPnL1ggEoBqR13aaW9HA==", + "dZg5w8rFETMp9SgW7m0gfg==", + "dZgMquvZmfLqP4EcFaWCiA==", + "daBhAvmE9shDgmciDAC5eg==", + "dhTevyxTYAuKbdLWhG47Kw==", + "dihDsG7+6aocG6M9BWrCzQ==", + "dmAfbd9F0OJHRAhNMEkRsA==", + "dml2gqLPsKpbIZ93zTXwCQ==", + "dnvatwSEcl73ROwcZ4bbIQ==", + "dpSTNOCPFHN5yGoMpl1EUA==", + "dqVw2q2nhCvTcW82MT7z0g==", + "drfODfDI6GyMW7hzkmzQvA==", + "dsueq9eygFXILDC7ZpamuA==", + "dtnE401dC0zRWU0S/QOTAg==", + "duRFqmvqF93uf/vWn8aOmg==", + "dxWv00FN/2Cgmgq9U3NVDQ==", + "e/nWuo5YalCAFKsoJmFyFA==", + "e2xLFVavnZIUUtxJx+qa1g==", + "e369ZIQjxMZJtopA//G55Q==", + "e4B3HmWjW+6hQzcOLru6Xg==", + "e5KCqQ/1GAyVMRNgQpYf6g==", + "e5l9ZiNWXglpw6nVCtO8JQ==", + "e5txnNRcGs2a9+mBFcF1Qg==", + "e9GqAEnk8XI5ix6kJuieNQ==", + "eAOEgF5N80A/oDVnlZYRAw==", + "eBapvE+hdyFTsZ0y5yrahg==", + "eC/RcoCVQBlXdE9WtcgXIw==", + "eCy/T+a8kXggn1L8SQwgvA==", + "eDWsx4isnr2xPveBOGc7Hw==", + "eDcyiPaB954q5cPXcuxAQw==", + "eFimq+LuHi42byKnBeqnZQ==", + "eFkXKRd2dwu/KWI5ZFpEzw==", + "eJDUejE/Ez/7kV+S74PDYg==", + "eJFIQh/TR7JriMzYiTw4Sg==", + "eJLrGwPRa6NgWiOrw1pA7w==", + "eJlcN+gJnqAnctbWSIO9uA==", + "eKQCVzLuzoCLcB4im8147A==", + "eLYKLr4labZeLiRrDJ9mnA==", + "ePlsM/iOMme2jEUYwi15ng==", + "eQ45Mvf5in9xKrP6/qjYbg==", + "eRwaYiog2DdlGQyaltCMJg==", + "eS/vTdSlMUnpmnl1PbHjyw==", + "eTMPXa60OTGjSPmvR4IgGw==", + "eV+RwWPiGEB+76bqvw+hbA==", + "eWgLAqJOU+fdn8raHb9HCw==", + "eXFOya6x5inTdGwJx/xtUQ==", + "eYAQWuWZX2346VMCD6s7/A==", + "eYE9No9sN5kUZ5ePEyS3+Q==", + "eddhS+FkXxiUnbPoCd5JJw==", + "edlXkskLx287vOBZ9+gVYg==", + "ehfPlu6YctzzpQmFiQDxGA==", + "ehwc2vvwNUAI7MxU4MWQZw==", + "ejfikwrSPMqEHjZAk3DMkA==", + "emVLJVzha7ui5OFHPJzeRQ==", + "enj9VEzLbmeOyYugTmdGfQ==", + "epY+dsm5EMoXnZCnO4WSHw==", + "es/L9iW8wsyLeC5S4Q8t+g==", + "eshD40tvOA6bXb0Fs/cH3A==", + "etRjRvfL/IwceY/IJ1tgzQ==", + "euxzbIq4vfGYoY3s1QmLcw==", + "evaWFoxZNQcRszIRnxqB+A==", + "ewPT4dM12nDWEDoRfiZZnA==", + "ewe/P3pJLYu/kMb5tpvVog==", + "ezsm4aFd6+DO9FUxz0A8Pg==", + "f/BjtP5fmFw2dRHgocbFlg==", + "f07bdNVAe9x+cAMdF1bByQ==", + "f09F7+1LRolRL5nZTcfKGA==", + "f0H/AFSx2KLZi9kVx5BAZg==", + "f1+fHgR5rDPsCZOzqrHM7Q==", + "f1Gs++Iilgq9GHukcnBG3w==", + "f1h+Vp+xmdZsZIziHrB2+g==", + "f5Xo7F1uaiM760Qbt978iw==", + "f6Ye5F0Lkn34uLVDCzogFQ==", + "f6iLrMpxKhFxIlfRsFAuew==", + "f9ywiGXsz+PuEsLTV3zIbQ==", + "fAKFfwlCOyhtdBK6yNnsNg==", + "fDOUzPTU2ndpbH0vgkgrJQ==", + "fFvXa1dbMoOOoWZdHxPGjw==", + "fHL+fHtDxhALZFb9W/uHuw==", + "fHNpW230mNib08aB7IM3XQ==", + "fKalNdhsyxTt1w08bv9fJA==", + "fM5uYpkvJFArnYiQ3MrQnA==", + "fO0+6TsjL+45p9mSsMRiIg==", + "fOARCnIg/foF/6tm7m9+3w==", + "fQS0jnQMnHBn7+JZWkiE/g==", + "fS471/rN4K2m10mUwGFuLg==", + "fSANOaHD0Koaqg7AoieY9A==", + "fU32wmMeD44UsFSqFY0wBA==", + "fU5ZZ1bIVsV+eXxOpGWo/Q==", + "fUAy3f9bAglLvZWvkO2Lug==", + "fVCRaPsTCKEVLkoF4y3zEw==", + "fW3QZyq5UixIA1mP6eWgqQ==", + "fX4G68hFL7DmEmjbWlCBJQ==", + "fY9VATklOvceDfHZDDk57A==", + "fZrj3wGQSt8RXv0ykJROcQ==", + "fbTm027Ms0/tEzbGnKZMDA==", + "fdqt93OrpG13KAJ5cASvkg==", + "fgXfRuqFfAu8qxbTi4bmhA==", + "fgdUFvQPb5h+Rqz8pzLsmw==", + "fhcbn9xE/6zobqQ2niSBgA==", + "fiv0DJivQeqUkrzDNlluRw==", + "fmC+85h5WBuk8fDEUWPjtQ==", + "fo3JL+2kPgDWfP+CCrFlFw==", + "foPAmiABJ3IXBoed2EgQXA==", + "foXSDEUwMhfHWJSmSejsQg==", + "fpXijBOM3Ai1RkmHven5Ww==", + "fsW2DaKYTCC7gswCT+ByQQ==", + "fsoXIbq0T0nmSpW8b+bj+g==", + "fsrX00onlGvfsuiCc35pGg==", + "ftsf2qztw3NC78ep/CZXWQ==", + "fv/PW8oexJYWf5De30fdLQ==", + "fvm0IQfnbfZFETg9v3z/Fg==", + "fxg/vQq9WPpmQsqQ4RFYaA==", + "fy54Milpa7KZH/zgrDmMXQ==", + "fzkmVWKhJsxyCwiqB/ULnQ==", + "g/z9yk94XaeBRFj4hqPzdw==", + "g0GbRp2hFVIdc7ct7Ky7ag==", + "g0aTR8aJ0uVy3YvGYu5xrw==", + "g0kHTNRI7x/lAsr92EEppw==", + "g0lWrzEYMntVIahC7i0O2g==", + "g1ELwsk6hQ+RAY1BH640Pg==", + "g2nh2xENCFOpHZfdEXnoQA==", + "g5EzTJ0KA4sO3+Opss3LMg==", + "g6udffWh7qUnSIo1Ldn3eA==", + "g6zSo8BvLuKqdmBFM1ejLA==", + "g8TcogVxHpw7uhgNFt5VCQ==", + "gAoV4BZYdW1Wm712YXOhWQ==", + "gB8wkuIzvuDAIhDtNT1gyA==", + "gBgJF0PiGEfcUnXF0RO7/w==", + "gC7gUwGumN7GNlWwfIOjJQ==", + "gDLjxT7vm07arF4SRX5/Vg==", + "gDxqUdxxeXDYhJk9zcrNyA==", + "gEHGeR2F82OgBeAlnYhRSw==", + "gFEnTI8os2BfRGqx9p5x8w==", + "gGLz3Ss+amU7y6JF09jq7A==", + "gICaI06E9scnisonpvqCsA==", + "gK7dhke5ChQzlYc/bcIkcg==", + "gR0sgItXIH8hE4FVs9Q07w==", + "gR3B8usSEb0NLos51BmJQg==", + "gTB2zM3RPm27mUQRXc/YRg==", + "gTnsH3IzALFscTZ1JkA9pw==", + "gU3gu8Y5CYVPqHrZmLYHbQ==", + "gUNP5w7ANJm257qjFxSJrA==", + "gW0oKhtQQ7BxozxUWw5XvQ==", + "gXlb7bbRqHXusTE5deolGA==", + "gYGQBLo5TdMyXks0LsZhsQ==", + "gYgCu/qUpXWryubJauuPNw==", + "gYnznEt9r97haD/j2Cko7g==", + "gYvdNJCDDQmNhtJ6NKSuTA==", + "gZNJ1Qq6OcnwXqc+jXzMLQ==", + "gZWTFt5CuLqMz6OhWL+hqQ==", + "gaEtlJtD6ZjF5Ftx0IFt0A==", + "gf1Ypna/Tt+TZ08Y+GcvGg==", + "gfhkPuMvjoC3CGcnOvki3Q==", + "gfnbviaVhKvv1UvlRGznww==", + "ggIfX1J4dX3xQoHnHUI7VA==", + "gglLMohmJDPRGMY1XKndjQ==", + "ghp8sWGKWw20S/z1tbTxFg==", + "ginkFyNVMwkZLE49AbfqfA==", + "gkrg0NR0iCaL7edq0vtewA==", + "glnqaRfwm6NxivtB2nySzw==", + "gnAIpoCyl3mQytLFgBEgGA==", + "gnez1VrH+UHT8C/SB9qGdA==", + "gnkadeCgjdmLdlu/AjBZJg==", + "goSgZ8N5UbT5NMnW3PjIlQ==", + "gqehq46BhFX2YLknuMv02w==", + "gsC/mWD8KFblxB0JxNuqJw==", + "gvvyX5ATi4q9NhnwxRxC8w==", + "gwyVIrTk5o0YMKQq4lpJ+Q==", + "gxwbqZDHLbQVqXjaq42BCg==", + "h+KRDKIvyVUBmRjv1LcCyg==", + "h0MH5NGFfChgmRJ3E/R3HQ==", + "h13Xuonj+0dD1xH86IhSyQ==", + "h1NNwMy0RjQmLloSw1hvdg==", + "h2B0ty0GobQhDnFqmKOpKQ==", + "h2cnQQF2/R3Mq2hWdDdrTg==", + "h3vYYI9yhpSZV2MQMJtwFQ==", + "h5HsEsObPuPFqREfynVblw==", + "h7Fc+eT/GuC8iWI+YTD0UQ==", + "hCzsi1yDv9ja5/o7t94j9Q==", + "hDGa2yLwNvgBd/v6mxmQaQ==", + "hDILjSpTLqJpiSSSGu445A==", + "hIABph+vhtSF5kkZQtOCTA==", + "hIJA+1QGuKEj+3ijniyBSQ==", + "hIjgi20+km+Ks23NJ4VQ6Q==", + "hJ8leLNuJ6DK5V8scnDaZQ==", + "hJSP7CostefBkJrwVEjKHA==", + "hK8KhTFcR06onlIJjTji/Q==", + "hKOsXOBoFTl/K4xE+RNHDA==", + "hN9bmMHfmnVBVr+7Ibd2Ng==", + "hNHqznsrIVRSQdII6crkww==", + "hP7dSa8lLn9KTE/Z0s4GVQ==", + "hPnPQOhz4QKhZi02KD6C+A==", + "hRxbdeniAVFgKUgB9Q3Y+g==", + "hSNZWNKUtDtMo6otkXA/DA==", + "hSkY45CeB6Ilvh0Io4W6cg==", + "hUWqqG1QwYgGC5uXJpCvJw==", + "hW9DJA1YCxHmVUAF7rhSmQ==", + "hWoxz5HhE50oYBNRoPp1JQ==", + "hY82j+sUQQRpCi6CCGea5A==", + "hZlX6qOfwxW5SPfqtRqaMw==", + "hdzol5dk//Q6tCm4+OndIA==", + "hf9HFxWRNX2ucH8FLS7ytA==", + "hfcH5Az2M7rp+EjtVpPwsg==", + "hiYg+aVzdBUDCG0CXz9kCw==", + "hkOBNoHbno2iNR7t3/d4vg==", + "hlMumZ7RJFpILuKs09ABtw==", + "hlu7os0KtAkpBTBV6D2jyQ==", + "hlvtFGW8r0PkbUAYXEM+Hw==", + "hnCUnoxofUiqQvrxl73M8w==", + "hq35Fjgvrcx6I9e6egWS4w==", + "hqeSvwu8eqA072iidlJBAw==", + "htDbVu1xGhCRd8qoMlBoMg==", + "htNVAogFakQkTX6GHoCVXg==", + "hv5GrLEIjPb4bGOi8RSO0w==", + "hvsZ5JmVevK1zclFYmxHaw==", + "hy303iin+Wm7JA6MeelwiQ==", + "i2sSvrTh/RdLJX0uKhbrew==", + "i42XumprV/aDT5R0HcmfIQ==", + "i6ZYpFwsyWyMJNgqUMSV1A==", + "i6r+mZfyhZyqlYv56o0H+w==", + "i8XXN7jcrmhnrOVDV8a2Hw==", + "i9IRqAqKjBTppsxtPB7rdw==", + "iANKiuMqWzrHSk9nbPe3bQ==", + "iCF+GWw9/YGQXsOOPAnPHQ==", + "iCnm5fPmSmxsIzuRK6osrA==", + "iFtadcw8v6betKka9yaJfg==", + "iGI9uqMoBBAjPszpxjZBWQ==", + "iGuY4VxcotHvMFXuXum7KA==", + "iGykaF+h4p46HhrWqL8Ffg==", + "iIWxFdolLcnXqIjPMg+5kQ==", + "iIm8c9uDotr87Aij+4vnMw==", + "iJ2nT8w8LuK11IXYqBK+YA==", + "iK0dWKHjVVexuXvMWJV9pg==", + "iPwX3SbbG9ez9HoHsrHbKw==", + "iQ304I1hmLZktA1d1cuOJA==", + "iS9wumBV5ktCTefFzKYfkA==", + "iSeH0JFSGK73F470Rhtesw==", + "iUsUCB0mfRsE9KPEQctIzw==", + "iVDd2Zk7vwmEh97LkOONpQ==", + "iWNlSnwrtCmVF89B+DZqOQ==", + "ibsb1ncaLZXAYgGkMO7tjQ==", + "ieEAgvK9LsWh2t6DsQOpWA==", + "ifZM0gBm9g9L09YlL+vXBg==", + "ifuJCv9ZA84Vz1FYAPsyEA==", + "ilBBNK/IV69xKTShvI94fQ==", + "imZ+mwiT22sW2M9alcUFfg==", + "inrUwXyKikpOW0y2Kl1wGw==", + "ionqS0piAOY2LeSReAz4zg==", + "ipPPjxpXHS1tcykXmrHPMQ==", + "irnD9K8bsT+up/JUrxPw6A==", + "iruDC5MeywV4yA8o1tw/KQ==", + "isep9d+Q7DEUf0W7CJJYzw==", + "itPtn+JaO4i7wz2wOPOmDQ==", + "iu5csar0IQQBOTgw5OvJwQ==", + "iujlt9fXcUXEYc+T2s5UjA==", + "iwKBOGDTFzV4aXgDGfyUkw==", + "izeyFvXOumNgVyLrbKW45g==", + "j+8/VARfbQSYhHzj0KPurQ==", + "j+lDhAnWAyso+1N8cm85hQ==", + "j4FBMnNfdBwx0VsDeTvhFg==", + "j8nMH8mK/0Aae7ZkqyPgdg==", + "j8to4gtSIRYpCogv2TESuQ==", + "jCgdKXsBCgf7giUKnr6paQ==", + "jEdanvXKyZdZJG6mj/3FWw==", + "jEqP0dyHKHiUjZ9dNNGTlQ==", + "jGHMJqbj6X1NdTDyWmXYAQ==", + "jHOoSl3ldFYr9YErEBnD3w==", + "jKJn4czwUl/6wtZklcMsSg==", + "jLI3XpVfjJ6IzrwOc4g9Pw==", + "jLkmUZ6fV56GfhC0nkh4GA==", + "jMZKSMP2THqwpWqJNJRWdw==", + "jNJQ6otieHBYIXA9LjXprg==", + "jNcMS2zX1iSZN9uYnb2EIg==", + "jOPdd330tB6+7C29a9wn0Q==", + "jQVlDU+HjZ2OHSDBidxX5A==", + "jQjyjWCEo9nWFjP4O8lehw==", + "jS0JuioLGAVaHdo/96JFoQ==", + "jTg9Y6EfpON4CRFOq0QovA==", + "jTmPbq+wh30+yJ/dRXk1cA==", + "jV/D2B11NLXZRH77sG9lBw==", + "jWsC7kdp2YmIZpfXGUimiA==", + "jZMDIu95ITTjaUX0pk4V5g==", + "jd6IpPJwOJW1otHKtKZ5Gw==", + "jdRzkUJrWxrqoyNH9paHfQ==", + "jdVMQqApseHH3fd91NFhxg==", + "jfegbZSZWkDoPulFomVntA==", + "jgNijyoj2JrQNSlUv4gk4A==", + "ji+1YHlRvzevs3q5Uw1gfA==", + "ji306HRiq965zb8EZD2uig==", + "jiV+b/1EFMnHG6J0hHpzBg==", + "jjNMPXbmpFNsCpWY0cv3eg==", + "jkUpkLoIXuu7aSH8ZghIAQ==", + "joDXdLpXvRjOqkRiYaD/Sw==", + "jon1y9yMEGfiIBjsDeeJdA==", + "jp5Em/0Ml4Txr1ptTUQjpg==", + "jpNUgFnanr9Sxvj2xbBXZw==", + "jpjpNjL1IKzJdGqWujhxCw==", + "jqPQ0aOuvOJte/ghI1RVng==", + "jrRH0aTUYCOpPLZwzwPRfQ==", + "jrfRznO0nAz6tZM1mHOKIA==", + "jt9Ocr9D8EwGRgrXVz//aQ==", + "jx7rpxbm1NaUMcE2ktg5sA==", + "jz7QlwxCIzysP39Cgro8jg==", + "k+IBS52XdOe5/hLp28ufnA==", + "k/Aou2Jmyh8Bu3k8/+ndsQ==", + "k/OVIllJvW6BefaLEPq7DA==", + "k/pBSWE2BvUsvJhA9Zl5uw==", + "k0XIjxp2vFG7sTrKcfAihA==", + "k1DPiH6NkOFXP/r3N12GyA==", + "k2KP9oPMnHmFlZO6u6tgyw==", + "k6OmSlaSZ5CB0i7SD9LczQ==", + "k8eZxqwxiN/ievXdLSEL/w==", + "kBAB2PSjXwqoQOXNrv80AA==", + "kFrRjz7Cf2KvLtz9X6oD+w==", + "kGeXrHEN6o7h5qJYcThCPw==", + "kHcBZXoxnFJ+GMwBZ/xhfQ==", + "kIGxCUxSlNgsKZ45Al1lWw==", + "kJdY3XEdJS/hyHdR+IN0GA==", + "kMUdiwM7WR8KGOucLK4Brw==", + "kNGIV3+jQmJlZDTXy1pnyA==", + "kRnBEH6ILR5GNSmjHYOclw==", + "kSUectNPXpXNg+tIveTFRw==", + "kTCHqcb3Cos51o8cL+MXcg==", + "kUhyc3G8Zvx8+q5q5nVEhw==", + "kUudvRfA33uJDzHIShQd3Q==", + "kWPUUi7x9kKKa6nJ+FDR5Q==", + "kZ/mZZg9YSDmk2rCGChYAg==", + "kZ0D191c/uv4YMG15yVLDw==", + "kZkmDatUOdIqs7GzH3nI1A==", + "ka7pMp8eSiv92WgAsz2vdA==", + "kcJ1acgBv6FtUhV8KuWoow==", + "kgKWQJJQKLUuD2VYKIKvxA==", + "kggaIvN2tlbZdZRI8S5Apw==", + "kgyUtd8MFe0tuuxDEUZA9w==", + "kh51WUI5TRnKhur6ZEpRTQ==", + "kj5WqpRCjWAfjM7ULMcuPQ==", + "kjWYVC7Eok2w2YT4rrI+IA==", + "kkbX+a00dfiTgbMI+aJpMg==", + "kly/2kE4/7ffbO34WTgoGg==", + "knYKU74onR6NkGVjQLezZg==", + "kq26VyDyJTH/eM6QvS2cMw==", + "kr8tw1+3NxoPExnAtTmfxg==", + "ksOFI9C7IrDNk4OP6SpPgw==", + "kuWGANwzNRpG4XmY7KjjNg==", + "kvAaIJb+aRAfKK104dxFAA==", + "kwlAQhR2jPMmfLTAwcmoxw==", + "kydoXVaNcx1peR5g6i588g==", + "kzGNkWh3fz27cZer4BspUQ==", + "kzTl7WH/JXsX1fqgnuTOgw==", + "kzXsrxWRnWhkA82LsLRYog==", + "kzYddqiMsY3EYrpxve2/CQ==", + "l+x2QhxG8wb5AQbcRxXlmA==", + "l0E0U/CJsyCVSTsXW4Fp+w==", + "l2NppPcweAtmA1V2CNdk2Q==", + "l2ZB9TvT68rn8AAN4MdxWw==", + "l2mAbuFF3QBIUILDODiUHQ==", + "l4ddTxbTCW5UmZW+KRmx6A==", + "l5f3I6osM9oxLRAwnUnc5A==", + "l6QHU5JsJExNoOnqxBPVbw==", + "l6Ssc04/CnsqUua9ELu2iQ==", + "l8/KMItWaW3n4g1Yot/rcQ==", + "lC5EumoIcctvxYqwELqIqw==", + "lFUq6PGk9dBRtUuiEW7Cug==", + "lHN2dn2cUKJ8ocVL3vEhUQ==", + "lJFPmPWcDzDp5B2S8Ad8AA==", + "lK2xe+OuPutp4os0ZAZx5w==", + "lM/EhwTsbivA7MDecaVTPw==", + "lMaO8Yf+6YNowGyhDkPhQA==", + "lMjip5hbCjkD9JQjuhewDg==", + "lNF8PvUIN02NattcGi5u4g==", + "lON3WM0uMJ30F8poBMvAjQ==", + "lOPJhHqCtMRFZfWMX/vFZQ==", + "lTE6u9G/RzvmbuAzq2J2/Q==", + "lV70RNlE++04G1KFB3BMXA==", + "lY+tivtsfvU0LJzBQ6itYQ==", + "lacCCRiWdquNm4YRO7FoKA==", + "leDlMcM+B1mDE8k5SWtUeg==", + "lf1fwA0YoWUZaEybE+LyMQ==", + "lfOLLyZNbsWQgHRhicr4ag==", + "lffapwUUgaQOIqLz2QPbAg==", + "lhAOM81Ej6YZYBu45pQYgg==", + "lizovLQxu6L9sbafNQuShQ==", + "lkl6XkrTMUpXi46dPxTPxg==", + "lkzFdvtBx5bV6xZO0cxK7g==", + "ll2M0QQzBsj5OFi02fv3Yg==", + "llOvGOUDVfX68jKnAlvVRA==", + "llujnWE17U8MIHmx4SbrSA==", + "lqhgbgEqROAdfzEnJ17eXA==", + "lsBTMnse2BgPS6wvPbe7JA==", + "luO1R8dUM9gy1E2lojRQoA==", + "luR/kvHLwA6tSdLeTM4TzA==", + "lwYQm2ynA3ik2gE1m11IEg==", + "lyfqic/AbEJbCiw+wA01FA==", + "lz+SeifYXxamOLs1FsFmSQ==", + "lzUQ1o7JAbdJYpmEqi6KnQ==", + "m+eh+ZqS74w2q0vejBkjaw==", + "m/Lp4U75AQyk9c8cX14HJg==", + "m06wctjNc3o7iyBHDMZs2w==", + "m3XYojKO+I6PXlVRUQBC3w==", + "m416yrrAlv+YPClGvGh+qQ==", + "m5JIUETVXcRza4VL4xlJbg==", + "m6get5wjq5j1i5abnpXuZQ==", + "m6srF+pMehggHB1tdoxlPg==", + "m9iuy4UtsjmyPzy6FTTZvw==", + "mAiD16zf+rCc7Qzxjd5buA==", + "mAzsVkijuqihhmhNTTz65g==", + "mDXHuOmI4ayjy2kLSHku1Q==", + "mI0eT4Rlr7QerMIngcu/ng==", + "mMLhjdWNnZ8zts9q+a2v3g==", + "mMfn8OaKBxtetweulho+xQ==", + "mNlYGAOPc6KIMW8ITyBzIg==", + "mNv2Q67zePjk/jbQuvkAFA==", + "mPk1IsU5DmDFA/Ym5+1ojw==", + "mPwCyD0yrIDonVi+fhXyEQ==", + "mS99D+CXhwyfVt8xJ+dJZA==", + "mSJF9dJnxZ15lTC6ilbJ2A==", + "mSstwJq7IkJ0JBJ5T8xDKg==", + "mTAqtg6oi0iytHQCaSVUsA==", + "mTLBkP+yGHsdk5g7zLjVUw==", + "mU4CqbAwpwqegxJaOz9ofQ==", + "mUek9NkXm8HiVhQ6YXiyzA==", + "mVT74Eht+gAowINoMKV7IQ==", + "mW6TCje9Zg2Ep7nzmDjSYQ==", + "mXBfDUt/sBW5OUZs2sihvw==", + "mXPtbPaoNAAlGmUMmJEWBQ==", + "mXZ4JeBwT2WJQL4a/Tm4jQ==", + "mXycPfF5zOvcj1p4hnikWw==", + "mc45FSMtzdw2PTcEBwHWPw==", + "md6zNd7ZBn3qArYqQz7/fw==", + "me61ST+JrXM5k3/a11gRAA==", + "meHzY9dIF7llDpFQo1gyMg==", + "miiOqnhtef1ODjFzMHnxjA==", + "mjFBVRJ7TgnJx+Q74xllPg==", + "mjQS8CpyGnsZIDOIEdYUxg==", + "mk1CKDah7EzDJEdhL22B7w==", + "mmRob7iyTkTLDu8ObmTPow==", + "mnalaO6xJucSiZ0+99r3Cg==", + "mpOtwBvle+nyY6lUBwTemw==", + "mpWNaUH9kn4WY26DWNAh3Q==", + "mr1qjhliRfl87wPOrJbFQg==", + "mrinv7KooPQPrLCNTRWCFg==", + "mrxlFD3FBqpSZr1kuuwxGg==", + "msstzxq++XO0AqNTmA7Bmg==", + "mxug34EekabLz0JynutfBg==", + "myzvc+2MfxGD9uuvZYdnqQ==", + "n+xYzfKmMoB3lWkdZ+D3rg==", + "n1M2dgFPpmaICP+JwxHUug==", + "n1ixvP7SfwYT3L2iWpJg6A==", + "n5GA+pA9mO/f4RN9NL9lNg==", + "n6QVaozMGniCO0PCwGQZ6w==", + "n7Bns42aTungqxKkRfQ5OQ==", + "n7KL1Kv027TSxBVwzt9qeA==", + "n7h9v2N1gOcvMuBEf8uThw==", + "nDAsSla+9XfAlQSPsXtzPA==", + "nE72uQToQFVLOzcu/nMjww==", + "nFBXCPeiwxK9mLXPScXzTA==", + "nFPDZGZowr3XXLmDVpo7hg==", + "nGzPc0kI/EduVjiK7bzM6Q==", + "nHTsDl0xeQPC5zNRnoa0Rw==", + "nHUpYmfV59fe3RWaXhPs3Q==", + "nL4iEd3b5v4Y9fHWDs+Lrw==", + "nMuMtK/Zkb3Xr34oFuX/Lg==", + "nNaGqigseHw30DaAhjBU3g==", + "nOiwBFnXxCBfPCHYITgqNg==", + "nR3ACzeVF5YcLX6Gj6AGyQ==", + "nULSbtw2dXbfVjZh33pDiA==", + "nUgYO7/oVNSX8fJqP2dbdg==", + "nVDxVhaa2o38gd1XJgE3aw==", + "nW3zZshjZEoM8KVJoVfnuQ==", + "nY/H7vThZ+dDxoPRyql+Cg==", + "neQoa8pvETr07blVMN3pgA==", + "nf8x+F03kOpMhsCSUWEhVg==", + "ng1Q0A7ljho3TUWWYl46sw==", + "nhAnHuCGXcYlqzOxrrEe1g==", + "nkbLVLvh3ClKED97+nH+7Q==", + "nkedTagkmf6YE4tEY+0fKw==", + "nknBKPgb7US42v8A0fTl/w==", + "nmD7fEU4u7/4+W/pkC4/0Q==", + "nqpKfidczdgrNaAyPi7BOQ==", + "nqtQI1bSM7DCO9P1jGV97Q==", + "nsnX3tKkN1elr18E31tXDw==", + "nvLEpj6ZZF3LWH3wUB6lKg==", + "nvUKoKfC6j8fz3gEDQrc/w==", + "nvmBgp0YlUrdZ05INsEE8Q==", + "nwtCsN1xEYaHvEOPzBv+qQ==", + "nx/U4Tode5ILux4DSR+QMg==", + "nxDGRpePV3H4NChn4eLwag==", + "nyaekSYTKzfSeSfPrB114Q==", + "nykEOLL/o7h0cs0yvdeT2g==", + "o+areESiXgSO0Lby56cBeg==", + "o+nYS4TqJc6XOiuUzEpC3A==", + "o/Y4U6rWfsUCXJ72p5CUGw==", + "o1uhaQg5/zfne84BFAINUQ==", + "o1zeXHJEKevURAAbUE/Vog==", + "o5XVEpdP4OXH0NEO4Yfc/A==", + "o64LDtKq/Fulf1PkVfFcyg==", + "o7y4zQXQAryST2cak4gVbw==", + "o9tdzmIu+3J/EYU4YWyTkA==", + "oAHVGBSJ2cf4dVnb/KEYmw==", + "oDca3JEdRb4vONT9GUUsaQ==", + "oFNMOKbQXcydxnp8fUNOHw==", + "oFanDWdePmmZN0xqwpUukA==", + "oGH7SMLI2/qjd9Vnhi3s0A==", + "oIU19xAvLJwQSZzIH577aA==", + "oIWwTbkVS5DDL47mY9/1KQ==", + "oKt57TPe4PogmsGssc3Cbg==", + "oLWWIn/2AbKRHnddr2og9g==", + "oMJLQTH1wW7LvOV0KRx/dw==", + "oNOI17POQCAkDwj6lJsYOA==", + "oONlXCW4aAqGczQ/bUllBw==", + "oPcxgoismve6+jXyIKK6AQ==", + "oPlhC4ebXdkIDazeMSn1fQ==", + "oQjugfjraFziga1BcwRLRA==", + "oR8rvIZoeoaZ/ufpo0htfQ==", + "oSnrpW4UmmVXtUGWqLq+tQ==", + "oUqO4HrBvkpSL781qAC9+w==", + "oVlG+0rjrg2tdFImxIeVBA==", + "oad5SwflzN0vfNcyEyF4EA==", + "obW3kzv2KBvuckU7F+tfjA==", + "ocRh5LR1ZIN9Johnht8fhQ==", + "ocpLRASvTgqfkY20YlVFHQ==", + "ocvA1/NbyxM0hanwwY6EiA==", + "odGhKtO4bDW5R8SYiI5yCg==", + "ogcuGHUZJkmv+vCz567a2g==", + "ohK6EftXOqBzIMI+5XnESw==", + "ojZY7Gi2QJXE/fp6Wy31iA==", + "ojf6uL85EuEYgLvHoGhUrw==", + "ojugpLIfzflgU2lonfdGxA==", + "ol9xhVTG9e1wNo50JdZbOA==", + "olTSlmirL9MFhKORiOKYkQ==", + "omAjyj1l6gyQAlBGfdxJTw==", + "onFcHOO1c3pDdfCb5N4WkQ==", + "oqlkgrYe9aCOwHXddxuyag==", + "oxoZP897lgMg/KLcZAtkAg==", + "oyYtf08AkWLR52bXm5+sKw==", + "ozVqYsmUueKifb4lDyVyrg==", + "p+bx+/WQWALXEBCTnIMr4w==", + "p/48hurJ1kh2FFPpyChzJg==", + "p/7qM5+Lwzw1/lIPY91YxQ==", + "p0eNK7zJd7D/HEGaVOrtrQ==", + "p2JPOX8yDQ0agG+tUyyT/g==", + "p3V7NfveB6cNxFW7+XQNeQ==", + "p48i7AfSSAyTdJSyHvOONw==", + "p73gSu4d+4T/ZNNkIv9Nlw==", + "p8W1LgFuW6JSOKjHkx3+aA==", + "pCQmlnn3BxhsV2GwqjRhXg==", + "pFKzcRHSUBqSMtkEJvrR1Q==", + "pGQEWJ38hb/ZYy2P1+FIuw==", + "pHo1O5zrCHCiLvopP2xaWw==", + "pHozgRyMiEmyzThtJnY4MQ==", + "pKaTI+TfcV3p/sxbd2e7YQ==", + "pT1raq2fChffFSIBX3fRiA==", + "pUfWmRXo70yGkUD/x5oIvA==", + "pVG1hL96/+hQ+58rJJy6/A==", + "pVgjGg4TeTNhKimyOu3AAw==", + "pW4gDKtVLj48gNz6V17QdA==", + "pZfn6IiG+V28fN8E2hawDQ==", + "pa8nkpAAzDKUldWjIvYMYg==", + "pcoBh5ic7baSD4TZWb3BSw==", + "pdPwUHauXOowaq9hpL2yFw==", + "pdaY6kZ8+QqkMOInvvACNA==", + "peMW+rpwmXrSwplVuB/gTA==", + "pfGcaa49SM3S6yJIPk/EJQ==", + "plXHHzA8X9QGwWzlJxhLRw==", + "pnJnBzAJlO4j3IRqcfmhkQ==", + "prCOYlboBnzmLEBG/OeVrQ==", + "prOsOG0adI4o+oz50moipw==", + "pulldyBt2sw6QDvTrCh6zw==", + "pv/m2mA/RJiEQu2Qyfv9RA==", + "pvXHwJ3dwf9GDzfDD9JI3g==", + "pw1jplCdTC+b0ThX0FXOjw==", + "pxuSWn1u+bHtRjyh2Z8veA==", + "pyrUqiZ98gVXxlXQNXv5fA==", + "pzC8Y0Vj9MPBy3YXR32z6w==", + "q/siBRjx6wNu+OTvpFKDwA==", + "q4z6A4l3nhX3smTmXr+Sig==", + "q5g3c8tnQTW2EjNfb2sukw==", + "q6LG0VzO1oxiogAAU63hyg==", + "q7m/EtZySBjZNBjQ5m1hKw==", + "q8YF9G2jqydAxSqwyyys5Q==", + "qA0sTaeNPNIiQbjIe1bOgQ==", + "qCPfJTR8ecTw6u6b1yHibA==", + "qE/h/Z+6buZWf+cmPdhxog==", + "qIFpKKwUmztsBpJgMaVvSg==", + "qIUJPanWmGzTD1XxvHp+6w==", + "qNOSm15bdkIDSc/iUr+UTQ==", + "qNyy6Fc0b8oOMWqqaliZ/w==", + "qO4HlyHMK5ygX+6HbwQe8w==", + "qOEIUWtGm5vx/+fg4tuazg==", + "qP1cCE4zsKGTPhjbcpczMw==", + "qQQwJ/aF87BbnLu3okXxaw==", + "qYHdgFAXhF/XcW4lxqfvWQ==", + "qYuo5vY8V3tZx41Kh9/4Dw==", + "qZ2q5j2gH3O56xqxkNhlIA==", + "qaTdVEeZ6S8NMOxfm+wOMA==", + "qcpeZWUlPllQYZU6mHVwUw==", + "qenHZKKlTUiEFv6goKM/Mw==", + "qkvEep4vvXhc2ZJ6R449Mg==", + "qngzBJbiTB4fivrdnE5gOg==", + "qnkFUlJ8QT322JuCI3LQgg==", + "qnsBdl050y9cUaWxbCczRw==", + "qnzWszsyJhYtx8wkMN6b1g==", + "qoK2keBg3hdbn7Q24kkVXg==", + "qpFJZqzkklby+u1UT3c1iA==", + "qt5CsMts2aD4lw/4Q6bHYQ==", + "qxALQrqHoDq9d91nU0DckA==", + "qyRmvxh8p4j4f+61c10ZFQ==", + "r/b5px/UImGNjT/X5sYjuA==", + "r0QffVKB9OD9yGsOtqzlhA==", + "r0hAwlS0mPZVfCSB+2G6uQ==", + "r1VGXWeqGeGbfKjigaAS+Q==", + "r2f2MyT+ww1g9uEBzdYI1w==", + "r36kVMpF+9J+sfI3GeGqow==", + "r3lQAYOYhwlLnDWQIunKqg==", + "r95wJtP5rsTExKMS7QhHcw==", + "rBt6L/KLT7eybxKt5wtFdg==", + "rCxoo4TP/+fupXMuIM0sDA==", + "rHagXw+CkF3uEWPWDKXvog==", + "rIMXaCaozDvrdpvpWvyZOQ==", + "rJ9qVn8/2nOxexWzqIHlcQ==", + "rJCuanCy51ydVD4nInf9IQ==", + "rKAQxu80Q8g1EEhW5Wh8tg==", + "rKb3TBM4EPx/RErFOFVCnQ==", + "rLZII1R6EGus+tYCiUtm6g==", + "rM/BOovNgnvebKMxZQdk7g==", + "rMm9bHK69h0fcMkMdGgeeA==", + "rOYeIcB+Rg5V6JG2k4zS2w==", + "rSvhrHyIlnIBlfNJqemEbw==", + "rTwJggSxTbwIYdp07ly0LA==", + "rUp5Mfc57+A8Q29SPcvH/Q==", + "rWliqgfZ3/uCRBOZ9sMmdA==", + "rXGWY/Gq+ZEsmvBHUfFMmQ==", + "rXSbbRABEf4Ymtda45w8Fw==", + "rXfWkabSPN+23Ei1bdxfmQ==", + "rXtGpN17Onx8LnccJnXwJQ==", + "rZKD8oJnIj5fSNGiccfcvA==", + "raKMXnnX6PFFsbloDqyVzQ==", + "raYifKqev8pASjjuV+UTKQ==", + "rcY4Ot40678ByCfqvGOGdg==", + "rdeftHE7gwAT67wwhCmkYQ==", + "rfPTskbnoh3hRJH6ZAzQRg==", + "rgcXxjx3pDLotH7TTfAoZw==", + "rh7bzsTQ1UZjG7amysr0Gg==", + "rhgtLQh0F9bRA6IllM7AGw==", + "ri4AOITPdB1YHyXV+5S51g==", + "rkeLYwMZ1/pW2EmIibALfA==", + "rlXt6zKE7DswUl0oWGOQUQ==", + "rqHKB91H3qVuQAm+Ym5cUA==", + "rqucO37p86LpzehR/asCSQ==", + "rs2QrN4qzAHCHhkcrAvIfA==", + "rtJdfki8fG6CB36CADp0QA==", + "rtd6mqFgGe98mqO0pFGbSw==", + "rueNryrchijjmWaA3kljYg==", + "rvE64KQGkVkbl07y7JwBqw==", + "rwplpbNJz0ADUHTmzAj15Q==", + "rwtF86ZAbWyKI6kLn4+KBw==", + "rxfACPLtKXbYua18l3WlUw==", + "rzj6mjHCcMEouL66083BAg==", + "s+eHg5K9zZ2Jozu5Oya9ZQ==", + "s/BZAhh1cTV3JCDUQsV8mA==", + "s2AKVTwrY65/SWqQxDGJQg==", + "s5+78jS4hQYrFtxqTW3g1Q==", + "s5RUHVRNAoKMuPR/Jkfc2Q==", + "s7iW1M6gkAMp+D/3jHY58w==", + "s8NpalwgPdHPla7Zi9FJ3w==", + "sBpytpE38xz0zYeT+0qc2A==", + "sC11Rf/mau3FG5SnON4+vQ==", + "sCLMrLjEUQ6P1L8tz90Kxg==", + "sEeblUmISi1HK4omrWuPTA==", + "sGLPmr568+SalaQr8SE/PA==", + "sLJrshdEANp0qk2xOUtTnQ==", + "sLdxIKap0ZfC3GpUk3gjog==", + "sNmW2b2Ud7dZi3qOF8O8EQ==", + "sQAxqWXeiu/Su0pnnXgI9A==", + "sQskMBELEq86o1SJGQqfzg==", + "sQzCwNDlRsSH7iB9cTbBcg==", + "sS6QcitMPdvUBLiMXkWQkw==", + "sWLcS+m4aWk31BiBF+vfJQ==", + "sXlFMSTBFnq0STHj6cS/8w==", + "sa2DECaqYH1z1/AFhpHi+g==", + "saEpnDGBSZWqeXSJm34eOA==", + "scCQPl0em2Zmv/RQYar60g==", + "sfIClgTMtZo9CM9MHaoqhQ==", + "sfowXUMdN2mCoBVrUzulZg==", + "sfte/o9vVNyida/yLvqADA==", + "siHwJx6EgeB1gBT9z/vTyw==", + "skrQRB9xbOsiSA19YgAdIQ==", + "snGTzo540cCqgBjxrfNpKw==", + "soBA65OmZdfBGJkBmY/4Iw==", + "spHVvA/pc7nF9Q4ON020+w==", + "spJI3xFUlpCDqzg0XCxopA==", + "sr3UXbMg5zzkRduFx/as7g==", + "sw+bmpzqsM4gEQtnqocQLQ==", + "swJhrPwllq5JORWiP5EkDA==", + "swsVVsPi/5aPFBGP+jmPIw==", + "syeBfQBUmkXNWCZ1GV8xSA==", + "t+bYn9UqrzKiuxAYGF7RLA==", + "t0WN8TwMLgi8UVEImoFXKg==", + "t2EkpUsLOEOsrnep0nZSmA==", + "t2vWMIh2BvfDSQaz5T1TZw==", + "t3Txxjq43e/CtQmfQTKwWg==", + "t5U+VMsTtlWAAWSW+00SfQ==", + "t5wh9JGSkQO78QoQoEqvXA==", + "t7HaNlXL16fVwjgSXmeOAQ==", + "t8pjhdyNJirkvYgWIO/eKg==", + "tBQDfy48FnIOZI04rxfdcA==", + "tFMJRXfWE9g78O1uBUxeqQ==", + "tFmWYH82I3zb+ymk5dhepA==", + "tG+rpfJBXlyGXxTmkceiKA==", + "tHDbi43e6k6uBgO0hA+Uiw==", + "tIqwBotg052wGBL65DZ+yA==", + "tJt6VDdAPEemBUvnoc4viA==", + "tOdlnsE3L3XCBDJRmb/OqA==", + "tOkYq1BZY152/7IJ6ZYKUg==", + "tU31r8zla146sqczdKXufg==", + "tVhXk9Ff3wAg56FbdNtcFg==", + "tVvWdA+JqH0HR2OlNVRoag==", + "tVw8U1AsslIFmQs4H1xshg==", + "tX8X8KoxUQ8atFSCxgwE1Q==", + "tXVb5f90k9l3e1oK2NGXog==", + "tXuu7YpZOuMLTv87NjKerA==", + "tY916jrSySzrL+YTcVmYKQ==", + "tYeIZjIm0tVEsYxH1iIiUQ==", + "tb5+2dmYALJibez1W4zXgA==", + "td7nDgTDmKPSODRusMcupw==", + "tdgI9v7cqJsgCAeW1Fii1A==", + "tdiTXKrkqxstDasT0D5BPA==", + "tejpAZp7y32SO2+o4OGvwQ==", + "tfgO55QqUyayjDfQh+Zo1Q==", + "tj2rWvF2Fl+XIccctj8Mhw==", + "tnUtJ/DQX9WaVJyTgemsUA==", + "tq5xUJt8GtjDIh1b48SthQ==", + "tr+U/vt+MIGXPRQYYWJfRg==", + "trjM81KANPZrg9iSThWx6Q==", + "tsiqwelcBAMU/HpLGBtMGw==", + "twPn6wTGqI0aR//0wP3xtA==", + "twjiDKJM7528oIu/el4Zbg==", + "tzV7ixFH37ze4zuLILTlfA==", + "u/QxrP1NOM/bOJlJlsi/jQ==", + "u2WQlcMxOACy6VbJXK4FwA==", + "u5cUPxM6/spLIV8VidPrAA==", + "uC2lzm7HaMAoczJO6Z/IhQ==", + "uChFnF0oCwARhAOz/d47eA==", + "uESeJe/nYrHCq4RQbrNpGA==", + "uExgqZkkJnZj252l5dKAGg==", + "uIkVijg7RPi/1j7c18G1qA==", + "uJZGw3IY2nCcdVeWW1geNQ==", + "uMq8cDVWFD+tpn8aeP8Pqg==", + "uNWFZlP7DA96sf+LWiAhtQ==", + "uNzpptKjihEfKRo5A1nWmw==", + "uO+uK1DntCxVRr1KttfUIw==", + "uOHrw37yF9oLLVd16nUpeg==", + "uOkMpYy/7DYYoethJdixfQ==", + "uPdjKJIGzN7pbGZDZdCGaA==", + "uPi8TsGY3vQsMVo/nsbgVQ==", + "uPm+cF4Jq08S5pQhYFjU8A==", + "uPnL9tboMZo0Kl2fe24CmA==", + "uQs79rbD/wEakMUxqMI48A==", + "uSIiF1r9F18avZczmlEuMQ==", + "uT6WRh5UpVdeABssoP2VTg==", + "uTA0XbiH3fTeVV7u5z0b3w==", + "uTHBqApdKOAgdwX3cjrCYQ==", + "uU1TX5DoDg6EcFKgFcn0GA==", + "uXuPA/2KJbb7ZX+NymN3dw==", + "uXvr6vi5kazZ9BCg2PWPJA==", + "uZ2gUA74/7Q33tI2TcGQlg==", + "ucLMWnNDSqE4NOCGWvcGWw==", + "udU65VtsvJspYmamiOsgXw==", + "ueODvMv/f9ZD8O0aIHn4sg==", + "ugY8rTtJkN4CXWMVcRZiZw==", + "uhT12XY79CtbwhcSfAmAXQ==", + "ulLuTZqhEDkX0EJ3xwRP9A==", + "ulpDxLeQnIRPnq6oaah2AA==", + "up2MVDi9ve+s83/nwNtZ7Q==", + "uqe3rFveJ2JIkcZQ3ZMXHQ==", + "uqp92lAqjec8UQYfyjaEZw==", + "ur9JDCVNwzSH4q4ngDlHNQ==", + "uu+ncs63SdQIvG6z4r7Q3Q==", + "uuiJ+yB7JLDh2ulthM0mjg==", + "uvKYnKE01D5r7kR9UQyo5A==", + "uvzmRcvgepW6mZbMfYgcNw==", + "uwA6N5LptSXqIBkTO0Jd7Q==", + "uwGivY3/C9WK+dirRPJZ4A==", + "uzEgwx1iAXAvWPKSVwYSeQ==", + "uzkNhmo2d08tv5AmnyqkoQ==", + "v/PshI6JjkL9nojLlMNfhg==", + "v0Bvws1WYVoEgDt8xmVKew==", + "v1AWe5qb5y3vSKFb7ADeEw==", + "v4xIYrfPGILEbD/LwVDDzA==", + "v6jZicMNM3ysm3U5xu0HoQ==", + "v7BrkRmK0FfWSHunTRHQFQ==", + "vCekQ2nOQKiN/q8Be/qwZg==", + "vFFzkWgGyw6OPADONtEojQ==", + "vFox1d3llOeBeCUZGvTy0A==", + "vFtC0B2oe1gck28JOM1dyg==", + "vGKknndb4j6VTV8DxeT4fQ==", + "vHGjRRSlZHJIliCwIkCAmQ==", + "vHVXsAMQqc0qp7HA5Q+YkA==", + "vHmQUl4WHXs1E/Shh+TeyA==", + "vIORTYSHFIXk5E2NyIvWcQ==", + "vMuaLvAntJB5o7lmt/kVXA==", + "vOJ55zFdgPPauPyFYBf01w==", + "vRgkZZGVN7YZrlml0vxrKA==", + "vSKsa0JhLCe9QFZKkcj58Q==", + "vTAmgfq3GxL4+ubXpzwk5w==", + "vUC0HlTTHj6qNHwfviDtAw==", + "vUE8Iw3NyWXURpXyoNJdaw==", + "vWn9OPnrJgfPavg4D6T/HQ==", + "vX7RIhatQeXAMr1+OjzhZw==", + "vZtL0yWpSIA+9v8i23bZSg==", + "vb6Agwzk4JG0Nn7qRPPFMQ==", + "vbyiKeDCQ4q9dDRI1Q0Ong==", + "vg3jozLXEmAnmJwdfcEN0g==", + "vhdFtKVH4bVatb4n8KzeXw==", + "vjrSYGUpeKOtJ2cNgLFg2g==", + "vljJciS+uuIvL7XXm5688g==", + "vmqfGJE6r4yDahtU/HLrxw==", + "vnOJ3e9Zd4wPx8PX7QgZzQ==", + "voO3krg4sdy4Iu+MZEr8+g==", + "vqYHQ3MnHrAIAr1QHwfIag==", + "vsRNZx4thFFFPneubKq1Fw==", + "vvEH5A39TTe1AOC11rRCLA==", + "vvh9vAIrXjIwLVkuJb5oDQ==", + "vwno3vugCvt6ooT3CD4qIQ==", + "w+jzM0I5DRzoUiLS/9QIMQ==", + "w0PKdssv+Zc5J/BbphoxpA==", + "w1zN28mSrI/gqHsgs4ME3A==", + "w3G+qXXqqKi8F5s+qvkBUg==", + "w5N/aHbtOIKzcvG3GlMjGA==", + "wDiGoFEfIVEDyyc4VpwhWQ==", + "wEJDulZafLuXCvcqBYioFQ==", + "wHA+D5cObfV3kGORCdEknw==", + "wI7JrSPQwYHpv2lRsQu9nQ==", + "wIfvvLKC61gOpsddUFjVog==", + "wJ4uCrl4DPg70ltw1dZO3w==", + "wJKFMqh6MGctWfasjHrPEg==", + "wJpepvmtQQ3sz3tVFDnFqw==", + "wK6Srd83eLigZ11Q20XGrg==", + "wM8tnXO4PDlLVHspZFcjYw==", + "wMOE/pEKVIklE75xjt6b6w==", + "wMum67lfk5E1ohUObJgrOg==", + "wMyJLQJdmrC2TSeFkIuSvQ==", + "wOc4TbwQGUwOC1B3BEZ4OQ==", + "wOhbpTzmFla8R0kI9OiHaA==", + "wPhJcp7U7IVX83szbIOOxQ==", + "wQKL8Ga6JQkpZ7yymDkC3w==", + "wR2Gxb07nkaPcZHlEjr8iA==", + "wRqaDZVHHurp5whOQ1kDbQ==", + "wTO49YX/ePHMWtcoxUAHpw==", + "wUYhs4j3W9nIywu1HIv2JA==", + "wVfSZYjMjbTsD2gaSbwuqQ==", + "wX2URK6eDDHeEOF3cgPgHA==", + "wX70jKLKJApHnhyK0r6t3A==", + "wajwXfWz2J+O+NVaj6j2UQ==", + "wc+8ohFWgOF4VlSYiZIGwQ==", + "wdRyYjaM11VmqkkxV/5bsA==", + "wfwuxn+Vja1DNwiDwL2pcQ==", + "wgH1GlUxWi6/yLLFzE76uQ==", + "who8uUamlHWHXnBf7dwy4A==", + "wlWxtQDJ+siGhN2fJn3qtw==", + "wnfYUctNK+UPwefX5y4/Rw==", + "wpZqFkKafFpLcykN2IISqg==", + "wqUJ1Gq1Yz2cXFkbcCmzHQ==", + "wqWqe0KRjZlUIrGgEOG9Mg==", + "wrewZ0hoHODf7qmoGcOd7g==", + "wsp+vmW8sEqXYVURd/gjHA==", + "wt+qDLU38kzNU75ZYi3Hbw==", + "wtyAZIfhomcHe9dLbYoSvA==", + "wux5Y8AipBnc5tJapTzgEQ==", + "wv4NC9CIpwuGf/nOQYe/oA==", + "wxkb8evGEaGf/rg/1XUWiA==", + "wy/Z8505o4sVovk4UuBp1A==", + "wyqmQGB6vgRVrYtmB2vB7w==", + "wyx5mnUMgP5wjykjAfTO7w==", + "x+8rwkqKCv0juoT5m1A4eg==", + "x/BIDm6TKMhqu/gtb3kGyw==", + "x/MpsQvziUpW40nNUHDS5Q==", + "x0eIHCvQLd2jdDaXwSWTYQ==", + "x1A74vg/hwwjAx6GrkU8zw==", + "x2NpqNnqRihktNzpxmepkQ==", + "x2nSgcTjA3oGgI8mMgiqjw==", + "x5lyMArsv1MuJmEFlWCnNw==", + "x5zMDuW66467ofgL3spLUQ==", + "x6M66krXSi0EhppwmDmsxA==", + "x6lNRGgJcRxgKTlzhc1WPg==", + "x8kRVzohTdhkryvYeMvkMw==", + "x9TIZ9Ua++3BX+MpjgTuWA==", + "x9VwDdFPp/rJ+SF16ooWYg==", + "xAAipGfHTGTjp9Qk1MR8RQ==", + "xJi0T+psHOXMivSOVpMWeQ==", + "xLm/bJBonpTs0PwsF0DvRg==", + "xMIHeno2qj3V8q9H1xezeg==", + "xNilc7UOu1kyP0+nK5MrLw==", + "xPe76nHyHmald6kmMQsKdg==", + "xQpYjaAmrQudWgsdu24J0A==", + "xTizUioizbMQxD0T6fy/EQ==", + "xUXEE7OBBCudsQnuj5ycOA==", + "xWYecfzAtXT9WyQ8NYY/hw==", + "xX6atcCApI08oVLjjLteLg==", + "xYD8jrCDmuQna+p1ebnKDQ==", + "xbBxUP9JyY0wDgHDipBHeg==", + "xdCCdP8SNBOK3IsX6PiPQA==", + "xdmY+qyoxxuRZa9kuNpDEg==", + "xfYZ6qhWNBqqJ0PdWRjOwA==", + "xfjBQk3CrNjhufdPIhr91A==", + "xiFlcSfa/gnPiO+LwbixcQ==", + "xiyRfVG0EfBA+rCk+tgWRQ==", + "xjA21QjNdThLW3VV7SCnrg==", + "xjTMO2mvtpvwQrounD4e8g==", + "xktOghh1S9nIX6fXWnT+Ug==", + "xmGgK3W5y+oCd0K2u8XjZQ==", + "xmsYnsJq78/f9xuKuQ2pBQ==", + "xoPSM86Se+1hHX0y3hhdkw==", + "xs8J3cesq7lDhP/dNltqOw==", + "xsCZVhCk2qJmOqvUjK3Y8Q==", + "xsf0m31Am0W9eLhopAkfnA==", + "xukOAM0QVsA72qEy0yku9A==", + "xvipmmwKdYt4eoKvvRnjEg==", + "xweGAZf+Yb3TtwR/sGmGIA==", + "xzGzN5Hhbh0m/KezjNvXbQ==", + "y+1I05LDAYJ09tKMs3zW6g==", + "y+cl1/Knb9MZPz8nBB0M+w==", + "y/e3HSdg7T19FanRpJ7+7Q==", + "y1J+o6DC2sETFsySgpDZyA==", + "y2JOIoIiT9cV1VxplZPraQ==", + "y2Tn2gmhKs5WKc01ce74rg==", + "y4/HohCJxtt+cT7nLJB08w==", + "y4Y4mSSTw/WrIdRpktc5Hw==", + "y4iBxAMn/KzMmaWShdYiIw==", + "y4mfEDerrhaqApDdhP5vjA==", + "y7yS9x3yshVhMpDbQtfYOQ==", + "yCu+DVU/ceMTOZ5h/7wQTg==", + "yD3Dd4ToRrl53k/2NSCJiw==", + "yDrAd1ot38soBk7zKdnT8A==", + "yKLLiqzxfrCsr6+Rm6kx1Q==", + "yKrsKX4/1B1C0TyvciNz5w==", + "yL1DwlIIREPuyuCFULi0uw==", + "yLAhLNezvqVHmN1SfMRrPw==", + "yOE90OHQdyOfrAgwDvn2gA==", + "yPIeWcW8+3HjDagegrN8bw==", + "yQCLV9IoPyXEOaj3IdFMWw==", + "yQmNZnp/JZywbBiZs3gecA==", + "yS/yMnJDHW0iaOsbj4oPTg==", + "yTVJKBn72RjakMBXDoBKHg==", + "yTgN5xFIdz1MzFS6xMl5uQ==", + "yU3N0HMSP5etuHPNrVkZtg==", + "yV3IbbTWAbHMhMGVvgb/ZQ==", + "yYBIS9PZbKo7Gram7IXWPA==", + "yYVW07lOZHdgtX42xJONIA==", + "yYmnM/WOgi+48Rw7foGyXA==", + "yYp4iuI5f/y/l1AEJxYolQ==", + "ybpTgPr3SjJ12Rj5lC/IMA==", + "ycjv4XkS5O7zcF3sqq9MwQ==", + "yctId8ltkl3+xqi9bj+RqA==", + "ydVj2odhergi+2zGUwK4/A==", + "yf06Slv9l3IZEjVqvxP2aA==", + "yfAaL0MMtSXPQ37pBdmHxQ==", + "yhI5jHlfFJxu4eV5VJO2zQ==", + "yhRi5M9Etuu9HSu4d24i3w==", + "yhexr/OFKfZl0o3lS70e4w==", + "ylA6sU7Kaf9fMNIx1+sIlw==", + "ymtA8EMPMgmMcimWZZ0A1Q==", + "ynaj4XjU27b7XbqPyxI8Ig==", + "yqQPU4jT9XvRABZgNQXjgg==", + "yqtj8GfLaUHYv/BsdjxIVw==", + "ysRQ+7Aq7eVLOp88KnFVMA==", + "ytDXLDBqWiU1w3sTurYmaw==", + "yteeQr3ub2lDXgLziZV+DQ==", + "yxCyBXqGWA735JEyljDP7Q==", + "z+1oDVy8GJ5u/UDF+bIQdA==", + "z/e5M2lE9qh3bzB97jZCKA==", + "z0BU//aSjYHAkGGk3ZSGNg==", + "z20AAnvj7WsfJeOu3vemlA==", + "z3L2BNjQOMOfTVBUxcpnRA==", + "z4Bft++f72QeDh4PWGr/sw==", + "z4oKy2wKH+sbNSgGjbdHGw==", + "z5DveTu377UW8IHnsiUGZg==", + "z920R8eahJPiTsifrPYdxA==", + "z9cd+Qj+ueX34Zf3997MNQ==", + "zCRZgVsHbQZcVMHd9pGD3A==", + "zCpibjrZOA3FQ4lYt0WoVA==", + "zDSQ3NJuUGkVOlvVCATRwA==", + "zDUZCzQesFjO1JI3PwDjfg==", + "zEzWZ6l7EKoVUxvk/l78Mw==", + "zJ7ScHNxr2leCDNNcuDApA==", + "zNLlWGW/aKBhUwQZ4DZWoQ==", + "zVupSPz7cD0v/mD/eUIIjg==", + "zZtYkKU50PPEj6qSbO5/Sw==", + "za4rzveYVMFe3Gw531DQJQ==", + "zaqyy3GaJ7cp8qDoLJWcTw==", + "zbjXhZaeyMfdTb2zxvmRMg==", + "zeELfk015D5krExLKRUYtg==", + "zeHF6fdeqcOId3fRUGscRw==", + "zgEyxj/sCs63O98sZS94Yw==", + "zi04Yc01ZheuFAQc59E45A==", + "zirOtGUXeRL22ezfotZfQg==", + "zm+z+OOyHhljV2TjA3U9zw==", + "zrZWcqQsUE3ocWE0fG+SOA==", + "ztULoqHvCOE6qV7ocqa4/w==", + "zwQ/3MzTJ9rfBmrANIh14w==", + "zwY6tCjjya/bgrYaCncaag==", + "zxsSqovedB3HT99jVblCnQ==", + "zyA9f5J7mw5InjhcfeumAQ==", +]); diff --git a/browser/components/newtab/lib/HighlightsFeed.sys.mjs b/browser/components/newtab/lib/HighlightsFeed.sys.mjs new file mode 100644 index 0000000000..c603b886da --- /dev/null +++ b/browser/components/newtab/lib/HighlightsFeed.sys.mjs @@ -0,0 +1,322 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { actionTypes as at } from "resource://activity-stream/common/Actions.sys.mjs"; + +import { shortURL } from "resource://activity-stream/lib/ShortURL.sys.mjs"; +import { + TOP_SITES_DEFAULT_ROWS, + TOP_SITES_MAX_SITES_PER_ROW, +} from "resource://activity-stream/common/Reducers.sys.mjs"; +import { Dedupe } from "resource://activity-stream/common/Dedupe.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + DownloadsManager: "resource://activity-stream/lib/DownloadsManager.sys.mjs", + FilterAdult: "resource://activity-stream/lib/FilterAdult.sys.mjs", + LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + Screenshots: "resource://activity-stream/lib/Screenshots.sys.mjs", + SectionsManager: "resource://activity-stream/lib/SectionsManager.sys.mjs", +}); + +const HIGHLIGHTS_MAX_LENGTH = 16; + +export const MANY_EXTRA_LENGTH = + HIGHLIGHTS_MAX_LENGTH * 5 + + TOP_SITES_DEFAULT_ROWS * TOP_SITES_MAX_SITES_PER_ROW; + +export const SECTION_ID = "highlights"; +export const SYNC_BOOKMARKS_FINISHED_EVENT = "weave:engine:sync:applied"; +export const BOOKMARKS_RESTORE_SUCCESS_EVENT = "bookmarks-restore-success"; +export const BOOKMARKS_RESTORE_FAILED_EVENT = "bookmarks-restore-failed"; +const RECENT_DOWNLOAD_THRESHOLD = 36 * 60 * 60 * 1000; + +export class HighlightsFeed { + constructor() { + this.dedupe = new Dedupe(this._dedupeKey); + this.linksCache = new lazy.LinksCache( + lazy.NewTabUtils.activityStreamLinks, + "getHighlights", + ["image"] + ); + lazy.PageThumbs.addExpirationFilter(this); + this.downloadsManager = new lazy.DownloadsManager(); + } + + _dedupeKey(site) { + // Treat bookmarks, pocket, and downloaded items as un-dedupable, otherwise show one of a url + return ( + site && + (site.pocket_id || site.type === "bookmark" || site.type === "download" + ? {} + : site.url) + ); + } + + init() { + Services.obs.addObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT); + Services.obs.addObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT); + Services.obs.addObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT); + lazy.SectionsManager.onceInitialized(this.postInit.bind(this)); + } + + postInit() { + lazy.SectionsManager.enableSection(SECTION_ID, true /* isStartup */); + this.fetchHighlights({ broadcast: true, isStartup: true }); + this.downloadsManager.init(this.store); + } + + uninit() { + lazy.SectionsManager.disableSection(SECTION_ID); + lazy.PageThumbs.removeExpirationFilter(this); + Services.obs.removeObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT); + Services.obs.removeObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT); + Services.obs.removeObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT); + } + + observe(subject, topic, data) { + // When we receive a notification that a sync has happened for bookmarks, + // or Places finished importing or restoring bookmarks, refresh highlights + const manyBookmarksChanged = + (topic === SYNC_BOOKMARKS_FINISHED_EVENT && data === "bookmarks") || + topic === BOOKMARKS_RESTORE_SUCCESS_EVENT || + topic === BOOKMARKS_RESTORE_FAILED_EVENT; + if (manyBookmarksChanged) { + this.fetchHighlights({ broadcast: true }); + } + } + + filterForThumbnailExpiration(callback) { + const state = this.store + .getState() + .Sections.find(section => section.id === SECTION_ID); + + callback( + state && state.initialized + ? state.rows.reduce((acc, site) => { + // Screenshots call in `fetchImage` will search for preview_image_url or + // fallback to URL, so we prevent both from being expired. + acc.push(site.url); + if (site.preview_image_url) { + acc.push(site.preview_image_url); + } + return acc; + }, []) + : [] + ); + } + + /** + * Chronologically sort highlights of all types except 'visited'. Then just append + * the rest at the end of highlights. + * @param {Array} pages The full list of links to order. + * @return {Array} A sorted array of highlights + */ + _orderHighlights(pages) { + const splitHighlights = { chronologicalCandidates: [], visited: [] }; + for (let page of pages) { + if (page.type === "history") { + splitHighlights.visited.push(page); + } else { + splitHighlights.chronologicalCandidates.push(page); + } + } + + return splitHighlights.chronologicalCandidates + .sort((a, b) => a.date_added < b.date_added) + .concat(splitHighlights.visited); + } + + /** + * Refresh the highlights data for content. + * @param {bool} options.broadcast Should the update be broadcasted. + */ + async fetchHighlights(options = {}) { + // If TopSites are enabled we need them for deduping, so wait for + // TOP_SITES_UPDATED. We also need the section to be registered to update + // state, so wait for postInit triggered by lazy.SectionsManager initializing. + if ( + (!this.store.getState().TopSites.initialized && + this.store.getState().Prefs.values["feeds.system.topsites"] && + this.store.getState().Prefs.values["feeds.topsites"]) || + !this.store.getState().Sections.length + ) { + return; + } + + // We broadcast when we want to force an update, so get fresh links + if (options.broadcast) { + this.linksCache.expire(); + } + + // Request more than the expected length to allow for items being removed by + // deduping against Top Sites or multiple history from the same domain, etc. + const manyPages = await this.linksCache.request({ + numItems: MANY_EXTRA_LENGTH, + excludeBookmarks: + !this.store.getState().Prefs.values[ + "section.highlights.includeBookmarks" + ], + excludeHistory: + !this.store.getState().Prefs.values[ + "section.highlights.includeVisited" + ], + excludePocket: + !this.store.getState().Prefs.values["section.highlights.includePocket"], + }); + + if ( + this.store.getState().Prefs.values["section.highlights.includeDownloads"] + ) { + // We only want 1 download that is less than 36 hours old, and the file currently exists + let results = await this.downloadsManager.getDownloads( + RECENT_DOWNLOAD_THRESHOLD, + { numItems: 1, onlySucceeded: true, onlyExists: true } + ); + if (results.length) { + // We only want 1 download, the most recent one + manyPages.push({ + ...results[0], + type: "download", + }); + } + } + + const orderedPages = this._orderHighlights(manyPages); + + // Remove adult highlights if we need to + const checkedAdult = lazy.FilterAdult.filter(orderedPages); + + // Remove any Highlights that are in Top Sites already + const [, deduped] = this.dedupe.group( + this.store.getState().TopSites.rows, + checkedAdult + ); + + // Keep all "bookmark"s and at most one (most recent) "history" per host + const highlights = []; + const hosts = new Set(); + for (const page of deduped) { + const hostname = shortURL(page); + // Skip this history page if we already something from the same host + if (page.type === "history" && hosts.has(hostname)) { + continue; + } + + // If we already have the image for the card, use that immediately. Else + // asynchronously fetch the image. NEVER fetch a screenshot for downloads + if (!page.image && page.type !== "download") { + this.fetchImage(page, options.isStartup); + } + + // Adjust the type for 'history' items that are also 'bookmarked' when we + // want to include bookmarks + if ( + page.type === "history" && + page.bookmarkGuid && + this.store.getState().Prefs.values[ + "section.highlights.includeBookmarks" + ] + ) { + page.type = "bookmark"; + } + + // We want the page, so update various fields for UI + Object.assign(page, { + hasImage: page.type !== "download", // Downloads do not have an image - all else types fall back to a screenshot + hostname, + type: page.type, + pocket_id: page.pocket_id, + }); + + // Add the "bookmark", "pocket", or not-skipped "history" + highlights.push(page); + hosts.add(hostname); + + // Remove internal properties that might be updated after dispatch + delete page.__sharedCache; + + // Skip the rest if we have enough items + if (highlights.length === HIGHLIGHTS_MAX_LENGTH) { + break; + } + } + + const { initialized } = this.store + .getState() + .Sections.find(section => section.id === SECTION_ID); + // Broadcast when required or if it is the first update. + const shouldBroadcast = options.broadcast || !initialized; + + lazy.SectionsManager.updateSection( + SECTION_ID, + { rows: highlights }, + shouldBroadcast, + options.isStartup + ); + } + + /** + * Fetch an image for a given highlight and update the card with it. If no + * image is available then fallback to fetching a screenshot. + */ + fetchImage(page, isStartup = false) { + // Request a screenshot if we don't already have one pending + const { preview_image_url: imageUrl, url } = page; + return lazy.Screenshots.maybeCacheScreenshot( + page, + imageUrl || url, + "image", + image => { + lazy.SectionsManager.updateSectionCard( + SECTION_ID, + url, + { image }, + true, + isStartup + ); + } + ); + } + + onAction(action) { + // Relay the downloads actions to DownloadsManager - it is a child of HighlightsFeed + this.downloadsManager.onAction(action); + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.SYSTEM_TICK: + case at.TOP_SITES_UPDATED: + this.fetchHighlights({ + broadcast: false, + isStartup: !!action.meta?.isStartup, + }); + break; + case at.PREF_CHANGED: + // Update existing pages when the user changes what should be shown + if (action.data.name.startsWith("section.highlights.include")) { + this.fetchHighlights({ broadcast: true }); + } + break; + case at.PLACES_HISTORY_CLEARED: + case at.PLACES_LINK_BLOCKED: + case at.DOWNLOAD_CHANGED: + case at.POCKET_LINK_DELETED_OR_ARCHIVED: + this.fetchHighlights({ broadcast: true }); + break; + case at.PLACES_LINKS_CHANGED: + case at.PLACES_SAVED_TO_POCKET: + this.linksCache.expire(); + this.fetchHighlights({ broadcast: false }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} diff --git a/browser/components/newtab/lib/LinksCache.sys.mjs b/browser/components/newtab/lib/LinksCache.sys.mjs new file mode 100644 index 0000000000..0dfb89e74e --- /dev/null +++ b/browser/components/newtab/lib/LinksCache.sys.mjs @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This should be slightly less than SYSTEM_TICK_INTERVAL as timer +// comparisons are too exact while the async/await functionality will make the +// last recorded time a little bit later. This causes the comparasion to skip +// updates. +// It should be 10% less than SYSTEM_TICK to update at least once every 5 mins. +// https://github.com/mozilla/activity-stream/pull/3695#discussion_r144678214 +const EXPIRATION_TIME = 4.5 * 60 * 1000; // 4.5 minutes + +/** + * Cache link results from a provided object property and refresh after some + * amount of time has passed. Allows for migrating data from previously cached + * links to the new links with the same url. + */ +export class LinksCache { + /** + * Create a links cache for a given object property. + * + * @param {object} linkObject Object containing the link property + * @param {string} linkProperty Name of property on object to access + * @param {array} properties Optional properties list to migrate to new links. + * @param {function} shouldRefresh Optional callback receiving the old and new + * options to refresh even when not expired. + */ + constructor( + linkObject, + linkProperty, + properties = [], + shouldRefresh = () => {} + ) { + this.clear(); + + // Allow getting links from both methods and array properties + this.linkGetter = options => { + const ret = linkObject[linkProperty]; + return typeof ret === "function" ? ret.call(linkObject, options) : ret; + }; + + // Always migrate the shared cache data in addition to any custom properties + this.migrateProperties = ["__sharedCache", ...properties]; + this.shouldRefresh = shouldRefresh; + } + + /** + * Clear the cached data. + */ + clear() { + this.cache = Promise.resolve([]); + this.lastOptions = {}; + this.expire(); + } + + /** + * Force the next request to update the cache. + */ + expire() { + delete this.lastUpdate; + } + + /** + * Request data and update the cache if necessary. + * + * @param {object} options Optional data to pass to the underlying method. + * @returns {promise(array)} Links array with objects that can be modified. + */ + async request(options = {}) { + // Update the cache if the data has been expired + const now = Date.now(); + if ( + this.lastUpdate === undefined || + now > this.lastUpdate + EXPIRATION_TIME || + // Allow custom rules around refreshing based on options + this.shouldRefresh(this.lastOptions, options) + ) { + // Update request state early so concurrent requests can refer to it + this.lastOptions = options; + this.lastUpdate = now; + + // Save a promise before awaits, so other requests wait for correct data + // eslint-disable-next-line no-async-promise-executor + this.cache = new Promise(async (resolve, reject) => { + try { + // Allow fast lookup of old links by url that might need to migrate + const toMigrate = new Map(); + for (const oldLink of await this.cache) { + if (oldLink) { + toMigrate.set(oldLink.url, oldLink); + } + } + + // Update the cache with migrated links without modifying source objects + resolve( + (await this.linkGetter(options)).map(link => { + // Keep original array hole positions + if (!link) { + return link; + } + + // Migrate data to the new link copy if we have an old link + const newLink = Object.assign({}, link); + const oldLink = toMigrate.get(newLink.url); + if (oldLink) { + for (const property of this.migrateProperties) { + const oldValue = oldLink[property]; + if (oldValue !== undefined) { + newLink[property] = oldValue; + } + } + } else { + // Share data among link copies and new links from future requests + newLink.__sharedCache = {}; + } + // Provide a helper to update the cached link + newLink.__sharedCache.updateLink = (property, value) => { + newLink[property] = value; + }; + + return newLink; + }) + ); + } catch (error) { + reject(error); + } + }); + } + + // Provide a shallow copy of the cached link objects for callers to modify + return (await this.cache).map(link => link && Object.assign({}, link)); + } +} diff --git a/browser/components/newtab/lib/NewTabInit.sys.mjs b/browser/components/newtab/lib/NewTabInit.sys.mjs new file mode 100644 index 0000000000..db30e009ec --- /dev/null +++ b/browser/components/newtab/lib/NewTabInit.sys.mjs @@ -0,0 +1,55 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +/** + * NewTabInit - A placeholder for now. This will send a copy of the state to all + * newly opened tabs. + */ +export class NewTabInit { + constructor() { + this._repliedEarlyTabs = new Map(); + } + + reply(target) { + // Skip this reply if we already replied to an early tab + if (this._repliedEarlyTabs.get(target)) { + return; + } + + const action = { + type: at.NEW_TAB_INITIAL_STATE, + data: this.store.getState(), + }; + this.store.dispatch(ac.AlsoToOneContent(action, target)); + + // Remember that this early tab has already gotten a rehydration response in + // case it thought we lost its initial REQUEST and asked again + if (this._repliedEarlyTabs.has(target)) { + this._repliedEarlyTabs.set(target, true); + } + } + + onAction(action) { + switch (action.type) { + case at.NEW_TAB_STATE_REQUEST: + this.reply(action.meta.fromTarget); + break; + case at.NEW_TAB_INIT: + // Initialize data for early tabs that might REQUEST twice + if (action.data.simulated) { + this._repliedEarlyTabs.set(action.data.portID, false); + } + break; + case at.NEW_TAB_UNLOAD: + // Clean up for any tab (no-op if not an early tab) + this._repliedEarlyTabs.delete(action.meta.fromTarget); + break; + } + } +} diff --git a/browser/components/newtab/lib/PersistentCache.sys.mjs b/browser/components/newtab/lib/PersistentCache.sys.mjs new file mode 100644 index 0000000000..1db9ca102e --- /dev/null +++ b/browser/components/newtab/lib/PersistentCache.sys.mjs @@ -0,0 +1,90 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * A file (disk) based persistent cache of a JSON serializable object. + */ +export class PersistentCache { + /** + * Create a cache object based on a name. + * + * @param {string} name Name of the cache. It will be used to create the filename. + * @param {boolean} preload (optional). Whether the cache should be preloaded from file. Defaults to false. + */ + constructor(name, preload = false) { + this.name = name; + this._filename = `activity-stream.${name}.json`; + if (preload) { + this._load(); + } + } + + /** + * Set a value to be cached with the specified key. + * + * @param {string} key The cache key. + * @param {object} value The data to be cached. + */ + async set(key, value) { + const data = await this._load(); + data[key] = value; + await this._persist(data); + } + + /** + * Get a value from the cache. + * + * @param {string} key (optional) The cache key. If not provided, we return the full cache. + * @returns {object} The cached data. + */ + async get(key) { + const data = await this._load(); + return key ? data[key] : data; + } + + /** + * Load the cache into memory if it isn't already. + */ + _load() { + return ( + this._cache || + // eslint-disable-next-line no-async-promise-executor + (this._cache = new Promise(async (resolve, reject) => { + let filepath; + try { + filepath = PathUtils.join(PathUtils.localProfileDir, this._filename); + } catch (error) { + reject(error); + return; + } + + let data = {}; + try { + data = await IOUtils.readJSON(filepath); + } catch (error) { + if ( + // isInstance() is not available in node unit test. It should be safe to use instanceof as it's directly from IOUtils. + // eslint-disable-next-line mozilla/use-isInstance + !(error instanceof DOMException) || + error.name !== "NotFoundError" + ) { + console.error(`Failed to parse ${this._filename}:`, error.message); + } + } + + resolve(data); + })) + ); + } + + /** + * Persist the cache to file. + */ + async _persist(data) { + const filepath = PathUtils.join(PathUtils.localProfileDir, this._filename); + await IOUtils.writeJSON(filepath, data, { + tmpPath: `${filepath}.tmp`, + }); + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.mjs b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.mjs new file mode 100644 index 0000000000..d5930e3147 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.mjs @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export class NaiveBayesTextTagger { + constructor(model, toksToTfIdfVector) { + this.model = model; + this.toksToTfIdfVector = toksToTfIdfVector; + } + + /** + * Determines if the tokenized text belongs to class according to binary naive Bayes + * classifier. Returns an object containing the class label ("label"), and + * the log probability ("logProb") that the text belongs to that class. If + * the positive class is more likely, then "label" is the positive class + * label. If the negative class is matched, then "label" is set to null. + */ + tagTokens(tokens) { + let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs); + + let bestLogProb = null; + let bestClassId = -1; + let bestClassLabel = null; + let logSumExp = 0.0; // will be P(x). Used to create a proper probability + for (let classId = 0; classId < this.model.classes.length; classId++) { + let classModel = this.model.classes[classId]; + let classLogProb = classModel.log_prior; + + // dot fv with the class model + for (let pair of Object.values(fv)) { + let [termId, tfidf] = pair; + classLogProb += tfidf * classModel.feature_log_probs[termId]; + } + + if (bestLogProb === null || classLogProb > bestLogProb) { + bestLogProb = classLogProb; + bestClassId = classId; + } + logSumExp += Math.exp(classLogProb); + } + + // now normalize the probability by dividing by P(x) + logSumExp = Math.log(logSumExp); + bestLogProb -= logSumExp; + if (bestClassId === this.model.positive_class_id) { + bestClassLabel = this.model.positive_class_label; + } else { + bestClassLabel = null; + } + + let confident = + bestClassId === this.model.positive_class_id && + bestLogProb > this.model.positive_class_threshold_log_prob; + return { + label: bestClassLabel, + logProb: bestLogProb, + confident, + }; + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.mjs b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.mjs new file mode 100644 index 0000000000..5c77152d8d --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.mjs @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export class NmfTextTagger { + constructor(model, toksToTfIdfVector) { + this.model = model; + this.toksToTfIdfVector = toksToTfIdfVector; + } + + /** + * A multiclass classifier that scores tokenized text against several classes through + * inference of a nonnegative matrix factorization of TF-IDF vectors and + * class labels. Returns a map of class labels as string keys to scores. + * (Higher is more confident.) All classes get scored, so it is up to + * consumer of this data determine what classes are most valuable. + */ + tagTokens(tokens) { + let fv = this.toksToTfIdfVector(tokens, this.model.vocab_idfs); + let fve = Object.values(fv); + + // normalize by the sum of the vector + let sum = 0.0; + for (let pair of fve) { + // eslint-disable-next-line prefer-destructuring + sum += pair[1]; + } + for (let i = 0; i < fve.length; i++) { + // eslint-disable-next-line prefer-destructuring + fve[i][1] /= sum; + } + + // dot the document with each topic vector so that we can transform it into + // the latent space + let toksInLatentSpace = []; + for (let topicVect of this.model.topic_word) { + let fvDotTwv = 0; + // dot fv with each topic word vector + for (let pair of fve) { + let [termId, tfidf] = pair; + fvDotTwv += tfidf * topicVect[termId]; + } + toksInLatentSpace.push(fvDotTwv); + } + + // now project toksInLatentSpace back into class space + let predictions = {}; + Object.keys(this.model.document_topic).forEach(topic => { + let score = 0; + for (let i = 0; i < toksInLatentSpace.length; i++) { + score += toksInLatentSpace[i] * this.model.document_topic[topic][i]; + } + predictions[topic] = score; + }); + + return predictions; + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.sys.mjs b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.sys.mjs new file mode 100644 index 0000000000..406a0fa200 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.sys.mjs @@ -0,0 +1,277 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BasePromiseWorker: "resource://gre/modules/PromiseWorker.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + Utils: "resource://services-settings/Utils.sys.mjs", +}); + +const RECIPE_NAME = "personality-provider-recipe"; +const MODELS_NAME = "personality-provider-models"; + +export class PersonalityProvider { + constructor(modelKeys) { + this.modelKeys = modelKeys; + this.onSync = this.onSync.bind(this); + this.setup(); + } + + setScores(scores) { + this.scores = scores || {}; + this.interestConfig = this.scores.interestConfig; + this.interestVector = this.scores.interestVector; + } + + get personalityProviderWorker() { + if (this._personalityProviderWorker) { + return this._personalityProviderWorker; + } + + this._personalityProviderWorker = new lazy.BasePromiseWorker( + "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.worker.mjs", + { type: "module" } + ); + + return this._personalityProviderWorker; + } + + get baseAttachmentsURL() { + // Returning a promise, so we can have an async getter. + return this._getBaseAttachmentsURL(); + } + + async _getBaseAttachmentsURL() { + if (this._baseAttachmentsURL) { + return this._baseAttachmentsURL; + } + const server = lazy.Utils.SERVER_URL; + const serverInfo = await ( + await fetch(`${server}/`, { + credentials: "omit", + }) + ).json(); + const { + capabilities: { + attachments: { base_url }, + }, + } = serverInfo; + this._baseAttachmentsURL = base_url; + return this._baseAttachmentsURL; + } + + setup() { + this.setupSyncAttachment(RECIPE_NAME); + this.setupSyncAttachment(MODELS_NAME); + } + + teardown() { + this.teardownSyncAttachment(RECIPE_NAME); + this.teardownSyncAttachment(MODELS_NAME); + if (this._personalityProviderWorker) { + this._personalityProviderWorker.terminate(); + } + } + + setupSyncAttachment(collection) { + lazy.RemoteSettings(collection).on("sync", this.onSync); + } + + teardownSyncAttachment(collection) { + lazy.RemoteSettings(collection).off("sync", this.onSync); + } + + onSync(event) { + this.personalityProviderWorker.post("onSync", [event]); + } + + /** + * Gets contents of the attachment if it already exists on file, + * and if not attempts to download it. + */ + getAttachment(record) { + return this.personalityProviderWorker.post("getAttachment", [record]); + } + + /** + * Returns a Recipe from remote settings to be consumed by a RecipeExecutor. + * A Recipe is a set of instructions on how to processes a RecipeExecutor. + */ + async getRecipe() { + if (!this.recipes || !this.recipes.length) { + const result = await lazy.RemoteSettings(RECIPE_NAME).get(); + this.recipes = await Promise.all( + result.map(async record => ({ + ...(await this.getAttachment(record)), + recordKey: record.key, + })) + ); + } + return this.recipes[0]; + } + + /** + * Grabs a slice of browse history for building a interest vector + */ + async fetchHistory(columns, beginTimeSecs, endTimeSecs) { + let sql = `SELECT url, title, visit_count, frecency, last_visit_date, description + FROM moz_places + WHERE last_visit_date >= ${beginTimeSecs * 1000000} + AND last_visit_date < ${endTimeSecs * 1000000}`; + columns.forEach(requiredColumn => { + sql += ` AND IFNULL(${requiredColumn}, '') <> ''`; + }); + sql += " LIMIT 30000"; + + const { activityStreamProvider } = lazy.NewTabUtils; + const history = await activityStreamProvider.executePlacesQuery(sql, { + columns, + params: {}, + }); + + return history; + } + + /** + * Handles setup and metrics of history fetch. + */ + async getHistory() { + let endTimeSecs = new Date().getTime() / 1000; + let beginTimeSecs = endTimeSecs - this.interestConfig.history_limit_secs; + if ( + !this.interestConfig || + !this.interestConfig.history_required_fields || + !this.interestConfig.history_required_fields.length + ) { + return []; + } + let history = await this.fetchHistory( + this.interestConfig.history_required_fields, + beginTimeSecs, + endTimeSecs + ); + + return history; + } + + async setBaseAttachmentsURL() { + await this.personalityProviderWorker.post("setBaseAttachmentsURL", [ + await this.baseAttachmentsURL, + ]); + } + + async setInterestConfig() { + this.interestConfig = this.interestConfig || (await this.getRecipe()); + await this.personalityProviderWorker.post("setInterestConfig", [ + this.interestConfig, + ]); + } + + async setInterestVector() { + await this.personalityProviderWorker.post("setInterestVector", [ + this.interestVector, + ]); + } + + async fetchModels() { + const models = await lazy.RemoteSettings(MODELS_NAME).get(); + return this.personalityProviderWorker.post("fetchModels", [models]); + } + + async generateTaggers() { + await this.personalityProviderWorker.post("generateTaggers", [ + this.modelKeys, + ]); + } + + async generateRecipeExecutor() { + await this.personalityProviderWorker.post("generateRecipeExecutor"); + } + + async createInterestVector() { + const history = await this.getHistory(); + + const interestVectorResult = await this.personalityProviderWorker.post( + "createInterestVector", + [history] + ); + + return interestVectorResult; + } + + async init(callback) { + await this.setBaseAttachmentsURL(); + await this.setInterestConfig(); + if (!this.interestConfig) { + return; + } + + // We always generate a recipe executor, no cache used here. + // This is because the result of this is an object with + // functions (taggers) so storing it in cache is not possible. + // Thus we cannot use it to rehydrate anything. + const fetchModelsResult = await this.fetchModels(); + // If this fails, log an error and return. + if (!fetchModelsResult.ok) { + return; + } + await this.generateTaggers(); + await this.generateRecipeExecutor(); + + // If we don't have a cached vector, create a new one. + if (!this.interestVector) { + const interestVectorResult = await this.createInterestVector(); + // If that failed, log an error and return. + if (!interestVectorResult.ok) { + return; + } + this.interestVector = interestVectorResult.interestVector; + } + + // This happens outside the createInterestVector call above, + // because create can be skipped if rehydrating from cache. + // In that case, the interest vector is provided and not created, so we just set it. + await this.setInterestVector(); + + this.initialized = true; + if (callback) { + callback(); + } + } + + async calculateItemRelevanceScore(pocketItem) { + if (!this.initialized) { + return pocketItem.item_score || 1; + } + const itemRelevanceScore = await this.personalityProviderWorker.post( + "calculateItemRelevanceScore", + [pocketItem] + ); + if (!itemRelevanceScore) { + return -1; + } + const { scorableItem, rankingVector } = itemRelevanceScore; + // Put the results on the item for debugging purposes. + pocketItem.scorableItem = scorableItem; + pocketItem.rankingVector = rankingVector; + return rankingVector.score; + } + + /** + * Returns an object holding the personalization scores of this provider instance. + */ + getScores() { + return { + // We cannot return taggers here. + // What we return here goes into persistent cache, and taggers have functions on it. + // If we attempted to save taggers into persistent cache, it would store it to disk, + // and the next time we load it, it would start thowing function is not defined. + interestConfig: this.interestConfig, + interestVector: this.interestVector, + }; + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.worker.mjs b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.worker.mjs new file mode 100644 index 0000000000..49797f4f2b --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.worker.mjs @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { PersonalityProviderWorker } from "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.mjs"; + +import { PromiseWorker } from "resource://gre/modules/workers/PromiseWorker.mjs"; + +const personalityProviderWorker = new PersonalityProviderWorker(); + +// This is boiler plate worker stuff that connects it to the main thread PromiseWorker. +const worker = new PromiseWorker.AbstractWorker(); +worker.dispatch = function (method, args = []) { + return personalityProviderWorker[method](...args); +}; +worker.postMessage = function (message, ...transfers) { + self.postMessage(message, ...transfers); +}; +worker.close = function () { + self.close(); +}; + +self.addEventListener("message", msg => worker.handleMessage(msg)); +self.addEventListener("unhandledrejection", function (error) { + throw error.reason; +}); diff --git a/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.mjs b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.mjs new file mode 100644 index 0000000000..372c061dd9 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.mjs @@ -0,0 +1,306 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + tokenize, + toksToTfIdfVector, +} from "resource://activity-stream/lib/PersonalityProvider/Tokenize.mjs"; +import { NaiveBayesTextTagger } from "resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.mjs"; +import { NmfTextTagger } from "resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.mjs"; +import { RecipeExecutor } from "resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.mjs"; + +// A helper function to create a hash out of a file. +async function _getFileHash(filepath) { + const data = await IOUtils.read(filepath); + // File is an instance of Uint8Array + const digest = await crypto.subtle.digest("SHA-256", data); + const uint8 = new Uint8Array(digest); + // return the two-digit hexadecimal code for a byte + const toHex = b => b.toString(16).padStart(2, "0"); + return Array.from(uint8, toHex).join(""); +} + +/** + * V2 provider builds and ranks an interest profile (also called an “interest vector”) off the browse history. + * This allows Firefox to classify pages into topics, by examining the text found on the page. + * It does this by looking at the history text content, title, and description. + */ +export class PersonalityProviderWorker { + async getPersonalityProviderDir() { + const personalityProviderDir = PathUtils.join( + await PathUtils.getLocalProfileDir(), + "personality-provider" + ); + + // Cache this so we don't need to await again. + this.getPersonalityProviderDir = () => + Promise.resolve(personalityProviderDir); + return personalityProviderDir; + } + + setBaseAttachmentsURL(url) { + this.baseAttachmentsURL = url; + } + + setInterestConfig(interestConfig) { + this.interestConfig = interestConfig; + } + + setInterestVector(interestVector) { + this.interestVector = interestVector; + } + + onSync(event) { + const { + data: { created, updated, deleted }, + } = event; + // Remove every removed attachment. + const toRemove = deleted.concat(updated.map(u => u.old)); + toRemove.forEach(record => this.deleteAttachment(record)); + + // Download every new/updated attachment. + const toDownload = created.concat(updated.map(u => u.new)); + // maybeDownloadAttachment is async but we don't care inside onSync. + toDownload.forEach(record => this.maybeDownloadAttachment(record)); + } + + /** + * Attempts to download the attachment, but only if it doesn't already exist. + */ + async maybeDownloadAttachment(record, retries = 3) { + const { + attachment: { filename, hash, size }, + } = record; + await IOUtils.makeDirectory(await this.getPersonalityProviderDir()); + const localFilePath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + let retry = 0; + while ( + retry++ < retries && + // exists is an issue for perf because I might not need to call it. + (!(await IOUtils.exists(localFilePath)) || + (await IOUtils.stat(localFilePath)).size !== size || + (await _getFileHash(localFilePath)) !== hash) + ) { + await this._downloadAttachment(record); + } + } + + /** + * Downloads the attachment to disk assuming the dir already exists + * and any existing files matching the filename are clobbered. + */ + async _downloadAttachment(record) { + const { + attachment: { location: loc, filename }, + } = record; + const remoteFilePath = this.baseAttachmentsURL + loc; + const localFilePath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + const xhr = new XMLHttpRequest(); + // Set false here for a synchronous request, because we're in a worker. + xhr.open("GET", remoteFilePath, false); + xhr.setRequestHeader("Accept-Encoding", "gzip"); + xhr.responseType = "arraybuffer"; + xhr.withCredentials = false; + xhr.send(null); + + if (xhr.status !== 200) { + console.error(`Failed to fetch ${remoteFilePath}: ${xhr.statusText}`); + return; + } + + const buffer = xhr.response; + const bytes = new Uint8Array(buffer); + + await IOUtils.write(localFilePath, bytes, { + tmpPath: `${localFilePath}.tmp`, + }); + } + + async deleteAttachment(record) { + const { + attachment: { filename }, + } = record; + await IOUtils.makeDirectory(await this.getPersonalityProviderDir()); + const path = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + await IOUtils.remove(path, { ignoreAbsent: true }); + // Cleanup the directory if it is empty, do nothing if it is not empty. + try { + await IOUtils.remove(await this.getPersonalityProviderDir(), { + ignoreAbsent: true, + }); + } catch (e) { + // This is likely because the directory is not empty, so we don't care. + } + } + + /** + * Gets contents of the attachment if it already exists on file, + * and if not attempts to download it. + */ + async getAttachment(record) { + const { + attachment: { filename }, + } = record; + const filepath = PathUtils.join( + await this.getPersonalityProviderDir(), + filename + ); + + try { + await this.maybeDownloadAttachment(record); + return await IOUtils.readJSON(filepath); + } catch (error) { + console.error(`Failed to load ${filepath}: ${error.message}`); + } + return {}; + } + + async fetchModels(models) { + this.models = await Promise.all( + models.map(async record => ({ + ...(await this.getAttachment(record)), + recordKey: record.key, + })) + ); + if (!this.models.length) { + return { + ok: false, + }; + } + return { + ok: true, + }; + } + + generateTaggers(modelKeys) { + if (!this.taggers) { + let nbTaggers = []; + let nmfTaggers = {}; + + for (let model of this.models) { + if (!modelKeys.includes(model.recordKey)) { + continue; + } + if (model.model_type === "nb") { + nbTaggers.push(new NaiveBayesTextTagger(model, toksToTfIdfVector)); + } else if (model.model_type === "nmf") { + nmfTaggers[model.parent_tag] = new NmfTextTagger( + model, + toksToTfIdfVector + ); + } + } + this.taggers = { nbTaggers, nmfTaggers }; + } + } + + /** + * Sets and generates a Recipe Executor. + * A Recipe Executor is a set of actions that can be consumed by a Recipe. + * The Recipe determines the order and specifics of which the actions are called. + */ + generateRecipeExecutor() { + const recipeExecutor = new RecipeExecutor( + this.taggers.nbTaggers, + this.taggers.nmfTaggers, + tokenize + ); + this.recipeExecutor = recipeExecutor; + } + + /** + * Examines the user's browse history and returns an interest vector that + * describes the topics the user frequently browses. + */ + createInterestVector(historyObj) { + let interestVector = {}; + + for (let historyRec of historyObj) { + let ivItem = this.recipeExecutor.executeRecipe( + historyRec, + this.interestConfig.history_item_builder + ); + if (ivItem === null) { + continue; + } + interestVector = this.recipeExecutor.executeCombinerRecipe( + interestVector, + ivItem, + this.interestConfig.interest_combiner + ); + if (interestVector === null) { + return null; + } + } + + const finalResult = this.recipeExecutor.executeRecipe( + interestVector, + this.interestConfig.interest_finalizer + ); + + return { + ok: true, + interestVector: finalResult, + }; + } + + /** + * Calculates a score of a Pocket item when compared to the user's interest + * vector. Returns the score. Higher scores are better. Assumes this.interestVector + * is populated. + */ + calculateItemRelevanceScore(pocketItem) { + const { personalization_models } = pocketItem; + let scorableItem; + + // If the server provides some models, we can just use them, + // and skip generating them. + if (personalization_models && Object.keys(personalization_models).length) { + scorableItem = { + id: pocketItem.id, + item_tags: personalization_models, + item_score: pocketItem.item_score, + item_sort_id: 1, + }; + } else { + scorableItem = this.recipeExecutor.executeRecipe( + pocketItem, + this.interestConfig.item_to_rank_builder + ); + if (scorableItem === null) { + return null; + } + } + + // We're doing a deep copy on an object. + let rankingVector = JSON.parse(JSON.stringify(this.interestVector)); + + Object.keys(scorableItem).forEach(key => { + rankingVector[key] = scorableItem[key]; + }); + + rankingVector = this.recipeExecutor.executeRecipe( + rankingVector, + this.interestConfig.item_ranker + ); + + if (rankingVector === null) { + return null; + } + + return { scorableItem, rankingVector }; + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.mjs b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.mjs new file mode 100644 index 0000000000..4f420c0812 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.mjs @@ -0,0 +1,1119 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * RecipeExecutor is the core feature engineering pipeline for the in-browser + * personalization work. These pipelines are called "recipes". A recipe is an + * array of objects that define a "step" in the recipe. A step is simply an + * object with a field "function" that specifies what is being done in the step + * along with other fields that are semantically defined for that step. + * + * There are two types of recipes "builder" recipes and "combiner" recipes. Builder + * recipes mutate an object until it matches some set of critera. Combiner + * recipes take two objects, (a "left" and a "right"), and specify the steps + * to merge the right object into the left object. + * + * A short nonsense example recipe is: + * [ {"function": "get_url_domain", "path_length": 1, "field": "url", "dest": "url_domain"}, + * {"function": "nb_tag", "fields": ["title", "description"]}, + * {"function": "conditionally_nmf_tag", "fields": ["title", "description"]} ] + * + * Recipes are sandboxed by the fact that the step functions must be explicitly + * allowed. Functions allowed for builder recipes are specifed in the + * RecipeExecutor.ITEM_BUILDER_REGISTRY, while combiner functions are allowed + * in RecipeExecutor.ITEM_COMBINER_REGISTRY . + */ +export class RecipeExecutor { + constructor(nbTaggers, nmfTaggers, tokenize) { + this.ITEM_BUILDER_REGISTRY = { + nb_tag: this.naiveBayesTag, + conditionally_nmf_tag: this.conditionallyNmfTag, + accept_item_by_field_value: this.acceptItemByFieldValue, + tokenize_url: this.tokenizeUrl, + get_url_domain: this.getUrlDomain, + tokenize_field: this.tokenizeField, + copy_value: this.copyValue, + keep_top_k: this.keepTopK, + scalar_multiply: this.scalarMultiply, + elementwise_multiply: this.elementwiseMultiply, + vector_multiply: this.vectorMultiply, + scalar_add: this.scalarAdd, + vector_add: this.vectorAdd, + make_boolean: this.makeBoolean, + allow_fields: this.allowFields, + filter_by_value: this.filterByValue, + l2_normalize: this.l2Normalize, + prob_normalize: this.probNormalize, + set_default: this.setDefault, + lookup_value: this.lookupValue, + copy_to_map: this.copyToMap, + scalar_multiply_tag: this.scalarMultiplyTag, + apply_softmax_tags: this.applySoftmaxTags, + }; + this.ITEM_COMBINER_REGISTRY = { + combiner_add: this.combinerAdd, + combiner_max: this.combinerMax, + combiner_collect_values: this.combinerCollectValues, + }; + this.nbTaggers = nbTaggers; + this.nmfTaggers = nmfTaggers; + this.tokenize = tokenize; + } + + /** + * Determines the type of a field. Valid types are: + * string + * number + * array + * map (strings to anything) + */ + _typeOf(data) { + let t = typeof data; + if (t === "object") { + if (data === null) { + return "null"; + } + if (Array.isArray(data)) { + return "array"; + } + return "map"; + } + return t; + } + + /** + * Returns a scalar, either because it was a constant, or by + * looking it up from the item. Allows for a default value if the lookup + * fails. + */ + _lookupScalar(item, k, dfault) { + if (this._typeOf(k) === "number") { + return k; + } else if ( + this._typeOf(k) === "string" && + k in item && + this._typeOf(item[k]) === "number" + ) { + return item[k]; + } + return dfault; + } + + /** + * Simply appends all the strings from a set fields together. If the field + * is a list, then the cells of the list are append. + */ + _assembleText(item, fields) { + let textArr = []; + for (let field of fields) { + if (field in item) { + let type = this._typeOf(item[field]); + if (type === "string") { + textArr.push(item[field]); + } else if (type === "array") { + for (let ele of item[field]) { + textArr.push(String(ele)); + } + } else { + textArr.push(String(item[field])); + } + } + } + return textArr.join(" "); + } + + /** + * Runs the naive bayes text taggers over a set of text fields. Stores the + * results in new fields: + * nb_tags: a map of text strings to probabilites + * nb_tokens: the tokenized text that was tagged + * + * Config: + * fields: an array containing a list of fields to concatenate and tag + */ + naiveBayesTag(item, config) { + let text = this._assembleText(item, config.fields); + let tokens = this.tokenize(text); + let tags = {}; + let extended_tags = {}; + + for (let nbTagger of this.nbTaggers) { + let result = nbTagger.tagTokens(tokens); + if (result.label !== null && result.confident) { + extended_tags[result.label] = result; + tags[result.label] = Math.exp(result.logProb); + } + } + item.nb_tags = tags; + item.nb_tags_extended = extended_tags; + item.nb_tokens = tokens; + return item; + } + + /** + * Selectively runs NMF text taggers depending on which tags were found + * by the naive bayes taggers. Writes the results in into new fields: + * nmf_tags_parent_weights: map of pareent tags to probabilites of those parent tags + * nmf_tags: map of strings to maps of strings to probabilities + * nmf_tags_parent map of child tags to parent tags + * + * Config: + * Not configurable + */ + conditionallyNmfTag(item, config) { + let nestedNmfTags = {}; + let parentTags = {}; + let parentWeights = {}; + + if (!("nb_tags" in item) || !("nb_tokens" in item)) { + return null; + } + + Object.keys(item.nb_tags).forEach(parentTag => { + let nmfTagger = this.nmfTaggers[parentTag]; + if (nmfTagger !== undefined) { + nestedNmfTags[parentTag] = {}; + parentWeights[parentTag] = item.nb_tags[parentTag]; + let nmfTags = nmfTagger.tagTokens(item.nb_tokens); + Object.keys(nmfTags).forEach(nmfTag => { + nestedNmfTags[parentTag][nmfTag] = nmfTags[nmfTag]; + parentTags[nmfTag] = parentTag; + }); + } + }); + + item.nmf_tags = nestedNmfTags; + item.nmf_tags_parent = parentTags; + item.nmf_tags_parent_weights = parentWeights; + + return item; + } + + /** + * Checks a field's value against another value (either from another field + * or a constant). If the test passes, then the item is emitted, otherwise + * the pipeline is aborted. + * + * Config: + * field Field to read the value to test. Left side of operator. + * op one of ==, !=, <, <=, >, >= + * rhsValue Constant value to compare against. Right side of operator. + * rhsField Field to read value to compare against. Right side of operator. + * + * NOTE: rhsValue takes precidence over rhsField. + */ + acceptItemByFieldValue(item, config) { + if (!(config.field in item)) { + return null; + } + let rhs = null; + if ("rhsValue" in config) { + rhs = config.rhsValue; + } else if ("rhsField" in config && config.rhsField in item) { + rhs = item[config.rhsField]; + } + if (rhs === null) { + return null; + } + + if ( + // eslint-disable-next-line eqeqeq + (config.op === "==" && item[config.field] == rhs) || + // eslint-disable-next-line eqeqeq + (config.op === "!=" && item[config.field] != rhs) || + (config.op === "<" && item[config.field] < rhs) || + (config.op === "<=" && item[config.field] <= rhs) || + (config.op === ">" && item[config.field] > rhs) || + (config.op === ">=" && item[config.field] >= rhs) + ) { + return item; + } + + return null; + } + + /** + * Splits a URL into text-like tokens. + * + * Config: + * field Field containing a URL + * dest Field to write the tokens to as an array of strings + * + * NOTE: Any initial 'www' on the hostname is removed. + */ + tokenizeUrl(item, config) { + if (!(config.field in item)) { + return null; + } + + let url = new URL(item[config.field]); + let domain = url.hostname; + if (domain.startsWith("www.")) { + domain = domain.substring(4); + } + let toks = this.tokenize(domain); + let pathToks = this.tokenize( + decodeURIComponent(url.pathname.replace(/\+/g, " ")) + ); + for (let tok of pathToks) { + toks.push(tok); + } + for (let pair of url.searchParams.entries()) { + let k = this.tokenize(decodeURIComponent(pair[0].replace(/\+/g, " "))); + for (let tok of k) { + toks.push(tok); + } + if (pair[1] !== null && pair[1] !== "") { + let v = this.tokenize(decodeURIComponent(pair[1].replace(/\+/g, " "))); + for (let tok of v) { + toks.push(tok); + } + } + } + item[config.dest] = toks; + + return item; + } + + /** + * Gets the hostname (minus any initial "www." along with the left most + * directories on the path. + * + * Config: + * field Field containing the URL + * dest Field to write the array of strings to + * path_length OPTIONAL (DEFAULT: 0) Number of leftmost subdirectories to include + */ + getUrlDomain(item, config) { + if (!(config.field in item)) { + return null; + } + + let url = new URL(item[config.field]); + let domain = url.hostname.toLocaleLowerCase(); + if (domain.startsWith("www.")) { + domain = domain.substring(4); + } + item[config.dest] = domain; + let pathLength = 0; + if ("path_length" in config) { + pathLength = config.path_length; + } + if (pathLength > 0) { + item[config.dest] += url.pathname + .toLocaleLowerCase() + .split("/") + .slice(0, pathLength + 1) + .join("/"); + } + + return item; + } + + /** + * Splits a field into tokens. + * Config: + * field Field containing a string to tokenize + * dest Field to write the array of strings to + */ + tokenizeField(item, config) { + if (!(config.field in item)) { + return null; + } + + item[config.dest] = this.tokenize(item[config.field]); + + return item; + } + + /** + * Deep copy from one field to another. + * Config: + * src Field to read from + * dest Field to write to + */ + copyValue(item, config) { + if (!(config.src in item)) { + return null; + } + + item[config.dest] = JSON.parse(JSON.stringify(item[config.src])); + + return item; + } + + /** + * Converts a field containing a map of strings to a map of strings + * to numbers, to a map of strings to numbers containing at most k elements. + * This operation is performed by first, promoting all the subkeys up one + * level, and then taking the top (or bottom) k values. + * + * Config: + * field Points to a map of strings to a map of strings to numbers + * k Maximum number of items to keep + * descending OPTIONAL (DEFAULT: True) Sorts score in descending order + * (i.e. keeps maximum) + */ + keepTopK(item, config) { + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, 1048576); + let descending = !("descending" in config) || config.descending !== false; + + // we can't sort by the values in the map, so we have to convert this + // to an array, and then sort. + let sortable = []; + Object.keys(item[config.field]).forEach(outerKey => { + let innerType = this._typeOf(item[config.field][outerKey]); + if (innerType === "map") { + Object.keys(item[config.field][outerKey]).forEach(innerKey => { + sortable.push({ + key: innerKey, + value: item[config.field][outerKey][innerKey], + }); + }); + } else { + sortable.push({ key: outerKey, value: item[config.field][outerKey] }); + } + }); + + sortable.sort((a, b) => { + if (descending) { + return b.value - a.value; + } + return a.value - b.value; + }); + + // now take the top k + let newMap = {}; + let i = 0; + for (let pair of sortable) { + if (i >= k) { + break; + } + newMap[pair.key] = pair.value; + i++; + } + item[config.field] = newMap; + + return item; + } + + /** + * Scalar multiplies a vector by some constant + * + * Config: + * field Points to: + * a map of strings to numbers + * an array of numbers + * a number + * k Either a number, or a string. If it's a number then This + * is the scalar value to multiply by. If it's a string, + * the value in the pointed to field is used. + * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric + * value is found, then use this value. + */ + scalarMultiply(item, config) { + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, config.dfault); + + let fieldType = this._typeOf(item[config.field]); + if (fieldType === "number") { + item[config.field] *= k; + } else if (fieldType === "array") { + for (let i = 0; i < item[config.field].length; i++) { + item[config.field][i] *= k; + } + } else if (fieldType === "map") { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] *= k; + }); + } else { + return null; + } + + return item; + } + + /** + * Elementwise multiplies either two maps or two arrays together, storing + * the result in left. If left and right are of the same type, results in an + * error. + * + * Maps are special case. For maps the left must be a nested map such as: + * { k1: { k11: 1, k12: 2}, k2: { k21: 3, k22: 4 } } and right needs to be + * simple map such as: { k1: 5, k2: 6} . The operation is then to mulitply + * every value of every right key, to every value every subkey where the + * parent keys match. Using the previous examples, the result would be: + * { k1: { k11: 5, k12: 10 }, k2: { k21: 18, k22: 24 } } . + * + * Config: + * left + * right + */ + elementwiseMultiply(item, config) { + if (!(config.left in item) || !(config.right in item)) { + return null; + } + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + item[config.left][i] *= item[config.right][i]; + } + } else if (leftType === "map") { + Object.keys(item[config.left]).forEach(outerKey => { + let r = 0.0; + if (outerKey in item[config.right]) { + r = item[config.right][outerKey]; + } + Object.keys(item[config.left][outerKey]).forEach(innerKey => { + item[config.left][outerKey][innerKey] *= r; + }); + }); + } else if (leftType === "number") { + item[config.left] *= item[config.right]; + } else { + return null; + } + + return item; + } + + /** + * Vector multiplies (i.e. dot products) two vectors and stores the result in + * third field. Both vectors must either by maps, or arrays of numbers with + * the same length. + * + * Config: + * left A field pointing to either a map of strings to numbers, + * or an array of numbers + * right A field pointing to either a map of strings to numbers, + * or an array of numbers + * dest The field to store the dot product. + */ + vectorMultiply(item, config) { + if (!(config.left in item) || !(config.right in item)) { + return null; + } + + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + + let destVal = 0.0; + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + destVal += item[config.left][i] * item[config.right][i]; + } + } else if (leftType === "map") { + Object.keys(item[config.left]).forEach(key => { + if (key in item[config.right]) { + destVal += item[config.left][key] * item[config.right][key]; + } + }); + } else { + return null; + } + + item[config.dest] = destVal; + return item; + } + + /** + * Adds a constant value to all elements in the field. Mathematically, + * this is the same as taking a 1-vector, scalar multiplying it by k, + * and then vector adding it to a field. + * + * Config: + * field A field pointing to either a map of strings to numbers, + * or an array of numbers + * k Either a number, or a string. If it's a number then This + * is the scalar value to multiply by. If it's a string, + * the value in the pointed to field is used. + * default OPTIONAL (DEFAULT: 0), If k is a string, and no numeric + * value is found, then use this value. + */ + scalarAdd(item, config) { + let k = this._lookupScalar(item, config.k, config.dfault); + if (!(config.field in item)) { + return null; + } + + let fieldType = this._typeOf(item[config.field]); + if (fieldType === "array") { + for (let i = 0; i < item[config.field].length; i++) { + item[config.field][i] += k; + } + } else if (fieldType === "map") { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] += k; + }); + } else if (fieldType === "number") { + item[config.field] += k; + } else { + return null; + } + + return item; + } + + /** + * Adds two vectors together and stores the result in left. + * + * Config: + * left A field pointing to either a map of strings to numbers, + * or an array of numbers + * right A field pointing to either a map of strings to numbers, + * or an array of numbers + */ + vectorAdd(item, config) { + if (!(config.left in item)) { + return this.copyValue(item, { src: config.right, dest: config.left }); + } + if (!(config.right in item)) { + return null; + } + + let leftType = this._typeOf(item[config.left]); + if (leftType !== this._typeOf(item[config.right])) { + return null; + } + if (leftType === "array") { + if (item[config.left].length !== item[config.right].length) { + return null; + } + for (let i = 0; i < item[config.left].length; i++) { + item[config.left][i] += item[config.right][i]; + } + return item; + } else if (leftType === "map") { + Object.keys(item[config.right]).forEach(key => { + let v = 0; + if (key in item[config.left]) { + v = item[config.left][key]; + } + item[config.left][key] = v + item[config.right][key]; + }); + return item; + } + + return null; + } + + /** + * Converts a vector from real values to boolean integers. (i.e. either 1/0 + * or 1/-1). + * + * Config: + * field Field containing either a map of strings to numbers or + * an array of numbers to convert. + * threshold OPTIONAL (DEFAULT: 0) Values above this will be replaced + * with 1.0. Those below will be converted to 0. + * keep_negative OPTIONAL (DEFAULT: False) If true, values below the + * threshold will be converted to -1 instead of 0. + */ + makeBoolean(item, config) { + if (!(config.field in item)) { + return null; + } + let threshold = this._lookupScalar(item, config.threshold, 0.0); + let type = this._typeOf(item[config.field]); + if (type === "array") { + for (let i = 0; i < item[config.field].length; i++) { + if (item[config.field][i] > threshold) { + item[config.field][i] = 1.0; + } else if (config.keep_negative) { + item[config.field][i] = -1.0; + } else { + item[config.field][i] = 0.0; + } + } + } else if (type === "map") { + Object.keys(item[config.field]).forEach(key => { + let value = item[config.field][key]; + if (value > threshold) { + item[config.field][key] = 1.0; + } else if (config.keep_negative) { + item[config.field][key] = -1.0; + } else { + item[config.field][key] = 0.0; + } + }); + } else if (type === "number") { + let value = item[config.field]; + if (value > threshold) { + item[config.field] = 1.0; + } else if (config.keep_negative) { + item[config.field] = -1.0; + } else { + item[config.field] = 0.0; + } + } else { + return null; + } + + return item; + } + + /** + * Removes all keys from the item except for the ones specified. + * + * fields An array of strings indicating the fields to keep + */ + allowFields(item, config) { + let newItem = {}; + for (let ele of config.fields) { + if (ele in item) { + newItem[ele] = item[ele]; + } + } + return newItem; + } + + /** + * Removes all keys whose value does not exceed some threshold. + * + * Config: + * field Points to a map of strings to numbers + * threshold Values must exceed this value, otherwise they are removed. + */ + filterByValue(item, config) { + if (!(config.field in item)) { + return null; + } + let threshold = this._lookupScalar(item, config.threshold, 0.0); + let filtered = {}; + Object.keys(item[config.field]).forEach(key => { + let value = item[config.field][key]; + if (value > threshold) { + filtered[key] = value; + } + }); + item[config.field] = filtered; + + return item; + } + + /** + * Rewrites a field so that its values are now L2 normed. + * + * Config: + * field Points to a map of strings to numbers, or an array of numbers + */ + l2Normalize(item, config) { + if (!(config.field in item)) { + return null; + } + let data = item[config.field]; + let type = this._typeOf(data); + if (type === "array") { + let norm = 0.0; + for (let datum of data) { + norm += datum * datum; + } + norm = Math.sqrt(norm); + if (norm !== 0) { + for (let i = 0; i < data.length; i++) { + data[i] /= norm; + } + } + } else if (type === "map") { + let norm = 0.0; + Object.keys(data).forEach(key => { + norm += data[key] * data[key]; + }); + norm = Math.sqrt(norm); + if (norm !== 0) { + Object.keys(data).forEach(key => { + data[key] /= norm; + }); + } + } else { + return null; + } + + item[config.field] = data; + + return item; + } + + /** + * Rewrites a field so that all of its values sum to 1.0 + * + * Config: + * field Points to a map of strings to numbers, or an array of numbers + */ + probNormalize(item, config) { + if (!(config.field in item)) { + return null; + } + let data = item[config.field]; + let type = this._typeOf(data); + if (type === "array") { + let norm = 0.0; + for (let datum of data) { + norm += datum; + } + if (norm !== 0) { + for (let i = 0; i < data.length; i++) { + data[i] /= norm; + } + } + } else if (type === "map") { + let norm = 0.0; + Object.keys(item[config.field]).forEach(key => { + norm += item[config.field][key]; + }); + if (norm !== 0) { + Object.keys(item[config.field]).forEach(key => { + item[config.field][key] /= norm; + }); + } + } else { + return null; + } + + return item; + } + + /** + * Stores a value, if it is not already present + * + * Config: + * field field to write to if it is missing + * value value to store in that field + */ + setDefault(item, config) { + let val = this._lookupScalar(item, config.value, config.value); + if (!(config.field in item)) { + item[config.field] = val; + } + + return item; + } + + /** + * Selctively promotes an value from an inner map up to the outer map + * + * Config: + * haystack Points to a map of strings to values + * needle Key inside the map we should promote up + * dest Where we should write the value of haystack[needle] + */ + lookupValue(item, config) { + if (config.haystack in item && config.needle in item[config.haystack]) { + item[config.dest] = item[config.haystack][config.needle]; + } + + return item; + } + + /** + * Demotes a field into a map + * + * Config: + * src Field to copy + * dest_map Points to a map + * dest_key Key inside dest_map to copy src to + */ + copyToMap(item, config) { + if (config.src in item) { + if (!(config.dest_map in item)) { + item[config.dest_map] = {}; + } + item[config.dest_map][config.dest_key] = item[config.src]; + } + + return item; + } + + /** + * Config: + * field Points to a string to number map + * k Scalar to multiply the values by + * log_scale Boolean, if true, then the values will be transformed + * by a logrithm prior to multiplications + */ + scalarMultiplyTag(item, config) { + let EPSILON = 0.000001; + if (!(config.field in item)) { + return null; + } + let k = this._lookupScalar(item, config.k, 1); + let type = this._typeOf(item[config.field]); + if (type === "map") { + Object.keys(item[config.field]).forEach(parentKey => { + Object.keys(item[config.field][parentKey]).forEach(key => { + let v = item[config.field][parentKey][key]; + if (config.log_scale) { + v = Math.log(v + EPSILON); + } + item[config.field][parentKey][key] = v * k; + }); + }); + } else { + return null; + } + + return item; + } + + /** + * Independently applies softmax across all subtags. + * + * Config: + * field Points to a map of strings with values being another map of strings + */ + applySoftmaxTags(item, config) { + let type = this._typeOf(item[config.field]); + if (type !== "map") { + return null; + } + + let abort = false; + let softmaxSum = {}; + Object.keys(item[config.field]).forEach(tag => { + if (this._typeOf(item[config.field][tag]) !== "map") { + abort = true; + return; + } + if (abort) { + return; + } + softmaxSum[tag] = 0; + Object.keys(item[config.field][tag]).forEach(subtag => { + if (this._typeOf(item[config.field][tag][subtag]) !== "number") { + abort = true; + return; + } + let score = item[config.field][tag][subtag]; + softmaxSum[tag] += Math.exp(score); + }); + }); + if (abort) { + return null; + } + + Object.keys(item[config.field]).forEach(tag => { + Object.keys(item[config.field][tag]).forEach(subtag => { + item[config.field][tag][subtag] = + Math.exp(item[config.field][tag][subtag]) / softmaxSum[tag]; + }); + }); + + return item; + } + + /** + * Vector adds a field and stores the result in left. + * + * Config: + * field The field to vector add + */ + combinerAdd(left, right, config) { + if (!(config.field in right)) { + return left; + } + let type = this._typeOf(right[config.field]); + if (!(config.field in left)) { + if (type === "map") { + left[config.field] = {}; + } else if (type === "array") { + left[config.field] = []; + } else if (type === "number") { + left[config.field] = 0; + } else { + return null; + } + } + if (type !== this._typeOf(left[config.field])) { + return null; + } + if (type === "map") { + Object.keys(right[config.field]).forEach(key => { + if (!(key in left[config.field])) { + left[config.field][key] = 0; + } + left[config.field][key] += right[config.field][key]; + }); + } else if (type === "array") { + for (let i = 0; i < right[config.field].length; i++) { + if (i < left[config.field].length) { + left[config.field][i] += right[config.field][i]; + } else { + left[config.field].push(right[config.field][i]); + } + } + } else if (type === "number") { + left[config.field] += right[config.field]; + } else { + return null; + } + + return left; + } + + /** + * Stores the maximum value of the field in left. + * + * Config: + * field The field to vector add + */ + combinerMax(left, right, config) { + if (!(config.field in right)) { + return left; + } + let type = this._typeOf(right[config.field]); + if (!(config.field in left)) { + if (type === "map") { + left[config.field] = {}; + } else if (type === "array") { + left[config.field] = []; + } else if (type === "number") { + left[config.field] = 0; + } else { + return null; + } + } + if (type !== this._typeOf(left[config.field])) { + return null; + } + if (type === "map") { + Object.keys(right[config.field]).forEach(key => { + if ( + !(key in left[config.field]) || + right[config.field][key] > left[config.field][key] + ) { + left[config.field][key] = right[config.field][key]; + } + }); + } else if (type === "array") { + for (let i = 0; i < right[config.field].length; i++) { + if (i < left[config.field].length) { + if (left[config.field][i] < right[config.field][i]) { + left[config.field][i] = right[config.field][i]; + } + } else { + left[config.field].push(right[config.field][i]); + } + } + } else if (type === "number") { + if (left[config.field] < right[config.field]) { + left[config.field] = right[config.field]; + } + } else { + return null; + } + + return left; + } + + /** + * Associates a value in right with another value in right. This association + * is then stored in a map in left. + * + * For example: If a sequence of rights is: + * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 41 } + * { 'tags': {}, 'url_domain': 'mbusa.com/mercedes', 'time': 21 } + * { 'tags': {}, 'url_domain': 'maseratiusa.com/maserati', 'time': 34 } + * + * Then assuming a 'sum' operation, left can build a map that would look like: + * { + * 'maseratiusa.com/maserati': 75, + * 'mbusa.com/mercedes': 21, + * } + * + * Fields: + * left_field field in the left to store / update the map + * right_key_field Field in the right to use as a key + * right_value_field Field in the right to use as a value + * operation One of "sum", "max", "overwrite", "count" + */ + combinerCollectValues(left, right, config) { + let op; + if (config.operation === "sum") { + op = (a, b) => a + b; + } else if (config.operation === "max") { + op = (a, b) => (a > b ? a : b); + } else if (config.operation === "overwrite") { + op = (a, b) => b; + } else if (config.operation === "count") { + op = (a, b) => a + 1; + } else { + return null; + } + if (!(config.left_field in left)) { + left[config.left_field] = {}; + } + if ( + !(config.right_key_field in right) || + !(config.right_value_field in right) + ) { + return left; + } + + let key = right[config.right_key_field]; + let rightValue = right[config.right_value_field]; + let leftValue = 0.0; + if (key in left[config.left_field]) { + leftValue = left[config.left_field][key]; + } + + left[config.left_field][key] = op(leftValue, rightValue); + + return left; + } + + /** + * Executes a recipe. Returns an object on success, or null on failure. + */ + executeRecipe(item, recipe) { + let newItem = item; + if (recipe) { + for (let step of recipe) { + let op = this.ITEM_BUILDER_REGISTRY[step.function]; + if (op === undefined) { + return null; + } + newItem = op.call(this, newItem, step); + if (newItem === null) { + break; + } + } + } + return newItem; + } + + /** + * Executes a recipe. Returns an object on success, or null on failure. + */ + executeCombinerRecipe(item1, item2, recipe) { + let newItem1 = item1; + for (let step of recipe) { + let op = this.ITEM_COMBINER_REGISTRY[step.function]; + if (op === undefined) { + return null; + } + newItem1 = op.call(this, newItem1, item2, step); + if (newItem1 === null) { + break; + } + } + + return newItem1; + } +} diff --git a/browser/components/newtab/lib/PersonalityProvider/Tokenize.mjs b/browser/components/newtab/lib/PersonalityProvider/Tokenize.mjs new file mode 100644 index 0000000000..740b2fc541 --- /dev/null +++ b/browser/components/newtab/lib/PersonalityProvider/Tokenize.mjs @@ -0,0 +1,83 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Unicode specifies certain mnemonics for code pages and character classes. +// They call them "character properties" https://en.wikipedia.org/wiki/Unicode_character_property . +// These mnemonics are have been adopted by many regular expression libraries, +// however the standard Javascript regexp system doesn't support unicode +// character properties, so we have to define these ourself. +// +// Each of these sections contains the characters values / ranges for specific +// character property: Whitespace, Symbol (S), Punctuation (P), Number (N), +// Mark (M), and Letter (L). +const UNICODE_SPACE = + "\x20\xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000"; +const UNICODE_SYMBOL = + "\\x24\\x2B\x3C-\x3E\\x5E\x60\\x7C\x7E\xA2-\xA6\xA8\xA9\xAC\xAE-\xB1\xB4\xB8\xD7\xF7\u02C2-\u02C5\u02D2-\u02DF\u02E5-\u02EB\u02ED\u02EF-\u02FF\u0375\u0384\u0385\u03F6\u0482\u058D-\u058F\u0606-\u0608\u060B\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09F2\u09F3\u09FA\u09FB\u0AF1\u0B70\u0BF3-\u0BFA\u0C7F\u0D4F\u0D79\u0E3F\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399\u17DB\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u1FBD\u1FBF-\u1FC1\u1FCD-\u1FCF\u1FDD-\u1FDF\u1FED-\u1FEF\u1FFD\u1FFE\u2044\u2052\u207A-\u207C\u208A-\u208C\u20A0-\u20BE\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116-\u2118\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u2140-\u2144\u214A-\u214D\u214F\u218A\u218B\u2190-\u2307\u230C-\u2328\u232B-\u23FE\u2400-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u2767\u2794-\u27C4\u27C7-\u27E5\u27F0-\u2982\u2999-\u29D7\u29DC-\u29FB\u29FE-\u2B73\u2B76-\u2B95\u2B98-\u2BB9\u2BBD-\u2BC8\u2BCA-\u2BD1\u2BEC-\u2BEF\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u309B\u309C\u3190\u3191\u3196-\u319F\u31C0-\u31E3\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF\uA490-\uA4C6\uA700-\uA716\uA720\uA721\uA789\uA78A\uA828-\uA82B\uA836-\uA839\uAA77-\uAA79\uAB5B\uFB29\uFBB2-\uFBC1\uFDFC\uFDFD\uFE62\uFE64-\uFE66\uFE69\uFF04\uFF0B\uFF1C-\uFF1E\uFF3E\uFF40\uFF5C\uFF5E\uFFE0-\uFFE6\uFFE8-\uFFEE\uFFFC\uFFFD"; +const UNICODE_PUNCT = + "\x21-\x23\x25-\\x2A\x2C-\x2F\x3A\x3B\\x3F\x40\\x5B-\\x5D\x5F\\x7B\x7D\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u0AF0\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166D\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E44\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65"; + +const UNICODE_NUMBER = + "0-9\xB2\xB3\xB9\xBC-\xBE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D58-\u0D5E\u0D66-\u0D78\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19"; +const UNICODE_MARK = + "\u0300-\u036F\u0483-\u0489\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED\u0711\u0730-\u074A\u07A6-\u07B0\u07EB-\u07F3\u0816-\u0819\u081B-\u0823\u0825-\u0827\u0829-\u082D\u0859-\u085B\u08D4-\u08E1\u08E3-\u0903\u093A-\u093C\u093E-\u094F\u0951-\u0957\u0962\u0963\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7\u09C8\u09CB-\u09CD\u09D7\u09E2\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A70\u0A71\u0A75\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AE2\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B56\u0B57\u0B62\u0B63\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7\u0C00-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C62\u0C63\u0C81-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CE2\u0CE3\u0D01-\u0D03\u0D3E-\u0D44\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D62\u0D63\u0D82\u0D83\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2\u0DF3\u0E31\u0E34-\u0E3A\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB\u0EBC\u0EC8-\u0ECD\u0F18\u0F19\u0F35\u0F37\u0F39\u0F3E\u0F3F\u0F71-\u0F84\u0F86\u0F87\u0F8D-\u0F97\u0F99-\u0FBC\u0FC6\u102B-\u103E\u1056-\u1059\u105E-\u1060\u1062-\u1064\u1067-\u106D\u1071-\u1074\u1082-\u108D\u108F\u109A-\u109D\u135D-\u135F\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17B4-\u17D3\u17DD\u180B-\u180D\u1885\u1886\u18A9\u1920-\u192B\u1930-\u193B\u1A17-\u1A1B\u1A55-\u1A5E\u1A60-\u1A7C\u1A7F\u1AB0-\u1ABE\u1B00-\u1B04\u1B34-\u1B44\u1B6B-\u1B73\u1B80-\u1B82\u1BA1-\u1BAD\u1BE6-\u1BF3\u1C24-\u1C37\u1CD0-\u1CD2\u1CD4-\u1CE8\u1CED\u1CF2-\u1CF4\u1CF8\u1CF9\u1DC0-\u1DF5\u1DFB-\u1DFF\u20D0-\u20F0\u2CEF-\u2CF1\u2D7F\u2DE0-\u2DFF\u302A-\u302F\u3099\u309A\uA66F-\uA672\uA674-\uA67D\uA69E\uA69F\uA6F0\uA6F1\uA802\uA806\uA80B\uA823-\uA827\uA880\uA881\uA8B4-\uA8C5\uA8E0-\uA8F1\uA926-\uA92D\uA947-\uA953\uA980-\uA983\uA9B3-\uA9C0\uA9E5\uAA29-\uAA36\uAA43\uAA4C\uAA4D\uAA7B-\uAA7D\uAAB0\uAAB2-\uAAB4\uAAB7\uAAB8\uAABE\uAABF\uAAC1\uAAEB-\uAAEF\uAAF5\uAAF6\uABE3-\uABEA\uABEC\uABED\uFB1E\uFE00-\uFE0F\uFE20-\uFE2F"; +const UNICODE_LETTER = + "A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u08B6-\u08BD\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C80-\u1C88\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AE\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC"; + +const REGEXP_SPLITS = new RegExp( + `[${UNICODE_SPACE}${UNICODE_SYMBOL}${UNICODE_PUNCT}]+` +); +// Match all token characters, so okay for regex to split multiple code points +// eslint-disable-next-line no-misleading-character-class +const REGEXP_ALPHANUMS = new RegExp( + `^[${UNICODE_NUMBER}${UNICODE_MARK}${UNICODE_LETTER}]+$` +); + +/** + * Downcases the text, and splits it into consecutive alphanumeric characters. + * This is locale aware, and so will not strip accents. This uses "word + * breaks", and os is not appropriate for languages without them + * (e.g. Chinese). + */ +export function tokenize(text) { + return text + .toLocaleLowerCase() + .split(REGEXP_SPLITS) + .filter(tok => tok.match(REGEXP_ALPHANUMS)); +} + +/** + * Converts a sequence of tokens into an L2 normed TF-IDF. Any terms that are + * not preindexed (i.e. does have a computed inverse document frequency) will + * be dropped. + */ +export function toksToTfIdfVector(tokens, vocab_idfs) { + let tfidfs = {}; + + // calcualte the term frequencies + for (let tok of tokens) { + if (!(tok in vocab_idfs)) { + continue; + } + if (!(tok in tfidfs)) { + tfidfs[tok] = [vocab_idfs[tok][0], 1]; + } else { + tfidfs[tok][1]++; + } + } + + // now multiply by the log inverse document frequencies, then take + // the L2 norm of this. + let l2Norm = 0.0; + Object.keys(tfidfs).forEach(tok => { + tfidfs[tok][1] *= vocab_idfs[tok][1]; + l2Norm += tfidfs[tok][1] * tfidfs[tok][1]; + }); + l2Norm = Math.sqrt(l2Norm); + Object.keys(tfidfs).forEach(tok => { + tfidfs[tok][1] /= l2Norm; + }); + + return tfidfs; +} diff --git a/browser/components/newtab/lib/PlacesFeed.sys.mjs b/browser/components/newtab/lib/PlacesFeed.sys.mjs new file mode 100644 index 0000000000..70011412f8 --- /dev/null +++ b/browser/components/newtab/lib/PlacesFeed.sys.mjs @@ -0,0 +1,572 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionCreators as ac, + actionTypes as at, + actionUtils as au, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +import { shortURL } from "resource://activity-stream/lib/ShortURL.sys.mjs"; + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// AboutNewTab, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { AboutNewTab } = ChromeUtils.importESModule( + "resource:///modules/AboutNewTab.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PartnerLinkAttribution: "resource:///modules/PartnerLinkAttribution.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", +}); + +const LINK_BLOCKED_EVENT = "newtab-linkBlocked"; +const PLACES_LINKS_CHANGED_DELAY_TIME = 1000; // time in ms to delay timer for places links changed events + +// The pref to store the blocked sponsors of the sponsored Top Sites. +// The value of this pref is an array (JSON serialized) of hostnames of the +// blocked sponsors. +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; + +/** + * PlacesObserver - observes events from PlacesUtils.observers + */ +class PlacesObserver { + constructor(dispatch) { + this.dispatch = dispatch; + this.QueryInterface = ChromeUtils.generateQI(["nsISupportsWeakReference"]); + this.handlePlacesEvent = this.handlePlacesEvent.bind(this); + } + + handlePlacesEvent(events) { + const removedPages = []; + const removedBookmarks = []; + + for (const { + itemType, + source, + dateAdded, + guid, + title, + url, + isRemovedFromStore, + isTagging, + type, + } of events) { + switch (type) { + case "history-cleared": + this.dispatch({ type: at.PLACES_HISTORY_CLEARED }); + break; + case "page-removed": + if (isRemovedFromStore) { + removedPages.push(url); + } + break; + case "bookmark-added": + // Skips items that are not bookmarks (like folders), about:* pages or + // default bookmarks, added when the profile is created. + if ( + isTagging || + itemType !== lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK || + source === lazy.PlacesUtils.bookmarks.SOURCES.IMPORT || + source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE || + source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP || + source === lazy.PlacesUtils.bookmarks.SOURCES.SYNC || + (!url.startsWith("http://") && !url.startsWith("https://")) + ) { + return; + } + + this.dispatch({ type: at.PLACES_LINKS_CHANGED }); + this.dispatch({ + type: at.PLACES_BOOKMARK_ADDED, + data: { + bookmarkGuid: guid, + bookmarkTitle: title, + dateAdded: dateAdded * 1000, + url, + }, + }); + break; + case "bookmark-removed": + if ( + isTagging || + (itemType === lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK && + source !== lazy.PlacesUtils.bookmarks.SOURCES.IMPORT && + source !== lazy.PlacesUtils.bookmarks.SOURCES.RESTORE && + source !== + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP && + source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC) + ) { + removedBookmarks.push(url); + } + break; + } + } + + if (removedPages.length || removedBookmarks.length) { + this.dispatch({ type: at.PLACES_LINKS_CHANGED }); + } + + if (removedPages.length) { + this.dispatch({ + type: at.PLACES_LINKS_DELETED, + data: { urls: removedPages }, + }); + } + + if (removedBookmarks.length) { + this.dispatch({ + type: at.PLACES_BOOKMARKS_REMOVED, + data: { urls: removedBookmarks }, + }); + } + } +} + +export class PlacesFeed { + constructor() { + this.placesChangedTimer = null; + this.customDispatch = this.customDispatch.bind(this); + this.placesObserver = new PlacesObserver(this.customDispatch); + } + + addObservers() { + lazy.PlacesUtils.observers.addListener( + ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"], + this.placesObserver.handlePlacesEvent + ); + + Services.obs.addObserver(this, LINK_BLOCKED_EVENT); + } + + /** + * setTimeout - A custom function that creates an nsITimer that can be cancelled + * + * @param {func} callback A function to be executed after the timer expires + * @param {int} delay The time (in ms) the timer should wait before the function is executed + */ + setTimeout(callback, delay) { + let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + timer.initWithCallback(callback, delay, Ci.nsITimer.TYPE_ONE_SHOT); + return timer; + } + + customDispatch(action) { + // If we are changing many links at once, delay this action and only dispatch + // one action at the end + if (action.type === at.PLACES_LINKS_CHANGED) { + if (this.placesChangedTimer) { + this.placesChangedTimer.delay = PLACES_LINKS_CHANGED_DELAY_TIME; + } else { + this.placesChangedTimer = this.setTimeout(() => { + this.placesChangedTimer = null; + this.store.dispatch(ac.OnlyToMain(action)); + }, PLACES_LINKS_CHANGED_DELAY_TIME); + } + } else { + // To avoid blocking Places notifications on expensive work, run it at the + // next tick of the events loop. + Services.tm.dispatchToMainThread(() => + this.store.dispatch(ac.BroadcastToContent(action)) + ); + } + } + + removeObservers() { + if (this.placesChangedTimer) { + this.placesChangedTimer.cancel(); + this.placesChangedTimer = null; + } + lazy.PlacesUtils.observers.removeListener( + ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"], + this.placesObserver.handlePlacesEvent + ); + Services.obs.removeObserver(this, LINK_BLOCKED_EVENT); + } + + /** + * observe - An observer for the LINK_BLOCKED_EVENT. + * Called when a link is blocked. + * Links can be blocked outside of newtab, + * which is why we need to listen to this + * on such a generic level. + * + * @param {null} subject + * @param {str} topic The name of the event + * @param {str} value The data associated with the event + */ + observe(subject, topic, value) { + if (topic === LINK_BLOCKED_EVENT) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PLACES_LINK_BLOCKED, + data: { url: value }, + }) + ); + } + } + + /** + * Open a link in a desired destination defaulting to action's event. + */ + openLink(action, where = "", isPrivate = false) { + const params = { + private: isPrivate, + targetBrowser: action._target.browser, + forceForeground: false, // This ensure we maintain user preference for how to open new tabs. + globalHistoryOptions: { + triggeringSponsoredURL: action.data.sponsored_tile_id + ? action.data.url + : undefined, + }, + }; + + // Always include the referrer (even for http links) if we have one + const { event, referrer, typedBonus } = action.data; + if (referrer) { + const ReferrerInfo = Components.Constructor( + "@mozilla.org/referrer-info;1", + "nsIReferrerInfo", + "init" + ); + params.referrerInfo = new ReferrerInfo( + Ci.nsIReferrerInfo.UNSAFE_URL, + true, + Services.io.newURI(referrer) + ); + } + + // Pocket gives us a special reader URL to open their stories in + const urlToOpen = + action.data.type === "pocket" ? action.data.open_url : action.data.url; + + try { + let uri = Services.io.newURI(urlToOpen); + if (!["http", "https"].includes(uri.scheme)) { + throw new Error( + `Can't open link using ${uri.scheme} protocol from the new tab page.` + ); + } + } catch (e) { + console.error(e); + return; + } + + // Mark the page as typed for frecency bonus before opening the link + if (typedBonus) { + lazy.PlacesUtils.history.markPageAsTyped(Services.io.newURI(urlToOpen)); + } + + const win = action._target.browser.ownerGlobal; + win.openTrustedLinkIn( + urlToOpen, + where || win.whereToOpenLink(event), + params + ); + + // If there's an original URL e.g. using the unprocessed %YYYYMMDDHH% tag, + // add a visit for that so it may become a frecent top site. + if (action.data.original_url) { + lazy.PlacesUtils.history.insert({ + url: action.data.original_url, + visits: [{ transition: lazy.PlacesUtils.history.TRANSITION_TYPED }], + }); + } + } + + async saveToPocket(site, browser) { + const sendToPocket = + lazy.NimbusFeatures.pocketNewtab.getVariable("sendToPocket"); + // An experiment to send the user directly to Pocket's signup page. + if (sendToPocket && !lazy.pktApi.isUserLoggedIn()) { + const pocketNewtabExperiment = lazy.ExperimentAPI.getExperiment({ + featureId: "pocketNewtab", + }); + const pocketSiteHost = Services.prefs.getStringPref( + "extensions.pocket.site" + ); // getpocket.com + let utmSource = "firefox_newtab_save_button"; + // We want to know if the user is in a Pocket newtab related experiment. + let utmCampaign = pocketNewtabExperiment?.slug; + let utmContent = pocketNewtabExperiment?.branch?.slug; + + const url = new URL(`https://${pocketSiteHost}/signup`); + url.searchParams.append("utm_source", utmSource); + if (utmCampaign && utmContent) { + url.searchParams.append("utm_campaign", utmCampaign); + url.searchParams.append("utm_content", utmContent); + } + + const win = browser.ownerGlobal; + win.openTrustedLinkIn(url.href, "tab"); + return; + } + + const { url, title } = site; + try { + let data = await lazy.NewTabUtils.activityStreamLinks.addPocketEntry( + url, + title, + browser + ); + if (data) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PLACES_SAVED_TO_POCKET, + data: { + url, + open_url: data.item.open_url, + title, + pocket_id: data.item.item_id, + }, + }) + ); + } + } catch (err) { + console.error(err); + } + } + + /** + * Deletes an item from a user's saved to Pocket feed + * @param {int} itemID + * The unique ID given by Pocket for that item; used to look the item up when deleting + */ + async deleteFromPocket(itemID) { + try { + await lazy.NewTabUtils.activityStreamLinks.deletePocketEntry(itemID); + this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED }); + } catch (err) { + console.error(err); + } + } + + /** + * Archives an item from a user's saved to Pocket feed + * @param {int} itemID + * The unique ID given by Pocket for that item; used to look the item up when archiving + */ + async archiveFromPocket(itemID) { + try { + await lazy.NewTabUtils.activityStreamLinks.archivePocketEntry(itemID); + this.store.dispatch({ type: at.POCKET_LINK_DELETED_OR_ARCHIVED }); + } catch (err) { + console.error(err); + } + } + + /** + * Sends an attribution request for Top Sites interactions. + * @param {object} data + * Attribution paramters from a Top Site. + */ + makeAttributionRequest(data) { + let args = Object.assign( + { + campaignID: Services.prefs.getStringPref( + "browser.partnerlink.campaign.topsites" + ), + }, + data + ); + lazy.PartnerLinkAttribution.makeRequest(args); + } + + async fillSearchTopSiteTerm({ _target, data }) { + const searchEngine = await Services.search.getEngineByAlias(data.label); + _target.browser.ownerGlobal.gURLBar.search(data.label, { + searchEngine, + searchModeEntry: "topsites_newtab", + }); + } + + _getDefaultSearchEngine(isPrivateWindow) { + return Services.search[ + isPrivateWindow ? "defaultPrivateEngine" : "defaultEngine" + ]; + } + + handoffSearchToAwesomebar(action) { + const { _target, data, meta } = action; + const searchEngine = this._getDefaultSearchEngine( + lazy.PrivateBrowsingUtils.isBrowserPrivate(_target.browser) + ); + const urlBar = _target.browser.ownerGlobal.gURLBar; + let isFirstChange = true; + + const newtabSession = AboutNewTab.activityStream.store.feeds + .get("feeds.telemetry") + ?.sessions.get(au.getPortIdOfSender(action)); + if (!data || !data.text) { + urlBar.setHiddenFocus(); + } else { + urlBar.handoff(data.text, searchEngine, newtabSession?.session_id); + isFirstChange = false; + } + + const checkFirstChange = () => { + // Check if this is the first change since we hidden focused. If it is, + // remove hidden focus styles, prepend the search alias and hide the + // in-content search. + if (isFirstChange) { + isFirstChange = false; + urlBar.removeHiddenFocus(true); + urlBar.handoff("", searchEngine, newtabSession?.session_id); + this.store.dispatch( + ac.OnlyToOneContent({ type: at.DISABLE_SEARCH }, meta.fromTarget) + ); + urlBar.removeEventListener("compositionstart", checkFirstChange); + urlBar.removeEventListener("paste", checkFirstChange); + } + }; + + const onKeydown = ev => { + // Check if the keydown will cause a value change. + if (ev.key.length === 1 && !ev.altKey && !ev.ctrlKey && !ev.metaKey) { + checkFirstChange(); + } + // If the Esc button is pressed, we are done. Show in-content search and cleanup. + if (ev.key === "Escape") { + onDone(); // eslint-disable-line no-use-before-define + } + }; + + const onDone = ev => { + // We are done. Show in-content search again and cleanup. + this.store.dispatch( + ac.OnlyToOneContent({ type: at.SHOW_SEARCH }, meta.fromTarget) + ); + + const forceSuppressFocusBorder = ev?.type === "mousedown"; + urlBar.removeHiddenFocus(forceSuppressFocusBorder); + + urlBar.removeEventListener("keydown", onKeydown); + urlBar.removeEventListener("mousedown", onDone); + urlBar.removeEventListener("blur", onDone); + urlBar.removeEventListener("compositionstart", checkFirstChange); + urlBar.removeEventListener("paste", checkFirstChange); + }; + + urlBar.addEventListener("keydown", onKeydown); + urlBar.addEventListener("mousedown", onDone); + urlBar.addEventListener("blur", onDone); + urlBar.addEventListener("compositionstart", checkFirstChange); + urlBar.addEventListener("paste", checkFirstChange); + } + + /** + * Add the hostnames of the given urls to the Top Sites sponsor blocklist. + * + * @param {array} urls + * An array of the objects structured as `{ url }` + */ + addToBlockedTopSitesSponsors(urls) { + const blockedPref = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + const merged = new Set([...blockedPref, ...urls.map(url => shortURL(url))]); + + Services.prefs.setStringPref( + TOP_SITES_BLOCKED_SPONSORS_PREF, + JSON.stringify([...merged]) + ); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + // Briefly avoid loading services for observing for better startup timing + Services.tm.dispatchToMainThread(() => this.addObservers()); + break; + case at.UNINIT: + this.removeObservers(); + break; + case at.ABOUT_SPONSORED_TOP_SITES: { + const url = `${Services.urlFormatter.formatURLPref( + "app.support.baseURL" + )}sponsor-privacy`; + const win = action._target.browser.ownerGlobal; + win.openTrustedLinkIn(url, "tab"); + break; + } + case at.BLOCK_URL: { + if (action.data) { + let sponsoredTopSites = []; + action.data.forEach(site => { + const { url, pocket_id, isSponsoredTopSite } = site; + lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id }); + if (isSponsoredTopSite) { + sponsoredTopSites.push({ url }); + } + }); + if (sponsoredTopSites.length) { + this.addToBlockedTopSitesSponsors(sponsoredTopSites); + } + } + break; + } + case at.BOOKMARK_URL: + lazy.NewTabUtils.activityStreamLinks.addBookmark( + action.data, + action._target.browser.ownerGlobal + ); + break; + case at.DELETE_BOOKMARK_BY_ID: + lazy.NewTabUtils.activityStreamLinks.deleteBookmark(action.data); + break; + case at.DELETE_HISTORY_URL: { + const { url, forceBlock, pocket_id } = action.data; + lazy.NewTabUtils.activityStreamLinks.deleteHistoryEntry(url); + if (forceBlock) { + lazy.NewTabUtils.activityStreamLinks.blockURL({ url, pocket_id }); + } + break; + } + case at.OPEN_NEW_WINDOW: + this.openLink(action, "window"); + break; + case at.OPEN_PRIVATE_WINDOW: + this.openLink(action, "window", true); + break; + case at.SAVE_TO_POCKET: + this.saveToPocket(action.data.site, action._target.browser); + break; + case at.DELETE_FROM_POCKET: + this.deleteFromPocket(action.data.pocket_id); + break; + case at.ARCHIVE_FROM_POCKET: + this.archiveFromPocket(action.data.pocket_id); + break; + case at.FILL_SEARCH_TERM: + this.fillSearchTopSiteTerm(action); + break; + case at.HANDOFF_SEARCH_TO_AWESOMEBAR: + this.handoffSearchToAwesomebar(action); + break; + case at.OPEN_LINK: { + this.openLink(action); + break; + } + case at.PARTNER_LINK_ATTRIBUTION: + this.makeAttributionRequest(action.data); + break; + } + } +} + +// Exported for testing only +PlacesFeed.PlacesObserver = PlacesObserver; diff --git a/browser/components/newtab/lib/PrefsFeed.sys.mjs b/browser/components/newtab/lib/PrefsFeed.sys.mjs new file mode 100644 index 0000000000..1c6f9b0d45 --- /dev/null +++ b/browser/components/newtab/lib/PrefsFeed.sys.mjs @@ -0,0 +1,273 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.sys.mjs"; +import { Prefs } from "resource://activity-stream/lib/ActivityStreamPrefs.sys.mjs"; + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// AppConstants, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", +}); + +export class PrefsFeed { + constructor(prefMap) { + this._prefMap = prefMap; + this._prefs = new Prefs(); + this.onExperimentUpdated = this.onExperimentUpdated.bind(this); + this.onPocketExperimentUpdated = this.onPocketExperimentUpdated.bind(this); + } + + onPrefChanged(name, value) { + const prefItem = this._prefMap.get(name); + if (prefItem) { + let action = "BroadcastToContent"; + if (prefItem.skipBroadcast) { + action = "OnlyToMain"; + if (prefItem.alsoToPreloaded) { + action = "AlsoToPreloaded"; + } + } + + this.store.dispatch( + ac[action]({ + type: at.PREF_CHANGED, + data: { name, value }, + }) + ); + } + } + + _setStringPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getStringPref); + } + + _setBoolPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getBoolPref); + } + + _setIntPref(values, key, defaultValue) { + this._setPref(values, key, defaultValue, Services.prefs.getIntPref); + } + + _setPref(values, key, defaultValue, getPrefFunction) { + let value = getPrefFunction( + `browser.newtabpage.activity-stream.${key}`, + defaultValue + ); + values[key] = value; + this._prefMap.set(key, { value }); + } + + /** + * Handler for when experiment data updates. + */ + onExperimentUpdated(event, reason) { + const value = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { + name: "featureConfig", + value, + }, + }) + ); + } + + /** + * Handler for Pocket specific experiment data updates. + */ + onPocketExperimentUpdated(event, reason) { + const value = lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}; + // Loaded experiments are set up inside init() + if ( + reason !== "feature-experiment-loaded" && + reason !== "feature-rollout-loaded" + ) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { + name: "pocketConfig", + value, + }, + }) + ); + } + } + + init() { + this._prefs.observeBranch(this); + lazy.NimbusFeatures.newtab.onUpdate(this.onExperimentUpdated); + lazy.NimbusFeatures.pocketNewtab.onUpdate(this.onPocketExperimentUpdated); + + this._storage = this.store.dbStorage.getDbTable("sectionPrefs"); + + // Get the initial value of each activity stream pref + const values = {}; + for (const name of this._prefMap.keys()) { + values[name] = this._prefs.get(name); + } + + // These are not prefs, but are needed to determine stuff in content that can only be + // computed in main process + values.isPrivateBrowsingEnabled = lazy.PrivateBrowsingUtils.enabled; + values.platform = AppConstants.platform; + + // Save the geo pref if we have it + if (lazy.Region.home) { + values.region = lazy.Region.home; + this.geo = values.region; + } else if (this.geo !== "") { + // Watch for geo changes and use a dummy value for now + Services.obs.addObserver(this, lazy.Region.REGION_TOPIC); + this.geo = ""; + } + + // Get the firefox accounts url for links and to send firstrun metrics to. + values.fxa_endpoint = Services.prefs.getStringPref( + "browser.newtabpage.activity-stream.fxaccounts.endpoint", + "https://accounts.firefox.com" + ); + + // Get the firefox update channel with values as default, nightly, beta or release + values.appUpdateChannel = Services.prefs.getStringPref( + "app.update.channel", + "" + ); + + // Read the pref for search shortcuts top sites experiment from firefox.js and store it + // in our internal list of prefs to watch + let searchTopSiteExperimentPrefValue = Services.prefs.getBoolPref( + "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts" + ); + values["improvesearch.topSiteSearchShortcuts"] = + searchTopSiteExperimentPrefValue; + this._prefMap.set("improvesearch.topSiteSearchShortcuts", { + value: searchTopSiteExperimentPrefValue, + }); + + values.mayHaveSponsoredTopSites = Services.prefs.getBoolPref( + "browser.topsites.useRemoteSetting" + ); + + // Read the pref for search hand-off from firefox.js and store it + // in our internal list of prefs to watch + let handoffToAwesomebarPrefValue = Services.prefs.getBoolPref( + "browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar" + ); + values["improvesearch.handoffToAwesomebar"] = handoffToAwesomebarPrefValue; + this._prefMap.set("improvesearch.handoffToAwesomebar", { + value: handoffToAwesomebarPrefValue, + }); + + // Add experiment values and default values + values.featureConfig = lazy.NimbusFeatures.newtab.getAllVariables() || {}; + values.pocketConfig = + lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}; + this._setBoolPref(values, "logowordmark.alwaysVisible", false); + this._setBoolPref(values, "feeds.section.topstories", false); + this._setBoolPref(values, "discoverystream.enabled", false); + this._setBoolPref( + values, + "discoverystream.sponsored-collections.enabled", + false + ); + this._setBoolPref(values, "discoverystream.isCollectionDismissible", false); + this._setBoolPref(values, "discoverystream.hardcoded-basic-layout", false); + this._setBoolPref(values, "discoverystream.personalization.enabled", false); + this._setBoolPref(values, "discoverystream.personalization.override"); + this._setStringPref( + values, + "discoverystream.personalization.modelKeys", + "" + ); + this._setStringPref(values, "discoverystream.spocs-endpoint", ""); + this._setStringPref(values, "discoverystream.spocs-endpoint-query", ""); + this._setStringPref(values, "newNewtabExperience.colors", ""); + + // Set the initial state of all prefs in redux + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREFS_INITIAL_VALUES, + data: values, + meta: { + isStartup: true, + }, + }) + ); + } + + uninit() { + this.removeListeners(); + } + + removeListeners() { + this._prefs.ignoreBranch(this); + lazy.NimbusFeatures.newtab.offUpdate(this.onExperimentUpdated); + lazy.NimbusFeatures.pocketNewtab.offUpdate(this.onPocketExperimentUpdated); + if (this.geo === "") { + Services.obs.removeObserver(this, lazy.Region.REGION_TOPIC); + } + } + + async _setIndexedDBPref(id, value) { + const name = id === "topsites" ? id : `feeds.section.${id}`; + try { + await this._storage.set(name, value); + } catch (e) { + console.error("Could not set section preferences."); + } + } + + observe(subject, topic, data) { + switch (topic) { + case lazy.Region.REGION_TOPIC: + this.store.dispatch( + ac.BroadcastToContent({ + type: at.PREF_CHANGED, + data: { name: "region", value: lazy.Region.home }, + }) + ); + break; + } + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.uninit(); + break; + case at.CLEAR_PREF: + Services.prefs.clearUserPref(this._prefs._branchStr + action.data.name); + break; + case at.SET_PREF: + this._prefs.set(action.data.name, action.data.value); + break; + case at.UPDATE_SECTION_PREFS: + this._setIndexedDBPref(action.data.id, action.data.value); + break; + } + } +} diff --git a/browser/components/newtab/lib/RecommendationProvider.sys.mjs b/browser/components/newtab/lib/RecommendationProvider.sys.mjs new file mode 100644 index 0000000000..03e976544f --- /dev/null +++ b/browser/components/newtab/lib/RecommendationProvider.sys.mjs @@ -0,0 +1,291 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + PersistentCache: "resource://activity-stream/lib/PersistentCache.sys.mjs", + PersonalityProvider: + "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.sys.mjs", +}); + +import { + actionTypes as at, + actionCreators as ac, +} from "resource://activity-stream/common/Actions.sys.mjs"; + +const CACHE_KEY = "personalization"; +const PREF_PERSONALIZATION_MODEL_KEYS = + "discoverystream.personalization.modelKeys"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories"; +const PREF_PERSONALIZATION = "discoverystream.personalization.enabled"; +const MIN_PERSONALIZATION_UPDATE_TIME = 12 * 60 * 60 * 1000; // 12 hours +const PREF_PERSONALIZATION_OVERRIDE = + "discoverystream.personalization.override"; + +// The main purpose of this class is to handle interactions with the recommendation provider. +// A recommendation provider scores a list of stories, currently this is a personality provider. +// So all calls to the provider, anything involved with the setup of the provider, +// accessing prefs for the provider, or updaing devtools with provider state, is contained in here. +export class RecommendationProvider { + constructor() { + // Persistent cache for remote endpoint data. + this.cache = new lazy.PersistentCache(CACHE_KEY, true); + } + + async setProvider(isStartup = false, scores) { + // A provider is already set. This can happen when new stories come in + // and we need to update their scores. + // We can use the existing one, a fresh one is created after startup. + // Using the existing one might be a bit out of date, + // but it's fine for now. We can rely on restarts for updates. + // See bug 1629931 for improvements to this. + if (!this.provider) { + this.provider = new lazy.PersonalityProvider(this.modelKeys); + this.provider.setScores(scores); + } + + if (this.provider && this.provider.init) { + await this.provider.init(); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_INIT, + meta: { + isStartup, + }, + }) + ); + } + } + + async enable(isStartup) { + await this.loadPersonalizationScoresCache(isStartup); + Services.obs.addObserver(this, "idle-daily"); + this.loaded = true; + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_UPDATED, + meta: { + isStartup, + }, + }) + ); + } + + get showStories() { + // Combine user-set stories opt-out with Mozilla-set config + return ( + this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] && + this.store.getState().Prefs.values[PREF_USER_TOPSTORIES] + ); + } + + get personalized() { + // If stories are not displayed, no point in trying to personalize them. + if (!this.showStories) { + return false; + } + const spocsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized; + const recsPersonalized = + this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized; + const personalization = + this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + + // There is a server sent flag to keep personalization on. + // If the server stops sending this, we turn personalization off, + // until the server starts returning the signal. + const overrideState = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_OVERRIDE]; + + return ( + personalization && + !overrideState && + (spocsPersonalized || recsPersonalized) + ); + } + + get modelKeys() { + if (!this._modelKeys) { + this._modelKeys = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_MODEL_KEYS]; + } + + return this._modelKeys; + } + + /* + * This creates a new recommendationProvider using fresh data, + * It's run on a last updated timer. This is the opposite of loadPersonalizationScoresCache. + * This is also much slower so we only trigger this in the background on idle-daily. + * It causes new profiles to pick up personalization slowly because the first time + * a new profile is run you don't have any old cache to use, so it needs to wait for the first + * idle-daily. Older profiles can rely on cache during the idle-daily gap. Idle-daily is + * usually run once every 24 hours. + */ + async updatePersonalizationScores() { + if ( + !this.personalized || + Date.now() - this.personalizationLastUpdated < + MIN_PERSONALIZATION_UPDATE_TIME + ) { + return; + } + + await this.setProvider(); + + const personalization = { scores: this.provider.getScores() }; + this.personalizationLastUpdated = Date.now(); + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, + data: { + lastUpdated: this.personalizationLastUpdated, + }, + }) + ); + personalization._timestamp = this.personalizationLastUpdated; + this.cache.set("personalization", personalization); + } + + /* + * This just re hydrates the provider from cache. + * We can call this on startup because it's generally fast. + * It reports to devtools the last time the data in the cache was updated. + */ + async loadPersonalizationScoresCache(isStartup = false) { + const cachedData = (await this.cache.get()) || {}; + const { personalization } = cachedData; + + if (this.personalized && personalization?.scores) { + await this.setProvider(isStartup, personalization.scores); + + this.personalizationLastUpdated = personalization._timestamp; + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_LAST_UPDATED, + data: { + lastUpdated: this.personalizationLastUpdated, + }, + meta: { + isStartup, + }, + }) + ); + } + } + + // This turns personalization on/off if the server sends the override command. + // The server sends a true signal to keep personalization on. So a malfunctioning + // server would more likely mistakenly turn off personalization, and not turn it on. + // This is safer, because the override is for cases where personalization is causing issues. + // So having it mistakenly go off is safe, but it mistakenly going on could be bad. + personalizationOverride(overrideCommand) { + // Are we currently in an override state. + // This is useful to know if we want to do a cleanup. + const overrideState = + this.store.getState().Prefs.values[PREF_PERSONALIZATION_OVERRIDE]; + + // Is this profile currently set to be personalized. + const personalization = + this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + + // If we have an override command, profile is currently personalized, + // and is not currently being overridden, we can set the override pref. + if (overrideCommand && personalization && !overrideState) { + this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION_OVERRIDE, true)); + } + + // This is if we need to revert an override and do cleanup. + // We do this if we are in an override state, + // but not currently receiving the override signal. + if (!overrideCommand && overrideState) { + this.store.dispatch({ + type: at.CLEAR_PREF, + data: { name: PREF_PERSONALIZATION_OVERRIDE }, + }); + } + } + + async calculateItemRelevanceScore(item) { + if (this.provider) { + const scoreResult = await this.provider.calculateItemRelevanceScore(item); + if (scoreResult === 0 || scoreResult) { + item.score = scoreResult; + } + } + } + + teardown() { + if (this.provider && this.provider.teardown) { + // This removes any in memory listeners if available. + this.provider.teardown(); + } + if (this.loaded) { + Services.obs.removeObserver(this, "idle-daily"); + } + this.loaded = false; + } + + async resetState() { + this._modelKeys = null; + this.personalizationLastUpdated = null; + this.provider = null; + await this.cache.set("personalization", {}); + this.store.dispatch( + ac.OnlyToMain({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_RESET, + }) + ); + } + + async observe(subject, topic, data) { + switch (topic) { + case "idle-daily": + await this.updatePersonalizationScores(); + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_PERSONALIZATION_UPDATED, + }) + ); + break; + } + } + + async onAction(action) { + switch (action.type) { + case at.INIT: + await this.enable(true /* isStartup */); + break; + case at.DISCOVERY_STREAM_CONFIG_CHANGE: + this.teardown(); + await this.resetState(); + await this.enable(); + break; + case at.DISCOVERY_STREAM_DEV_IDLE_DAILY: + Services.obs.notifyObservers(null, "idle-daily"); + break; + case at.PREF_CHANGED: + switch (action.data.name) { + case PREF_PERSONALIZATION_MODEL_KEYS: + this.store.dispatch( + ac.BroadcastToContent({ + type: at.DISCOVERY_STREAM_CONFIG_RESET, + }) + ); + break; + } + break; + case at.DISCOVERY_STREAM_PERSONALIZATION_TOGGLE: + let enabled = this.store.getState().Prefs.values[PREF_PERSONALIZATION]; + this.store.dispatch(ac.SetPref(PREF_PERSONALIZATION, !enabled)); + break; + case at.DISCOVERY_STREAM_PERSONALIZATION_OVERRIDE: + this.personalizationOverride(action.data.override); + break; + } + } +} diff --git a/browser/components/newtab/lib/Screenshots.sys.mjs b/browser/components/newtab/lib/Screenshots.sys.mjs new file mode 100644 index 0000000000..e5423bd52f --- /dev/null +++ b/browser/components/newtab/lib/Screenshots.sys.mjs @@ -0,0 +1,140 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// XPCOMUtils, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + BackgroundPageThumbs: "resource://gre/modules/BackgroundPageThumbs.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", +}); + +const GREY_10 = "#F9F9FA"; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "gPrivilegedAboutProcessEnabled", + "browser.tabs.remote.separatePrivilegedContentProcess", + false +); + +export const Screenshots = { + /** + * Get a screenshot / thumbnail for a url. Either returns the disk cached + * image or initiates a background request for the url. + * + * @param url {string} The url to get a thumbnail + * @return {Promise} Resolves a custom object or null if failed + */ + async getScreenshotForURL(url) { + try { + await lazy.BackgroundPageThumbs.captureIfMissing(url, { + backgroundColor: GREY_10, + }); + + // The privileged about content process is able to use the moz-page-thumb + // protocol, so if it's enabled, send that down. + if (lazy.gPrivilegedAboutProcessEnabled) { + return lazy.PageThumbs.getThumbnailURL(url); + } + + // Otherwise, for normal content processes, we fallback to using + // Blob URIs for the screenshots. + const imgPath = lazy.PageThumbs.getThumbnailPath(url); + + const filePathResponse = await fetch(`file://${imgPath}`); + const fileContents = await filePathResponse.blob(); + + // Check if the file is empty, which indicates there isn't actually a + // thumbnail, so callers can show a failure state. + if (fileContents.size === 0) { + return null; + } + + return { path: imgPath, data: fileContents }; + } catch (err) { + console.error(`getScreenshot(${url}) failed:`, err); + } + + // We must have failed to get the screenshot, so persist the failure by + // storing an empty file. Future calls will then skip requesting and return + // failure, so do the same thing here. The empty file should not expire with + // the usual filtering process to avoid repeated background requests, which + // can cause unwanted high CPU, network and memory usage - Bug 1384094 + try { + await lazy.PageThumbs._store(url, url, null, true); + } catch (err) { + // Probably failed to create the empty file, but not much more we can do. + } + return null; + }, + + /** + * Checks if all the open windows are private browsing windows. If so, we do not + * want to collect screenshots. If there exists at least 1 non-private window, + * we are ok to collect screenshots. + */ + _shouldGetScreenshots() { + for (let win of Services.wm.getEnumerator("navigator:browser")) { + if (!lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + // As soon as we encounter 1 non-private window, screenshots are fair game. + return true; + } + } + return false; + }, + + /** + * Conditionally get a screenshot for a link if there's no existing pending + * screenshot. Updates the cached link's desired property with the result. + * + * @param link {object} Link object to update + * @param url {string} Url to get a screenshot of + * @param property {string} Name of property on object to set + @ @param onScreenshot {function} Callback for when the screenshot loads + */ + async maybeCacheScreenshot(link, url, property, onScreenshot) { + // If there are only private windows open, do not collect screenshots + if (!this._shouldGetScreenshots()) { + return; + } + // __sharedCache may not exist yet for links from default top sites that + // don't have a default tippy top icon. + if (!link.__sharedCache) { + link.__sharedCache = { + updateLink(prop, val) { + link[prop] = val; + }, + }; + } + const cache = link.__sharedCache; + // Nothing to do if we already have a pending screenshot or + // if a previous request failed and returned null. + if (cache.fetchingScreenshot || link[property] !== undefined) { + return; + } + + // Save the promise to the cache so other links get it immediately + cache.fetchingScreenshot = this.getScreenshotForURL(url); + + // Clean up now that we got the screenshot + const screenshot = await cache.fetchingScreenshot; + delete cache.fetchingScreenshot; + + // Update the cache for future links and call back for existing content + cache.updateLink(property, screenshot); + onScreenshot(screenshot); + }, +}; diff --git a/browser/components/newtab/lib/SearchShortcuts.sys.mjs b/browser/components/newtab/lib/SearchShortcuts.sys.mjs new file mode 100644 index 0000000000..1448f87ca4 --- /dev/null +++ b/browser/components/newtab/lib/SearchShortcuts.sys.mjs @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// List of sites we match against Topsites in order to identify sites +// that should be converted to search Topsites +export const SEARCH_SHORTCUTS = [ + { keyword: "@amazon", shortURL: "amazon", url: "https://amazon.com" }, + { keyword: "@\u767E\u5EA6", shortURL: "baidu", url: "https://baidu.com" }, + { keyword: "@google", shortURL: "google", url: "https://google.com" }, + { + keyword: "@\u044F\u043D\u0434\u0435\u043A\u0441", + shortURL: "yandex", + url: "https://yandex.com", + }, +]; + +// These can be added via the editor but will not be added organically +export const CUSTOM_SEARCH_SHORTCUTS = [ + ...SEARCH_SHORTCUTS, + { keyword: "@bing", shortURL: "bing", url: "https://bing.com" }, + { + keyword: "@duckduckgo", + shortURL: "duckduckgo", + url: "https://duckduckgo.com", + }, + { keyword: "@ebay", shortURL: "ebay", url: "https://ebay.com" }, + { keyword: "@twitter", shortURL: "twitter", url: "https://twitter.com" }, + { + keyword: "@wikipedia", + shortURL: "wikipedia", + url: "https://wikipedia.org", + }, +]; + +// Note: you must add the activity stream branch to the beginning of this if using outside activity stream +export const SEARCH_SHORTCUTS_EXPERIMENT = + "improvesearch.topSiteSearchShortcuts"; + +export const SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF = + "improvesearch.topSiteSearchShortcuts.searchEngines"; + +export const SEARCH_SHORTCUTS_HAVE_PINNED_PREF = + "improvesearch.topSiteSearchShortcuts.havePinned"; + +export function getSearchProvider(candidateShortURL) { + return ( + SEARCH_SHORTCUTS.filter(match => candidateShortURL === match.shortURL)[0] || + null + ); +} + +// Get the search form URL for a given search keyword. This allows us to pick +// different tippytop icons for the different variants. Sush as yandex.com vs. yandex.ru. +// See more details in bug 1643523. +export async function getSearchFormURL(keyword) { + const engine = await Services.search.getEngineByAlias(keyword); + return engine?.wrappedJSObject._searchForm; +} + +// Check topsite against predefined list of valid search engines +// https://searchfox.org/mozilla-central/rev/ca869724246f4230b272ed1c8b9944596e80d920/toolkit/components/search/nsSearchService.js#939 +export async function checkHasSearchEngine(keyword) { + try { + return !!(await Services.search.getAppProvidedEngines()).find( + e => e.aliases.includes(keyword) && !e.hidden + ); + } catch { + // When the search service has not successfully initialized, + // there will be no search engines ready. + return false; + } +} diff --git a/browser/components/newtab/lib/SectionsManager.sys.mjs b/browser/components/newtab/lib/SectionsManager.sys.mjs new file mode 100644 index 0000000000..96bba0c9ea --- /dev/null +++ b/browser/components/newtab/lib/SectionsManager.sys.mjs @@ -0,0 +1,715 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// EventEmitter, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { EventEmitter } = ChromeUtils.importESModule( + "resource://gre/modules/EventEmitter.sys.mjs" +); +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.sys.mjs"; +import { getDefaultOptions } from "resource://activity-stream/lib/ActivityStreamStorage.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +/* + * Generators for built in sections, keyed by the pref name for their feed. + * Built in sections may depend on options stored as serialised JSON in the pref + * `${feed_pref_name}.options`. + */ + +const BUILT_IN_SECTIONS = ({ newtab, pocketNewtab }) => ({ + "feeds.section.topstories": options => ({ + id: "topstories", + pref: { + titleString: { + id: "home-prefs-recommended-by-header-generic", + }, + descString: { + id: "home-prefs-recommended-by-description-generic", + }, + nestedPrefs: [ + ...(Services.prefs.getBoolPref( + "browser.newtabpage.activity-stream.system.showSponsored", + true + ) + ? [ + { + name: "showSponsored", + titleString: + "home-prefs-recommended-by-option-sponsored-stories", + icon: "icon-info", + eventSource: "POCKET_SPOCS", + }, + ] + : []), + ...(pocketNewtab.recentSavesEnabled + ? [ + { + name: "showRecentSaves", + titleString: "home-prefs-recommended-by-option-recent-saves", + icon: "icon-info", + eventSource: "POCKET_RECENT_SAVES", + }, + ] + : []), + ], + learnMore: { + link: { + href: "https://getpocket.com/firefox/new_tab_learn_more", + id: "home-prefs-recommended-by-learn-more", + }, + }, + }, + shouldHidePref: options.hidden, + eventSource: "TOP_STORIES", + icon: options.provider_icon, + title: { + id: "newtab-section-header-stories", + }, + learnMore: { + link: { + href: "https://getpocket.com/firefox/new_tab_learn_more", + message: { id: "newtab-pocket-learn-more" }, + }, + }, + compactCards: false, + rowsPref: "section.topstories.rows", + maxRows: 4, + availableLinkMenuOptions: [ + "CheckBookmarkOrArchive", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + ], + emptyState: { + message: { + id: "newtab-empty-section-topstories-generic", + }, + icon: "check", + }, + shouldSendImpressionStats: true, + dedupeFrom: ["highlights"], + }), + "feeds.section.highlights": options => ({ + id: "highlights", + pref: { + titleString: { + id: "home-prefs-recent-activity-header", + }, + descString: { + id: "home-prefs-recent-activity-description", + }, + nestedPrefs: [ + { + name: "section.highlights.includeVisited", + titleString: "home-prefs-highlights-option-visited-pages", + }, + { + name: "section.highlights.includeBookmarks", + titleString: "home-prefs-highlights-options-bookmarks", + }, + { + name: "section.highlights.includeDownloads", + titleString: "home-prefs-highlights-option-most-recent-download", + }, + { + name: "section.highlights.includePocket", + titleString: "home-prefs-highlights-option-saved-to-pocket", + hidden: !Services.prefs.getBoolPref( + "extensions.pocket.enabled", + true + ), + }, + ], + }, + shouldHidePref: false, + eventSource: "HIGHLIGHTS", + icon: "chrome://global/skin/icons/highlights.svg", + title: { + id: "newtab-section-header-recent-activity", + }, + compactCards: true, + rowsPref: "section.highlights.rows", + maxRows: 4, + emptyState: { + message: { id: "newtab-empty-section-highlights" }, + icon: "chrome://global/skin/icons/highlights.svg", + }, + shouldSendImpressionStats: false, + }), +}); + +export const SectionsManager = { + ACTIONS_TO_PROXY: ["WEBEXT_CLICK", "WEBEXT_DISMISS"], + CONTEXT_MENU_PREFS: { CheckSavedToPocket: "extensions.pocket.enabled" }, + CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES: { + history: [ + "CheckBookmark", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + "DeleteUrl", + ], + bookmark: [ + "CheckBookmark", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + "DeleteUrl", + ], + pocket: [ + "ArchiveFromPocket", + "CheckSavedToPocket", + "Separator", + "OpenInNewWindow", + "OpenInPrivateWindow", + "Separator", + "BlockUrl", + ], + download: [ + "OpenFile", + "ShowFile", + "Separator", + "GoToDownloadPage", + "CopyDownloadLink", + "Separator", + "RemoveDownload", + "BlockUrl", + ], + }, + initialized: false, + sections: new Map(), + async init(prefs = {}, storage) { + this._storage = storage; + const featureConfig = { + newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {}, + pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}, + }; + + for (const feedPrefName of Object.keys(BUILT_IN_SECTIONS(featureConfig))) { + const optionsPrefName = `${feedPrefName}.options`; + await this.addBuiltInSection(feedPrefName, prefs[optionsPrefName]); + + this._dedupeConfiguration = []; + this.sections.forEach(section => { + if (section.dedupeFrom) { + this._dedupeConfiguration.push({ + id: section.id, + dedupeFrom: section.dedupeFrom, + }); + } + }); + } + + Object.keys(this.CONTEXT_MENU_PREFS).forEach(k => + Services.prefs.addObserver(this.CONTEXT_MENU_PREFS[k], this) + ); + + this.initialized = true; + this.emit(this.INIT); + }, + observe(subject, topic, data) { + switch (topic) { + case "nsPref:changed": + for (const pref of Object.keys(this.CONTEXT_MENU_PREFS)) { + if (data === this.CONTEXT_MENU_PREFS[pref]) { + this.updateSections(); + } + } + break; + } + }, + updateSectionPrefs(id, collapsed) { + const section = this.sections.get(id); + if (!section) { + return; + } + + const updatedSection = Object.assign({}, section, { + pref: Object.assign({}, section.pref, collapsed), + }); + this.updateSection(id, updatedSection, true); + }, + async addBuiltInSection(feedPrefName, optionsPrefValue = "{}") { + let options; + let storedPrefs; + const featureConfig = { + newtab: lazy.NimbusFeatures.newtab.getAllVariables() || {}, + pocketNewtab: lazy.NimbusFeatures.pocketNewtab.getAllVariables() || {}, + }; + try { + options = JSON.parse(optionsPrefValue); + } catch (e) { + options = {}; + console.error(`Problem parsing options pref for ${feedPrefName}`); + } + try { + storedPrefs = (await this._storage.get(feedPrefName)) || {}; + } catch (e) { + storedPrefs = {}; + console.error(`Problem getting stored prefs for ${feedPrefName}`); + } + const defaultSection = + BUILT_IN_SECTIONS(featureConfig)[feedPrefName](options); + const section = Object.assign({}, defaultSection, { + pref: Object.assign( + {}, + defaultSection.pref, + getDefaultOptions(storedPrefs) + ), + }); + section.pref.feed = feedPrefName; + this.addSection(section.id, Object.assign(section, { options })); + }, + addSection(id, options) { + this.updateLinkMenuOptions(options, id); + this.sections.set(id, options); + this.emit(this.ADD_SECTION, id, options); + }, + removeSection(id) { + this.emit(this.REMOVE_SECTION, id); + this.sections.delete(id); + }, + enableSection(id, isStartup = false) { + this.updateSection(id, { enabled: true }, true, isStartup); + this.emit(this.ENABLE_SECTION, id); + }, + disableSection(id) { + this.updateSection( + id, + { enabled: false, rows: [], initialized: false }, + true + ); + this.emit(this.DISABLE_SECTION, id); + }, + updateSections() { + this.sections.forEach((section, id) => + this.updateSection(id, section, true) + ); + }, + updateSection(id, options, shouldBroadcast, isStartup = false) { + this.updateLinkMenuOptions(options, id); + if (this.sections.has(id)) { + const optionsWithDedupe = Object.assign({}, options, { + dedupeConfigurations: this._dedupeConfiguration, + }); + this.sections.set(id, Object.assign(this.sections.get(id), options)); + this.emit( + this.UPDATE_SECTION, + id, + optionsWithDedupe, + shouldBroadcast, + isStartup + ); + } + }, + + /** + * Save metadata to places db and add a visit for that URL. + */ + updateBookmarkMetadata({ url }) { + this.sections.forEach((section, id) => { + if (id === "highlights") { + // Skip Highlights cards, we already have that metadata. + return; + } + if (section.rows) { + section.rows.forEach(card => { + if ( + card.url === url && + card.description && + card.title && + card.image + ) { + lazy.PlacesUtils.history.update({ + url: card.url, + title: card.title, + description: card.description, + previewImageURL: card.image, + }); + // Highlights query skips bookmarks with no visits. + lazy.PlacesUtils.history.insert({ + url, + title: card.title, + visits: [{}], + }); + } + }); + } + }); + }, + + /** + * Sets the section's context menu options. These are all available context menu + * options minus the ones that are tied to a pref (see CONTEXT_MENU_PREFS) set + * to false. + * + * @param options section options + * @param id section ID + */ + updateLinkMenuOptions(options, id) { + if (options.availableLinkMenuOptions) { + options.contextMenuOptions = options.availableLinkMenuOptions.filter( + o => + !this.CONTEXT_MENU_PREFS[o] || + Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o]) + ); + } + + // Once we have rows, we can give each card it's own context menu based on it's type. + // We only want to do this for highlights because those have different data types. + // All other sections (built by the web extension API) will have the same context menu per section + if (options.rows && id === "highlights") { + this._addCardTypeLinkMenuOptions(options.rows); + } + }, + + /** + * Sets each card in highlights' context menu options based on the card's type. + * (See types.js for a list of types) + * + * @param rows section rows containing a type for each card + */ + _addCardTypeLinkMenuOptions(rows) { + for (let card of rows) { + if (!this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[card.type]) { + console.error( + `No context menu for highlight type ${card.type} is configured` + ); + } else { + card.contextMenuOptions = + this.CONTEXT_MENU_OPTIONS_FOR_HIGHLIGHT_TYPES[card.type]; + + // Remove any options that shouldn't be there based on CONTEXT_MENU_PREFS. + // For example: If the Pocket extension is disabled, we should remove the CheckSavedToPocket option + // for each card that has it + card.contextMenuOptions = card.contextMenuOptions.filter( + o => + !this.CONTEXT_MENU_PREFS[o] || + Services.prefs.getBoolPref(this.CONTEXT_MENU_PREFS[o]) + ); + } + } + }, + + /** + * Update a specific section card by its url. This allows an action to be + * broadcast to all existing pages to update a specific card without having to + * also force-update the rest of the section's cards and state on those pages. + * + * @param id The id of the section with the card to be updated + * @param url The url of the card to update + * @param options The options to update for the card + * @param shouldBroadcast Whether or not to broadcast the update + * @param isStartup If this update is during startup. + */ + updateSectionCard(id, url, options, shouldBroadcast, isStartup = false) { + if (this.sections.has(id)) { + const card = this.sections.get(id).rows.find(elem => elem.url === url); + if (card) { + Object.assign(card, options); + } + this.emit( + this.UPDATE_SECTION_CARD, + id, + url, + options, + shouldBroadcast, + isStartup + ); + } + }, + removeSectionCard(sectionId, url) { + if (!this.sections.has(sectionId)) { + return; + } + const rows = this.sections + .get(sectionId) + .rows.filter(row => row.url !== url); + this.updateSection(sectionId, { rows }, true); + }, + onceInitialized(callback) { + if (this.initialized) { + callback(); + } else { + this.once(this.INIT, callback); + } + }, + uninit() { + Object.keys(this.CONTEXT_MENU_PREFS).forEach(k => + Services.prefs.removeObserver(this.CONTEXT_MENU_PREFS[k], this) + ); + SectionsManager.initialized = false; + }, +}; + +for (const action of [ + "ACTION_DISPATCHED", + "ADD_SECTION", + "REMOVE_SECTION", + "ENABLE_SECTION", + "DISABLE_SECTION", + "UPDATE_SECTION", + "UPDATE_SECTION_CARD", + "INIT", + "UNINIT", +]) { + SectionsManager[action] = action; +} + +EventEmitter.decorate(SectionsManager); + +export class SectionsFeed { + constructor() { + this.init = this.init.bind(this); + this.onAddSection = this.onAddSection.bind(this); + this.onRemoveSection = this.onRemoveSection.bind(this); + this.onUpdateSection = this.onUpdateSection.bind(this); + this.onUpdateSectionCard = this.onUpdateSectionCard.bind(this); + } + + init() { + SectionsManager.on(SectionsManager.ADD_SECTION, this.onAddSection); + SectionsManager.on(SectionsManager.REMOVE_SECTION, this.onRemoveSection); + SectionsManager.on(SectionsManager.UPDATE_SECTION, this.onUpdateSection); + SectionsManager.on( + SectionsManager.UPDATE_SECTION_CARD, + this.onUpdateSectionCard + ); + // Catch any sections that have already been added + SectionsManager.sections.forEach((section, id) => + this.onAddSection( + SectionsManager.ADD_SECTION, + id, + section, + true /* isStartup */ + ) + ); + } + + uninit() { + SectionsManager.uninit(); + SectionsManager.emit(SectionsManager.UNINIT); + SectionsManager.off(SectionsManager.ADD_SECTION, this.onAddSection); + SectionsManager.off(SectionsManager.REMOVE_SECTION, this.onRemoveSection); + SectionsManager.off(SectionsManager.UPDATE_SECTION, this.onUpdateSection); + SectionsManager.off( + SectionsManager.UPDATE_SECTION_CARD, + this.onUpdateSectionCard + ); + } + + onAddSection(event, id, options, isStartup = false) { + if (options) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.SECTION_REGISTER, + data: Object.assign({ id }, options), + meta: { + isStartup, + }, + }) + ); + + // Make sure the section is in sectionOrder pref. Otherwise, prepend it. + const orderedSections = this.orderedSectionIds; + if (!orderedSections.includes(id)) { + orderedSections.unshift(id); + this.store.dispatch( + ac.SetPref("sectionOrder", orderedSections.join(",")) + ); + } + } + } + + onRemoveSection(event, id) { + this.store.dispatch( + ac.BroadcastToContent({ type: at.SECTION_DEREGISTER, data: id }) + ); + } + + onUpdateSection( + event, + id, + options, + shouldBroadcast = false, + isStartup = false + ) { + if (options) { + const action = { + type: at.SECTION_UPDATE, + data: Object.assign(options, { id }), + meta: { + isStartup, + }, + }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + } + + onUpdateSectionCard( + event, + id, + url, + options, + shouldBroadcast = false, + isStartup = false + ) { + if (options) { + const action = { + type: at.SECTION_UPDATE_CARD, + data: { id, url, options }, + meta: { + isStartup, + }, + }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + } + + get orderedSectionIds() { + return this.store.getState().Prefs.values.sectionOrder.split(","); + } + + get enabledSectionIds() { + let sections = this.store + .getState() + .Sections.filter(section => section.enabled) + .map(s => s.id); + // Top Sites is a special case. Append if the feed is enabled. + if (this.store.getState().Prefs.values["feeds.topsites"]) { + sections.push("topsites"); + } + return sections; + } + + moveSection(id, direction) { + const orderedSections = this.orderedSectionIds; + const enabledSections = this.enabledSectionIds; + let index = orderedSections.indexOf(id); + orderedSections.splice(index, 1); + if (direction > 0) { + // "Move Down" + while (index < orderedSections.length) { + // If the section at the index is enabled/visible, insert moved section after. + // Otherwise, move on to the next spot and check it. + if (enabledSections.includes(orderedSections[index++])) { + break; + } + } + } else { + // "Move Up" + while (index > 0) { + // If the section at the previous index is enabled/visible, insert moved section there. + // Otherwise, move on to the previous spot and check it. + index--; + if (enabledSections.includes(orderedSections[index])) { + break; + } + } + } + + orderedSections.splice(index, 0, id); + this.store.dispatch(ac.SetPref("sectionOrder", orderedSections.join(","))); + } + + async onAction(action) { + switch (action.type) { + case at.INIT: + SectionsManager.onceInitialized(this.init); + break; + // Wait for pref values, as some sections have options stored in prefs + case at.PREFS_INITIAL_VALUES: + SectionsManager.init( + action.data, + this.store.dbStorage.getDbTable("sectionPrefs") + ); + break; + case at.PREF_CHANGED: { + if (action.data) { + const matched = action.data.name.match( + /^(feeds.section.(\S+)).options$/i + ); + if (matched) { + await SectionsManager.addBuiltInSection( + matched[1], + action.data.value + ); + this.store.dispatch({ + type: at.SECTION_OPTIONS_CHANGED, + data: matched[2], + }); + } + } + break; + } + case at.UPDATE_SECTION_PREFS: + SectionsManager.updateSectionPrefs(action.data.id, action.data.value); + break; + case at.PLACES_BOOKMARK_ADDED: + SectionsManager.updateBookmarkMetadata(action.data); + break; + case at.WEBEXT_DISMISS: + if (action.data) { + SectionsManager.removeSectionCard( + action.data.source, + action.data.url + ); + } + break; + case at.SECTION_DISABLE: + SectionsManager.disableSection(action.data); + break; + case at.SECTION_ENABLE: + SectionsManager.enableSection(action.data); + break; + case at.SECTION_MOVE: + this.moveSection(action.data.id, action.data.direction); + break; + case at.UNINIT: + this.uninit(); + break; + } + if ( + SectionsManager.ACTIONS_TO_PROXY.includes(action.type) && + SectionsManager.sections.size > 0 + ) { + SectionsManager.emit( + SectionsManager.ACTION_DISPATCHED, + action.type, + action.data + ); + } + } +} diff --git a/browser/components/newtab/lib/ShortURL.sys.mjs b/browser/components/newtab/lib/ShortURL.sys.mjs new file mode 100644 index 0000000000..6ee34c20dd --- /dev/null +++ b/browser/components/newtab/lib/ShortURL.sys.mjs @@ -0,0 +1,88 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// XPCOMUtils, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +const lazy = {}; + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "IDNService", + "@mozilla.org/network/idn-service;1", + "nsIIDNService" +); + +/** + * Properly convert internationalized domain names. + * @param {string} host Domain hostname. + * @returns {string} Hostname suitable to be displayed. + */ +function handleIDNHost(hostname) { + try { + return lazy.IDNService.convertToDisplayIDN(hostname, {}); + } catch (e) { + // If something goes wrong (e.g. host is an IP address) just fail back + // to the full domain. + return hostname; + } +} + +/** + * Get the effective top level domain of a host. + * @param {string} host The host to be analyzed. + * @return {str} The suffix or empty string if there's no suffix. + */ +export function getETLD(host) { + try { + return Services.eTLD.getPublicSuffixFromHost(host); + } catch (err) { + return ""; + } +} + +/** + * shortURL - Creates a short version of a link's url, used for display purposes + * e.g. {url: http://www.foosite.com} => "foosite" + * + * @param {obj} link A link object + * {str} link.url (required)- The url of the link + * @return {str} A short url + */ +export function shortURL({ url }) { + if (!url) { + return ""; + } + + // Make sure we have a valid / parseable url + let parsed; + try { + parsed = new URL(url); + } catch (ex) { + // Not entirely sure what we have, but just give it back + return url; + } + + // Clean up the url (lowercase hostname via URL and remove www.) + const hostname = parsed.hostname.replace(/^www\./i, ""); + + // Remove the eTLD (e.g., com, net) and the preceding period from the hostname + const eTLD = getETLD(hostname); + const eTLDExtra = eTLD.length ? -(eTLD.length + 1) : Infinity; + + // Ideally get the short eTLD-less host but fall back to longer url parts + return ( + handleIDNHost(hostname.slice(0, eTLDExtra) || hostname) || + parsed.pathname || + parsed.href + ); +} diff --git a/browser/components/newtab/lib/SiteClassifier.sys.mjs b/browser/components/newtab/lib/SiteClassifier.sys.mjs new file mode 100644 index 0000000000..64c7309bf5 --- /dev/null +++ b/browser/components/newtab/lib/SiteClassifier.sys.mjs @@ -0,0 +1,103 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on this module. This +// is because the Karma test environment already stubs out +// RemoteSettings, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). + +// eslint-disable-next-line mozilla/use-static-import +const { RemoteSettings } = ChromeUtils.importESModule( + "resource://services-settings/remote-settings.sys.mjs" +); + +// Returns whether the passed in params match the criteria. +// To match, they must contain all the params specified in criteria and the values +// must match if a value is provided in criteria. +function _hasParams(criteria, params) { + for (let param of criteria) { + const val = params.get(param.key); + if ( + val === null || + (param.value && param.value !== val) || + (param.prefix && !val.startsWith(param.prefix)) + ) { + return false; + } + } + return true; +} + +/** + * classifySite + * Classifies a given URL into a category based on classification data from RemoteSettings. + * The data from remote settings can match a category by one of the following: + * - match the exact URL + * - match the hostname or second level domain (sld) + * - match query parameter(s), and optionally their values or prefixes + * - match both (hostname or sld) and query parameter(s) + * + * The data looks like: + * [{ + * "type": "hostname-and-params-match", + * "criteria": [ + * { + * "url": "https://matchurl.com", + * "hostname": "matchhostname.com", + * "sld": "secondleveldomain", + * "params": [ + * { + * "key": "matchparam", + * "value": "matchvalue", + * "prefix": "matchpPrefix", + * }, + * ], + * }, + * ], + * "weight": 300, + * },...] + */ +export async function classifySite(url, RS = RemoteSettings) { + let category = "other"; + let parsedURL; + + // Try to parse the url. + for (let _url of [url, `https://${url}`]) { + try { + parsedURL = new URL(_url); + break; + } catch (e) {} + } + + if (parsedURL) { + // If we parsed successfully, find a match. + const hostname = parsedURL.hostname.replace(/^www\./i, ""); + const params = parsedURL.searchParams; + // NOTE: there will be an initial/default local copy of the data in m-c. + // Therefore, this should never return an empty list []. + const siteTypes = await RS("sites-classification").get(); + const sortedSiteTypes = siteTypes.sort( + (x, y) => (y.weight || 0) - (x.weight || 0) + ); + for (let type of sortedSiteTypes) { + for (let criteria of type.criteria) { + if (criteria.url && criteria.url !== url) { + continue; + } + if (criteria.hostname && criteria.hostname !== hostname) { + continue; + } + if (criteria.sld && criteria.sld !== hostname.split(".")[0]) { + continue; + } + if (criteria.params && !_hasParams(criteria.params, params)) { + continue; + } + return type.type; + } + } + } + return category; +} diff --git a/browser/components/newtab/lib/Store.sys.mjs b/browser/components/newtab/lib/Store.sys.mjs new file mode 100644 index 0000000000..3a4fdfa98d --- /dev/null +++ b/browser/components/newtab/lib/Store.sys.mjs @@ -0,0 +1,188 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { ActivityStreamMessageChannel } from "resource://activity-stream/lib/ActivityStreamMessageChannel.sys.mjs"; +import { ActivityStreamStorage } from "resource://activity-stream/lib/ActivityStreamStorage.sys.mjs"; +import { Prefs } from "resource://activity-stream/lib/ActivityStreamPrefs.sys.mjs"; +import { reducers } from "resource://activity-stream/common/Reducers.sys.mjs"; +import { redux } from "resource://activity-stream/vendor/Redux.sys.mjs"; + +/** + * Store - This has a similar structure to a redux store, but includes some extra + * functionality to allow for routing of actions between the Main processes + * and child processes via a ActivityStreamMessageChannel. + * It also accepts an array of "Feeds" on inititalization, which + * can listen for any action that is dispatched through the store. + */ +export class Store { + /** + * constructor - The redux store and message manager are created here, + * but no listeners are added until "init" is called. + */ + constructor() { + this._middleware = this._middleware.bind(this); + // Bind each redux method so we can call it directly from the Store. E.g., + // store.dispatch() will call store._store.dispatch(); + for (const method of ["dispatch", "getState", "subscribe"]) { + this[method] = (...args) => this._store[method](...args); + } + this.feeds = new Map(); + this._prefs = new Prefs(); + this._messageChannel = new ActivityStreamMessageChannel({ + dispatch: this.dispatch, + }); + this._store = redux.createStore( + redux.combineReducers(reducers), + redux.applyMiddleware(this._middleware, this._messageChannel.middleware) + ); + this.storage = null; + } + + /** + * _middleware - This is redux middleware consumed by redux.createStore. + * it calls each feed's .onAction method, if one + * is defined. + */ + _middleware() { + return next => action => { + next(action); + for (const store of this.feeds.values()) { + if (store.onAction) { + store.onAction(action); + } + } + }; + } + + /** + * initFeed - Initializes a feed by calling its constructor function + * + * @param {string} feedName The name of a feed, as defined in the object + * passed to Store.init + * @param {Action} initAction An optional action to initialize the feed + */ + initFeed(feedName, initAction) { + const feed = this._feedFactories.get(feedName)(); + feed.store = this; + this.feeds.set(feedName, feed); + if (initAction && feed.onAction) { + feed.onAction(initAction); + } + } + + /** + * uninitFeed - Removes a feed and calls its uninit function if defined + * + * @param {string} feedName The name of a feed, as defined in the object + * passed to Store.init + * @param {Action} uninitAction An optional action to uninitialize the feed + */ + uninitFeed(feedName, uninitAction) { + const feed = this.feeds.get(feedName); + if (!feed) { + return; + } + if (uninitAction && feed.onAction) { + feed.onAction(uninitAction); + } + this.feeds.delete(feedName); + } + + /** + * onPrefChanged - Listener for handling feed changes. + */ + onPrefChanged(name, value) { + if (this._feedFactories.has(name)) { + if (value) { + this.initFeed(name, this._initAction); + } else { + this.uninitFeed(name, this._uninitAction); + } + } + } + + /** + * init - Initializes the ActivityStreamMessageChannel channel, and adds feeds. + * + * Note that it intentionally initializes the TelemetryFeed first so that the + * addon is able to report the init errors from other feeds. + * + * @param {Map} feedFactories A Map of feeds with the name of the pref for + * the feed as the key and a function that + * constructs an instance of the feed. + * @param {Action} initAction An optional action that will be dispatched + * to feeds when they're created. + * @param {Action} uninitAction An optional action for when feeds uninit. + */ + async init(feedFactories, initAction, uninitAction) { + this._feedFactories = feedFactories; + this._initAction = initAction; + this._uninitAction = uninitAction; + + const telemetryKey = "feeds.telemetry"; + if (feedFactories.has(telemetryKey) && this._prefs.get(telemetryKey)) { + this.initFeed(telemetryKey); + } + + await this._initIndexedDB(telemetryKey); + + for (const pref of feedFactories.keys()) { + if (pref !== telemetryKey && this._prefs.get(pref)) { + this.initFeed(pref); + } + } + + this._prefs.observeBranch(this); + + // Dispatch an initial action after all enabled feeds are ready + if (initAction) { + this.dispatch(initAction); + } + + // Dispatch NEW_TAB_INIT/NEW_TAB_LOAD events after INIT event. + this._messageChannel.simulateMessagesForExistingTabs(); + } + + async _initIndexedDB(telemetryKey) { + // "snippets" is the name of one storage space, but these days it is used + // not for snippet-related data (snippets were removed in bug 1715158), + // but storage for impression or session data for all ASRouter messages. + // + // We keep the name "snippets" to avoid having to do an IndexedDB database + // migration. + this.dbStorage = new ActivityStreamStorage({ + storeNames: ["sectionPrefs", "snippets"], + }); + // Accessing the db causes the object stores to be created / migrated. + // This needs to happen before other instances try to access the db, which + // would update only a subset of the stores to the latest version. + try { + await this.dbStorage.db; // eslint-disable-line no-unused-expressions + } catch (e) { + this.dbStorage.telemetry = null; + } + } + + /** + * uninit - Uninitalizes each feed, clears them, and destroys the message + * manager channel. + * + * @return {type} description + */ + uninit() { + if (this._uninitAction) { + this.dispatch(this._uninitAction); + } + this._prefs.ignoreBranch(this); + this.feeds.clear(); + this._feedFactories = null; + } + + /** + * getMessageChannel - Used by the AboutNewTabParent actor to get the message channel. + */ + getMessageChannel() { + return this._messageChannel; + } +} diff --git a/browser/components/newtab/lib/SystemTickFeed.sys.mjs b/browser/components/newtab/lib/SystemTickFeed.sys.mjs new file mode 100644 index 0000000000..d87860fab2 --- /dev/null +++ b/browser/components/newtab/lib/SystemTickFeed.sys.mjs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { actionTypes as at } from "resource://activity-stream/common/Actions.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + clearInterval: "resource://gre/modules/Timer.sys.mjs", + setInterval: "resource://gre/modules/Timer.sys.mjs", +}); + +// Frequency at which SYSTEM_TICK events are fired +export const SYSTEM_TICK_INTERVAL = 5 * 60 * 1000; + +export class SystemTickFeed { + init() { + this._idleService = Cc["@mozilla.org/widget/useridleservice;1"].getService( + Ci.nsIUserIdleService + ); + this._hasObserver = false; + this.setTimer(); + } + + setTimer() { + this.intervalId = lazy.setInterval(() => { + if (this._idleService.idleTime > SYSTEM_TICK_INTERVAL) { + this.cancelTimer(); + Services.obs.addObserver(this, "user-interaction-active"); + this._hasObserver = true; + return; + } + this.dispatchTick(); + }, SYSTEM_TICK_INTERVAL); + } + + cancelTimer() { + lazy.clearInterval(this.intervalId); + this.intervalId = null; + } + + observe() { + this.dispatchTick(); + Services.obs.removeObserver(this, "user-interaction-active"); + this._hasObserver = false; + this.setTimer(); + } + + dispatchTick() { + ChromeUtils.idleDispatch(() => + this.store.dispatch({ type: at.SYSTEM_TICK }) + ); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + break; + case at.UNINIT: + this.cancelTimer(); + if (this._hasObserver) { + Services.obs.removeObserver(this, "user-interaction-active"); + this._hasObserver = false; + } + break; + } + } +} diff --git a/browser/components/newtab/lib/TelemetryFeed.sys.mjs b/browser/components/newtab/lib/TelemetryFeed.sys.mjs new file mode 100644 index 0000000000..99bed168a8 --- /dev/null +++ b/browser/components/newtab/lib/TelemetryFeed.sys.mjs @@ -0,0 +1,1122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// We use importESModule here instead of static import so that +// the Karma test environment won't choke on these module. This +// is because the Karma test environment already stubs out +// XPCOMUtils, and overrides importESModule to be a no-op (which +// can't be done for a static import statement). MESSAGE_TYPES_HASH / msg +// isn't something that the tests for this module seem to rely on in the +// Karma environment, but if that ever becomes the case, we should import +// those into unit-entry like we do for the ASRouter tests. + +// eslint-disable-next-line mozilla/use-static-import +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +// eslint-disable-next-line mozilla/use-static-import +const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule( + "resource:///modules/asrouter/ActorConstants.sys.mjs" +); + +import { + actionTypes as at, + actionUtils as au, +} from "resource://activity-stream/common/Actions.sys.mjs"; +import { Prefs } from "resource://activity-stream/lib/ActivityStreamPrefs.sys.mjs"; +import { classifySite } from "resource://activity-stream/lib/SiteClassifier.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AboutNewTab: "resource:///modules/AboutNewTab.sys.mjs", + AboutWelcomeTelemetry: + "resource:///modules/aboutwelcome/AboutWelcomeTelemetry.sys.mjs", + ClientID: "resource://gre/modules/ClientID.sys.mjs", + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + ExtensionSettingsStore: + "resource://gre/modules/ExtensionSettingsStore.sys.mjs", + HomePage: "resource:///modules/HomePage.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs", + UTEventReporting: "resource://activity-stream/lib/UTEventReporting.sys.mjs", + UpdateUtils: "resource://gre/modules/UpdateUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); +ChromeUtils.defineLazyGetter( + lazy, + "Telemetry", + () => new lazy.AboutWelcomeTelemetry() +); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "handoffToAwesomebarPrefValue", + "browser.newtabpage.activity-stream.improvesearch.handoffToAwesomebar", + false, + (preference, previousValue, new_value) => + Glean.newtabHandoffPreference.enabled.set(new_value) +); + +// This is a mapping table between the user preferences and its encoding code +export const USER_PREFS_ENCODING = { + showSearch: 1 << 0, + "feeds.topsites": 1 << 1, + "feeds.section.topstories": 1 << 2, + "feeds.section.highlights": 1 << 3, + showSponsored: 1 << 5, + "asrouter.userprefs.cfr.addons": 1 << 6, + "asrouter.userprefs.cfr.features": 1 << 7, + showSponsoredTopSites: 1 << 8, +}; + +export const PREF_IMPRESSION_ID = "impressionId"; +export const TELEMETRY_PREF = "telemetry"; +export const EVENTS_TELEMETRY_PREF = "telemetry.ut.events"; + +// Used as the missing value for timestamps in the session ping +const TIMESTAMP_MISSING_VALUE = -1; + +// Page filter for onboarding telemetry, any value other than these will +// be set as "other" +const ONBOARDING_ALLOWED_PAGE_VALUES = [ + "about:welcome", + "about:home", + "about:newtab", +]; + +ChromeUtils.defineLazyGetter( + lazy, + "browserSessionId", + () => lazy.TelemetrySession.getMetadata("").sessionId +); + +// The scalar category for TopSites of Contextual Services +const SCALAR_CATEGORY_TOPSITES = "contextual.services.topsites"; +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +ChromeUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream."; +const NEWTAB_PING_PREFS = { + showSearch: Glean.newtabSearch.enabled, + "feeds.topsites": Glean.topsites.enabled, + showSponsoredTopSites: Glean.topsites.sponsoredEnabled, + "feeds.section.topstories": Glean.pocket.enabled, + showSponsored: Glean.pocket.sponsoredStoriesEnabled, + topSitesRows: Glean.topsites.rows, +}; +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; + +export class TelemetryFeed { + constructor() { + this.sessions = new Map(); + this._prefs = new Prefs(); + this._impressionId = this.getOrCreateImpressionId(); + this._aboutHomeSeen = false; + this._classifySite = classifySite; + this._browserOpenNewtabStart = null; + } + + get telemetryEnabled() { + return this._prefs.get(TELEMETRY_PREF); + } + + get eventTelemetryEnabled() { + return this._prefs.get(EVENTS_TELEMETRY_PREF); + } + + get telemetryClientId() { + Object.defineProperty(this, "telemetryClientId", { + value: lazy.ClientID.getClientID(), + }); + return this.telemetryClientId; + } + + get processStartTs() { + let startupInfo = Services.startup.getStartupInfo(); + let processStartTs = startupInfo.process.getTime(); + + Object.defineProperty(this, "processStartTs", { + value: processStartTs, + }); + return this.processStartTs; + } + + init() { + this._beginObservingNewtabPingPrefs(); + Services.obs.addObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + // Set two scalars for the "deletion-request" ping (See bug 1602064 and 1729474) + Services.telemetry.scalarSet( + "deletion.request.impression_id", + this._impressionId + ); + Services.telemetry.scalarSet("deletion.request.context_id", lazy.contextId); + Glean.newtab.locale.set(Services.locale.appLocaleAsBCP47); + Glean.newtabHandoffPreference.enabled.set( + lazy.handoffToAwesomebarPrefValue + ); + } + + getOrCreateImpressionId() { + let impressionId = this._prefs.get(PREF_IMPRESSION_ID); + if (!impressionId) { + impressionId = String(Services.uuid.generateUUID()); + this._prefs.set(PREF_IMPRESSION_ID, impressionId); + } + return impressionId; + } + + browserOpenNewtabStart() { + let now = Cu.now(); + this._browserOpenNewtabStart = Math.round(this.processStartTs + now); + + ChromeUtils.addProfilerMarker( + "UserTiming", + now, + "browser-open-newtab-start" + ); + } + + setLoadTriggerInfo(port) { + // XXX note that there is a race condition here; we're assuming that no + // other tab will be interleaving calls to browserOpenNewtabStart and + // when at.NEW_TAB_INIT gets triggered by RemotePages and calls this + // method. For manually created windows, it's hard to imagine us hitting + // this race condition. + // + // However, for session restore, where multiple windows with multiple tabs + // might be restored much closer together in time, it's somewhat less hard, + // though it should still be pretty rare. + // + // The fix to this would be making all of the load-trigger notifications + // return some data with their notifications, and somehow propagate that + // data through closures into the tab itself so that we could match them + // + // As of this writing (very early days of system add-on perf telemetry), + // the hypothesis is that hitting this race should be so rare that makes + // more sense to live with the slight data inaccuracy that it would + // introduce, rather than doing the correct but complicated thing. It may + // well be worth reexamining this hypothesis after we have more experience + // with the data. + + let data_to_save; + try { + if (!this._browserOpenNewtabStart) { + throw new Error("No browser-open-newtab-start recorded."); + } + data_to_save = { + load_trigger_ts: this._browserOpenNewtabStart, + load_trigger_type: "menu_plus_or_keyboard", + }; + } catch (e) { + // if no mark was returned, we have nothing to save + return; + } + this.saveSessionPerfData(port, data_to_save); + } + + /** + * Lazily initialize UTEventReporting to send pings + */ + get utEvents() { + Object.defineProperty(this, "utEvents", { + value: new lazy.UTEventReporting(), + }); + return this.utEvents; + } + + /** + * Get encoded user preferences, multiple prefs will be combined via bitwise OR operator + */ + get userPreferences() { + let prefs = 0; + + for (const pref of Object.keys(USER_PREFS_ENCODING)) { + if (this._prefs.get(pref)) { + prefs |= USER_PREFS_ENCODING[pref]; + } + } + return prefs; + } + + /** + * Check if it is in the CFR experiment cohort by querying against the + * experiment manager of Messaging System + * + * @return {bool} + */ + get isInCFRCohort() { + const experimentData = lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "cfr", + }); + if (experimentData && experimentData.slug) { + return true; + } + + return false; + } + + /** + * addSession - Start tracking a new session + * + * @param {string} id the portID of the open session + * @param {string} the URL being loaded for this session (optional) + * @return {obj} Session object + */ + addSession(id, url) { + // XXX refactor to use setLoadTriggerInfo or saveSessionPerfData + + // "unexpected" will be overwritten when appropriate + let load_trigger_type = "unexpected"; + let load_trigger_ts; + + if (!this._aboutHomeSeen && url === "about:home") { + this._aboutHomeSeen = true; + + // XXX note that this will be incorrectly set in the following cases: + // session_restore following by clicking on the toolbar button, + // or someone who has changed their default home page preference to + // something else and later clicks the toolbar. It will also be + // incorrectly unset if someone changes their "Home Page" preference to + // about:newtab. + // + // That said, the ratio of these mistakes to correct cases should + // be very small, and these issues should follow away as we implement + // the remaining load_trigger_type values for about:home in issue 3556. + // + // XXX file a bug to implement remaining about:home cases so this + // problem will go away and link to it here. + load_trigger_type = "first_window_opened"; + + // The real perceived trigger of first_window_opened is the OS-level + // clicking of the icon. We express this by using the process start + // absolute timestamp. + load_trigger_ts = this.processStartTs; + } + + const session = { + session_id: String(Services.uuid.generateUUID()), + // "unknown" will be overwritten when appropriate + page: url ? url : "unknown", + perf: { + load_trigger_type, + is_preloaded: false, + }, + }; + + if (load_trigger_ts) { + session.perf.load_trigger_ts = load_trigger_ts; + } + + this.sessions.set(id, session); + return session; + } + + /** + * endSession - Stop tracking a session + * + * @param {string} portID the portID of the session that just closed + */ + endSession(portID) { + const session = this.sessions.get(portID); + + if (!session) { + // It's possible the tab was never visible – in which case, there was no user session. + return; + } + + Glean.newtab.closed.record({ newtab_visit_id: session.session_id }); + if ( + this.telemetryEnabled && + (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) + ) { + GleanPings.newtab.submit("newtab_session_end"); + } + + if (session.perf.visibility_event_rcvd_ts) { + let absNow = this.processStartTs + Cu.now(); + session.session_duration = Math.round( + absNow - session.perf.visibility_event_rcvd_ts + ); + + // Rounding all timestamps in perf to ease the data processing on the backend. + // NB: use `TIMESTAMP_MISSING_VALUE` if the value is missing. + session.perf.visibility_event_rcvd_ts = Math.round( + session.perf.visibility_event_rcvd_ts + ); + session.perf.load_trigger_ts = Math.round( + session.perf.load_trigger_ts || TIMESTAMP_MISSING_VALUE + ); + session.perf.topsites_first_painted_ts = Math.round( + session.perf.topsites_first_painted_ts || TIMESTAMP_MISSING_VALUE + ); + } else { + // This session was never shown (i.e. the hidden preloaded newtab), there was no user session either. + this.sessions.delete(portID); + return; + } + + let sessionEndEvent = this.createSessionEndEvent(session); + this.sendUTEvent(sessionEndEvent, this.utEvents.sendSessionEndEvent); + this.sessions.delete(portID); + } + + /** + * handleNewTabInit - Handle NEW_TAB_INIT, which creates a new session and sets the a flag + * for session.perf based on whether or not this new tab is preloaded + * + * @param {obj} action the Action object + */ + handleNewTabInit(action) { + const session = this.addSession( + au.getPortIdOfSender(action), + action.data.url + ); + session.perf.is_preloaded = + action.data.browser.getAttribute("preloadedState") === "preloaded"; + } + + /** + * createPing - Create a ping with common properties + * + * @param {string} id The portID of the session, if a session is relevant (optional) + * @return {obj} A telemetry ping + */ + createPing(portID) { + const ping = { + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + user_prefs: this.userPreferences, + }; + + // If the ping is part of a user session, add session-related info + if (portID) { + const session = this.sessions.get(portID) || this.addSession(portID); + Object.assign(ping, { session_id: session.session_id }); + + if (session.page) { + Object.assign(ping, { page: session.page }); + } + } + return ping; + } + + createUserEvent(action) { + return Object.assign( + this.createPing(au.getPortIdOfSender(action)), + action.data, + { action: "activity_stream_user_event" } + ); + } + + createSessionEndEvent(session) { + return Object.assign(this.createPing(), { + session_id: session.session_id, + page: session.page, + session_duration: session.session_duration, + action: "activity_stream_session", + perf: session.perf, + profile_creation_date: + lazy.TelemetryEnvironment.currentEnvironment.profile.resetDate || + lazy.TelemetryEnvironment.currentEnvironment.profile.creationDate, + }); + } + + /** + * Create a ping for AS router event. The client_id is set to "n/a" by default, + * different component can override this by its own telemetry collection policy. + */ + async createASRouterEvent(action) { + let event = { + ...action.data, + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + }; + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (event.event_context && typeof event.event_context === "object") { + event.event_context = JSON.stringify(event.event_context); + } + switch (event.action) { + case "cfr_user_event": + event = await this.applyCFRPolicy(event); + break; + case "badge_user_event": + case "whats-new-panel_user_event": + event = await this.applyWhatsNewPolicy(event); + break; + case "infobar_user_event": + event = await this.applyInfoBarPolicy(event); + break; + case "spotlight_user_event": + event = await this.applySpotlightPolicy(event); + break; + case "toast_notification_user_event": + event = await this.applyToastNotificationPolicy(event); + break; + case "moments_user_event": + event = await this.applyMomentsPolicy(event); + break; + case "onboarding_user_event": + event = await this.applyOnboardingPolicy(event, session); + break; + case "asrouter_undesired_event": + event = this.applyUndesiredEventPolicy(event); + break; + default: + event = { ping: event }; + break; + } + return event; + } + + /** + * Per Bug 1484035, CFR metrics comply with following policies: + * 1). In release, it collects impression_id and bucket_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + * 4). In Private Browsing windows, unless in experiment, collects impression_id and bucket_id + */ + async applyCFRPolicy(ping) { + if ( + (lazy.UpdateUtils.getUpdateChannel(true) === "release" || + ping.is_private) && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + delete ping.is_private; + return { ping, pingType: "cfr" }; + } + + /** + * Per Bug 1482134, all the metrics for What's New panel use client_id in + * all the release channels + */ + async applyWhatsNewPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + delete ping.action; + return { ping, pingType: "whats-new-panel" }; + } + + async applyInfoBarPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "infobar" }; + } + + async applySpotlightPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "spotlight" }; + } + + async applyToastNotificationPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "toast_notification" }; + } + + /** + * Per Bug 1484035, Moments metrics comply with following policies: + * 1). In release, it collects impression_id, and treats bucket_id as message_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + */ + async applyMomentsPolicy(ping) { + if ( + lazy.UpdateUtils.getUpdateChannel(true) === "release" && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + return { ping, pingType: "moments" }; + } + + /** + * Per Bug 1482134, all the metrics for Onboarding in AS router use client_id in + * all the release channels + */ + async applyOnboardingPolicy(ping, session) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + if (ping.action === "onboarding_user_event" && session && session.page) { + let event_context; + + try { + event_context = ping.event_context + ? JSON.parse(ping.event_context) + : {}; + } catch (e) { + // If `ping.event_context` is not a JSON serialized string, then we create a `value` + // key for it + event_context = { value: ping.event_context }; + } + + if (ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page)) { + event_context.page = session.page; + } else { + console.error(`Invalid 'page' for Onboarding event: ${session.page}`); + } + ping.event_context = JSON.stringify(event_context); + } + delete ping.action; + return { ping, pingType: "onboarding" }; + } + + applyUndesiredEventPolicy(ping) { + ping.impression_id = this._impressionId; + delete ping.action; + return { ping, pingType: "undesired-events" }; + } + + sendUTEvent(event_object, eventFunction) { + if (this.telemetryEnabled && this.eventTelemetryEnabled) { + eventFunction(event_object); + } + } + + handleTopSitesSponsoredImpressionStats(action) { + const { data } = action; + const { + type, + position, + source, + advertiser: advertiser_name, + tile_id, + } = data; + // Legacy telemetry expects 1-based tile positions. + const legacyTelemetryPosition = position + 1; + + let pingType; + + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (type === "impression") { + pingType = "topsites-impression"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.impression`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.impression.record({ + advertiser_name, + tile_id, + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else if (type === "click") { + pingType = "topsites-click"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.click`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.click.record({ + advertiser_name, + tile_id, + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else { + console.error("Unknown ping type for sponsored TopSites impression"); + return; + } + + Glean.topSites.pingType.set(pingType); + Glean.topSites.position.set(legacyTelemetryPosition); + Glean.topSites.source.set(source); + Glean.topSites.tileId.set(tile_id); + if (data.reporting_url) { + Glean.topSites.reportingUrl.set(data.reporting_url); + } + Glean.topSites.advertiser.set(advertiser_name); + Glean.topSites.contextId.set(lazy.contextId); + GleanPings.topSites.submit(); + } + + handleTopSitesOrganicImpressionStats(action) { + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (!session) { + return; + } + + switch (action.data?.type) { + case "impression": + Glean.topsites.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + case "click": + Glean.topsites.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + default: + break; + } + } + + handleUserEvent(action) { + let userEvent = this.createUserEvent(action); + this.sendUTEvent(userEvent, this.utEvents.sendUserEvent); + } + + handleDiscoveryStreamUserEvent(action) { + const pocket_logged_in_status = lazy.pktApi.isUserLoggedIn(); + Glean.pocket.isSignedIn.set(pocket_logged_in_status); + this.handleUserEvent({ + ...action, + data: { + ...(action.data || {}), + value: { + ...(action.data?.value || {}), + pocket_logged_in_status, + }, + }, + }); + const session = this.sessions.get(au.getPortIdOfSender(action)); + switch (action.data?.event) { + case "CLICK": + const { card_type, topic, recommendation_id, tile_id, shim } = + action.data.value ?? {}; + if ( + action.data.source === "POPULAR_TOPICS" || + card_type === "topics_widget" + ) { + Glean.pocket.topicClick.record({ + newtab_visit_id: session.session_id, + topic, + }); + } else if (["spoc", "organic"].includes(card_type)) { + Glean.pocket.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: card_type === "spoc", + position: action.data.action_position, + recommendation_id, + tile_id, + }); + if (shim) { + Glean.pocket.shim.set(shim); + GleanPings.spoc.submit("click"); + } + } + break; + case "SAVE_TO_POCKET": + Glean.pocket.save.record({ + newtab_visit_id: session.session_id, + is_sponsored: action.data.value?.card_type === "spoc", + position: action.data.action_position, + recommendation_id: action.data.value?.recommendation_id, + tile_id: action.data.value?.tile_id, + }); + if (action.data.value?.shim) { + Glean.pocket.shim.set(action.data.value.shim); + GleanPings.spoc.submit("save"); + } + break; + } + } + + async handleASRouterUserEvent(action) { + const { ping, pingType } = await this.createASRouterEvent(action); + if (!pingType) { + console.error("Unknown ping type for ASRouter telemetry"); + return; + } + + // Now that the action has become a ping, we can echo it to Glean. + if (this.telemetryEnabled) { + lazy.Telemetry.submitGleanPingForPing({ ...ping, pingType }); + } + } + + /** + * This function is used by ActivityStreamStorage to report errors + * trying to access IndexedDB. + */ + SendASRouterUndesiredEvent(data) { + this.handleASRouterUserEvent({ + data: { ...data, action: "asrouter_undesired_event" }, + }); + } + + async sendPageTakeoverData() { + if (this.telemetryEnabled) { + const value = {}; + let homeAffected = false; + let newtabCategory = "disabled"; + let homePageCategory = "disabled"; + + // Check whether or not about:home and about:newtab are set to a custom URL. + // If so, classify them. + if (Services.prefs.getBoolPref("browser.newtabpage.enabled")) { + newtabCategory = "enabled"; + if ( + lazy.AboutNewTab.newTabURLOverridden && + !lazy.AboutNewTab.newTabURL.startsWith("moz-extension://") + ) { + value.newtab_url_category = await this._classifySite( + lazy.AboutNewTab.newTabURL + ); + newtabCategory = value.newtab_url_category; + } + } + // Check if the newtab page setting is controlled by an extension. + await lazy.ExtensionSettingsStore.initialize(); + const newtabExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "url_overrides", + "newTabURL" + ); + if (newtabExtensionInfo && newtabExtensionInfo.id) { + value.newtab_extension_id = newtabExtensionInfo.id; + newtabCategory = "extension"; + } + + const homePageURL = lazy.HomePage.get(); + if ( + !["about:home", "about:blank"].includes(homePageURL) && + !homePageURL.startsWith("moz-extension://") + ) { + value.home_url_category = await this._classifySite(homePageURL); + homeAffected = true; + homePageCategory = value.home_url_category; + } + const homeExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "prefs", + "homepage_override" + ); + if (homeExtensionInfo && homeExtensionInfo.id) { + value.home_extension_id = homeExtensionInfo.id; + homeAffected = true; + homePageCategory = "extension"; + } + if (!homeAffected && !lazy.HomePage.overridden) { + homePageCategory = "enabled"; + } + + Glean.newtab.newtabCategory.set(newtabCategory); + Glean.newtab.homepageCategory.set(homePageCategory); + if (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) { + GleanPings.newtab.submit("component_init"); + } + } + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.sendPageTakeoverData(); + break; + case at.NEW_TAB_INIT: + this.handleNewTabInit(action); + break; + case at.NEW_TAB_UNLOAD: + this.endSession(au.getPortIdOfSender(action)); + break; + case at.SAVE_SESSION_PERF_DATA: + this.saveSessionPerfData(au.getPortIdOfSender(action), action.data); + break; + case at.DISCOVERY_STREAM_IMPRESSION_STATS: + this.handleDiscoveryStreamImpressionStats( + au.getPortIdOfSender(action), + action.data + ); + break; + case at.DISCOVERY_STREAM_USER_EVENT: + this.handleDiscoveryStreamUserEvent(action); + break; + case at.TELEMETRY_USER_EVENT: + this.handleUserEvent(action); + break; + // The next few action types come from ASRouter, which doesn't use + // Actions from Actions.jsm, but uses these other custom strings. + case msg.TOOLBAR_BADGE_TELEMETRY: + // Intentional fall-through + case msg.TOOLBAR_PANEL_TELEMETRY: + // Intentional fall-through + case msg.MOMENTS_PAGE_TELEMETRY: + // Intentional fall-through + case msg.DOORHANGER_TELEMETRY: + // Intentional fall-through + case msg.INFOBAR_TELEMETRY: + // Intentional fall-through + case msg.SPOTLIGHT_TELEMETRY: + // Intentional fall-through + case msg.TOAST_NOTIFICATION_TELEMETRY: + // Intentional fall-through + case at.AS_ROUTER_TELEMETRY_USER_EVENT: + this.handleASRouterUserEvent(action); + break; + case at.TOP_SITES_SPONSORED_IMPRESSION_STATS: + this.handleTopSitesSponsoredImpressionStats(action); + break; + case at.TOP_SITES_ORGANIC_IMPRESSION_STATS: + this.handleTopSitesOrganicImpressionStats(action); + break; + case at.UNINIT: + this.uninit(); + break; + case at.ABOUT_SPONSORED_TOP_SITES: + this.handleAboutSponsoredTopSites(action); + break; + case at.BLOCK_URL: + this.handleBlockUrl(action); + break; + } + } + + handleBlockUrl(action) { + const session = this.sessions.get(au.getPortIdOfSender(action)); + // TODO: Do we want to not send this unless there's a newtab_visit_id? + if (!session) { + return; + } + + // Despite the action name, this is actually a bulk dismiss action: + // it can be applied to multiple topsites simultaneously. + const { data } = action; + for (const datum of data) { + if (datum.is_pocket_card) { + // There is no instrumentation for Pocket dismissals (yet). + continue; + } + const { position, advertiser_name, tile_id, isSponsoredTopSite } = datum; + Glean.topsites.dismiss.record({ + advertiser_name, + tile_id, + newtab_visit_id: session.session_id, + is_sponsored: !!isSponsoredTopSite, + position, + }); + } + } + + handleAboutSponsoredTopSites(action) { + const session = this.sessions.get(au.getPortIdOfSender(action)); + const { data } = action; + const { position, advertiser_name, tile_id } = data; + + if (session) { + Glean.topsites.showPrivacyClick.record({ + advertiser_name, + tile_id, + newtab_visit_id: session.session_id, + position, + }); + } + } + + /** + * Handle impression stats actions from Discovery Stream. + * + * @param {String} port The session port with which this is associated + * @param {Object} data The impression data structured as {source: "SOURCE", tiles: [{id: 123}]} + * + */ + handleDiscoveryStreamImpressionStats(port, data) { + let session = this.sessions.get(port); + + if (!session) { + throw new Error("Session does not exist."); + } + + const { tiles } = data; + tiles.forEach(tile => { + Glean.pocket.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: tile.type === "spoc", + position: tile.pos, + recommendation_id: tile.recommendation_id, + tile_id: tile.id, + }); + if (tile.shim) { + Glean.pocket.shim.set(tile.shim); + GleanPings.spoc.submit("impression"); + } + }); + } + + /** + * Take all enumerable members of the data object and merge them into + * the session.perf object for the given port, so that it is sent to the + * server when the session ends. All members of the data object should + * be valid values of the perf object, as defined in pings.js and the + * data*.md documentation. + * + * @note Any existing keys with the same names already in the + * session perf object will be overwritten by values passed in here. + * + * @param {String} port The session with which this is associated + * @param {Object} data The perf data to be + */ + saveSessionPerfData(port, data) { + // XXX should use try/catch and send a bad state indicator if this + // get blows up. + let session = this.sessions.get(port); + + // XXX Partial workaround for #3118; avoids the worst incorrect associations + // of times with browsers, by associating the load trigger with the + // visibility event as the user is most likely associating the trigger to + // the tab just shown. This helps avoid associating with a preloaded + // browser as those don't get the event until shown. Better fix for more + // cases forthcoming. + // + // XXX the about:home check (and the corresponding test) should go away + // once the load_trigger stuff in addSession is refactored into + // setLoadTriggerInfo. + // + if (data.visibility_event_rcvd_ts && session.page !== "about:home") { + this.setLoadTriggerInfo(port); + } + + let timestamp = data.topsites_first_painted_ts; + + if ( + timestamp && + session.page === "about:home" && + !lazy.HomePage.overridden && + Services.prefs.getIntPref("browser.startup.page") === 1 + ) { + lazy.AboutNewTab.maybeRecordTopsitesPainted(timestamp); + } + + Object.assign(session.perf, data); + + if (data.visibility_event_rcvd_ts && !session.newtabOpened) { + session.newtabOpened = true; + const source = ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page) + ? session.page + : "other"; + Glean.newtab.opened.record({ + newtab_visit_id: session.session_id, + source, + }); + } + } + + _beginObservingNewtabPingPrefs() { + Services.prefs.addObserver(ACTIVITY_STREAM_PREF_BRANCH, this); + + for (const pref of Object.keys(NEWTAB_PING_PREFS)) { + const fullPrefName = ACTIVITY_STREAM_PREF_BRANCH + pref; + this._setNewtabPrefMetrics(fullPrefName, false); + } + Glean.pocket.isSignedIn.set(lazy.pktApi.isUserLoggedIn()); + + Services.prefs.addObserver(TOP_SITES_BLOCKED_SPONSORS_PREF, this); + this._setBlockedSponsorsMetrics(); + } + + _stopObservingNewtabPingPrefs() { + Services.prefs.removeObserver(ACTIVITY_STREAM_PREF_BRANCH, this); + Services.prefs.removeObserver(TOP_SITES_BLOCKED_SPONSORS_PREF, this); + } + + observe(subject, topic, data) { + if (data === TOP_SITES_BLOCKED_SPONSORS_PREF) { + this._setBlockedSponsorsMetrics(); + } else { + this._setNewtabPrefMetrics(data, true); + } + } + + _setNewtabPrefMetrics(fullPrefName, isChanged) { + const pref = fullPrefName.slice(ACTIVITY_STREAM_PREF_BRANCH.length); + if (!Object.hasOwn(NEWTAB_PING_PREFS, pref)) { + return; + } + const metric = NEWTAB_PING_PREFS[pref]; + switch (Services.prefs.getPrefType(fullPrefName)) { + case Services.prefs.PREF_BOOL: + metric.set(Services.prefs.getBoolPref(fullPrefName)); + break; + + case Services.prefs.PREF_INT: + metric.set(Services.prefs.getIntPref(fullPrefName)); + break; + } + if (isChanged) { + switch (fullPrefName) { + case `${ACTIVITY_STREAM_PREF_BRANCH}feeds.topsites`: + case `${ACTIVITY_STREAM_PREF_BRANCH}showSponsoredTopSites`: + Glean.topsites.prefChanged.record({ + pref_name: fullPrefName, + new_value: Services.prefs.getBoolPref(fullPrefName), + }); + break; + } + } + } + + _setBlockedSponsorsMetrics() { + let blocklist; + try { + blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + } catch (e) {} + if (blocklist) { + Glean.newtab.blockedSponsors.set(blocklist); + } + } + + uninit() { + this._stopObservingNewtabPingPrefs(); + + try { + Services.obs.removeObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + } catch (e) { + // Operation can fail when uninit is called before + // init has finished setting up the observer + } + + // Only uninit if the getter has initialized it + if (Object.prototype.hasOwnProperty.call(this, "utEvents")) { + this.utEvents.uninit(); + } + + // TODO: Send any unfinished sessions + } +} diff --git a/browser/components/newtab/lib/TippyTopProvider.sys.mjs b/browser/components/newtab/lib/TippyTopProvider.sys.mjs new file mode 100644 index 0000000000..8f32516119 --- /dev/null +++ b/browser/components/newtab/lib/TippyTopProvider.sys.mjs @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const TIPPYTOP_PATH = "chrome://activity-stream/content/data/content/tippytop/"; +const TIPPYTOP_JSON_PATH = + "chrome://activity-stream/content/data/content/tippytop/top_sites.json"; + +/* + * Get a domain from a url optionally stripping subdomains. + */ +export function getDomain(url, strip = "www.") { + let domain = ""; + try { + domain = new URL(url).hostname; + } catch (ex) {} + if (strip === "*") { + try { + domain = Services.eTLD.getBaseDomainFromHost(domain); + } catch (ex) {} + } else if (domain.startsWith(strip)) { + domain = domain.slice(strip.length); + } + return domain; +} + +export class TippyTopProvider { + constructor() { + this._sitesByDomain = new Map(); + this.initialized = false; + } + + async init() { + // Load the Tippy Top sites from the json manifest. + try { + for (const site of await ( + await fetch(TIPPYTOP_JSON_PATH, { + credentials: "omit", + }) + ).json()) { + for (const domain of site.domains) { + this._sitesByDomain.set(domain, site); + } + } + this.initialized = true; + } catch (error) { + console.error("Failed to load tippy top manifest."); + } + } + + processSite(site, strip) { + const tippyTop = this._sitesByDomain.get(getDomain(site.url, strip)); + if (tippyTop) { + site.tippyTopIcon = TIPPYTOP_PATH + tippyTop.image_url; + site.smallFavicon = TIPPYTOP_PATH + tippyTop.favicon_url; + site.backgroundColor = tippyTop.background_color; + } + return site; + } +} diff --git a/browser/components/newtab/lib/TopSitesFeed.sys.mjs b/browser/components/newtab/lib/TopSitesFeed.sys.mjs new file mode 100644 index 0000000000..db21411fdd --- /dev/null +++ b/browser/components/newtab/lib/TopSitesFeed.sys.mjs @@ -0,0 +1,2007 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionCreators as ac, + actionTypes as at, +} from "resource://activity-stream/common/Actions.sys.mjs"; +import { TippyTopProvider } from "resource://activity-stream/lib/TippyTopProvider.sys.mjs"; +import { + insertPinned, + TOP_SITES_MAX_SITES_PER_ROW, +} from "resource://activity-stream/common/Reducers.sys.mjs"; +import { Dedupe } from "resource://activity-stream/common/Dedupe.sys.mjs"; +import { shortURL } from "resource://activity-stream/lib/ShortURL.sys.mjs"; +import { getDefaultOptions } from "resource://activity-stream/lib/ActivityStreamStorage.sys.mjs"; + +import { + CUSTOM_SEARCH_SHORTCUTS, + SEARCH_SHORTCUTS_EXPERIMENT, + SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF, + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + checkHasSearchEngine, + getSearchProvider, + getSearchFormURL, +} from "resource://activity-stream/lib/SearchShortcuts.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + FilterAdult: "resource://activity-stream/lib/FilterAdult.sys.mjs", + LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs", + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs", + Region: "resource://gre/modules/Region.sys.mjs", + RemoteSettings: "resource://services-settings/remote-settings.sys.mjs", + Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs", + Screenshots: "resource://activity-stream/lib/Screenshots.sys.mjs", +}); + +ChromeUtils.defineLazyGetter(lazy, "log", () => { + const { Logger } = ChromeUtils.importESModule( + "resource://messaging-system/lib/Logger.sys.mjs" + ); + return new Logger("TopSitesFeed"); +}); + +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +ChromeUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +const DEFAULT_SITES_PREF = "default.sites"; +const SHOWN_ON_NEWTAB_PREF = "feeds.topsites"; +export const DEFAULT_TOP_SITES = []; +const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages) +const MIN_FAVICON_SIZE = 96; +const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"]; +const PINNED_FAVICON_PROPS_TO_MIGRATE = [ + "favicon", + "faviconRef", + "faviconSize", +]; +const SECTION_ID = "topsites"; +const ROWS_PREF = "topSitesRows"; +const SHOW_SPONSORED_PREF = "showSponsoredTopSites"; +// The default total number of sponsored top sites to fetch from Contile +// and Pocket. +const MAX_NUM_SPONSORED = 2; +// Nimbus variable for the total number of sponsored top sites including +// both Contile and Pocket sources. +// The default will be `MAX_NUM_SPONSORED` if this variable is unspecified. +const NIMBUS_VARIABLE_MAX_SPONSORED = "topSitesMaxSponsored"; +// Nimbus variable to allow more than two sponsored tiles from Contile to be +//considered for Top Sites. +const NIMBUS_VARIABLE_ADDITIONAL_TILES = + "topSitesUseAdditionalTilesFromContile"; +// Nimbus variable to enable the SOV feature for sponsored tiles. +const NIMBUS_VARIABLE_CONTILE_SOV_ENABLED = "topSitesContileSovEnabled"; +// Nimbu variable for the total number of sponsor topsite that come from Contile +// The default will be `CONTILE_MAX_NUM_SPONSORED` if variable is unspecified. +const NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED = "topSitesContileMaxSponsored"; + +// Search experiment stuff +const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile"; +const SEARCH_FILTERS = [ + "google", + "search.yahoo", + "yahoo", + "bing", + "ask", + "duckduckgo", +]; + +const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting"; +const DEFAULT_SITES_OVERRIDE_PREF = + "browser.newtabpage.activity-stream.default.sites"; +const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment."; + +// Mozilla Tiles Service (Contile) prefs +// Nimbus variable for the Contile integration. It falls back to the pref: +// `browser.topsites.contile.enabled`. +const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled"; +const NIMBUS_VARIABLE_CONTILE_POSITIONS = "contileTopsitesPositions"; +const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint"; +const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes +// The maximum number of sponsored top sites to fetch from Contile. +const CONTILE_MAX_NUM_SPONSORED = 2; +const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; +const CONTILE_CACHE_PREF = "browser.topsites.contile.cachedTiles"; +const CONTILE_CACHE_VALID_FOR_PREF = "browser.topsites.contile.cacheValidFor"; +const CONTILE_CACHE_LAST_FETCH_PREF = "browser.topsites.contile.lastFetch"; + +// Partners of sponsored tiles. +const SPONSORED_TILE_PARTNER_AMP = "amp"; +const SPONSORED_TILE_PARTNER_MOZ_SALES = "moz-sales"; +const SPONSORED_TILE_PARTNERS = new Set([ + SPONSORED_TILE_PARTNER_AMP, + SPONSORED_TILE_PARTNER_MOZ_SALES, +]); + +const DISPLAY_FAIL_REASON_OVERSOLD = "oversold"; +const DISPLAY_FAIL_REASON_DISMISSED = "dismissed"; +const DISPLAY_FAIL_REASON_UNRESOLVED = "unresolved"; + +function getShortURLForCurrentSearch() { + const url = shortURL({ url: Services.search.defaultEngine.searchForm }); + return url; +} + +class TopSitesTelemetry { + constructor() { + this.allSponsoredTiles = {}; + this.sponsoredTilesConfigured = 0; + } + + _tileProviderForTiles(tiles) { + // Assumption: the list of tiles is from a single provider + return tiles && tiles.length ? this._tileProvider(tiles[0]) : null; + } + + _tileProvider(tile) { + return tile.partner || SPONSORED_TILE_PARTNER_AMP; + } + + _buildPropertyKey(tile) { + let provider = this._tileProvider(tile); + return provider + shortURL(tile); + } + + // Returns an array of strings indicating the property name (based on the + // provider and brand) of tiles that have been filtered e.g. ["moz-salesbrand1"] + // currentTiles: The list of tiles remaining and may be displayed in new tab. + // this.allSponsoredTiles: The original list of tiles set via setTiles prior to any filtering + // The returned list indicated the difference between these two lists (excluding any previously filtered tiles). + _getFilteredTiles(currentTiles) { + let notPreviouslyFilteredTiles = Object.assign( + {}, + ...Object.entries(this.allSponsoredTiles) + .filter( + ([k, v]) => + v.display_fail_reason === null || + v.display_fail_reason === undefined + ) + .map(([k, v]) => ({ [k]: v })) + ); + + // Get the property names of the newly filtered list. + let remainingTiles = currentTiles.map(el => { + return this._buildPropertyKey(el); + }); + + // Get the property names of the tiles that were filtered. + let tilesToUpdate = Object.keys(notPreviouslyFilteredTiles).filter( + element => !remainingTiles.includes(element) + ); + return tilesToUpdate; + } + + setSponsoredTilesConfigured() { + const maxSponsored = + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_MAX_SPONSORED + ) ?? MAX_NUM_SPONSORED; + + this.sponsoredTilesConfigured = maxSponsored; + Glean.topsites.sponsoredTilesConfigured.set(maxSponsored); + } + + clearTilesForProvider(provider) { + Object.entries(this.allSponsoredTiles) + .filter(([k, v]) => k.startsWith(provider)) + .map(([k, v]) => delete this.allSponsoredTiles[k]); + } + + _getAdvertiser(tile) { + let label = tile.label || null; + let title = tile.title || null; + + return label ?? title ?? shortURL(tile); + } + + setTiles(tiles) { + // Assumption: the list of tiles is from a single provider, + // should be called once per tile source. + if (tiles && tiles.length) { + let tile_provider = this._tileProviderForTiles(tiles); + this.clearTilesForProvider(tile_provider); + + for (let sponsoredTile of tiles) { + this.allSponsoredTiles[this._buildPropertyKey(sponsoredTile)] = { + advertiser: this._getAdvertiser(sponsoredTile).toLowerCase(), + provider: tile_provider, + display_position: null, + display_fail_reason: null, + }; + } + } + } + + _setDisplayFailReason(filteredTiles, reason) { + for (let tile of filteredTiles) { + if (tile in this.allSponsoredTiles) { + let tileToUpdate = this.allSponsoredTiles[tile]; + tileToUpdate.display_position = null; + tileToUpdate.display_fail_reason = reason; + } + } + } + + determineFilteredTilesAndSetToOversold(nonOversoldTiles) { + let filteredTiles = this._getFilteredTiles(nonOversoldTiles); + this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_OVERSOLD); + } + + determineFilteredTilesAndSetToDismissed(nonDismissedTiles) { + let filteredTiles = this._getFilteredTiles(nonDismissedTiles); + this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_DISMISSED); + } + + _setTilePositions(currentTiles) { + // This function performs many loops over a small dataset. The size of + // dataset is limited by the number of sponsored tiles displayed on + // the newtab instance. + if (this.allSponsoredTiles) { + let tilePositionsAssigned = []; + // processing the currentTiles parameter, assigns a position to the + // corresponding property in this.allSponsoredTiles + currentTiles.forEach(item => { + let tile = this.allSponsoredTiles[this._buildPropertyKey(item)]; + if ( + tile && + (tile.display_fail_reason === undefined || + tile.display_fail_reason === null) + ) { + tile.display_position = item.sponsored_position; + // Track assigned tile slots. + tilePositionsAssigned.push(item.sponsored_position); + } + }); + + // Need to check if any objects in this.allSponsoredTiles do not + // have either a display_fail_reason or a display_position set. + // This can happen if the tiles list was updated before the + // metric is written to Glean. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1877197 + let tilesMissingPosition = []; + Object.keys(this.allSponsoredTiles).forEach(property => { + let tile = this.allSponsoredTiles[property]; + if (!tile.display_fail_reason && !tile.display_position) { + tilesMissingPosition.push(property); + } + }); + + if (tilesMissingPosition.length) { + // Determine if any available slots exist based on max number of tiles + // and the list of tiles already used and assign to a tile with missing + // value. + for (let i = 1; i <= this.sponsoredTilesConfigured; i++) { + if (!tilePositionsAssigned.includes(i)) { + let tileProperty = tilesMissingPosition.shift(); + this.allSponsoredTiles[tileProperty].display_position = i; + } + } + } + + // At this point we might still have a few unresolved states. These + // rows will be tagged with a display_fail_reason `unresolved`. + this._detectErrorConditionAndSetUnresolved(); + } + } + + // Checks the data for inconsistent state and updates the display_fail_reason + _detectErrorConditionAndSetUnresolved() { + Object.keys(this.allSponsoredTiles).forEach(property => { + let tile = this.allSponsoredTiles[property]; + if ( + (!tile.display_fail_reason && !tile.display_position) || + (tile.display_fail_reason && tile.display_position) + ) { + tile.display_position = null; + tile.display_fail_reason = DISPLAY_FAIL_REASON_UNRESOLVED; + } + }); + } + + finalizeNewtabPingFields(currentTiles) { + this._setTilePositions(currentTiles); + Glean.topsites.sponsoredTilesReceived.set( + JSON.stringify({ + sponsoredTilesReceived: Object.values(this.allSponsoredTiles), + }) + ); + } +} + +export class ContileIntegration { + constructor(topSitesFeed) { + this._topSitesFeed = topSitesFeed; + this._lastPeriodicUpdate = 0; + this._sites = []; + // The Share-of-Voice object managed by Shepherd and sent via Contile. + this._sov = null; + } + + get sites() { + return this._sites; + } + + get sov() { + return this._sov; + } + + periodicUpdate() { + let now = Date.now(); + if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) { + this._lastPeriodicUpdate = now; + this.refresh(); + } + } + + async refresh() { + let updateDefaultSites = await this._fetchSites(); + await this._topSitesFeed.allocatePositions(); + if (updateDefaultSites) { + this._topSitesFeed._readDefaults(); + } + } + + /** + * Clear Contile Cache Prefs. + */ + _resetContileCachePrefs() { + Services.prefs.clearUserPref(CONTILE_CACHE_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_LAST_FETCH_PREF); + Services.prefs.clearUserPref(CONTILE_CACHE_VALID_FOR_PREF); + } + + /** + * Filter the tiles whose sponsor is on the Top Sites sponsor blocklist. + * + * @param {array} tiles + * An array of the tile objects + */ + _filterBlockedSponsors(tiles) { + const blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + return tiles.filter(tile => !blocklist.includes(shortURL(tile))); + } + + /** + * Calculate the time Contile response is valid for based on cache-control header + * + * @param {string} cacheHeader + * string value of the Contile resposne cache-control header + */ + _extractCacheValidFor(cacheHeader) { + if (!cacheHeader) { + lazy.log.warn("Contile response cache control header is empty"); + return 0; + } + const [, staleIfError] = cacheHeader.match(/stale-if-error=\s*([0-9]+)/i); + const [, maxAge] = cacheHeader.match(/max-age=\s*([0-9]+)/i); + const validFor = + Number.parseInt(staleIfError, 10) + Number.parseInt(maxAge, 10); + return isNaN(validFor) ? 0 : validFor; + } + + /** + * Load Tiles from Contile Cache Prefs + */ + _loadTilesFromCache() { + lazy.log.info("Contile client is trying to load tiles from local cache."); + const now = Math.round(Date.now() / 1000); + const lastFetch = Services.prefs.getIntPref( + CONTILE_CACHE_LAST_FETCH_PREF, + 0 + ); + const validFor = Services.prefs.getIntPref(CONTILE_CACHE_VALID_FOR_PREF, 0); + this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured(); + if (now <= lastFetch + validFor) { + try { + let cachedTiles = JSON.parse( + Services.prefs.getStringPref(CONTILE_CACHE_PREF) + ); + this._topSitesFeed._telemetryUtility.setTiles(cachedTiles); + cachedTiles = this._filterBlockedSponsors(cachedTiles); + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed( + cachedTiles + ); + this._sites = cachedTiles; + lazy.log.info("Local cache loaded."); + return true; + } catch (error) { + lazy.log.warn(`Failed to load tiles from local cache: ${error}.`); + return false; + } + } + + return false; + } + + /** + * Determine number of Tiles to get from Contile + */ + _getMaxNumFromContile() { + return ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) ?? CONTILE_MAX_NUM_SPONSORED + ); + } + + async _fetchSites() { + if ( + !lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) || + !this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF] + ) { + if (this._sites.length) { + this._sites = []; + return true; + } + return false; + } + try { + let url = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF); + const response = await this._topSitesFeed.fetch(url, { + credentials: "omit", + }); + if (!response.ok) { + lazy.log.warn( + `Contile endpoint returned unexpected status: ${response.status}` + ); + if (response.status === 304 || response.status >= 500) { + return this._loadTilesFromCache(); + } + } + + const lastFetch = Math.round(Date.now() / 1000); + Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, lastFetch); + this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured(); + + // Contile returns 204 indicating there is no content at the moment. + // If this happens, it will clear `this._sites` reset the cached tiles + // to an empty array. + if (response.status === 204) { + this._topSitesFeed._telemetryUtility.clearTilesForProvider( + SPONSORED_TILE_PARTNER_AMP + ); + if (this._sites.length) { + this._sites = []; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + return true; + } + return false; + } + const body = await response.json(); + + if (body?.sov) { + this._sov = JSON.parse(atob(body.sov)); + } + if (body?.tiles && Array.isArray(body.tiles)) { + const useAdditionalTiles = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_ADDITIONAL_TILES + ); + + const maxNumFromContile = this._getMaxNumFromContile(); + + let { tiles } = body; + this._topSitesFeed._telemetryUtility.setTiles(tiles); + if ( + useAdditionalTiles !== undefined && + !useAdditionalTiles && + tiles.length > maxNumFromContile + ) { + tiles.length = maxNumFromContile; + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold( + tiles + ); + } + tiles = this._filterBlockedSponsors(tiles); + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed( + tiles + ); + if (tiles.length > maxNumFromContile) { + lazy.log.info("Remove unused links from Contile"); + tiles.length = maxNumFromContile; + this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold( + tiles + ); + } + this._sites = tiles; + Services.prefs.setStringPref( + CONTILE_CACHE_PREF, + JSON.stringify(this._sites) + ); + Services.prefs.setIntPref( + CONTILE_CACHE_VALID_FOR_PREF, + this._extractCacheValidFor( + response.headers.get("cache-control") || + response.headers.get("Cache-Control") + ) + ); + + return true; + } + } catch (error) { + lazy.log.warn( + `Failed to fetch data from Contile server: ${error.message}` + ); + return this._loadTilesFromCache(); + } + return false; + } +} + +export class TopSitesFeed { + constructor() { + this._telemetryUtility = new TopSitesTelemetry(); + this._contile = new ContileIntegration(this); + this._tippyTopProvider = new TippyTopProvider(); + ChromeUtils.defineLazyGetter( + this, + "_currentSearchHostname", + getShortURLForCurrentSearch + ); + this.dedupe = new Dedupe(this._dedupeKey); + this.frecentCache = new lazy.LinksCache( + lazy.NewTabUtils.activityStreamLinks, + "getTopSites", + CACHED_LINK_PROPS_TO_MIGRATE, + (oldOptions, newOptions) => + // Refresh if no old options or requesting more items + !(oldOptions.numItems >= newOptions.numItems) + ); + this.pinnedCache = new lazy.LinksCache( + lazy.NewTabUtils.pinnedLinks, + "links", + [...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE] + ); + lazy.PageThumbs.addExpirationFilter(this); + this._nimbusChangeListener = this._nimbusChangeListener.bind(this); + } + + _nimbusChangeListener(event, reason) { + // The Nimbus API current doesn't specify the changed variable(s) in the + // listener callback, so we have to refresh unconditionally on every change + // of the `newtab` feature. It should be a manageable overhead given the + // current update cadence (6 hours) of Nimbus. + // + // Skip the experiment and rollout loading reasons since this feature has + // `isEarlyStartup` enabled, the feature variables are already available + // before the experiment or rollout loads. + if ( + !["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason) + ) { + this._contile.refresh(); + } + } + + init() { + // If the feed was previously disabled PREFS_INITIAL_VALUES was never received + this._readDefaults({ isStartup: true }); + this._storage = this.store.dbStorage.getDbTable("sectionPrefs"); + this._contile.refresh(); + Services.obs.addObserver(this, "browser-search-engine-modified"); + Services.obs.addObserver(this, "browser-region-updated"); + Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener); + } + + uninit() { + lazy.PageThumbs.removeExpirationFilter(this); + Services.obs.removeObserver(this, "browser-search-engine-modified"); + Services.obs.removeObserver(this, "browser-region-updated"); + Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this); + Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this); + lazy.NimbusFeatures.newtab.offUpdate(this._nimbusChangeListener); + } + + observe(subj, topic, data) { + switch (topic) { + case "browser-search-engine-modified": + // We should update the current top sites if the search engine has been changed since + // the search engine that gets filtered out of top sites has changed. + // We also need to drop search shortcuts when their engine gets removed / hidden. + if ( + data === "engine-default" && + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] + ) { + delete this._currentSearchHostname; + this._currentSearchHostname = getShortURLForCurrentSearch(); + } + this.refresh({ broadcast: true }); + break; + case "browser-region-updated": + this._readDefaults(); + break; + case "nsPref:changed": + if ( + data === REMOTE_SETTING_DEFAULTS_PREF || + data === DEFAULT_SITES_OVERRIDE_PREF || + data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH) + ) { + this._readDefaults(); + } + break; + } + } + + _dedupeKey(site) { + return site && site.hostname; + } + + /** + * _readDefaults - sets DEFAULT_TOP_SITES + */ + async _readDefaults({ isStartup = false } = {}) { + this._useRemoteSetting = false; + + if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) { + this.refreshDefaults( + this.store.getState().Prefs.values[DEFAULT_SITES_PREF], + { isStartup } + ); + return; + } + + // Try using default top sites from enterprise policies or tests. The pref + // is locked when set via enterprise policy. Tests have no default sites + // unless they set them via this pref. + if ( + Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) || + Cu.isInAutomation + ) { + let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, ""); + this.refreshDefaults(sites, { isStartup }); + return; + } + + // Clear out the array of any previous defaults. + DEFAULT_TOP_SITES.length = 0; + + // Read defaults from contile. + const contileEnabled = lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ); + + // Keep the number of positions in the array in sync with CONTILE_MAX_NUM_SPONSORED. + // sponsored_position is a 1-based index, and contilePositions is a 0-based index, + // so we need to add 1 to each of these. + // Also currently this does not work with SOV. + let contilePositions = lazy.NimbusFeatures.pocketNewtab + .getVariable(NIMBUS_VARIABLE_CONTILE_POSITIONS) + ?.split(",") + .map(item => parseInt(item, 10) + 1) + .filter(item => !Number.isNaN(item)); + if (!contilePositions || contilePositions.length === 0) { + contilePositions = [1, 2]; + } + + let hasContileTiles = false; + if (contileEnabled) { + let contilePositionIndex = 0; + // We need to loop through potential spocs and set their positions. + // If we run out of spocs or positions, we stop. + // First, we need to know which array is shortest. This is our exit condition. + const minLength = Math.min( + contilePositions.length, + this._contile.sites.length + ); + // Loop until we run out of spocs or positions. + for (let i = 0; i < minLength; i++) { + let site = this._contile.sites[i]; + let hostname = shortURL(site); + let link = { + isDefault: true, + url: site.url, + hostname, + sendAttributionRequest: false, + label: site.name, + show_sponsored_label: hostname !== "yandex", + sponsored_position: contilePositions[contilePositionIndex++], + sponsored_click_url: site.click_url, + sponsored_impression_url: site.impression_url, + sponsored_tile_id: site.id, + partner: SPONSORED_TILE_PARTNER_AMP, + }; + if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) { + // Only use the image from Contile if it's hi-res, otherwise, fallback + // to the built-in favicons. + link.favicon = site.image_url; + link.faviconSize = site.image_size; + } + DEFAULT_TOP_SITES.push(link); + } + hasContileTiles = contilePositionIndex > 0; + //This is to catch where we receive 3 tiles but reduce to 2 early in the filtering, before blocked list applied. + this._telemetryUtility.determineFilteredTilesAndSetToOversold( + DEFAULT_TOP_SITES + ); + } + + // Read defaults from remote settings. + this._useRemoteSetting = true; + let remoteSettingData = await this._getRemoteConfig(); + + const sponsoredBlocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + + for (let siteData of remoteSettingData) { + let hostname = shortURL(siteData); + // Drop default sites when Contile already provided a sponsored one with + // the same host name. + if ( + contileEnabled && + DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1 + ) { + continue; + } + // Also drop those sponsored sites that were blocked by the user before + // with the same hostname. + if ( + siteData.sponsored_position && + sponsoredBlocklist.includes(hostname) + ) { + continue; + } + let link = { + isDefault: true, + url: siteData.url, + hostname, + sendAttributionRequest: !!siteData.send_attribution_request, + }; + if (siteData.url_urlbar_override) { + link.url_urlbar = siteData.url_urlbar_override; + } + if (siteData.title) { + link.label = siteData.title; + } + if (siteData.search_shortcut) { + link = await this.topSiteToSearchTopSite(link); + } else if (siteData.sponsored_position) { + if (contileEnabled && hasContileTiles) { + continue; + } + const { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + } = siteData; + link = { + sponsored_position, + sponsored_tile_id, + sponsored_impression_url, + sponsored_click_url, + show_sponsored_label: link.hostname !== "yandex", + ...link, + }; + } + DEFAULT_TOP_SITES.push(link); + } + + this.refresh({ broadcast: true, isStartup }); + } + + refreshDefaults(sites, { isStartup = false } = {}) { + // Clear out the array of any previous defaults + DEFAULT_TOP_SITES.length = 0; + + // Add default sites if any based on the pref + if (sites) { + for (const url of sites.split(",")) { + const site = { + isDefault: true, + url, + }; + site.hostname = shortURL(site); + DEFAULT_TOP_SITES.push(site); + } + } + + this.refresh({ broadcast: true, isStartup }); + } + + async _getRemoteConfig(firstTime = true) { + if (!this._remoteConfig) { + this._remoteConfig = await lazy.RemoteSettings("top-sites"); + this._remoteConfig.on("sync", () => { + this._readDefaults(); + }); + } + + let result = []; + let failed = false; + try { + result = await this._remoteConfig.get(); + } catch (ex) { + console.error(ex); + failed = true; + } + if (!result.length) { + console.error("Received empty top sites configuration!"); + failed = true; + } + // If we failed, or the result is empty, try loading from the local dump. + if (firstTime && failed) { + await this._remoteConfig.db.clear(); + // Now call this again. + return this._getRemoteConfig(false); + } + + // Sort sites based on the "order" attribute. + result.sort((a, b) => a.order - b.order); + + result = result.filter(topsite => { + // Filter by region. + if (topsite.exclude_regions?.includes(lazy.Region.home)) { + return false; + } + if ( + topsite.include_regions?.length && + !topsite.include_regions.includes(lazy.Region.home) + ) { + return false; + } + + // Filter by locale. + if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) { + return false; + } + if ( + topsite.include_locales?.length && + !topsite.include_locales.includes(Services.locale.appLocaleAsBCP47) + ) { + return false; + } + + // Filter by experiment. + // Exclude this top site if any of the specified experiments are running. + if ( + topsite.exclude_experiments?.some(experimentID => + Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + // Exclude this top site if none of the specified experiments are running. + if ( + topsite.include_experiments?.length && + topsite.include_experiments.every( + experimentID => + !Services.prefs.getBoolPref( + DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID, + false + ) + ) + ) { + return false; + } + + return true; + }); + + return result; + } + + filterForThumbnailExpiration(callback) { + const { rows } = this.store.getState().TopSites; + callback( + rows.reduce((acc, site) => { + acc.push(site.url); + if (site.customScreenshotURL) { + acc.push(site.customScreenshotURL); + } + return acc; + }, []) + ); + } + + /** + * shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine? + * + * @param {string} hostname a top site hostname, such as "amazon" or "foo" + * @returns {bool} + */ + shouldFilterSearchTile(hostname) { + if ( + this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] && + (SEARCH_FILTERS.includes(hostname) || + hostname === this._currentSearchHostname) + ) { + return true; + } + return false; + } + + /** + * _maybeInsertSearchShortcuts - if the search shortcuts experiment is running, + * insert search shortcuts if needed + * @param {Array} plainPinnedSites (from the pinnedSitesCache) + * @returns {Boolean} Did we insert any search shortcuts? + */ + async _maybeInsertSearchShortcuts(plainPinnedSites) { + // Only insert shortcuts if the experiment is running + if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + // We don't want to insert shortcuts we've previously inserted + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",") + .filter(s => s); // Filter out empty strings + const newInsertedShortcuts = []; + + let shouldPin = this._useRemoteSetting + ? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname) + : this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(","); + shouldPin = shouldPin + .map(getSearchProvider) + .filter(s => s && s.shortURL !== this._currentSearchHostname); + + // If we've previously inserted all search shortcuts return early + if ( + shouldPin.every(shortcut => + prevInsertedShortcuts.includes(shortcut.shortURL) + ) + ) { + return false; + } + + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + + // The plainPinnedSites array is populated with pinned sites at their + // respective indices, and null everywhere else, but is not always the + // right length + const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0); + const pinnedSites = [...plainPinnedSites].concat( + Array(emptySlots).fill(null) + ); + + const tryToInsertSearchShortcut = async shortcut => { + const nextAvailable = pinnedSites.indexOf(null); + // Only add a search shortcut if the site isn't already pinned, we + // haven't previously inserted it, there's space to pin it, and the + // search engine is available in Firefox + if ( + !pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) && + !prevInsertedShortcuts.includes(shortcut.shortURL) && + nextAvailable > -1 && + (await checkHasSearchEngine(shortcut.keyword)) + ) { + const site = await this.topSiteToSearchTopSite({ url: shortcut.url }); + this._pinSiteAt(site, nextAvailable); + pinnedSites[nextAvailable] = site; + newInsertedShortcuts.push(shortcut.shortURL); + } + }; + + for (let shortcut of shouldPin) { + await tryToInsertSearchShortcut(shortcut); + } + + if (newInsertedShortcuts.length) { + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.concat(newInsertedShortcuts).join(",") + ) + ); + return true; + } + } + + return false; + } + + /** + * This thin wrapper around global.fetch makes it easier for us to write + * automated tests that simulate responses from this fetch. + */ + fetch(...args) { + return fetch(...args); + } + + /** + * Fetch topsites spocs from the DiscoveryStream feed. + * + * @returns {Array} An array of sponsored tile objects. + */ + fetchDiscoveryStreamSpocs() { + let sponsored = []; + const { DiscoveryStream } = this.store.getState(); + if (DiscoveryStream) { + const discoveryStreamSpocs = + DiscoveryStream.spocs.data["sponsored-topsites"]?.items || []; + // Find the first component of a type and remove it from layout + const findSponsoredTopsitesPositions = name => { + for (const row of DiscoveryStream.layout) { + for (const component of row.components) { + if (component.placement?.name === name) { + return component.spocs.positions; + } + } + } + return null; + }; + + // Get positions from layout for now. This could be improved if we store position data in state. + const discoveryStreamSpocPositions = + findSponsoredTopsitesPositions("sponsored-topsites"); + + if (discoveryStreamSpocPositions?.length) { + function reformatImageURL(url, width, height) { + // Change the image URL to request a size tailored for the parent container width + // Also: force JPEG, quality 60, no upscaling, no EXIF data + // Uses Thumbor: https://thumbor.readthedocs.io/en/latest/usage.html + // For now we wrap this in single quotes because this is being used in a url() css rule, and otherwise would cause a parsing error. + return `'https://img-getpocket.cdn.mozilla.net/${width}x${height}/filters:format(jpeg):quality(60):no_upscale():strip_exif()/${encodeURIComponent( + url + )}'`; + } + + // We need to loop through potential spocs and set their positions. + // If we run out of spocs or positions, we stop. + // First, we need to know which array is shortest. This is our exit condition. + const minLength = Math.min( + discoveryStreamSpocPositions.length, + discoveryStreamSpocs.length + ); + // Loop until we run out of spocs or positions. + for (let i = 0; i < minLength; i++) { + const positionIndex = discoveryStreamSpocPositions[i].index; + const spoc = discoveryStreamSpocs[i]; + const link = { + favicon: reformatImageURL(spoc.raw_image_src, 96, 96), + faviconSize: 96, + type: "SPOC", + label: spoc.title || spoc.sponsor, + title: spoc.title || spoc.sponsor, + url: spoc.url, + flightId: spoc.flight_id, + id: spoc.id, + guid: spoc.id, + shim: spoc.shim, + // For now we are assuming position based on intended position. + // Actual position can shift based on other content. + // We send the intended position in the ping. + pos: positionIndex, + // Set this so that SPOC topsites won't be shown in the URL bar. + // See Bug 1822027. Note that `sponsored_position` is 1-based. + sponsored_position: positionIndex + 1, + // This is used for topsites deduping. + hostname: shortURL({ url: spoc.url }), + partner: SPONSORED_TILE_PARTNER_MOZ_SALES, + }; + sponsored.push(link); + } + } + } + return sponsored; + } + + // eslint-disable-next-line max-statements + async getLinksWithDefaults(isStartup = false) { + const prefValues = this.store.getState().Prefs.values; + const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW; + const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT]; + // We must wait for search services to initialize in order to access default + // search engine properties without triggering a synchronous initialization + try { + await Services.search.init(); + } catch { + // We continue anyway because we want the user to see their sponsored, + // saved, or visited shortcut tiles even if search engines are not + // available. + } + + // Get all frecent sites from history. + let frecent = []; + const cache = await this.frecentCache.request({ + // We need to overquery due to the top 5 alexa search + default search possibly being removed + numItems: numItems + SEARCH_FILTERS.length + 1, + topsiteFrecency: FRECENCY_THRESHOLD, + }); + for (let link of cache) { + const hostname = shortURL(link); + if (!this.shouldFilterSearchTile(hostname)) { + frecent.push({ + ...(searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link), + hostname, + }); + } + } + + // Get defaults. + let contileSponsored = []; + let notBlockedDefaultSites = []; + for (let link of DEFAULT_TOP_SITES) { + // For sponsored Yandex links, default filtering is reversed: we only + // show them if Yandex is the default search engine. + if (link.sponsored_position && link.hostname === "yandex") { + if (link.hostname !== this._currentSearchHostname) { + continue; + } + } else if (this.shouldFilterSearchTile(link.hostname)) { + continue; + } + // Drop blocked default sites. + if ( + lazy.NewTabUtils.blockedLinks.isBlocked({ + url: link.url, + }) + ) { + continue; + } + // If we've previously blocked a search shortcut, remove the default top site + // that matches the hostname + const searchProvider = getSearchProvider(shortURL(link)); + if ( + searchProvider && + lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url }) + ) { + continue; + } + if (link.sponsored_position) { + if (!prefValues[SHOW_SPONSORED_PREF]) { + continue; + } + contileSponsored[link.sponsored_position - 1] = link; + + // Unpin search shortcut if present for the sponsored link to be shown + // instead. + this._unpinSearchShortcut(link.hostname); + } else { + notBlockedDefaultSites.push( + searchShortcutsExperiment + ? await this.topSiteToSearchTopSite(link) + : link + ); + } + } + this._telemetryUtility.determineFilteredTilesAndSetToDismissed( + contileSponsored + ); + + const discoverySponsored = this.fetchDiscoveryStreamSpocs(); + this._telemetryUtility.setTiles(discoverySponsored); + + const sponsored = this._mergeSponsoredLinks({ + [SPONSORED_TILE_PARTNER_AMP]: contileSponsored, + [SPONSORED_TILE_PARTNER_MOZ_SALES]: discoverySponsored, + }); + + this._maybeCapSponsoredLinks(sponsored); + + // This will set all extra tiles to oversold, including moz-sales. + this._telemetryUtility.determineFilteredTilesAndSetToOversold(sponsored); + + // Get pinned links augmented with desired properties + let plainPinned = await this.pinnedCache.request(); + + // Insert search shortcuts if we need to. + // _maybeInsertSearchShortcuts returns true if any search shortcuts are + // inserted, meaning we need to expire and refresh the pinnedCache + if (await this._maybeInsertSearchShortcuts(plainPinned)) { + this.pinnedCache.expire(); + plainPinned = await this.pinnedCache.request(); + } + + const pinned = await Promise.all( + plainPinned.map(async link => { + if (!link) { + return link; + } + + // Drop pinned search shortcuts when their engine has been removed / hidden. + if (link.searchTopSite) { + const searchProvider = getSearchProvider(shortURL(link)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return null; + } + } + + // Copy all properties from a frecent link and add more + const finder = other => other.url === link.url; + + // Remove frecent link's screenshot if pinned link has a custom one + const frecentSite = frecent.find(finder); + if (frecentSite && link.customScreenshotURL) { + delete frecentSite.screenshot; + } + // If the link is a frecent site, do not copy over 'isDefault', else check + // if the site is a default site + const copy = Object.assign( + {}, + frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) }, + link, + { hostname: shortURL(link) }, + { searchTopSite: !!link.searchTopSite } + ); + + // Add in favicons if we don't already have it + if (!copy.favicon) { + try { + lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI( + await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy]) + ); + + for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) { + copy.__sharedCache.updateLink(prop, copy[prop]); + } + } catch (e) { + // Some issue with favicon, so just continue without one + } + } + + return copy; + }) + ); + + // Remove any duplicates from frecent and default sites + const [, dedupedSponsored, dedupedFrecent, dedupedDefaults] = + this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites); + const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults]; + + // Remove adult sites if we need to + const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned); + + // Insert the original pinned sites into the deduped frecent and defaults. + let withPinned = insertPinned(checkedAdult, pinned); + // Insert sponsored sites at their desired position. + dedupedSponsored.forEach(link => { + if (!link) { + return; + } + let index = link.sponsored_position - 1; + if (index >= withPinned.length) { + withPinned[index] = link; + } else if (withPinned[index]?.sponsored_position) { + // We currently want DiscoveryStream spocs to replace existing spocs. + withPinned[index] = link; + } else { + withPinned.splice(index, 0, link); + } + }); + // Remove excess items after we inserted sponsored ones. + withPinned = withPinned.slice(0, numItems); + + // Now, get a tippy top icon, a rich icon, or screenshot for every item + for (const link of withPinned) { + if (link) { + // If there is a custom screenshot this is the only image we display + if (link.customScreenshotURL) { + this._fetchScreenshot(link, link.customScreenshotURL, isStartup); + } else if (link.searchTopSite && !link.isDefault) { + await this._attachTippyTopIconForSearchShortcut(link, link.label); + } else { + this._fetchIcon(link, isStartup); + } + + // Remove internal properties that might be updated after dispatch + delete link.__sharedCache; + + // Indicate that these links should get a frecency bonus when clicked + link.typedBonus = true; + } + } + + this._linksWithDefaults = withPinned; + + this._telemetryUtility.finalizeNewtabPingFields(dedupedSponsored); + return withPinned; + } + + /** + * Cap sponsored links if they're more than the specified maximum. + * + * @param {Array} links An array of sponsored links. Capping will be performed in-place. + */ + _maybeCapSponsoredLinks(links) { + // Set maximum sponsored top sites + const maxSponsored = + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_MAX_SPONSORED + ) ?? MAX_NUM_SPONSORED; + if (links.length > maxSponsored) { + links.length = maxSponsored; + } + } + + /** + * Merge sponsored links from all the partners using SOV if present. + * For each tile position, the user is assigned to one partner via stable sampling. + * If the chosen partner doesn't have a tile to serve, another tile from a different + * partner is used as the replacement. + * + * @param {Object} sponsoredLinks An object with sponsored links from all the partners. + * @returns {Array} An array of merged sponsored links. + */ + _mergeSponsoredLinks(sponsoredLinks) { + const { positions: allocatedPositions, ready: sovReady } = + this.store.getState().TopSites.sov || {}; + if ( + !this._contile.sov || + !sovReady || + !lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_SOV_ENABLED + ) + ) { + return Object.values(sponsoredLinks).flat(); + } + + // AMP links might have empty slots, remove them as SOV doesn't need those. + sponsoredLinks[SPONSORED_TILE_PARTNER_AMP] = + sponsoredLinks[SPONSORED_TILE_PARTNER_AMP].filter(Boolean); + + let sponsored = []; + let chosenPartners = []; + + for (const allocation of allocatedPositions) { + let link = null; + const { assignedPartner } = allocation; + if (assignedPartner) { + // Unknown partners are allowed so that new parters can be added to Shepherd + // sooner without waiting for client changes. + link = sponsoredLinks[assignedPartner]?.shift(); + } + + if (!link) { + // If the chosen partner doesn't have a tile for this postion, choose any + // one from another group. For simplicity, we do _not_ do resampling here + // against the remaining partners. + for (const partner of SPONSORED_TILE_PARTNERS) { + if ( + partner === assignedPartner || + sponsoredLinks[partner].length === 0 + ) { + continue; + } + link = sponsoredLinks[partner].shift(); + break; + } + + if (!link) { + // No more links to be added across all the partners, just return. + if (chosenPartners.length) { + Glean.newtab.sovAllocation.set( + chosenPartners.map(entry => JSON.stringify(entry)) + ); + } + return sponsored; + } + } + + // Update the position fields. Note that postion is also 1-based in SOV. + link.sponsored_position = allocation.position; + if (link.pos !== undefined) { + // Pocket `pos` is 0-based. + link.pos = allocation.position - 1; + } + sponsored.push(link); + + chosenPartners.push({ + pos: allocation.position, + assigned: assignedPartner, // The assigned partner based on SOV + chosen: link.partner, + }); + } + // Record chosen partners to glean + if (chosenPartners.length) { + Glean.newtab.sovAllocation.set( + chosenPartners.map(entry => JSON.stringify(entry)) + ); + } + + // add the remaining contile sponsoredLinks when nimbus variable present + if ( + lazy.NimbusFeatures.pocketNewtab.getVariable( + NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED + ) + ) { + return sponsored.concat(sponsoredLinks[SPONSORED_TILE_PARTNER_AMP]); + } + + return sponsored; + } + + /** + * Attach TippyTop icon to the given search shortcut + * + * Note that it queries the search form URL from search service For Yandex, + * and uses it to choose the best icon for its shortcut variants. + * + * @param {Object} link A link object with a `url` property + * @param {string} keyword Search keyword + */ + async _attachTippyTopIconForSearchShortcut(link, keyword) { + if ( + ["@\u044F\u043D\u0434\u0435\u043A\u0441", "@yandex"].includes(keyword) + ) { + let site = { url: link.url }; + site.url = (await getSearchFormURL(keyword)) || site.url; + this._tippyTopProvider.processSite(site); + link.tippyTopIcon = site.tippyTopIcon; + link.smallFavicon = site.smallFavicon; + link.backgroundColor = site.backgroundColor; + } else { + this._tippyTopProvider.processSite(link); + } + } + + /** + * Refresh the top sites data for content. + * @param {bool} options.broadcast Should the update be broadcasted. + * @param {bool} options.isStartup Being called while TopSitesFeed is initting. + */ + async refresh(options = {}) { + if (!this._startedUp && !options.isStartup) { + // Initial refresh still pending. + return; + } + this._startedUp = true; + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + const links = await this.getLinksWithDefaults({ + isStartup: options.isStartup, + }); + const newAction = { type: at.TOP_SITES_UPDATED, data: { links } }; + let storedPrefs; + try { + storedPrefs = (await this._storage.get(SECTION_ID)) || {}; + } catch (e) { + storedPrefs = {}; + console.error("Problem getting stored prefs for TopSites"); + } + newAction.data.pref = getDefaultOptions(storedPrefs); + + if (options.isStartup) { + newAction.meta = { + isStartup: true, + }; + } + + if (options.broadcast) { + // Broadcast an update to all open content pages + this.store.dispatch(ac.BroadcastToContent(newAction)); + } else { + // Don't broadcast only update the state and update the preloaded tab. + this.store.dispatch(ac.AlsoToPreloaded(newAction)); + } + } + + // Allocate ad positions to partners based on SOV via stable randomization. + async allocatePositions() { + // If the fetch to get sov fails for whatever reason, we can just return here. + // Code that uses this falls back to flattening allocations instead if this has failed. + if (!this._contile.sov) { + return; + } + // This sample input should ensure we return the same result for this allocation, + // even if called from other parts of the code. + const sampleInput = `${lazy.contextId}-${this._contile.sov.name}`; + const allocatedPositions = []; + for (const allocation of this._contile.sov.allocations) { + const allocatedPosition = { + position: allocation.position, + }; + allocatedPositions.push(allocatedPosition); + const ratios = allocation.allocation.map(alloc => alloc.percentage); + if (ratios.length) { + const index = await lazy.Sampling.ratioSample(sampleInput, ratios); + allocatedPosition.assignedPartner = + allocation.allocation[index].partner; + } + } + + this.store.dispatch( + ac.OnlyToMain({ + type: at.SOV_UPDATED, + data: { + ready: !!allocatedPositions.length, + positions: allocatedPositions, + }, + }) + ); + } + + async updateCustomSearchShortcuts(isStartup = false) { + if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) { + return; + } + + if (!this._tippyTopProvider.initialized) { + await this._tippyTopProvider.init(); + } + + // Populate the state with available search shortcuts + let searchShortcuts = []; + for (const engine of await Services.search.getAppProvidedEngines()) { + const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s => + engine.aliases.includes(s.keyword) + ); + if (shortcut) { + let clone = { ...shortcut }; + await this._attachTippyTopIconForSearchShortcut(clone, clone.keyword); + searchShortcuts.push(clone); + } + } + + this.store.dispatch( + ac.BroadcastToContent({ + type: at.UPDATE_SEARCH_SHORTCUTS, + data: { searchShortcuts }, + meta: { + isStartup, + }, + }) + ); + } + + async topSiteToSearchTopSite(site) { + const searchProvider = getSearchProvider(shortURL(site)); + if ( + !searchProvider || + !(await checkHasSearchEngine(searchProvider.keyword)) + ) { + return site; + } + return { + ...site, + searchTopSite: true, + label: searchProvider.keyword, + }; + } + + /** + * Get an image for the link preferring tippy top, rich favicon, screenshots. + */ + async _fetchIcon(link, isStartup = false) { + // Nothing to do if we already have a rich icon from the page + if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) { + return; + } + + // Nothing more to do if we can use a default tippy top icon + this._tippyTopProvider.processSite(link); + if (link.tippyTopIcon) { + return; + } + + // Make a request for a better icon + this._requestRichIcon(link.url); + + // Also request a screenshot if we don't have one yet + await this._fetchScreenshot(link, link.url, isStartup); + } + + /** + * Fetch, cache and broadcast a screenshot for a specific topsite. + * @param link cached topsite object + * @param url where to fetch the image from + * @param isStartup Whether the screenshot is fetched while initting TopSitesFeed. + */ + async _fetchScreenshot(link, url, isStartup = false) { + // We shouldn't bother caching screenshots if they won't be shown. + if ( + link.screenshot || + !this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF] + ) { + return; + } + await lazy.Screenshots.maybeCacheScreenshot( + link, + url, + "screenshot", + screenshot => + this.store.dispatch( + ac.BroadcastToContent({ + data: { screenshot, url: link.url }, + type: at.SCREENSHOT_UPDATED, + meta: { + isStartup, + }, + }) + ) + ); + } + + /** + * Dispatch screenshot preview to target or notify if request failed. + * @param customScreenshotURL {string} The URL used to capture the screenshot + * @param target {string} Id of content process where to dispatch the result + */ + async getScreenshotPreview(url, target) { + const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || ""; + this.store.dispatch( + ac.OnlyToOneContent( + { + data: { url, preview }, + type: at.PREVIEW_RESPONSE, + }, + target + ) + ); + } + + _requestRichIcon(url) { + this.store.dispatch({ + type: at.RICH_ICON_MISSING, + data: { url }, + }); + } + + updateSectionPrefs(collapsed) { + this.store.dispatch( + ac.BroadcastToContent({ + type: at.TOP_SITES_PREFS_UPDATED, + data: { pref: collapsed }, + }) + ); + } + + /** + * Inform others that top sites data has been updated due to pinned changes. + */ + _broadcastPinnedSitesUpdated() { + // Pinned data changed, so make sure we get latest + this.pinnedCache.expire(); + + // Refresh to update pinned sites with screenshots, trigger deduping, etc. + this.refresh({ broadcast: true }); + } + + /** + * Pin a site at a specific position saving only the desired keys. + * @param customScreenshotURL {string} User set URL of preview image for site + * @param label {string} User set string of custom site name + */ + async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) { + const toPin = { url }; + if (label) { + toPin.label = label; + } + if (customScreenshotURL) { + toPin.customScreenshotURL = customScreenshotURL; + } + if (searchTopSite) { + toPin.searchTopSite = searchTopSite; + } + lazy.NewTabUtils.pinnedLinks.pin(toPin, index); + + await this._clearLinkCustomScreenshot({ customScreenshotURL, url }); + } + + async _clearLinkCustomScreenshot(site) { + // If screenshot url changed or was removed we need to update the cached link obj + if (site.customScreenshotURL !== undefined) { + const pinned = await this.pinnedCache.request(); + const link = pinned.find(pin => pin && pin.url === site.url); + if (link && link.customScreenshotURL !== site.customScreenshotURL) { + link.__sharedCache.updateLink("screenshot", undefined); + } + } + } + + /** + * Handle a pin action of a site to a position. + */ + async pin(action) { + let { site, index } = action.data; + index = this._adjustPinIndexForSponsoredLinks(site, index); + // If valid index provided, pin at that position + if (index >= 0) { + await this._pinSiteAt(site, index); + this._broadcastPinnedSitesUpdated(); + } else { + // Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option, + // then we want to make sure to unblock that link if it has previously been + // blocked. We know if the site has been added because the index will be -1. + if (index === -1) { + lazy.NewTabUtils.blockedLinks.unblock({ url: site.url }); + this.frecentCache.expire(); + } + this.insert(action); + } + } + + /** + * Handle an unpin action of a site. + */ + unpin(action) { + const { site } = action.data; + lazy.NewTabUtils.pinnedLinks.unpin(site); + this._broadcastPinnedSitesUpdated(); + } + + unpinAllSearchShortcuts() { + Services.prefs.clearUserPref( + `browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}` + ); + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if (pinnedLink && pinnedLink.searchTopSite) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + } + } + this.pinnedCache.expire(); + } + + _unpinSearchShortcut(vendor) { + for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) { + if ( + pinnedLink && + pinnedLink.searchTopSite && + shortURL(pinnedLink) === vendor + ) { + lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink); + this.pinnedCache.expire(); + + const prevInsertedShortcuts = this.store + .getState() + .Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(","); + this.store.dispatch( + ac.SetPref( + SEARCH_SHORTCUTS_HAVE_PINNED_PREF, + prevInsertedShortcuts.filter(s => s !== vendor).join(",") + ) + ); + break; + } + } + } + + /** + * Reduces the given pinning index by the number of preceding sponsored + * sites, to accomodate for sponsored sites pushing pinned ones to the side, + * effectively increasing their index again. + */ + _adjustPinIndexForSponsoredLinks(site, index) { + if (!this._linksWithDefaults) { + return index; + } + // Adjust insertion index for sponsored sites since their position is + // fixed. + let adjustedIndex = index; + for (let i = 0; i < index; i++) { + const link = this._linksWithDefaults[i]; + if ( + link && + link.sponsored_position && + this._linksWithDefaults[i]?.url !== site.url + ) { + adjustedIndex--; + } + } + return adjustedIndex; + } + + /** + * Insert a site to pin at a position shifting over any other pinned sites. + */ + _insertPin(site, originalIndex, draggedFromIndex) { + let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex); + + // Don't insert any pins past the end of the visible top sites. Otherwise, + // we can end up with a bunch of pinned sites that can never be unpinned again + // from the UI. + const topSitesCount = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + if (index >= topSitesCount) { + return; + } + + let pinned = lazy.NewTabUtils.pinnedLinks.links; + if (!pinned[index]) { + this._pinSiteAt(site, index); + } else { + pinned[draggedFromIndex] = null; + // Find the hole to shift the pinned site(s) towards. We shift towards the + // hole left by the site being dragged. + let holeIndex = index; + const indexStep = index > draggedFromIndex ? -1 : 1; + while (pinned[holeIndex]) { + holeIndex += indexStep; + } + if (holeIndex >= topSitesCount || holeIndex < 0) { + // There are no holes, so we will effectively unpin the last slot and shifting + // towards it. This only happens when adding a new top site to an already + // fully pinned grid. + holeIndex = topSitesCount - 1; + } + + // Shift towards the hole. + const shiftingStep = holeIndex > index ? -1 : 1; + while (holeIndex !== index) { + const nextIndex = holeIndex + shiftingStep; + this._pinSiteAt(pinned[nextIndex], holeIndex); + holeIndex = nextIndex; + } + this._pinSiteAt(site, index); + } + } + + /** + * Handle an insert (drop/add) action of a site. + */ + async insert(action) { + let { index } = action.data; + // Treat invalid pin index values (e.g., -1, undefined) as insert in the first position + if (!(index > 0)) { + index = 0; + } + + // Inserting a top site pins it in the specified slot, pushing over any link already + // pinned in the slot (unless it's the last slot, then it replaces). + this._insertPin( + action.data.site, + index, + action.data.draggedFromIndex !== undefined + ? action.data.draggedFromIndex + : this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW + ); + + await this._clearLinkCustomScreenshot(action.data.site); + this._broadcastPinnedSitesUpdated(); + } + + updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) { + // Unpin the deletedShortcuts. + deletedShortcuts.forEach(({ url }) => { + lazy.NewTabUtils.pinnedLinks.unpin({ url }); + }); + + // Pin the addedShortcuts. + const numberOfSlots = + this.store.getState().Prefs.values[ROWS_PREF] * + TOP_SITES_MAX_SITES_PER_ROW; + addedShortcuts.forEach(shortcut => { + // Find first hole in pinnedLinks. + let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link); + if ( + index < 0 && + lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots + ) { + // pinnedLinks can have less slots than the total available. + index = lazy.NewTabUtils.pinnedLinks.links.length; + } + if (index >= 0) { + lazy.NewTabUtils.pinnedLinks.pin(shortcut, index); + } else { + // No slots available, we need to do an insert in first slot and push over other pinned links. + this._insertPin(shortcut, 0, numberOfSlots); + } + }); + + this._broadcastPinnedSitesUpdated(); + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.updateCustomSearchShortcuts(true /* isStartup */); + break; + case at.SYSTEM_TICK: + this.refresh({ broadcast: false }); + this._contile.periodicUpdate(); + break; + // All these actions mean we need new top sites + case at.PLACES_HISTORY_CLEARED: + case at.PLACES_LINKS_DELETED: + this.frecentCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PLACES_LINKS_CHANGED: + this.frecentCache.expire(); + this.refresh({ broadcast: false }); + break; + case at.PLACES_LINK_BLOCKED: + this.frecentCache.expire(); + this.pinnedCache.expire(); + this.refresh({ broadcast: true }); + break; + case at.PREF_CHANGED: + switch (action.data.name) { + case DEFAULT_SITES_PREF: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data.value); + } + break; + case ROWS_PREF: + case FILTER_DEFAULT_SEARCH_PREF: + case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF: + this.refresh({ broadcast: true }); + break; + case SHOW_SPONSORED_PREF: + if ( + lazy.NimbusFeatures.newtab.getVariable( + NIMBUS_VARIABLE_CONTILE_ENABLED + ) + ) { + this._contile.refresh(); + } else { + this.refresh({ broadcast: true }); + } + if (!action.data.value) { + this._contile._resetContileCachePrefs(); + } + + break; + case SEARCH_SHORTCUTS_EXPERIMENT: + if (action.data.value) { + this.updateCustomSearchShortcuts(); + } else { + this.unpinAllSearchShortcuts(); + } + this.refresh({ broadcast: true }); + } + break; + case at.UPDATE_SECTION_PREFS: + if (action.data.id === SECTION_ID) { + this.updateSectionPrefs(action.data.value); + } + break; + case at.PREFS_INITIAL_VALUES: + if (!this._useRemoteSetting) { + this.refreshDefaults(action.data[DEFAULT_SITES_PREF]); + } + break; + case at.TOP_SITES_PIN: + this.pin(action); + break; + case at.TOP_SITES_UNPIN: + this.unpin(action); + break; + case at.TOP_SITES_INSERT: + this.insert(action); + break; + case at.PREVIEW_REQUEST: + this.getScreenshotPreview(action.data.url, action.meta.fromTarget); + break; + case at.UPDATE_PINNED_SEARCH_SHORTCUTS: + this.updatePinnedSearchShortcuts(action.data); + break; + case at.DISCOVERY_STREAM_SPOCS_UPDATE: + // Refresh to update sponsored topsites. + this.refresh({ broadcast: true, isStartup: action.meta.isStartup }); + break; + case at.UNINIT: + this.uninit(); + break; + } + } +} diff --git a/browser/components/newtab/lib/TopStoriesFeed.sys.mjs b/browser/components/newtab/lib/TopStoriesFeed.sys.mjs new file mode 100644 index 0000000000..be030649dd --- /dev/null +++ b/browser/components/newtab/lib/TopStoriesFeed.sys.mjs @@ -0,0 +1,731 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + actionTypes as at, + actionCreators as ac, +} from "resource://activity-stream/common/Actions.sys.mjs"; +import { Prefs } from "resource://activity-stream/lib/ActivityStreamPrefs.sys.mjs"; +import { shortURL } from "resource://activity-stream/lib/ShortURL.sys.mjs"; +import { SectionsManager } from "resource://activity-stream/lib/SectionsManager.sys.mjs"; +import { PersistentCache } from "resource://activity-stream/lib/PersistentCache.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); + +export const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +export const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours +const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours +export const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour +export const SECTION_ID = "topstories"; +const IMPRESSION_SOURCE = "TOP_STORIES"; + +export const SPOC_IMPRESSION_TRACKING_PREF = + "feeds.section.topstories.spoc.impressions"; + +const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled"; +const DISCOVERY_STREAM_PREF_ENABLED_PATH = + "browser.newtabpage.activity-stream.discoverystream.enabled"; +export const REC_IMPRESSION_TRACKING_PREF = + "feeds.section.topstories.rec.impressions"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server +const DISCOVERY_STREAM_PREF = "discoverystream.config"; + +export class TopStoriesFeed { + constructor(ds) { + // Use discoverystream config pref default values for fast path and + // if needed lazy load activity stream top stories feed based on + // actual user preference when INIT and PREF_CHANGED is invoked + this.discoveryStreamEnabled = + ds && + ds.value && + JSON.parse(ds.value).enabled && + Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false); + if (!this.discoveryStreamEnabled) { + this.initializeProperties(); + } + } + + initializeProperties() { + this.contentUpdateQueue = []; + this.spocCampaignMap = new Map(); + this.cache = new PersistentCache(SECTION_ID, true); + this._prefs = new Prefs(); + this.propertiesInitialized = true; + } + + async onInit() { + SectionsManager.enableSection(SECTION_ID, true /* isStartup */); + if (this.discoveryStreamEnabled) { + return; + } + + try { + const { options } = SectionsManager.sections.get(SECTION_ID); + const apiKey = this.getApiKeyFromPref(options.api_key_pref); + this.stories_endpoint = this.produceFinalEndpointUrl( + options.stories_endpoint, + apiKey + ); + this.topics_endpoint = this.produceFinalEndpointUrl( + options.topics_endpoint, + apiKey + ); + this.read_more_endpoint = options.read_more_endpoint; + this.stories_referrer = options.stories_referrer; + this.show_spocs = options.show_spocs; + this.storiesLastUpdated = 0; + this.topicsLastUpdated = 0; + this.storiesLoaded = false; + this.dispatchPocketCta(this._prefs.get("pocketCta"), false); + + // Cache is used for new page loads, which shouldn't have changed data. + // If we have changed data, cache should be cleared, + // and last updated should be 0, and we can fetch. + let { stories, topics } = await this.loadCachedData(); + if (this.storiesLastUpdated === 0) { + stories = await this.fetchStories(); + } + if (this.topicsLastUpdated === 0) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, true); + this.storiesLoaded = true; + + // This is filtered so an update function can return true to retry on the next run + this.contentUpdateQueue = this.contentUpdateQueue.filter(update => + update() + ); + } catch (e) { + console.error(`Problem initializing top stories feed: ${e.message}`); + } + } + + init() { + SectionsManager.onceInitialized(this.onInit.bind(this)); + } + + async clearCache() { + await this.cache.set("stories", {}); + await this.cache.set("topics", {}); + await this.cache.set("spocs", {}); + } + + uninit() { + this.storiesLoaded = false; + SectionsManager.disableSection(SECTION_ID); + } + + getPocketState(target) { + const action = { + type: at.POCKET_LOGGED_IN, + data: lazy.pktApi.isUserLoggedIn(), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + dispatchPocketCta(data, shouldBroadcast) { + const action = { type: at.POCKET_CTA, data: JSON.parse(data) }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + + /** + * doContentUpdate - Updates topics and stories in the topstories section. + * + * Sections have one update action for the whole section. + * Redux creates a state race condition if you call the same action, + * twice, concurrently. Because of this, doContentUpdate is + * one place to update both topics and stories in a single action. + * + * Section updates used old topics if none are available, + * but clear stories if none are available. Because of this, if no + * stories are passed, we instead use the existing stories in state. + * + * @param {Object} This is an object with potential new stories or topics. + * @param {Boolean} shouldBroadcast If we should update existing tabs or not. For first page + * loads or pref changes, we want to update existing tabs, + * for system tick or other updates we do not. + */ + doContentUpdate({ stories, topics }, shouldBroadcast) { + let updateProps = {}; + if (stories) { + updateProps.rows = stories; + } else { + const { Sections } = this.store.getState(); + if (Sections && Sections.find) { + updateProps.rows = Sections.find(s => s.id === SECTION_ID).rows; + } + } + if (topics) { + Object.assign(updateProps, { + topics, + read_more_endpoint: this.read_more_endpoint, + }); + } + + // We should only be calling this once per init. + this.dispatchUpdateEvent(shouldBroadcast, updateProps); + } + + async fetchStories() { + if (!this.stories_endpoint) { + return null; + } + try { + const response = await fetch(this.stories_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Stories endpoint returned unexpected status: ${response.status}` + ); + } + + const body = await response.json(); + this.updateSettings(body.settings); + this.stories = this.rotate(this.transform(body.recommendations)); + this.cleanUpTopRecImpressionPref(); + + if (this.show_spocs && body.spocs) { + this.spocCampaignMap = new Map( + body.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(body.spocs); + this.cleanUpCampaignImpressionPref(); + } + this.storiesLastUpdated = Date.now(); + body._timestamp = this.storiesLastUpdated; + this.cache.set("stories", body); + } catch (error) { + console.error(`Failed to fetch content: ${error.message}`); + } + return this.stories; + } + + async loadCachedData() { + const data = await this.cache.get(); + let stories = data.stories && data.stories.recommendations; + let topics = data.topics && data.topics.topics; + + if (stories && !!stories.length && this.storiesLastUpdated === 0) { + this.updateSettings(data.stories.settings); + this.stories = this.rotate(this.transform(stories)); + this.storiesLastUpdated = data.stories._timestamp; + if (data.stories.spocs && data.stories.spocs.length) { + this.spocCampaignMap = new Map( + data.stories.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(data.stories.spocs); + this.cleanUpCampaignImpressionPref(); + } + } + if (topics && !!topics.length && this.topicsLastUpdated === 0) { + this.topics = topics; + this.topicsLastUpdated = data.topics._timestamp; + } + + return { topics: this.topics, stories: this.stories }; + } + + transform(items) { + if (!items) { + return []; + } + + const calcResult = items + .filter(s => !lazy.NewTabUtils.blockedLinks.isBlocked({ url: s.url })) + .map(s => { + let mapped = { + guid: s.id, + hostname: s.domain || shortURL(Object.assign({}, s, { url: s.url })), + type: + Date.now() - s.published_timestamp * 1000 <= STORIES_NOW_THRESHOLD + ? "now" + : "trending", + context: s.context, + icon: s.icon, + title: s.title, + description: s.excerpt, + image: this.normalizeUrl(s.image_src), + referrer: this.stories_referrer, + url: s.url, + score: s.item_score || 1, + spoc_meta: this.show_spocs + ? { campaign_id: s.campaign_id, caps: s.caps } + : {}, + }; + + // Very old cached spocs may not contain an `expiration_timestamp` property + if (s.expiration_timestamp) { + mapped.expiration_timestamp = s.expiration_timestamp; + } + + return mapped; + }) + .sort(this.compareScore); + + return calcResult; + } + + async fetchTopics() { + if (!this.topics_endpoint) { + return null; + } + try { + const response = await fetch(this.topics_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Topics endpoint returned unexpected status: ${response.status}` + ); + } + const body = await response.json(); + const { topics } = body; + if (topics) { + this.topics = topics; + this.topicsLastUpdated = Date.now(); + body._timestamp = this.topicsLastUpdated; + this.cache.set("topics", body); + } + } catch (error) { + console.error(`Failed to fetch topics: ${error.message}`); + } + return this.topics; + } + + dispatchUpdateEvent(shouldBroadcast, data) { + SectionsManager.updateSection(SECTION_ID, data, shouldBroadcast); + } + + compareScore(a, b) { + return b.score - a.score; + } + + updateSettings(settings = {}) { + this.spocsPerNewTabs = settings.spocsPerNewTabs || 1; // Probability of a new tab getting a spoc [0,1] + this.recsExpireTime = settings.recsExpireTime; + } + + // We rotate stories on the client so that + // active stories are at the front of the list, followed by stories that have expired + // impressions i.e. have been displayed for longer than recsExpireTime. + rotate(items) { + if (items.length <= 3) { + return items; + } + + const maxImpressionAge = Math.max( + this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, + DEFAULT_RECS_EXPIRE_TIME + ); + const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + const expired = []; + const active = []; + for (const item of items) { + if ( + impressions[item.guid] && + Date.now() - impressions[item.guid] >= maxImpressionAge + ) { + expired.push(item); + } else { + active.push(item); + } + } + return active.concat(expired); + } + + getApiKeyFromPref(apiKeyPref) { + if (!apiKeyPref) { + return apiKeyPref; + } + + return ( + this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref) + ); + } + + produceFinalEndpointUrl(url, apiKey) { + if (!url) { + return url; + } + if (url.includes("$apiKey") && !apiKey) { + throw new Error(`An API key was specified but none configured: ${url}`); + } + return url.replace("$apiKey", apiKey); + } + + // Need to remove parenthesis from image URLs as React will otherwise + // fail to render them properly as part of the card template. + normalizeUrl(url) { + if (url) { + return url.replace(/\(/g, "%28").replace(/\)/g, "%29"); + } + return url; + } + + shouldShowSpocs() { + return this.show_spocs && this.store.getState().Prefs.values.showSponsored; + } + + dispatchSpocDone(target) { + const action = { type: at.POCKET_WAITING_FOR_SPOC, data: false }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + filterSpocs() { + if (!this.shouldShowSpocs()) { + return []; + } + + if (Math.random() > this.spocsPerNewTabs) { + return []; + } + + if (!this.spocs || !this.spocs.length) { + // We have stories but no spocs so there's nothing to do and this update can be + // removed from the queue. + return []; + } + + // Filter spocs based on frequency caps + const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + let spocs = this.spocs.filter(s => + this.isBelowFrequencyCap(impressions, s) + ); + + // Filter out expired spocs based on `expiration_timestamp` + spocs = spocs.filter(spoc => { + // If cached data is so old it doesn't contain this property, assume the spoc is ok to show + if (!(`expiration_timestamp` in spoc)) { + return true; + } + // `expiration_timestamp` is the number of seconds elapsed since January 1, 1970 00:00:00 UTC + return spoc.expiration_timestamp * 1000 > Date.now(); + }); + + return spocs; + } + + maybeAddSpoc(target) { + const updateContent = () => { + let spocs = this.filterSpocs(); + + if (!spocs.length) { + this.dispatchSpocDone(target); + return false; + } + + // Create a new array with a spoc inserted at index 2 + const section = this.store + .getState() + .Sections.find(s => s.id === SECTION_ID); + let rows = section.rows.slice(0, this.stories.length); + rows.splice(2, 0, Object.assign(spocs[0], { pinned: true })); + + // Send a content update to the target tab + const action = { + type: at.SECTION_UPDATE, + data: Object.assign({ rows }, { id: SECTION_ID }), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + this.dispatchSpocDone(target); + return false; + }; + + if (this.storiesLoaded) { + updateContent(); + } else { + // Delay updating tab content until initial data has been fetched + this.contentUpdateQueue.push(updateContent); + } + } + + // Frequency caps are based on campaigns, which may include multiple spocs. + // We currently support two types of frequency caps: + // - lifetime: Indicates how many times spocs from a campaign can be shown in total + // - period: Indicates how many times spocs from a campaign can be shown within a period + // + // So, for example, the feed configuration below defines that for campaign 1 no more + // than 5 spocs can be show in total, and no more than 2 per hour. + // "campaign_id": 1, + // "caps": { + // "lifetime": 5, + // "campaign": { + // "count": 2, + // "period": 3600 + // } + // } + isBelowFrequencyCap(impressions, spoc) { + const campaignImpressions = impressions[spoc.spoc_meta.campaign_id]; + if (!campaignImpressions) { + return true; + } + + const lifeTimeCap = Math.min( + spoc.spoc_meta.caps && spoc.spoc_meta.caps.lifetime, + MAX_LIFETIME_CAP + ); + const lifeTimeCapExceeded = campaignImpressions.length >= lifeTimeCap; + if (lifeTimeCapExceeded) { + return false; + } + + const campaignCap = + (spoc.spoc_meta.caps && spoc.spoc_meta.caps.campaign) || {}; + const campaignCapExceeded = + campaignImpressions.filter( + i => Date.now() - i < campaignCap.period * 1000 + ).length >= campaignCap.count; + return !campaignCapExceeded; + } + + // Clean up campaign impression pref by removing all campaigns that are no + // longer part of the response, and are therefore considered inactive. + cleanUpCampaignImpressionPref() { + const campaignIds = new Set(this.spocCampaignMap.values()); + this.cleanUpImpressionPref( + id => !campaignIds.has(id), + SPOC_IMPRESSION_TRACKING_PREF + ); + } + + // Clean up rec impression pref by removing all stories that are no + // longer part of the response. + cleanUpTopRecImpressionPref() { + const activeStories = new Set(this.stories.map(s => `${s.guid}`)); + this.cleanUpImpressionPref( + id => !activeStories.has(id), + REC_IMPRESSION_TRACKING_PREF + ); + } + + /** + * Cleans up the provided impression pref (spocs or recs). + * + * @param isExpired predicate (boolean-valued function) that returns whether or not + * the impression for the given key is expired. + * @param pref the impression pref to clean up. + */ + cleanUpImpressionPref(isExpired, pref) { + const impressions = this.readImpressionsPref(pref); + let changed = false; + + Object.keys(impressions).forEach(id => { + if (isExpired(id)) { + changed = true; + delete impressions[id]; + } + }); + + if (changed) { + this.writeImpressionsPref(pref, impressions); + } + } + + // Sets a pref mapping campaign IDs to timestamp arrays. + // The timestamps represent impressions which are used to calculate frequency caps. + recordCampaignImpression(campaignId) { + let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + + const timeStamps = impressions[campaignId] || []; + timeStamps.push(Date.now()); + impressions = Object.assign(impressions, { [campaignId]: timeStamps }); + + this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions); + } + + // Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression). + // We use these timestamps to guarantee a story doesn't stay on top for longer than + // configured in the feed settings (settings.recsExpireTime). + recordTopRecImpressions(topItems) { + let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + let changed = false; + + topItems.forEach(t => { + if (!impressions[t]) { + changed = true; + impressions = Object.assign(impressions, { [t]: Date.now() }); + } + }); + + if (changed) { + this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions); + } + } + + readImpressionsPref(pref) { + const prefVal = this._prefs.get(pref); + return prefVal ? JSON.parse(prefVal) : {}; + } + + writeImpressionsPref(pref, impressions) { + this._prefs.set(pref, JSON.stringify(impressions)); + } + + async removeSpocs() { + // Quick hack so that SPOCS are removed from all open and preloaded tabs when + // they are disabled. The longer term fix should probably be to remove them + // in the Reducer. + await this.clearCache(); + this.uninit(); + this.init(); + } + + lazyLoadTopStories(options = {}) { + let { dsPref, userPref } = options; + if (!dsPref) { + dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF]; + } + if (!userPref) { + userPref = this.store.getState().Prefs.values[PREF_USER_TOPSTORIES]; + } + + try { + this.discoveryStreamEnabled = + JSON.parse(dsPref).enabled && + this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED]; + } catch (e) { + // Load activity stream top stories if fail to determine discovery stream state + this.discoveryStreamEnabled = false; + } + + // Return without invoking initialization if top stories are loaded, or preffed off. + if (this.storiesLoaded || !userPref) { + return; + } + + if (!this.discoveryStreamEnabled && !this.propertiesInitialized) { + this.initializeProperties(); + } + this.init(); + } + + handleDisabled(action) { + switch (action.type) { + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) { + this.lazyLoadTopStories(); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + break; + case at.UNINIT: + this.uninit(); + break; + } + } + + async onAction(action) { + if (this.discoveryStreamEnabled) { + this.handleDisabled(action); + return; + } + switch (action.type) { + // Check discoverystream pref and load activity stream top stories only if needed + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.SYSTEM_TICK: + let stories; + let topics; + if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) { + stories = await this.fetchStories(); + } + if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, false); + break; + case at.UNINIT: + this.uninit(); + break; + case at.NEW_TAB_REHYDRATED: + this.getPocketState(action.meta.fromTarget); + this.maybeAddSpoc(action.meta.fromTarget); + break; + case at.SECTION_OPTIONS_CHANGED: + if (action.data === SECTION_ID) { + await this.clearCache(); + this.uninit(); + this.init(); + } + break; + case at.PLACES_LINK_BLOCKED: + if (this.spocs) { + this.spocs = this.spocs.filter(s => s.url !== action.data.url); + } + break; + case at.TELEMETRY_IMPRESSION_STATS: { + // We want to make sure we only track impressions from Top Stories, + // otherwise unexpected things that are not properly handled can happen. + // Example: Impressions from spocs on Discovery Stream can cause the + // Top Stories impressions pref to continuously grow, see bug #1523408 + if (action.data.source === IMPRESSION_SOURCE) { + const payload = action.data; + const viewImpression = !( + "click" in payload || + "block" in payload || + "pocket" in payload + ); + if (payload.tiles && viewImpression) { + if (this.shouldShowSpocs()) { + payload.tiles.forEach(t => { + if (this.spocCampaignMap.has(t.id)) { + this.recordCampaignImpression(this.spocCampaignMap.get(t.id)); + } + }); + } + const topRecs = payload.tiles + .filter(t => !this.spocCampaignMap.has(t.id)) + .map(t => t.id); + this.recordTopRecImpressions(topRecs); + } + } + break; + } + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + // Check if spocs was disabled. Remove them if they were. + if (action.data.name === "showSponsored" && !action.data.value) { + await this.removeSpocs(); + } + if (action.data.name === "pocketCta") { + this.dispatchPocketCta(action.data.value, true); + } + break; + } + } +} diff --git a/browser/components/newtab/lib/UTEventReporting.sys.mjs b/browser/components/newtab/lib/UTEventReporting.sys.mjs new file mode 100644 index 0000000000..8da7824415 --- /dev/null +++ b/browser/components/newtab/lib/UTEventReporting.sys.mjs @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Note: the schema can be found in + * https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml + */ +const EXTRAS_FIELD_NAMES = [ + "addon_version", + "session_id", + "page", + "user_prefs", + "action_position", +]; + +export class UTEventReporting { + constructor() { + Services.telemetry.setEventRecordingEnabled("activity_stream", true); + this.sendUserEvent = this.sendUserEvent.bind(this); + this.sendSessionEndEvent = this.sendSessionEndEvent.bind(this); + } + + _createExtras(data) { + // Make a copy of the given data and delete/modify it as needed. + let utExtras = Object.assign({}, data); + for (let field of Object.keys(utExtras)) { + if (EXTRAS_FIELD_NAMES.includes(field)) { + utExtras[field] = String(utExtras[field]); + continue; + } + delete utExtras[field]; + } + return utExtras; + } + + sendUserEvent(data) { + let mainFields = ["event", "source"]; + let eventFields = mainFields.map(field => String(data[field]) || null); + + Services.telemetry.recordEvent( + "activity_stream", + "event", + ...eventFields, + this._createExtras(data) + ); + } + + sendSessionEndEvent(data) { + Services.telemetry.recordEvent( + "activity_stream", + "end", + "session", + String(data.session_duration), + this._createExtras(data) + ); + } + + uninit() { + Services.telemetry.setEventRecordingEnabled("activity_stream", false); + } +} diff --git a/browser/components/newtab/lib/cache.worker.js b/browser/components/newtab/lib/cache.worker.js new file mode 100644 index 0000000000..1195da05fa --- /dev/null +++ b/browser/components/newtab/lib/cache.worker.js @@ -0,0 +1,203 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* global ReactDOMServer, NewtabRenderUtils */ + +const PAGE_TEMPLATE_RESOURCE_PATH = + "resource://activity-stream/data/content/abouthomecache/page.html.template"; +const SCRIPT_TEMPLATE_RESOURCE_PATH = + "resource://activity-stream/data/content/abouthomecache/script.js.template"; + +// If we don't stub these functions out, React throws warnings in the console +// upon being loaded. +let window = self; +window.requestAnimationFrame = () => {}; +window.cancelAnimationFrame = () => {}; +window.ASRouterMessage = () => { + return Promise.resolve(); +}; +window.ASRouterAddParentListener = () => {}; +window.ASRouterRemoveParentListener = () => {}; + +/* import-globals-from /toolkit/components/workerloader/require.js */ +importScripts("resource://gre/modules/workers/require.js"); + +{ + let oldChromeUtils = ChromeUtils; + + // ChromeUtils is defined inside of a Worker, but we don't want the + // activity-stream.bundle.js to detect it when loading, since that results + // in it attempting to import JSMs on load, which is not allowed in + // a Worker. So we temporarily clear ChromeUtils so that activity-stream.bundle.js + // thinks its being loaded in content scope. + // + // eslint-disable-next-line no-implicit-globals, no-global-assign + ChromeUtils = undefined; + + /* import-globals-from ../vendor/react.js */ + /* import-globals-from ../vendor/react-dom.js */ + /* import-globals-from ../vendor/react-dom-server.js */ + /* import-globals-from ../vendor/redux.js */ + /* import-globals-from ../vendor/react-transition-group.js */ + /* import-globals-from ../vendor/prop-types.js */ + /* import-globals-from ../vendor/react-redux.js */ + /* import-globals-from ../data/content/activity-stream.bundle.js */ + importScripts( + "resource://activity-stream/vendor/react.js", + "resource://activity-stream/vendor/react-dom.js", + "resource://activity-stream/vendor/react-dom-server.js", + "resource://activity-stream/vendor/redux.js", + "resource://activity-stream/vendor/react-transition-group.js", + "resource://activity-stream/vendor/prop-types.js", + "resource://activity-stream/vendor/react-redux.js", + "resource://activity-stream/data/content/activity-stream.bundle.js" + ); + + // eslint-disable-next-line no-global-assign, no-implicit-globals + ChromeUtils = oldChromeUtils; +} + +let PromiseWorker = require("resource://gre/modules/workers/PromiseWorker.js"); + +let Agent = { + _templates: null, + + /** + * Synchronously loads the template files off of the file + * system, and returns them as an object. If the Worker has loaded + * these templates before, a cached copy of the templates is returned + * instead. + * + * @return Object + * An object with the following properties: + * + * pageTemplate (String): + * The template for the document markup. + * + * scriptTempate (String): + * The template for the script. + */ + getOrCreateTemplates() { + if (this._templates) { + return this._templates; + } + + const templateResources = new Map([ + ["pageTemplate", PAGE_TEMPLATE_RESOURCE_PATH], + ["scriptTemplate", SCRIPT_TEMPLATE_RESOURCE_PATH], + ]); + + this._templates = {}; + + for (let [name, path] of templateResources) { + const xhr = new XMLHttpRequest(); + // Using a synchronous XHR in a worker is fine. + xhr.open("GET", path, false); + xhr.responseType = "text"; + xhr.send(null); + this._templates[name] = xhr.responseText; + } + + return this._templates; + }, + + /** + * Constructs the cached about:home document using ReactDOMServer. This will + * be called when "construct" messages are sent to this PromiseWorker. + * + * @param state (Object) + * The most recent Activity Stream Redux state. + * @return Object + * An object with the following properties: + * + * page (String): + * The generated markup for the document. + * + * script (String): + * The generated script for the document. + */ + construct(state) { + // If anything in this function throws an exception, PromiseWorker + // runs the risk of leaving the Promise associated with this method + // forever unresolved. This is particularly bad when this method is + // called via AsyncShutdown, since the forever unresolved Promise can + // result in a AsyncShutdown timeout crash. + // + // To help ensure that no matter what, the Promise resolves with something, + // we wrap the whole operation in a try/catch. + try { + return this._construct(state); + } catch (e) { + console.error("about:home startup cache construction failed:", e); + return { page: null, script: null }; + } + }, + + /** + * Internal method that actually does the work of constructing the cached + * about:home document using ReactDOMServer. This should be called from + * `construct` only. + * + * @param state (Object) + * The most recent Activity Stream Redux state. + * @return Object + * An object with the following properties: + * + * page (String): + * The generated markup for the document. + * + * script (String): + * The generated script for the document. + */ + _construct(state) { + state.App.isForStartupCache = true; + + // ReactDOMServer.renderToString expects a Redux store to pull + // the state from, so we mock out a minimal store implementation. + let fakeStore = { + getState() { + return state; + }, + dispatch() {}, + }; + + let markup = ReactDOMServer.renderToString( + NewtabRenderUtils.NewTab({ + store: fakeStore, + isFirstrun: false, + }) + ); + + let { pageTemplate, scriptTemplate } = this.getOrCreateTemplates(); + let cacheTime = new Date().toUTCString(); + let page = pageTemplate + .replace("{{ MARKUP }}", markup) + .replace("{{ CACHE_TIME }}", cacheTime); + let script = scriptTemplate.replace( + "{{ STATE }}", + JSON.stringify(state, null, "\t") + ); + + return { page, script }; + }, +}; + +// This boilerplate connects the PromiseWorker to the Agent so +// that messages from the main thread map to methods on the +// Agent. +let worker = new PromiseWorker.AbstractWorker(); +worker.dispatch = function (method, args = []) { + return Agent[method](...args); +}; +worker.postMessage = function (result, ...transfers) { + self.postMessage(result, ...transfers); +}; +worker.close = function () { + self.close(); +}; + +self.addEventListener("message", msg => worker.handleMessage(msg)); +self.addEventListener("unhandledrejection", function (error) { + throw error.reason; +}); |