diff options
Diffstat (limited to 'browser/components/newtab/lib/TopStoriesFeed.jsm')
| -rw-r--r-- | browser/components/newtab/lib/TopStoriesFeed.jsm | 746 |
1 files changed, 746 insertions, 0 deletions
diff --git a/browser/components/newtab/lib/TopStoriesFeed.jsm b/browser/components/newtab/lib/TopStoriesFeed.jsm new file mode 100644 index 0000000000..aa51b2a4f8 --- /dev/null +++ b/browser/components/newtab/lib/TopStoriesFeed.jsm @@ -0,0 +1,746 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +"use strict"; + +const { actionTypes: at, actionCreators: ac } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { shortURL } = ChromeUtils.import( + "resource://activity-stream/lib/ShortURL.jsm" +); +const { SectionsManager } = ChromeUtils.import( + "resource://activity-stream/lib/SectionsManager.jsm" +); +const { PersistentCache } = ChromeUtils.importESModule( + "resource://activity-stream/lib/PersistentCache.sys.mjs" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); + +const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes +const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours +const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours +const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour +const SECTION_ID = "topstories"; +const IMPRESSION_SOURCE = "TOP_STORIES"; +const SPOC_IMPRESSION_TRACKING_PREF = + "feeds.section.topstories.spoc.impressions"; +const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled"; +const DISCOVERY_STREAM_PREF_ENABLED_PATH = + "browser.newtabpage.activity-stream.discoverystream.enabled"; +const REC_IMPRESSION_TRACKING_PREF = "feeds.section.topstories.rec.impressions"; +const PREF_USER_TOPSTORIES = "feeds.section.topstories"; +const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server +const DISCOVERY_STREAM_PREF = "discoverystream.config"; + +class TopStoriesFeed { + constructor(ds) { + // Use discoverystream config pref default values for fast path and + // if needed lazy load activity stream top stories feed based on + // actual user preference when INIT and PREF_CHANGED is invoked + this.discoveryStreamEnabled = + ds && + ds.value && + JSON.parse(ds.value).enabled && + Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false); + if (!this.discoveryStreamEnabled) { + this.initializeProperties(); + } + } + + initializeProperties() { + this.contentUpdateQueue = []; + this.spocCampaignMap = new Map(); + this.cache = new PersistentCache(SECTION_ID, true); + this._prefs = new Prefs(); + this.propertiesInitialized = true; + } + + async onInit() { + SectionsManager.enableSection(SECTION_ID, true /* isStartup */); + if (this.discoveryStreamEnabled) { + return; + } + + try { + const { options } = SectionsManager.sections.get(SECTION_ID); + const apiKey = this.getApiKeyFromPref(options.api_key_pref); + this.stories_endpoint = this.produceFinalEndpointUrl( + options.stories_endpoint, + apiKey + ); + this.topics_endpoint = this.produceFinalEndpointUrl( + options.topics_endpoint, + apiKey + ); + this.read_more_endpoint = options.read_more_endpoint; + this.stories_referrer = options.stories_referrer; + this.show_spocs = options.show_spocs; + this.storiesLastUpdated = 0; + this.topicsLastUpdated = 0; + this.storiesLoaded = false; + this.dispatchPocketCta(this._prefs.get("pocketCta"), false); + + // Cache is used for new page loads, which shouldn't have changed data. + // If we have changed data, cache should be cleared, + // and last updated should be 0, and we can fetch. + let { stories, topics } = await this.loadCachedData(); + if (this.storiesLastUpdated === 0) { + stories = await this.fetchStories(); + } + if (this.topicsLastUpdated === 0) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, true); + this.storiesLoaded = true; + + // This is filtered so an update function can return true to retry on the next run + this.contentUpdateQueue = this.contentUpdateQueue.filter(update => + update() + ); + } catch (e) { + console.error(`Problem initializing top stories feed: ${e.message}`); + } + } + + init() { + SectionsManager.onceInitialized(this.onInit.bind(this)); + } + + async clearCache() { + await this.cache.set("stories", {}); + await this.cache.set("topics", {}); + await this.cache.set("spocs", {}); + } + + uninit() { + this.storiesLoaded = false; + SectionsManager.disableSection(SECTION_ID); + } + + getPocketState(target) { + const action = { + type: at.POCKET_LOGGED_IN, + data: lazy.pktApi.isUserLoggedIn(), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + dispatchPocketCta(data, shouldBroadcast) { + const action = { type: at.POCKET_CTA, data: JSON.parse(data) }; + this.store.dispatch( + shouldBroadcast + ? ac.BroadcastToContent(action) + : ac.AlsoToPreloaded(action) + ); + } + + /** + * doContentUpdate - Updates topics and stories in the topstories section. + * + * Sections have one update action for the whole section. + * Redux creates a state race condition if you call the same action, + * twice, concurrently. Because of this, doContentUpdate is + * one place to update both topics and stories in a single action. + * + * Section updates used old topics if none are available, + * but clear stories if none are available. Because of this, if no + * stories are passed, we instead use the existing stories in state. + * + * @param {Object} This is an object with potential new stories or topics. + * @param {Boolean} shouldBroadcast If we should update existing tabs or not. For first page + * loads or pref changes, we want to update existing tabs, + * for system tick or other updates we do not. + */ + doContentUpdate({ stories, topics }, shouldBroadcast) { + let updateProps = {}; + if (stories) { + updateProps.rows = stories; + } else { + const { Sections } = this.store.getState(); + if (Sections && Sections.find) { + updateProps.rows = Sections.find(s => s.id === SECTION_ID).rows; + } + } + if (topics) { + Object.assign(updateProps, { + topics, + read_more_endpoint: this.read_more_endpoint, + }); + } + + // We should only be calling this once per init. + this.dispatchUpdateEvent(shouldBroadcast, updateProps); + } + + async fetchStories() { + if (!this.stories_endpoint) { + return null; + } + try { + const response = await fetch(this.stories_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Stories endpoint returned unexpected status: ${response.status}` + ); + } + + const body = await response.json(); + this.updateSettings(body.settings); + this.stories = this.rotate(this.transform(body.recommendations)); + this.cleanUpTopRecImpressionPref(); + + if (this.show_spocs && body.spocs) { + this.spocCampaignMap = new Map( + body.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(body.spocs); + this.cleanUpCampaignImpressionPref(); + } + this.storiesLastUpdated = Date.now(); + body._timestamp = this.storiesLastUpdated; + this.cache.set("stories", body); + } catch (error) { + console.error(`Failed to fetch content: ${error.message}`); + } + return this.stories; + } + + async loadCachedData() { + const data = await this.cache.get(); + let stories = data.stories && data.stories.recommendations; + let topics = data.topics && data.topics.topics; + + if (stories && !!stories.length && this.storiesLastUpdated === 0) { + this.updateSettings(data.stories.settings); + this.stories = this.rotate(this.transform(stories)); + this.storiesLastUpdated = data.stories._timestamp; + if (data.stories.spocs && data.stories.spocs.length) { + this.spocCampaignMap = new Map( + data.stories.spocs.map(s => [s.id, `${s.campaign_id}`]) + ); + this.spocs = this.transform(data.stories.spocs); + this.cleanUpCampaignImpressionPref(); + } + } + if (topics && !!topics.length && this.topicsLastUpdated === 0) { + this.topics = topics; + this.topicsLastUpdated = data.topics._timestamp; + } + + return { topics: this.topics, stories: this.stories }; + } + + transform(items) { + if (!items) { + return []; + } + + const calcResult = items + .filter(s => !lazy.NewTabUtils.blockedLinks.isBlocked({ url: s.url })) + .map(s => { + let mapped = { + guid: s.id, + hostname: s.domain || shortURL(Object.assign({}, s, { url: s.url })), + type: + Date.now() - s.published_timestamp * 1000 <= STORIES_NOW_THRESHOLD + ? "now" + : "trending", + context: s.context, + icon: s.icon, + title: s.title, + description: s.excerpt, + image: this.normalizeUrl(s.image_src), + referrer: this.stories_referrer, + url: s.url, + score: s.item_score || 1, + spoc_meta: this.show_spocs + ? { campaign_id: s.campaign_id, caps: s.caps } + : {}, + }; + + // Very old cached spocs may not contain an `expiration_timestamp` property + if (s.expiration_timestamp) { + mapped.expiration_timestamp = s.expiration_timestamp; + } + + return mapped; + }) + .sort(this.compareScore); + + return calcResult; + } + + async fetchTopics() { + if (!this.topics_endpoint) { + return null; + } + try { + const response = await fetch(this.topics_endpoint, { + credentials: "omit", + }); + if (!response.ok) { + throw new Error( + `Topics endpoint returned unexpected status: ${response.status}` + ); + } + const body = await response.json(); + const { topics } = body; + if (topics) { + this.topics = topics; + this.topicsLastUpdated = Date.now(); + body._timestamp = this.topicsLastUpdated; + this.cache.set("topics", body); + } + } catch (error) { + console.error(`Failed to fetch topics: ${error.message}`); + } + return this.topics; + } + + dispatchUpdateEvent(shouldBroadcast, data) { + SectionsManager.updateSection(SECTION_ID, data, shouldBroadcast); + } + + compareScore(a, b) { + return b.score - a.score; + } + + updateSettings(settings = {}) { + this.spocsPerNewTabs = settings.spocsPerNewTabs || 1; // Probability of a new tab getting a spoc [0,1] + this.recsExpireTime = settings.recsExpireTime; + } + + // We rotate stories on the client so that + // active stories are at the front of the list, followed by stories that have expired + // impressions i.e. have been displayed for longer than recsExpireTime. + rotate(items) { + if (items.length <= 3) { + return items; + } + + const maxImpressionAge = Math.max( + this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME, + DEFAULT_RECS_EXPIRE_TIME + ); + const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + const expired = []; + const active = []; + for (const item of items) { + if ( + impressions[item.guid] && + Date.now() - impressions[item.guid] >= maxImpressionAge + ) { + expired.push(item); + } else { + active.push(item); + } + } + return active.concat(expired); + } + + getApiKeyFromPref(apiKeyPref) { + if (!apiKeyPref) { + return apiKeyPref; + } + + return ( + this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref) + ); + } + + produceFinalEndpointUrl(url, apiKey) { + if (!url) { + return url; + } + if (url.includes("$apiKey") && !apiKey) { + throw new Error(`An API key was specified but none configured: ${url}`); + } + return url.replace("$apiKey", apiKey); + } + + // Need to remove parenthesis from image URLs as React will otherwise + // fail to render them properly as part of the card template. + normalizeUrl(url) { + if (url) { + return url.replace(/\(/g, "%28").replace(/\)/g, "%29"); + } + return url; + } + + shouldShowSpocs() { + return this.show_spocs && this.store.getState().Prefs.values.showSponsored; + } + + dispatchSpocDone(target) { + const action = { type: at.POCKET_WAITING_FOR_SPOC, data: false }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + } + + filterSpocs() { + if (!this.shouldShowSpocs()) { + return []; + } + + if (Math.random() > this.spocsPerNewTabs) { + return []; + } + + if (!this.spocs || !this.spocs.length) { + // We have stories but no spocs so there's nothing to do and this update can be + // removed from the queue. + return []; + } + + // Filter spocs based on frequency caps + const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + let spocs = this.spocs.filter(s => + this.isBelowFrequencyCap(impressions, s) + ); + + // Filter out expired spocs based on `expiration_timestamp` + spocs = spocs.filter(spoc => { + // If cached data is so old it doesn't contain this property, assume the spoc is ok to show + if (!(`expiration_timestamp` in spoc)) { + return true; + } + // `expiration_timestamp` is the number of seconds elapsed since January 1, 1970 00:00:00 UTC + return spoc.expiration_timestamp * 1000 > Date.now(); + }); + + return spocs; + } + + maybeAddSpoc(target) { + const updateContent = () => { + let spocs = this.filterSpocs(); + + if (!spocs.length) { + this.dispatchSpocDone(target); + return false; + } + + // Create a new array with a spoc inserted at index 2 + const section = this.store + .getState() + .Sections.find(s => s.id === SECTION_ID); + let rows = section.rows.slice(0, this.stories.length); + rows.splice(2, 0, Object.assign(spocs[0], { pinned: true })); + + // Send a content update to the target tab + const action = { + type: at.SECTION_UPDATE, + data: Object.assign({ rows }, { id: SECTION_ID }), + }; + this.store.dispatch(ac.OnlyToOneContent(action, target)); + this.dispatchSpocDone(target); + return false; + }; + + if (this.storiesLoaded) { + updateContent(); + } else { + // Delay updating tab content until initial data has been fetched + this.contentUpdateQueue.push(updateContent); + } + } + + // Frequency caps are based on campaigns, which may include multiple spocs. + // We currently support two types of frequency caps: + // - lifetime: Indicates how many times spocs from a campaign can be shown in total + // - period: Indicates how many times spocs from a campaign can be shown within a period + // + // So, for example, the feed configuration below defines that for campaign 1 no more + // than 5 spocs can be show in total, and no more than 2 per hour. + // "campaign_id": 1, + // "caps": { + // "lifetime": 5, + // "campaign": { + // "count": 2, + // "period": 3600 + // } + // } + isBelowFrequencyCap(impressions, spoc) { + const campaignImpressions = impressions[spoc.spoc_meta.campaign_id]; + if (!campaignImpressions) { + return true; + } + + const lifeTimeCap = Math.min( + spoc.spoc_meta.caps && spoc.spoc_meta.caps.lifetime, + MAX_LIFETIME_CAP + ); + const lifeTimeCapExceeded = campaignImpressions.length >= lifeTimeCap; + if (lifeTimeCapExceeded) { + return false; + } + + const campaignCap = + (spoc.spoc_meta.caps && spoc.spoc_meta.caps.campaign) || {}; + const campaignCapExceeded = + campaignImpressions.filter( + i => Date.now() - i < campaignCap.period * 1000 + ).length >= campaignCap.count; + return !campaignCapExceeded; + } + + // Clean up campaign impression pref by removing all campaigns that are no + // longer part of the response, and are therefore considered inactive. + cleanUpCampaignImpressionPref() { + const campaignIds = new Set(this.spocCampaignMap.values()); + this.cleanUpImpressionPref( + id => !campaignIds.has(id), + SPOC_IMPRESSION_TRACKING_PREF + ); + } + + // Clean up rec impression pref by removing all stories that are no + // longer part of the response. + cleanUpTopRecImpressionPref() { + const activeStories = new Set(this.stories.map(s => `${s.guid}`)); + this.cleanUpImpressionPref( + id => !activeStories.has(id), + REC_IMPRESSION_TRACKING_PREF + ); + } + + /** + * Cleans up the provided impression pref (spocs or recs). + * + * @param isExpired predicate (boolean-valued function) that returns whether or not + * the impression for the given key is expired. + * @param pref the impression pref to clean up. + */ + cleanUpImpressionPref(isExpired, pref) { + const impressions = this.readImpressionsPref(pref); + let changed = false; + + Object.keys(impressions).forEach(id => { + if (isExpired(id)) { + changed = true; + delete impressions[id]; + } + }); + + if (changed) { + this.writeImpressionsPref(pref, impressions); + } + } + + // Sets a pref mapping campaign IDs to timestamp arrays. + // The timestamps represent impressions which are used to calculate frequency caps. + recordCampaignImpression(campaignId) { + let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF); + + const timeStamps = impressions[campaignId] || []; + timeStamps.push(Date.now()); + impressions = Object.assign(impressions, { [campaignId]: timeStamps }); + + this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions); + } + + // Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression). + // We use these timestamps to guarantee a story doesn't stay on top for longer than + // configured in the feed settings (settings.recsExpireTime). + recordTopRecImpressions(topItems) { + let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF); + let changed = false; + + topItems.forEach(t => { + if (!impressions[t]) { + changed = true; + impressions = Object.assign(impressions, { [t]: Date.now() }); + } + }); + + if (changed) { + this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions); + } + } + + readImpressionsPref(pref) { + const prefVal = this._prefs.get(pref); + return prefVal ? JSON.parse(prefVal) : {}; + } + + writeImpressionsPref(pref, impressions) { + this._prefs.set(pref, JSON.stringify(impressions)); + } + + async removeSpocs() { + // Quick hack so that SPOCS are removed from all open and preloaded tabs when + // they are disabled. The longer term fix should probably be to remove them + // in the Reducer. + await this.clearCache(); + this.uninit(); + this.init(); + } + + lazyLoadTopStories(options = {}) { + let { dsPref, userPref } = options; + if (!dsPref) { + dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF]; + } + if (!userPref) { + userPref = this.store.getState().Prefs.values[PREF_USER_TOPSTORIES]; + } + + try { + this.discoveryStreamEnabled = + JSON.parse(dsPref).enabled && + this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED]; + } catch (e) { + // Load activity stream top stories if fail to determine discovery stream state + this.discoveryStreamEnabled = false; + } + + // Return without invoking initialization if top stories are loaded, or preffed off. + if (this.storiesLoaded || !userPref) { + return; + } + + if (!this.discoveryStreamEnabled && !this.propertiesInitialized) { + this.initializeProperties(); + } + this.init(); + } + + handleDisabled(action) { + switch (action.type) { + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) { + this.lazyLoadTopStories(); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + break; + case at.UNINIT: + this.uninit(); + break; + } + } + + async onAction(action) { + if (this.discoveryStreamEnabled) { + this.handleDisabled(action); + return; + } + switch (action.type) { + // Check discoverystream pref and load activity stream top stories only if needed + case at.INIT: + this.lazyLoadTopStories(); + break; + case at.SYSTEM_TICK: + let stories; + let topics; + if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) { + stories = await this.fetchStories(); + } + if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) { + topics = await this.fetchTopics(); + } + this.doContentUpdate({ stories, topics }, false); + break; + case at.UNINIT: + this.uninit(); + break; + case at.NEW_TAB_REHYDRATED: + this.getPocketState(action.meta.fromTarget); + this.maybeAddSpoc(action.meta.fromTarget); + break; + case at.SECTION_OPTIONS_CHANGED: + if (action.data === SECTION_ID) { + await this.clearCache(); + this.uninit(); + this.init(); + } + break; + case at.PLACES_LINK_BLOCKED: + if (this.spocs) { + this.spocs = this.spocs.filter(s => s.url !== action.data.url); + } + break; + case at.TELEMETRY_IMPRESSION_STATS: { + // We want to make sure we only track impressions from Top Stories, + // otherwise unexpected things that are not properly handled can happen. + // Example: Impressions from spocs on Discovery Stream can cause the + // Top Stories impressions pref to continuously grow, see bug #1523408 + if (action.data.source === IMPRESSION_SOURCE) { + const payload = action.data; + const viewImpression = !( + "click" in payload || + "block" in payload || + "pocket" in payload + ); + if (payload.tiles && viewImpression) { + if (this.shouldShowSpocs()) { + payload.tiles.forEach(t => { + if (this.spocCampaignMap.has(t.id)) { + this.recordCampaignImpression(this.spocCampaignMap.get(t.id)); + } + }); + } + const topRecs = payload.tiles + .filter(t => !this.spocCampaignMap.has(t.id)) + .map(t => t.id); + this.recordTopRecImpressions(topRecs); + } + } + break; + } + case at.PREF_CHANGED: + if (action.data.name === DISCOVERY_STREAM_PREF) { + this.lazyLoadTopStories({ dsPref: action.data.value }); + } + if (action.data.name === PREF_USER_TOPSTORIES) { + if (action.data.value) { + // init topstories if value if true. + this.lazyLoadTopStories({ userPref: action.data.value }); + } else { + this.uninit(); + } + } + // Check if spocs was disabled. Remove them if they were. + if (action.data.name === "showSponsored" && !action.data.value) { + await this.removeSpocs(); + } + if (action.data.name === "pocketCta") { + this.dispatchPocketCta(action.data.value, true); + } + break; + } + } +} + +const EXPORTED_SYMBOLS = [ + "TopStoriesFeed", + "STORIES_UPDATE_TIME", + "TOPICS_UPDATE_TIME", + "SECTION_ID", + "SPOC_IMPRESSION_TRACKING_PREF", + "REC_IMPRESSION_TRACKING_PREF", + "DEFAULT_RECS_EXPIRE_TIME", +]; |