diff options
Diffstat (limited to 'browser/components/newtab/lib/TelemetryFeed.jsm')
| -rw-r--r-- | browser/components/newtab/lib/TelemetryFeed.jsm | 1403 |
1 files changed, 1403 insertions, 0 deletions
diff --git a/browser/components/newtab/lib/TelemetryFeed.jsm b/browser/components/newtab/lib/TelemetryFeed.jsm new file mode 100644 index 0000000000..653603d809 --- /dev/null +++ b/browser/components/newtab/lib/TelemetryFeed.jsm @@ -0,0 +1,1403 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { MESSAGE_TYPE_HASH: msg } = ChromeUtils.importESModule( + "resource://activity-stream/common/ActorConstants.sys.mjs" +); + +const { actionTypes: at, actionUtils: au } = ChromeUtils.importESModule( + "resource://activity-stream/common/Actions.sys.mjs" +); +const { Prefs } = ChromeUtils.import( + "resource://activity-stream/lib/ActivityStreamPrefs.jsm" +); +const { classifySite } = ChromeUtils.import( + "resource://activity-stream/lib/SiteClassifier.jsm" +); + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "AboutNewTab", + "resource:///modules/AboutNewTab.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "PingCentre", + "resource:///modules/PingCentre.jsm" +); +ChromeUtils.defineESModuleGetters(lazy, { + ClientID: "resource://gre/modules/ClientID.sys.mjs", + ExperimentAPI: "resource://nimbus/ExperimentAPI.sys.mjs", + ExtensionSettingsStore: + "resource://gre/modules/ExtensionSettingsStore.sys.mjs", + NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs", + TelemetrySession: "resource://gre/modules/TelemetrySession.sys.mjs", + UTEventReporting: "resource://activity-stream/lib/UTEventReporting.sys.mjs", + UpdateUtils: "resource://gre/modules/UpdateUtils.sys.mjs", + pktApi: "chrome://pocket/content/pktApi.sys.mjs", +}); +ChromeUtils.defineModuleGetter( + lazy, + "HomePage", + "resource:///modules/HomePage.jsm" +); +XPCOMUtils.defineLazyModuleGetters(lazy, { + AboutWelcomeTelemetry: + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm", +}); +XPCOMUtils.defineLazyGetter( + lazy, + "Telemetry", + () => new lazy.AboutWelcomeTelemetry() +); + +const ACTIVITY_STREAM_ID = "activity-stream"; +const DOMWINDOW_OPENED_TOPIC = "domwindowopened"; +const DOMWINDOW_UNLOAD_TOPIC = "unload"; +const TAB_PINNED_EVENT = "TabPinned"; + +// This is a mapping table between the user preferences and its encoding code +const USER_PREFS_ENCODING = { + showSearch: 1 << 0, + "feeds.topsites": 1 << 1, + "feeds.section.topstories": 1 << 2, + "feeds.section.highlights": 1 << 3, + "feeds.snippets": 1 << 4, + showSponsored: 1 << 5, + "asrouter.userprefs.cfr.addons": 1 << 6, + "asrouter.userprefs.cfr.features": 1 << 7, + showSponsoredTopSites: 1 << 8, +}; + +const PREF_IMPRESSION_ID = "impressionId"; +const TELEMETRY_PREF = "telemetry"; +const EVENTS_TELEMETRY_PREF = "telemetry.ut.events"; +const STRUCTURED_INGESTION_ENDPOINT_PREF = + "telemetry.structuredIngestion.endpoint"; +// List of namespaces for the structured ingestion system. +// They are defined in https://github.com/mozilla-services/mozilla-pipeline-schemas +const STRUCTURED_INGESTION_NAMESPACE_AS = "activity-stream"; +const STRUCTURED_INGESTION_NAMESPACE_MS = "messaging-system"; +const STRUCTURED_INGESTION_NAMESPACE_CS = "contextual-services"; + +// Used as the missing value for timestamps in the session ping +const TIMESTAMP_MISSING_VALUE = -1; + +// Page filter for onboarding telemetry, any value other than these will +// be set as "other" +const ONBOARDING_ALLOWED_PAGE_VALUES = [ + "about:welcome", + "about:home", + "about:newtab", +]; + +XPCOMUtils.defineLazyGetter( + lazy, + "browserSessionId", + () => lazy.TelemetrySession.getMetadata("").sessionId +); + +// The scalar category for TopSites of Contextual Services +const SCALAR_CATEGORY_TOPSITES = "contextual.services.topsites"; +// `contextId` is a unique identifier used by Contextual Services +const CONTEXT_ID_PREF = "browser.contextual-services.contextId"; +XPCOMUtils.defineLazyGetter(lazy, "contextId", () => { + let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null); + if (!_contextId) { + _contextId = String(Services.uuid.generateUUID()); + Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId); + } + return _contextId; +}); + +class TelemetryFeed { + constructor() { + this.sessions = new Map(); + this._prefs = new Prefs(); + this._impressionId = this.getOrCreateImpressionId(); + this._aboutHomeSeen = false; + this._classifySite = classifySite; + this._addWindowListeners = this._addWindowListeners.bind(this); + this._browserOpenNewtabStart = null; + this.handleEvent = this.handleEvent.bind(this); + } + + get telemetryEnabled() { + return this._prefs.get(TELEMETRY_PREF); + } + + get eventTelemetryEnabled() { + return this._prefs.get(EVENTS_TELEMETRY_PREF); + } + + get structuredIngestionEndpointBase() { + return this._prefs.get(STRUCTURED_INGESTION_ENDPOINT_PREF); + } + + get telemetryClientId() { + Object.defineProperty(this, "telemetryClientId", { + value: lazy.ClientID.getClientID(), + }); + return this.telemetryClientId; + } + + get processStartTs() { + let startupInfo = Services.startup.getStartupInfo(); + let processStartTs = startupInfo.process.getTime(); + + Object.defineProperty(this, "processStartTs", { + value: processStartTs, + }); + return this.processStartTs; + } + + init() { + this._beginObservingNewtabPingPrefs(); + Services.obs.addObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + // Add pin tab event listeners on future windows + Services.obs.addObserver(this._addWindowListeners, DOMWINDOW_OPENED_TOPIC); + // Listen for pin tab events on all open windows + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._addWindowListeners(win); + } + // Set two scalars for the "deletion-request" ping (See bug 1602064 and 1729474) + Services.telemetry.scalarSet( + "deletion.request.impression_id", + this._impressionId + ); + Services.telemetry.scalarSet("deletion.request.context_id", lazy.contextId); + Glean.newtab.locale.set(Services.locale.appLocaleAsBCP47); + } + + handleEvent(event) { + switch (event.type) { + case TAB_PINNED_EVENT: + this.countPinnedTab(event.target); + break; + case DOMWINDOW_UNLOAD_TOPIC: + this._removeWindowListeners(event.target); + break; + } + } + + _removeWindowListeners(win) { + win.removeEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent); + win.removeEventListener(TAB_PINNED_EVENT, this.handleEvent); + } + + _addWindowListeners(win) { + win.addEventListener(DOMWINDOW_UNLOAD_TOPIC, this.handleEvent); + win.addEventListener(TAB_PINNED_EVENT, this.handleEvent); + } + + countPinnedTab(target, source = "TAB_CONTEXT_MENU") { + const win = target.ownerGlobal; + if (lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + return; + } + const event = Object.assign(this.createPing(), { + action: "activity_stream_user_event", + event: TAB_PINNED_EVENT.toUpperCase(), + value: { total_pinned_tabs: this.countTotalPinnedTabs() }, + source, + // These fields are required but not relevant for this ping + page: "n/a", + session_id: "n/a", + }); + this.sendEvent(event); + } + + countTotalPinnedTabs() { + let pinnedTabs = 0; + for (let win of Services.wm.getEnumerator("navigator:browser")) { + if (win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win)) { + continue; + } + for (let tab of win.gBrowser.tabs) { + pinnedTabs += tab.pinned ? 1 : 0; + } + } + + return pinnedTabs; + } + + getOrCreateImpressionId() { + let impressionId = this._prefs.get(PREF_IMPRESSION_ID); + if (!impressionId) { + impressionId = String(Services.uuid.generateUUID()); + this._prefs.set(PREF_IMPRESSION_ID, impressionId); + } + return impressionId; + } + + browserOpenNewtabStart() { + let now = Cu.now(); + this._browserOpenNewtabStart = Math.round(this.processStartTs + now); + + ChromeUtils.addProfilerMarker( + "UserTiming", + now, + "browser-open-newtab-start" + ); + } + + setLoadTriggerInfo(port) { + // XXX note that there is a race condition here; we're assuming that no + // other tab will be interleaving calls to browserOpenNewtabStart and + // when at.NEW_TAB_INIT gets triggered by RemotePages and calls this + // method. For manually created windows, it's hard to imagine us hitting + // this race condition. + // + // However, for session restore, where multiple windows with multiple tabs + // might be restored much closer together in time, it's somewhat less hard, + // though it should still be pretty rare. + // + // The fix to this would be making all of the load-trigger notifications + // return some data with their notifications, and somehow propagate that + // data through closures into the tab itself so that we could match them + // + // As of this writing (very early days of system add-on perf telemetry), + // the hypothesis is that hitting this race should be so rare that makes + // more sense to live with the slight data inaccuracy that it would + // introduce, rather than doing the correct but complicated thing. It may + // well be worth reexamining this hypothesis after we have more experience + // with the data. + + let data_to_save; + try { + if (!this._browserOpenNewtabStart) { + throw new Error("No browser-open-newtab-start recorded."); + } + data_to_save = { + load_trigger_ts: this._browserOpenNewtabStart, + load_trigger_type: "menu_plus_or_keyboard", + }; + } catch (e) { + // if no mark was returned, we have nothing to save + return; + } + this.saveSessionPerfData(port, data_to_save); + } + + /** + * Lazily initialize PingCentre for Activity Stream to send pings + */ + get pingCentre() { + Object.defineProperty(this, "pingCentre", { + value: new lazy.PingCentre({ topic: ACTIVITY_STREAM_ID }), + }); + return this.pingCentre; + } + + /** + * Lazily initialize UTEventReporting to send pings + */ + get utEvents() { + Object.defineProperty(this, "utEvents", { + value: new lazy.UTEventReporting(), + }); + return this.utEvents; + } + + /** + * Get encoded user preferences, multiple prefs will be combined via bitwise OR operator + */ + get userPreferences() { + let prefs = 0; + + for (const pref of Object.keys(USER_PREFS_ENCODING)) { + if (this._prefs.get(pref)) { + prefs |= USER_PREFS_ENCODING[pref]; + } + } + return prefs; + } + + /** + * Check if it is in the CFR experiment cohort by querying against the + * experiment manager of Messaging System + * + * @return {bool} + */ + get isInCFRCohort() { + const experimentData = lazy.ExperimentAPI.getExperimentMetaData({ + featureId: "cfr", + }); + if (experimentData && experimentData.slug) { + return true; + } + + return false; + } + + /** + * addSession - Start tracking a new session + * + * @param {string} id the portID of the open session + * @param {string} the URL being loaded for this session (optional) + * @return {obj} Session object + */ + addSession(id, url) { + // XXX refactor to use setLoadTriggerInfo or saveSessionPerfData + + // "unexpected" will be overwritten when appropriate + let load_trigger_type = "unexpected"; + let load_trigger_ts; + + if (!this._aboutHomeSeen && url === "about:home") { + this._aboutHomeSeen = true; + + // XXX note that this will be incorrectly set in the following cases: + // session_restore following by clicking on the toolbar button, + // or someone who has changed their default home page preference to + // something else and later clicks the toolbar. It will also be + // incorrectly unset if someone changes their "Home Page" preference to + // about:newtab. + // + // That said, the ratio of these mistakes to correct cases should + // be very small, and these issues should follow away as we implement + // the remaining load_trigger_type values for about:home in issue 3556. + // + // XXX file a bug to implement remaining about:home cases so this + // problem will go away and link to it here. + load_trigger_type = "first_window_opened"; + + // The real perceived trigger of first_window_opened is the OS-level + // clicking of the icon. We express this by using the process start + // absolute timestamp. + load_trigger_ts = this.processStartTs; + } + + const session = { + session_id: String(Services.uuid.generateUUID()), + // "unknown" will be overwritten when appropriate + page: url ? url : "unknown", + perf: { + load_trigger_type, + is_preloaded: false, + }, + }; + + if (load_trigger_ts) { + session.perf.load_trigger_ts = load_trigger_ts; + } + + this.sessions.set(id, session); + return session; + } + + /** + * endSession - Stop tracking a session + * + * @param {string} portID the portID of the session that just closed + */ + endSession(portID) { + const session = this.sessions.get(portID); + + if (!session) { + // It's possible the tab was never visible – in which case, there was no user session. + return; + } + + this.sendDiscoveryStreamLoadedContent(portID, session); + this.sendDiscoveryStreamImpressions(portID, session); + + Glean.newtab.closed.record({ newtab_visit_id: session.session_id }); + if ( + this.telemetryEnabled && + (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) + ) { + GleanPings.newtab.submit("newtab_session_end"); + } + + if (session.perf.visibility_event_rcvd_ts) { + let absNow = this.processStartTs + Cu.now(); + session.session_duration = Math.round( + absNow - session.perf.visibility_event_rcvd_ts + ); + + // Rounding all timestamps in perf to ease the data processing on the backend. + // NB: use `TIMESTAMP_MISSING_VALUE` if the value is missing. + session.perf.visibility_event_rcvd_ts = Math.round( + session.perf.visibility_event_rcvd_ts + ); + session.perf.load_trigger_ts = Math.round( + session.perf.load_trigger_ts || TIMESTAMP_MISSING_VALUE + ); + session.perf.topsites_first_painted_ts = Math.round( + session.perf.topsites_first_painted_ts || TIMESTAMP_MISSING_VALUE + ); + } else { + // This session was never shown (i.e. the hidden preloaded newtab), there was no user session either. + this.sessions.delete(portID); + return; + } + + let sessionEndEvent = this.createSessionEndEvent(session); + this.sendEvent(sessionEndEvent); + this.sendUTEvent(sessionEndEvent, this.utEvents.sendSessionEndEvent); + this.sessions.delete(portID); + } + + /** + * Send impression pings for Discovery Stream for a given session. + * + * @note the impression reports are stored in session.impressionSets for different + * sources, and will be sent separately accordingly. + * + * @param {String} port The session port with which this is associated + * @param {Object} session The session object + */ + sendDiscoveryStreamImpressions(port, session) { + const { impressionSets } = session; + + if (!impressionSets) { + return; + } + + Object.keys(impressionSets).forEach(source => { + const { tiles, window_inner_width, window_inner_height } = + impressionSets[source]; + const payload = this.createImpressionStats(port, { + source, + tiles, + window_inner_width, + window_inner_height, + }); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + }); + } + + /** + * Send loaded content pings for Discovery Stream for a given session. + * + * @note the loaded content reports are stored in session.loadedContentSets for different + * sources, and will be sent separately accordingly. + * + * @param {String} port The session port with which this is associated + * @param {Object} session The session object + */ + sendDiscoveryStreamLoadedContent(port, session) { + const { loadedContentSets } = session; + + if (!loadedContentSets) { + return; + } + + Object.keys(loadedContentSets).forEach(source => { + const tiles = loadedContentSets[source]; + const payload = this.createImpressionStats(port, { + source, + tiles, + loaded: tiles.length, + }); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + }); + } + + /** + * handleNewTabInit - Handle NEW_TAB_INIT, which creates a new session and sets the a flag + * for session.perf based on whether or not this new tab is preloaded + * + * @param {obj} action the Action object + */ + handleNewTabInit(action) { + const session = this.addSession( + au.getPortIdOfSender(action), + action.data.url + ); + session.perf.is_preloaded = + action.data.browser.getAttribute("preloadedState") === "preloaded"; + } + + /** + * createPing - Create a ping with common properties + * + * @param {string} id The portID of the session, if a session is relevant (optional) + * @return {obj} A telemetry ping + */ + createPing(portID) { + const ping = { + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + user_prefs: this.userPreferences, + }; + + // If the ping is part of a user session, add session-related info + if (portID) { + const session = this.sessions.get(portID) || this.addSession(portID); + Object.assign(ping, { session_id: session.session_id }); + + if (session.page) { + Object.assign(ping, { page: session.page }); + } + } + return ping; + } + + /** + * createImpressionStats - Create a ping for an impression stats + * + * @param {string} portID The portID of the open session + * @param {ob} data The data object to be included in the ping. + * @return {obj} A telemetry ping + */ + createImpressionStats(portID, data) { + let ping = Object.assign(this.createPing(portID), data, { + impression_id: this._impressionId, + }); + // Make sure `session_id` and `client_id` are not in the ping. + delete ping.session_id; + delete ping.client_id; + return ping; + } + + createUserEvent(action) { + return Object.assign( + this.createPing(au.getPortIdOfSender(action)), + action.data, + { action: "activity_stream_user_event" } + ); + } + + createSessionEndEvent(session) { + return Object.assign(this.createPing(), { + session_id: session.session_id, + page: session.page, + session_duration: session.session_duration, + action: "activity_stream_session", + perf: session.perf, + profile_creation_date: + lazy.TelemetryEnvironment.currentEnvironment.profile.resetDate || + lazy.TelemetryEnvironment.currentEnvironment.profile.creationDate, + }); + } + + /** + * Create a ping for AS router event. The client_id is set to "n/a" by default, + * different component can override this by its own telemetry collection policy. + */ + async createASRouterEvent(action) { + let event = { + ...action.data, + addon_version: Services.appinfo.appBuildID, + locale: Services.locale.appLocaleAsBCP47, + }; + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (event.event_context && typeof event.event_context === "object") { + event.event_context = JSON.stringify(event.event_context); + } + switch (event.action) { + case "cfr_user_event": + event = await this.applyCFRPolicy(event); + break; + case "snippets_local_testing_user_event": + case "snippets_user_event": + event = await this.applySnippetsPolicy(event); + break; + case "badge_user_event": + case "whats-new-panel_user_event": + event = await this.applyWhatsNewPolicy(event); + break; + case "infobar_user_event": + event = await this.applyInfoBarPolicy(event); + break; + case "spotlight_user_event": + event = await this.applySpotlightPolicy(event); + break; + case "toast_notification_user_event": + event = await this.applyToastNotificationPolicy(event); + break; + case "moments_user_event": + event = await this.applyMomentsPolicy(event); + break; + case "onboarding_user_event": + event = await this.applyOnboardingPolicy(event, session); + break; + case "asrouter_undesired_event": + event = this.applyUndesiredEventPolicy(event); + break; + default: + event = { ping: event }; + break; + } + return event; + } + + /** + * Per Bug 1484035, CFR metrics comply with following policies: + * 1). In release, it collects impression_id and bucket_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + * 4). In Private Browsing windows, unless in experiment, collects impression_id and bucket_id + */ + async applyCFRPolicy(ping) { + if ( + (lazy.UpdateUtils.getUpdateChannel(true) === "release" || + ping.is_private) && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + delete ping.is_private; + return { ping, pingType: "cfr" }; + } + + /** + * Per Bug 1482134, all the metrics for What's New panel use client_id in + * all the release channels + */ + async applyWhatsNewPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + delete ping.action; + return { ping, pingType: "whats-new-panel" }; + } + + async applyInfoBarPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "infobar" }; + } + + async applySpotlightPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "spotlight" }; + } + + async applyToastNotificationPolicy(ping) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + delete ping.action; + return { ping, pingType: "toast_notification" }; + } + + /** + * Per Bug 1484035, Moments metrics comply with following policies: + * 1). In release, it collects impression_id, and treats bucket_id as message_id + * 2). In prerelease, it collects client_id and message_id + * 3). In shield experiments conducted in release, it collects client_id and message_id + */ + async applyMomentsPolicy(ping) { + if ( + lazy.UpdateUtils.getUpdateChannel(true) === "release" && + !this.isInCFRCohort + ) { + ping.message_id = "n/a"; + ping.impression_id = this._impressionId; + } else { + ping.client_id = await this.telemetryClientId; + } + delete ping.action; + return { ping, pingType: "moments" }; + } + + /** + * Per Bug 1485069, all the metrics for Snippets in AS router use client_id in + * all the release channels + */ + async applySnippetsPolicy(ping) { + ping.client_id = await this.telemetryClientId; + delete ping.action; + return { ping, pingType: "snippets" }; + } + + /** + * Per Bug 1482134, all the metrics for Onboarding in AS router use client_id in + * all the release channels + */ + async applyOnboardingPolicy(ping, session) { + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + // Attach page info to `event_context` if there is a session associated with this ping + if (ping.action === "onboarding_user_event" && session && session.page) { + let event_context; + + try { + event_context = ping.event_context + ? JSON.parse(ping.event_context) + : {}; + } catch (e) { + // If `ping.event_context` is not a JSON serialized string, then we create a `value` + // key for it + event_context = { value: ping.event_context }; + } + + if (ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page)) { + event_context.page = session.page; + } else { + console.error(`Invalid 'page' for Onboarding event: ${session.page}`); + } + ping.event_context = JSON.stringify(event_context); + } + delete ping.action; + return { ping, pingType: "onboarding" }; + } + + applyUndesiredEventPolicy(ping) { + ping.impression_id = this._impressionId; + delete ping.action; + return { ping, pingType: "undesired-events" }; + } + + sendEvent(event_object) { + switch (event_object.action) { + case "activity_stream_user_event": + this.sendEventPing(event_object); + break; + case "activity_stream_session": + this.sendSessionPing(event_object); + break; + } + } + + async sendEventPing(ping) { + delete ping.action; + ping.client_id = await this.telemetryClientId; + ping.browser_session_id = lazy.browserSessionId; + if (ping.value && typeof ping.value === "object") { + ping.value = JSON.stringify(ping.value); + } + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_AS, + "events", + 1 + ); + } + + async sendSessionPing(ping) { + delete ping.action; + ping.client_id = await this.telemetryClientId; + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_AS, + "sessions", + 1 + ); + } + + sendUTEvent(event_object, eventFunction) { + if (this.telemetryEnabled && this.eventTelemetryEnabled) { + eventFunction(event_object); + } + } + + /** + * Generates an endpoint for Structured Ingestion telemetry pipeline. Note that + * Structured Ingestion requires a different endpoint for each ping. See more + * details about endpoint schema at: + * https://github.com/mozilla/gcp-ingestion/blob/master/docs/edge.md#postput-request + * + * @param {String} namespace Namespace of the ping, such as "activity-stream" or "messaging-system". + * @param {String} pingType Type of the ping, such as "impression-stats". + * @param {String} version Endpoint version for this ping type. + */ + _generateStructuredIngestionEndpoint(namespace, pingType, version) { + const uuid = Services.uuid.generateUUID().toString(); + // Structured Ingestion does not support the UUID generated by Services.uuid, + // because it contains leading and trailing braces. Need to trim them first. + const docID = uuid.slice(1, -1); + const extension = `${namespace}/${pingType}/${version}/${docID}`; + return `${this.structuredIngestionEndpointBase}/${extension}`; + } + + sendStructuredIngestionEvent(eventObject, namespace, pingType, version) { + if (this.telemetryEnabled) { + this.pingCentre.sendStructuredIngestionPing( + eventObject, + this._generateStructuredIngestionEndpoint(namespace, pingType, version), + namespace + ); + } + } + + handleImpressionStats(action) { + const payload = this.createImpressionStats( + au.getPortIdOfSender(action), + action.data + ); + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_AS, + "impression-stats", + "1" + ); + } + + handleTopSitesSponsoredImpressionStats(action) { + const { data } = action; + const { + type, + position, + source, + advertiser: advertiser_name, + tile_id, + } = data; + // Legacy telemetry (scalars and PingCentre payloads) expects 1-based tile + // positions. + const legacyTelemetryPosition = position + 1; + + let pingType; + + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (type === "impression") { + pingType = "topsites-impression"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.impression`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.impression.record({ + advertiser_name, + tile_id: tile_id.toString(), + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else if (type === "click") { + pingType = "topsites-click"; + Services.telemetry.keyedScalarAdd( + `${SCALAR_CATEGORY_TOPSITES}.click`, + `${source}_${legacyTelemetryPosition}`, + 1 + ); + if (session) { + Glean.topsites.click.record({ + advertiser_name, + tile_id: tile_id.toString(), + newtab_visit_id: session.session_id, + is_sponsored: true, + position, + }); + } + } else { + console.error("Unknown ping type for sponsored TopSites impression"); + return; + } + + let payload = { + ...data, + position: legacyTelemetryPosition, + context_id: lazy.contextId, + }; + delete payload.type; + this.sendStructuredIngestionEvent( + payload, + STRUCTURED_INGESTION_NAMESPACE_CS, + pingType, + "1" + ); + } + + handleTopSitesOrganicImpressionStats(action) { + const session = this.sessions.get(au.getPortIdOfSender(action)); + if (!session) { + return; + } + + switch (action.data?.type) { + case "impression": + Glean.topsites.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + case "click": + Glean.topsites.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: false, + position: action.data.position, + }); + break; + + default: + break; + } + } + + handleUserEvent(action) { + let userEvent = this.createUserEvent(action); + this.sendEvent(userEvent); + this.sendUTEvent(userEvent, this.utEvents.sendUserEvent); + } + + handleDiscoveryStreamUserEvent(action) { + const pocket_logged_in_status = lazy.pktApi.isUserLoggedIn(); + Glean.pocket.isSignedIn.set(pocket_logged_in_status); + this.handleUserEvent({ + ...action, + data: { + ...(action.data || {}), + value: { + ...(action.data?.value || {}), + pocket_logged_in_status, + }, + }, + }); + const session = this.sessions.get(au.getPortIdOfSender(action)); + switch (action.data?.event) { + case "CLICK": + if ( + action.data.source === "POPULAR_TOPICS" || + action.data.value?.card_type === "topics_widget" + ) { + Glean.pocket.topicClick.record({ + newtab_visit_id: session.session_id, + topic: action.data.value?.topic, + }); + } else if (["spoc", "organic"].includes(action.data.value?.card_type)) { + Glean.pocket.click.record({ + newtab_visit_id: session.session_id, + is_sponsored: action.data.value?.card_type === "spoc", + position: action.data.action_position, + }); + } + break; + case "SAVE_TO_POCKET": + Glean.pocket.save.record({ + newtab_visit_id: session.session_id, + is_sponsored: action.data.value?.card_type === "spoc", + position: action.data.action_position, + }); + break; + } + } + + async handleASRouterUserEvent(action) { + const { ping, pingType } = await this.createASRouterEvent(action); + if (!pingType) { + console.error("Unknown ping type for ASRouter telemetry"); + return; + } + + // Now that the action has become a ping, we can echo it to Glean. + if (this.telemetryEnabled) { + lazy.Telemetry.submitGleanPingForPing({ ...ping, pingType }); + } + + this.sendStructuredIngestionEvent( + ping, + STRUCTURED_INGESTION_NAMESPACE_MS, + pingType, + "1" + ); + } + + /** + * This function is used by ActivityStreamStorage to report errors + * trying to access IndexedDB. + */ + SendASRouterUndesiredEvent(data) { + this.handleASRouterUserEvent({ + data: { ...data, action: "asrouter_undesired_event" }, + }); + } + + async sendPageTakeoverData() { + if (this.telemetryEnabled) { + const value = {}; + let newtabAffected = false; + let homeAffected = false; + let newtabCategory = "disabled"; + let homePageCategory = "disabled"; + + // Check whether or not about:home and about:newtab are set to a custom URL. + // If so, classify them. + if (Services.prefs.getBoolPref("browser.newtabpage.enabled")) { + newtabCategory = "enabled"; + if ( + lazy.AboutNewTab.newTabURLOverridden && + !lazy.AboutNewTab.newTabURL.startsWith("moz-extension://") + ) { + value.newtab_url_category = await this._classifySite( + lazy.AboutNewTab.newTabURL + ); + newtabAffected = true; + newtabCategory = value.newtab_url_category; + } + } + // Check if the newtab page setting is controlled by an extension. + await lazy.ExtensionSettingsStore.initialize(); + const newtabExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "url_overrides", + "newTabURL" + ); + if (newtabExtensionInfo && newtabExtensionInfo.id) { + value.newtab_extension_id = newtabExtensionInfo.id; + newtabAffected = true; + newtabCategory = "extension"; + } + + const homePageURL = lazy.HomePage.get(); + if ( + !["about:home", "about:blank"].includes(homePageURL) && + !homePageURL.startsWith("moz-extension://") + ) { + value.home_url_category = await this._classifySite(homePageURL); + homeAffected = true; + homePageCategory = value.home_url_category; + } + const homeExtensionInfo = lazy.ExtensionSettingsStore.getSetting( + "prefs", + "homepage_override" + ); + if (homeExtensionInfo && homeExtensionInfo.id) { + value.home_extension_id = homeExtensionInfo.id; + homeAffected = true; + homePageCategory = "extension"; + } + if (!homeAffected && !lazy.HomePage.overridden) { + homePageCategory = "enabled"; + } + + let page; + if (newtabAffected && homeAffected) { + page = "both"; + } else if (newtabAffected) { + page = "about:newtab"; + } else if (homeAffected) { + page = "about:home"; + } + + if (page) { + const event = Object.assign(this.createPing(), { + action: "activity_stream_user_event", + event: "PAGE_TAKEOVER_DATA", + value, + page, + session_id: "n/a", + }); + this.sendEvent(event); + } + Glean.newtab.newtabCategory.set(newtabCategory); + Glean.newtab.homepageCategory.set(homePageCategory); + if (lazy.NimbusFeatures.glean.getVariable("newtabPingEnabled") ?? true) { + GleanPings.newtab.submit("component_init"); + } + } + } + + onAction(action) { + switch (action.type) { + case at.INIT: + this.init(); + this.sendPageTakeoverData(); + break; + case at.NEW_TAB_INIT: + this.handleNewTabInit(action); + break; + case at.NEW_TAB_UNLOAD: + this.endSession(au.getPortIdOfSender(action)); + break; + case at.SAVE_SESSION_PERF_DATA: + this.saveSessionPerfData(au.getPortIdOfSender(action), action.data); + break; + case at.TELEMETRY_IMPRESSION_STATS: + this.handleImpressionStats(action); + break; + case at.DISCOVERY_STREAM_IMPRESSION_STATS: + this.handleDiscoveryStreamImpressionStats( + au.getPortIdOfSender(action), + action.data + ); + break; + case at.DISCOVERY_STREAM_LOADED_CONTENT: + this.handleDiscoveryStreamLoadedContent( + au.getPortIdOfSender(action), + action.data + ); + break; + case at.DISCOVERY_STREAM_USER_EVENT: + this.handleDiscoveryStreamUserEvent(action); + break; + case at.TELEMETRY_USER_EVENT: + this.handleUserEvent(action); + break; + // The next few action types come from ASRouter, which doesn't use + // Actions from Actions.jsm, but uses these other custom strings. + case msg.TOOLBAR_BADGE_TELEMETRY: + // Intentional fall-through + case msg.TOOLBAR_PANEL_TELEMETRY: + // Intentional fall-through + case msg.MOMENTS_PAGE_TELEMETRY: + // Intentional fall-through + case msg.DOORHANGER_TELEMETRY: + // Intentional fall-through + case msg.INFOBAR_TELEMETRY: + // Intentional fall-through + case msg.SPOTLIGHT_TELEMETRY: + // Intentional fall-through + case msg.TOAST_NOTIFICATION_TELEMETRY: + // Intentional fall-through + case at.AS_ROUTER_TELEMETRY_USER_EVENT: + this.handleASRouterUserEvent(action); + break; + case at.TOP_SITES_SPONSORED_IMPRESSION_STATS: + this.handleTopSitesSponsoredImpressionStats(action); + break; + case at.TOP_SITES_ORGANIC_IMPRESSION_STATS: + this.handleTopSitesOrganicImpressionStats(action); + break; + case at.UNINIT: + this.uninit(); + break; + } + } + + /** + * Handle impression stats actions from Discovery Stream. The data will be + * stored into the session.impressionSets object for the given port, so that + * it is sent to the server when the session ends. + * + * @note session.impressionSets will be keyed on `source` of the `data`, + * all the data will be appended to an array for the same source. + * + * @param {String} port The session port with which this is associated + * @param {Object} data The impression data structured as {source: "SOURCE", tiles: [{id: 123}]} + * + */ + handleDiscoveryStreamImpressionStats(port, data) { + let session = this.sessions.get(port); + + if (!session) { + throw new Error("Session does not exist."); + } + + const { window_inner_width, window_inner_height, source, tiles } = data; + const impressionSets = session.impressionSets || {}; + const impressions = impressionSets[source] || { + tiles: [], + window_inner_width, + window_inner_height, + }; + // The payload might contain other properties, we need `id`, `pos` and potentially `shim` here. + tiles.forEach(tile => { + impressions.tiles.push({ + id: tile.id, + pos: tile.pos, + ...(tile.shim ? { shim: tile.shim } : {}), + }); + Glean.pocket.impression.record({ + newtab_visit_id: session.session_id, + is_sponsored: tile.type === "spoc", + position: tile.pos, + }); + }); + impressionSets[source] = impressions; + session.impressionSets = impressionSets; + } + + /** + * Handle loaded content actions from Discovery Stream. The data will be + * stored into the session.loadedContentSets object for the given port, so that + * it is sent to the server when the session ends. + * + * @note session.loadedContentSets will be keyed on `source` of the `data`, + * all the data will be appended to an array for the same source. + * + * @param {String} port The session port with which this is associated + * @param {Object} data The loaded content structured as {source: "SOURCE", tiles: [{id: 123}]} + * + */ + handleDiscoveryStreamLoadedContent(port, data) { + let session = this.sessions.get(port); + + if (!session) { + throw new Error("Session does not exist."); + } + + const loadedContentSets = session.loadedContentSets || {}; + const loadedContents = loadedContentSets[data.source] || []; + // The payload might contain other properties, we need `id` and `pos` here. + data.tiles.forEach(tile => + loadedContents.push({ id: tile.id, pos: tile.pos }) + ); + loadedContentSets[data.source] = loadedContents; + session.loadedContentSets = loadedContentSets; + } + + /** + * Take all enumerable members of the data object and merge them into + * the session.perf object for the given port, so that it is sent to the + * server when the session ends. All members of the data object should + * be valid values of the perf object, as defined in pings.js and the + * data*.md documentation. + * + * @note Any existing keys with the same names already in the + * session perf object will be overwritten by values passed in here. + * + * @param {String} port The session with which this is associated + * @param {Object} data The perf data to be + */ + saveSessionPerfData(port, data) { + // XXX should use try/catch and send a bad state indicator if this + // get blows up. + let session = this.sessions.get(port); + + // XXX Partial workaround for #3118; avoids the worst incorrect associations + // of times with browsers, by associating the load trigger with the + // visibility event as the user is most likely associating the trigger to + // the tab just shown. This helps avoid associating with a preloaded + // browser as those don't get the event until shown. Better fix for more + // cases forthcoming. + // + // XXX the about:home check (and the corresponding test) should go away + // once the load_trigger stuff in addSession is refactored into + // setLoadTriggerInfo. + // + if (data.visibility_event_rcvd_ts && session.page !== "about:home") { + this.setLoadTriggerInfo(port); + } + + let timestamp = data.topsites_first_painted_ts; + + if ( + timestamp && + session.page === "about:home" && + !lazy.HomePage.overridden && + Services.prefs.getIntPref("browser.startup.page") === 1 + ) { + lazy.AboutNewTab.maybeRecordTopsitesPainted(timestamp); + } + + Object.assign(session.perf, data); + + if (data.visibility_event_rcvd_ts && !session.newtabOpened) { + session.newtabOpened = true; + const source = ONBOARDING_ALLOWED_PAGE_VALUES.includes(session.page) + ? session.page + : "other"; + Glean.newtab.opened.record({ + newtab_visit_id: session.session_id, + source, + }); + } + } + + _beginObservingNewtabPingPrefs() { + const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors"; + const ACTIVITY_STREAM_PREF_BRANCH = "browser.newtabpage.activity-stream."; + const NEWTAB_PING_PREFS = { + showSearch: Glean.newtabSearch.enabled, + "feeds.topsites": Glean.topsites.enabled, + showSponsoredTopSites: Glean.topsites.sponsoredEnabled, + "feeds.section.topstories": Glean.pocket.enabled, + showSponsored: Glean.pocket.sponsoredStoriesEnabled, + topSitesRows: Glean.topsites.rows, + }; + const setNewtabPrefMetrics = fullPrefName => { + const pref = fullPrefName.slice(ACTIVITY_STREAM_PREF_BRANCH.length); + if (!Object.hasOwn(NEWTAB_PING_PREFS, pref)) { + return; + } + const metric = NEWTAB_PING_PREFS[pref]; + switch (Services.prefs.getPrefType(fullPrefName)) { + case Services.prefs.PREF_BOOL: + metric.set(Services.prefs.getBoolPref(fullPrefName)); + break; + + case Services.prefs.PREF_INT: + metric.set(Services.prefs.getIntPref(fullPrefName)); + break; + } + }; + Services.prefs.addObserver( + ACTIVITY_STREAM_PREF_BRANCH, + (subject, topic, data) => setNewtabPrefMetrics(data) + ); + for (const pref of Object.keys(NEWTAB_PING_PREFS)) { + const fullPrefName = ACTIVITY_STREAM_PREF_BRANCH + pref; + setNewtabPrefMetrics(fullPrefName); + } + Glean.pocket.isSignedIn.set(lazy.pktApi.isUserLoggedIn()); + + const setBlockedSponsorsMetrics = () => { + let blocklist; + try { + blocklist = JSON.parse( + Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]") + ); + } catch (e) {} + if (blocklist) { + Glean.newtab.blockedSponsors.set(blocklist); + } + }; + + Services.prefs.addObserver( + TOP_SITES_BLOCKED_SPONSORS_PREF, + setBlockedSponsorsMetrics + ); + setBlockedSponsorsMetrics(); + } + + uninit() { + try { + Services.obs.removeObserver( + this.browserOpenNewtabStart, + "browser-open-newtab-start" + ); + Services.obs.removeObserver( + this._addWindowListeners, + DOMWINDOW_OPENED_TOPIC + ); + } catch (e) { + // Operation can fail when uninit is called before + // init has finished setting up the observer + } + + // Only uninit if the getter has initialized it + if (Object.prototype.hasOwnProperty.call(this, "pingCentre")) { + this.pingCentre.uninit(); + } + if (Object.prototype.hasOwnProperty.call(this, "utEvents")) { + this.utEvents.uninit(); + } + + // TODO: Send any unfinished sessions + } +} + +const EXPORTED_SYMBOLS = [ + "TelemetryFeed", + "USER_PREFS_ENCODING", + "PREF_IMPRESSION_ID", + "TELEMETRY_PREF", + "EVENTS_TELEMETRY_PREF", + "STRUCTURED_INGESTION_ENDPOINT_PREF", +]; |