diff options
Diffstat (limited to 'toolkit/components/telemetry/app')
| -rw-r--r-- | toolkit/components/telemetry/app/ClientID.jsm | 498 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryArchive.jsm | 123 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryController.jsm | 45 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryControllerBase.jsm | 143 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryControllerContent.jsm | 108 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryControllerParent.jsm | 1450 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryEnvironment.jsm | 2128 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm | 584 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryScheduler.jsm | 434 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetrySend.jsm | 1600 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryStorage.jsm | 2221 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryTimestamps.jsm | 55 | ||||
| -rw-r--r-- | toolkit/components/telemetry/app/TelemetryUtils.jsm | 301 |
13 files changed, 9690 insertions, 0 deletions
diff --git a/toolkit/components/telemetry/app/ClientID.jsm b/toolkit/components/telemetry/app/ClientID.jsm new file mode 100644 index 0000000000..43d95ab9b8 --- /dev/null +++ b/toolkit/components/telemetry/app/ClientID.jsm @@ -0,0 +1,498 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["ClientID"]; + +const { XPCOMUtils } = ChromeUtils.import( + "resource://gre/modules/XPCOMUtils.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm"); +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +if (AppConstants.MOZ_GLEAN) { + Cu.importGlobalProperties(["Glean"]); +} + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "ClientID::"; +// Must match ID in TelemetryUtils +const CANARY_CLIENT_ID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"; + +ChromeUtils.defineModuleGetter( + this, + "CommonUtils", + "resource://services-common/utils.js" +); + +XPCOMUtils.defineLazyGetter(this, "CryptoHash", () => { + return Components.Constructor( + "@mozilla.org/security/hash;1", + "nsICryptoHash", + "initWithString" + ); +}); + +XPCOMUtils.defineLazyGetter(this, "gDatareportingPath", () => { + return PathUtils.join( + Services.dirsvc.get("ProfD", Ci.nsIFile).path, + "datareporting" + ); +}); + +XPCOMUtils.defineLazyGetter(this, "gStateFilePath", () => { + return PathUtils.join(gDatareportingPath, "state.json"); +}); + +const PREF_CACHED_CLIENTID = "toolkit.telemetry.cachedClientID"; + +const SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID = + "deletion.request.ecosystem_client_id"; + +/** + * Checks if client ID has a valid format. + * + * @param {String} id A string containing the client ID. + * @return {Boolean} True when the client ID has valid format, or False + * otherwise. + */ +function isValidClientID(id) { + const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + return UUID_REGEX.test(id); +} + +var ClientID = Object.freeze({ + /** + * This returns a promise resolving to the the stable client ID we use for + * data reporting (FHR & Telemetry). + * + * WARNING: This functionality is duplicated for Android (see GeckoProfile.getClientId + * for more). There are Java tests (TestGeckoProfile) to ensure the functionality is + * consistent and Gecko tests to come (bug 1249156). However, THIS IS NOT FOOLPROOF. + * Be careful when changing this code and, in particular, the underlying file format. + * + * @return {Promise<string>} The stable client ID. + */ + getClientID() { + return ClientIDImpl.getClientID(); + }, + + /** + * Returns a promise resolving to the ecosystem client ID, used in ecosystem + * pings as a stable identifier for this profile. + * + * @return {Promise<string>} The ecosystem client ID. + */ + getEcosystemClientID() { + return ClientIDImpl.getEcosystemClientID(); + }, + + /** + * This returns true if the client ID prior to the last client ID reset was a canary client ID. + * Android only. Always returns null on Desktop. + */ + wasCanaryClientID() { + if (AppConstants.platform == "android") { + return ClientIDImpl.wasCanaryClientID(); + } + + return null; + }, + + /** + * Get the client id synchronously without hitting the disk. + * This returns: + * - the current on-disk client id if it was already loaded + * - the client id that we cached into preferences (if any) + * - null otherwise + */ + getCachedClientID() { + return ClientIDImpl.getCachedClientID(); + }, + + /** + * Gets the in-memory cached ecosystem client ID if it was already loaded; + * `null` otherwise. + */ + getCachedEcosystemClientID() { + return ClientIDImpl.getCachedEcosystemClientID(); + }, + + async getClientIdHash() { + return ClientIDImpl.getClientIdHash(); + }, + + /** + * Sets the main and ecosystem client IDs to the canary (known) client ID, + * writing them to disk and updating the cached versions. + * + * Use `removeClientIDs` followed by `get{Ecosystem}ClientID` to clear the + * existing IDs and generate new, random ones if required. + * + * @return {Promise<void>} + */ + setCanaryClientIDs() { + return ClientIDImpl.setCanaryClientIDs(); + }, + + /** + * Sets the ecosystem client IDs to a new random value while leaving other IDs + * unchanged, writing the result to disk and updating the cached identifier. + * This can be used when a user signs out, to avoid linking telemetry between + * different accounts. + * + * Use `removeClientIDs` followed by `get{Ecosystem}ClientID` to reset *all* the + * identifiers rather than just the ecosystem client id. + * + * @return {Promise<void>} Resolves when the change has been saved to disk. + */ + resetEcosystemClientID() { + return ClientIDImpl.resetEcosystemClientID(); + }, + + /** + * Clears the main and ecosystem client IDs asynchronously, removing them + * from disk. Use `getClientID()` and `getEcosystemClientID()` to generate + * fresh IDs after calling this method. + * + * Should only be used if a reset is explicitly requested by the user. + * + * @return {Promise<void>} + */ + removeClientIDs() { + return ClientIDImpl.removeClientIDs(); + }, + + /** + * Only used for testing. Invalidates the cached client IDs so that they're + * read again from file, but doesn't remove the existing IDs from disk. + */ + _reset() { + return ClientIDImpl._reset(); + }, +}); + +var ClientIDImpl = { + _clientID: null, + _clientIDHash: null, + _ecosystemClientID: null, + _loadClientIdsTask: null, + _saveClientIdsTask: null, + _removeClientIdsTask: null, + _logger: null, + _wasCanary: null, + + _loadClientIDs() { + if (this._loadClientIdsTask) { + return this._loadClientIdsTask; + } + + this._loadClientIdsTask = this._doLoadClientIDs(); + let clear = () => (this._loadClientIdsTask = null); + this._loadClientIdsTask.then(clear, clear); + return this._loadClientIdsTask; + }, + + /** + * Load the client IDs (Telemetry Client ID and Ecosystem Client ID) from the + * DataReporting Service state file. If either ID is missing, we generate a + * new one. + */ + async _doLoadClientIDs() { + this._log.trace(`_doLoadClientIDs`); + // If there's a removal in progress, let's wait for it + await this._removeClientIdsTask; + + // Try to load the client id from the DRS state file. + let hasCurrentClientID = false; + let hasCurrentEcosystemClientID = false; + try { + let state = await CommonUtils.readJSON(gStateFilePath); + if (AppConstants.platform == "android" && state && "wasCanary" in state) { + this._wasCanary = state.wasCanary; + } + if (state) { + hasCurrentClientID = this.updateClientID(state.clientID); + hasCurrentEcosystemClientID = this.updateEcosystemClientID( + state.ecosystemClientID + ); + if (hasCurrentClientID && hasCurrentEcosystemClientID) { + this._log.trace(`_doLoadClientIDs: Client IDs loaded from state.`); + return { + clientID: this._clientID, + ecosystemClientID: this._ecosystemClientID, + }; + } + } + } catch (e) { + // fall through to next option + } + + // We're missing one or both IDs from the DRS state file, generate new ones. + if (!hasCurrentClientID) { + this.updateClientID(CommonUtils.generateUUID()); + } + if (!hasCurrentEcosystemClientID) { + this.updateEcosystemClientID(CommonUtils.generateUUID()); + } + this._saveClientIdsTask = this._saveClientIDs(); + + // Wait on persisting the id. Otherwise failure to save the ID would result in + // the client creating and subsequently sending multiple IDs to the server. + // This would appear as multiple clients submitting similar data, which would + // result in orphaning. + await this._saveClientIdsTask; + + this._log.trace("_doLoadClientIDs: New client IDs loaded and persisted."); + return { + clientID: this._clientID, + ecosystemClientID: this._ecosystemClientID, + }; + }, + + /** + * Save the client ID to the client ID file. + * + * @return {Promise} A promise resolved when the client ID is saved to disk. + */ + async _saveClientIDs() { + this._log.trace(`_saveClientIDs`); + let obj = { + clientID: this._clientID, + ecosystemClientID: this._ecosystemClientID, + }; + // We detected a canary client ID when resetting, storing this as a flag + if (AppConstants.platform == "android" && this._wasCanary) { + obj.wasCanary = true; + } + try { + await IOUtils.makeDirectory(gDatareportingPath); + } catch (ex) { + if (ex.name != "NotAllowedError") { + throw ex; + } + } + await CommonUtils.writeJSON(obj, gStateFilePath); + this._saveClientIdsTask = null; + }, + + /** + * This returns a promise resolving to the the stable client ID we use for + * data reporting (FHR & Telemetry). + * + * @return {Promise<string>} The stable client ID. + */ + async getClientID() { + if (!this._clientID) { + let { clientID } = await this._loadClientIDs(); + return clientID; + } + + return Promise.resolve(this._clientID); + }, + + async getEcosystemClientID() { + if (!this._ecosystemClientID) { + let { ecosystemClientID } = await this._loadClientIDs(); + return ecosystemClientID; + } + + return Promise.resolve(this._ecosystemClientID); + }, + + /** + * This returns true if the client ID prior to the last client ID reset was a canary client ID. + * Android only. Always returns null on Desktop. + */ + wasCanaryClientID() { + return this._wasCanary; + }, + + /** + * Get the client id synchronously without hitting the disk. + * This returns: + * - the current on-disk client id if it was already loaded + * - the client id that we cached into preferences (if any) + * - null otherwise + */ + getCachedClientID() { + if (this._clientID) { + // Already loaded the client id from disk. + return this._clientID; + } + + // If the client id cache contains a value of the wrong type, + // reset the pref. We need to do this before |getStringPref| since + // it will just return |null| in that case and we won't be able + // to distinguish between the missing pref and wrong type cases. + if ( + Services.prefs.prefHasUserValue(PREF_CACHED_CLIENTID) && + Services.prefs.getPrefType(PREF_CACHED_CLIENTID) != + Ci.nsIPrefBranch.PREF_STRING + ) { + this._log.error( + "getCachedClientID - invalid client id type in preferences, resetting" + ); + Services.prefs.clearUserPref(PREF_CACHED_CLIENTID); + } + + // Not yet loaded, return the cached client id if we have one. + let id = Services.prefs.getStringPref(PREF_CACHED_CLIENTID, null); + if (id === null) { + return null; + } + if (!isValidClientID(id)) { + this._log.error( + "getCachedClientID - invalid client id in preferences, resetting", + id + ); + Services.prefs.clearUserPref(PREF_CACHED_CLIENTID); + return null; + } + return id; + }, + + getCachedEcosystemClientID() { + return this._ecosystemClientID; + }, + + async getClientIdHash() { + if (!this._clientIDHash) { + let byteArr = new TextEncoder().encode(await this.getClientID()); + let hash = new CryptoHash("sha256"); + hash.update(byteArr, byteArr.length); + this._clientIDHash = CommonUtils.bytesAsHex(hash.finish(false)); + } + return this._clientIDHash; + }, + + /* + * Resets the provider. This is for testing only. + */ + async _reset() { + await this._loadClientIdsTask; + await this._saveClientIdsTask; + this._clientID = null; + this._clientIDHash = null; + this._ecosystemClientID = null; + }, + + async setCanaryClientIDs() { + this._log.trace("setCanaryClientIDs"); + this.updateClientID(CANARY_CLIENT_ID); + this.updateEcosystemClientID(CANARY_CLIENT_ID); + + this._saveClientIdsTask = this._saveClientIDs(); + await this._saveClientIdsTask; + return this._clientID; + }, + + async resetEcosystemClientID() { + this._log.trace("resetEcosystemClientID"); + this.updateEcosystemClientID(CommonUtils.generateUUID()); + this._saveClientIdsTask = this._saveClientIDs(); + await this._saveClientIdsTask; + return this._ecosystemClientID; + }, + + async _doRemoveClientIDs() { + this._log.trace("_doRemoveClientIDs"); + + // Reset the cached main and ecosystem client IDs. + this._clientID = null; + this._clientIDHash = null; + this._ecosystemClientID = null; + + // Clear the client id from the preference cache. + Services.prefs.clearUserPref(PREF_CACHED_CLIENTID); + + // Clear the old ecosystem client ID from the deletion request scalar store. + Services.telemetry.scalarSet( + SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID, + "" + ); + + // If there is a save in progress, wait for it to complete. + await this._saveClientIdsTask; + + // Remove the client id from disk + await IOUtils.remove(gStateFilePath); + }, + + async removeClientIDs() { + this._log.trace("removeClientIDs"); + let oldClientId = this._clientID; + + // Wait for the removal. + // Asynchronous calls to getClientID will also be blocked on this. + this._removeClientIdsTask = this._doRemoveClientIDs(); + let clear = () => (this._removeClientIdsTask = null); + this._removeClientIdsTask.then(clear, clear); + + await this._removeClientIdsTask; + + // On Android we detect resets after a canary client ID. + if (AppConstants.platform == "android") { + this._wasCanary = oldClientId == CANARY_CLIENT_ID; + } + }, + + /** + * Sets the client id to the given value and updates the value cached in + * preferences only if the given id is a valid. + * + * @param {String} id A string containing the client ID. + * @return {Boolean} True when the client ID has valid format, or False + * otherwise. + */ + updateClientID(id) { + if (!isValidClientID(id)) { + this._log.error("updateClientID - invalid client ID", id); + return false; + } + + this._clientID = id; + + if (AppConstants.MOZ_GLEAN) { + Glean.fogValidation.legacyTelemetryClientId.set(this._clientID); + } + + this._clientIDHash = null; + Services.prefs.setStringPref(PREF_CACHED_CLIENTID, this._clientID); + return true; + }, + + updateEcosystemClientID(id) { + if (!isValidClientID(id)) { + this._log.error( + "updateEcosystemClientID - invalid ecosystem client ID", + id + ); + return false; + } + this._ecosystemClientID = id; + Services.telemetry.scalarSet( + SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID, + id + ); + return true; + }, + + /** + * A helper for getting access to telemetry logger. + */ + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + + return this._logger; + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryArchive.jsm b/toolkit/components/telemetry/app/TelemetryArchive.jsm new file mode 100644 index 0000000000..bbbc6fce71 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryArchive.jsm @@ -0,0 +1,123 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryArchive"]; + +ChromeUtils.import("resource://gre/modules/Log.jsm", this); +ChromeUtils.import("resource://gre/modules/Preferences.jsm", this); +ChromeUtils.import("resource://gre/modules/osfile.jsm", this); +ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this); + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetryArchive::"; + +ChromeUtils.defineModuleGetter( + this, + "TelemetryStorage", + "resource://gre/modules/TelemetryStorage.jsm" +); + +var TelemetryArchive = { + /** + * Get a list of the archived pings, sorted by the creation date. + * Note that scanning the archived pings on disk is delayed on startup, + * use promizeInitialized() to access this after scanning. + * + * @return {Promise<sequence<Object>>} + * A list of the archived ping info in the form: + * { id: <string>, + * timestampCreated: <number>, + * type: <string> } + */ + promiseArchivedPingList() { + return TelemetryArchiveImpl.promiseArchivedPingList(); + }, + + /** + * Load an archived ping from disk by id, asynchronously. + * + * @param id {String} The pings UUID. + * @return {Promise<PingData>} A promise resolved with the pings data on success. + */ + promiseArchivedPingById(id) { + return TelemetryArchiveImpl.promiseArchivedPingById(id); + }, + + /** + * Archive a ping and persist it to disk. + * + * @param {object} ping The ping data to archive. + * @return {promise} Promise that is resolved when the ping is successfully archived. + */ + promiseArchivePing(ping) { + return TelemetryArchiveImpl.promiseArchivePing(ping); + }, +}; + +/** + * Checks if pings can be archived. Some products (e.g. Thunderbird) might not want + * to do that. + * @return {Boolean} True if pings should be archived, false otherwise. + */ +function shouldArchivePings() { + return Preferences.get(TelemetryUtils.Preferences.ArchiveEnabled, false); +} + +var TelemetryArchiveImpl = { + _logger: null, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + + return this._logger; + }, + + promiseArchivePing(ping) { + if (!shouldArchivePings()) { + this._log.trace("promiseArchivePing - archiving is disabled"); + return Promise.resolve(); + } + + for (let field of ["creationDate", "id", "type"]) { + if (!(field in ping)) { + this._log.warn("promiseArchivePing - missing field " + field); + return Promise.reject(new Error("missing field " + field)); + } + } + + return TelemetryStorage.saveArchivedPing(ping); + }, + + _buildArchivedPingList(archivedPingsMap) { + let list = Array.from(archivedPingsMap, p => ({ + id: p[0], + timestampCreated: p[1].timestampCreated, + type: p[1].type, + })); + + list.sort((a, b) => a.timestampCreated - b.timestampCreated); + + return list; + }, + + promiseArchivedPingList() { + this._log.trace("promiseArchivedPingList"); + + return TelemetryStorage.loadArchivedPingList().then(loadedInfo => { + return this._buildArchivedPingList(loadedInfo); + }); + }, + + promiseArchivedPingById(id) { + this._log.trace("promiseArchivedPingById - id: " + id); + return TelemetryStorage.loadArchivedPing(id); + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryController.jsm b/toolkit/components/telemetry/app/TelemetryController.jsm new file mode 100644 index 0000000000..7d0dcd3b76 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryController.jsm @@ -0,0 +1,45 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/** + * This module chooses the correct telemetry controller module to load + * based on the process: + * + * - TelemetryControllerParent is loaded only in the parent process, and + * contains code specific to the parent. + * - TelemetryControllerContent is loaded only in content processes, and + * contains code specific to them. + * + * Both the parent and the content modules load TelemetryControllerBase, + * which contains code which is common to all processes. + * + * This division is important for content process memory usage and + * startup time. The parent-specific code occupies tens of KB of memory + * which, multiplied by the number of content processes we have, adds up + * fast. + */ + +var EXPORTED_SYMBOLS = ["TelemetryController"]; + +// We can't use Services.appinfo here because tests stub out the appinfo +// service, and if we touch Services.appinfo now, the built-in version +// will be cached in place of the stub. +const isParentProcess = + // eslint-disable-next-line mozilla/use-services + Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime).processType === + Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT; + +var TelemetryController; +if (isParentProcess) { + ({ TelemetryController } = ChromeUtils.import( + "resource://gre/modules/TelemetryControllerParent.jsm" + )); +} else { + ({ TelemetryController } = ChromeUtils.import( + "resource://gre/modules/TelemetryControllerContent.jsm" + )); +} diff --git a/toolkit/components/telemetry/app/TelemetryControllerBase.jsm b/toolkit/components/telemetry/app/TelemetryControllerBase.jsm new file mode 100644 index 0000000000..4f17fac5fe --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryControllerBase.jsm @@ -0,0 +1,143 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryControllerBase"]; + +ChromeUtils.defineModuleGetter( + this, + "AppConstants", + "resource://gre/modules/AppConstants.jsm" +); +const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm"); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetryController::"; + +const PREF_BRANCH_LOG = "toolkit.telemetry.log."; +const PREF_LOG_LEVEL = "toolkit.telemetry.log.level"; +const PREF_LOG_DUMP = "toolkit.telemetry.log.dump"; + +const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled"; + +const Preferences = Object.freeze({ + OverridePreRelease: "toolkit.telemetry.testing.overridePreRelease", + Unified: "toolkit.telemetry.unified", +}); + +/** + * Setup Telemetry logging. This function also gets called when loggin related + * preferences change. + */ +var gLogger = null; +var gPrefixLogger = null; +var gLogAppenderDump = null; + +var TelemetryControllerBase = Object.freeze({ + // Whether the FHR/Telemetry unification features are enabled. + // Changing this pref requires a restart. + IS_UNIFIED_TELEMETRY: Services.prefs.getBoolPref(Preferences.Unified, false), + + Preferences, + + /** + * Returns the state of the Telemetry enabled preference, making sure + * it correctly evaluates to a boolean type. + */ + get isTelemetryEnabled() { + return Services.prefs.getBoolPref(PREF_TELEMETRY_ENABLED, false) === true; + }, + + get log() { + if (!gPrefixLogger) { + gPrefixLogger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + return gPrefixLogger; + }, + + configureLogging() { + if (!gLogger) { + gLogger = Log.repository.getLogger(LOGGER_NAME); + + // Log messages need to go to the browser console. + let consoleAppender = new Log.ConsoleAppender(new Log.BasicFormatter()); + gLogger.addAppender(consoleAppender); + + Services.prefs.addObserver(PREF_BRANCH_LOG, this.configureLogging); + } + + // Make sure the logger keeps up with the logging level preference. + gLogger.level = + Log.Level[Services.prefs.getStringPref(PREF_LOG_LEVEL, "Warn")]; + + // If enabled in the preferences, add a dump appender. + let logDumping = Services.prefs.getBoolPref(PREF_LOG_DUMP, false); + if (logDumping != !!gLogAppenderDump) { + if (logDumping) { + gLogAppenderDump = new Log.DumpAppender(new Log.BasicFormatter()); + gLogger.addAppender(gLogAppenderDump); + } else { + gLogger.removeAppender(gLogAppenderDump); + gLogAppenderDump = null; + } + } + }, + + /** + * Set the Telemetry core recording flag for Unified Telemetry. + */ + setTelemetryRecordingFlags() { + // Enable extended Telemetry on pre-release channels and disable it + // on Release/ESR. + let prereleaseChannels = ["nightly", "aurora", "beta"]; + if (!AppConstants.MOZILLA_OFFICIAL) { + // Turn extended telemetry for local developer builds. + prereleaseChannels.push("default"); + } + const isPrereleaseChannel = prereleaseChannels.includes( + AppConstants.MOZ_UPDATE_CHANNEL + ); + const isReleaseCandidateOnBeta = + AppConstants.MOZ_UPDATE_CHANNEL === "release" && + Services.prefs.getCharPref("app.update.channel", null) === "beta"; + Services.telemetry.canRecordBase = true; + Services.telemetry.canRecordExtended = + isPrereleaseChannel || + isReleaseCandidateOnBeta || + Services.prefs.getBoolPref(this.Preferences.OverridePreRelease, false); + }, + + /** + * Perform telemetry initialization for either chrome or content process. + * @return {Boolean} True if Telemetry is allowed to record at least base (FHR) data, + * false otherwise. + */ + enableTelemetryRecording: function enableTelemetryRecording() { + // Configure base Telemetry recording. + // Unified Telemetry makes it opt-out. If extended Telemetry is enabled, base recording + // is always on as well. + if (this.IS_UNIFIED_TELEMETRY) { + this.setTelemetryRecordingFlags(); + } else { + // We're not on unified Telemetry, stick to the old behaviour for + // supporting Fennec. + Services.telemetry.canRecordBase = Services.telemetry.canRecordExtended = this.isTelemetryEnabled; + } + + this.log.config( + "enableTelemetryRecording - canRecordBase:" + + Services.telemetry.canRecordBase + + ", canRecordExtended: " + + Services.telemetry.canRecordExtended + ); + + return Services.telemetry.canRecordBase; + }, +}); diff --git a/toolkit/components/telemetry/app/TelemetryControllerContent.jsm b/toolkit/components/telemetry/app/TelemetryControllerContent.jsm new file mode 100644 index 0000000000..7cc36deaa5 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryControllerContent.jsm @@ -0,0 +1,108 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { DeferredTask } = ChromeUtils.import( + "resource://gre/modules/DeferredTask.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { TelemetryControllerBase } = ChromeUtils.import( + "resource://gre/modules/TelemetryControllerBase.jsm" +); + +// Delay before intializing telemetry (ms) +const TELEMETRY_DELAY = + Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000; +// Delay before initializing telemetry if we're testing (ms) +const TELEMETRY_TEST_DELAY = 1; + +var EXPORTED_SYMBOLS = ["TelemetryController", "getTelemetryController"]; + +var TelemetryController = Object.freeze({ + /** + * Used only for testing purposes. + */ + testInitLogging() { + TelemetryControllerBase.configureLogging(); + }, + + /** + * Used only for testing purposes. + */ + testSetupContent() { + return Impl.setupContentTelemetry(true); + }, + + /** + * Send a notification. + */ + observe(aSubject, aTopic, aData) { + return Impl.observe(aSubject, aTopic, aData); + }, + + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), +}); + +var Impl = { + // This is true when running in the test infrastructure. + _testMode: false, + + get _log() { + return TelemetryControllerBase.log; + }, + + /** + * This triggers basic telemetry initialization for content processes. + * @param {Boolean} [testing=false] True if we are in test mode, false otherwise. + */ + setupContentTelemetry(testing = false) { + this._testMode = testing; + + // The thumbnail service also runs in a content process, even with e10s off. + // We need to check if e10s is on so we don't submit child payloads for it. + // We still need xpcshell child tests to work, so we skip this if test mode is enabled. + if (testing || Services.appinfo.browserTabsRemoteAutostart) { + // We call |enableTelemetryRecording| here to make sure that Telemetry.canRecord* flags + // are in sync between chrome and content processes. + if (!TelemetryControllerBase.enableTelemetryRecording()) { + this._log.trace( + "setupContentTelemetry - Content process recording disabled." + ); + return; + } + } + Services.telemetry.earlyInit(); + + // FIXME: This is a terrible abuse of DeferredTask. + let delayedTask = new DeferredTask( + () => { + Services.telemetry.delayedInit(); + }, + testing ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY, + testing ? 0 : undefined + ); + + delayedTask.arm(); + }, + + /** + * This observer drives telemetry. + */ + observe(aSubject, aTopic, aData) { + if (aTopic == "app-startup") { + TelemetryControllerBase.configureLogging(); + + this._log.trace(`observe - ${aTopic} notified.`); + + this.setupContentTelemetry(); + } + }, +}; + +// Used by service registration, which requires a callable function. +function getTelemetryController() { + return TelemetryController; +} diff --git a/toolkit/components/telemetry/app/TelemetryControllerParent.jsm b/toolkit/components/telemetry/app/TelemetryControllerParent.jsm new file mode 100644 index 0000000000..832b9e0b28 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryControllerParent.jsm @@ -0,0 +1,1450 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +const { PromiseUtils } = ChromeUtils.import( + "resource://gre/modules/PromiseUtils.jsm" +); +const { DeferredTask } = ChromeUtils.import( + "resource://gre/modules/DeferredTask.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { TelemetryUtils } = ChromeUtils.import( + "resource://gre/modules/TelemetryUtils.jsm" +); +const { TelemetryControllerBase } = ChromeUtils.import( + "resource://gre/modules/TelemetryControllerBase.jsm" +); +const { XPCOMUtils } = ChromeUtils.import( + "resource://gre/modules/XPCOMUtils.jsm" +); + +const Utils = TelemetryUtils; + +const PING_FORMAT_VERSION = 4; + +// Delay before intializing telemetry (ms) +const TELEMETRY_DELAY = + Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000; +// Delay before initializing telemetry if we're testing (ms) +const TELEMETRY_TEST_DELAY = 1; + +// How long to wait (ms) before sending the new profile ping on the first +// run of a new profile. +const NEWPROFILE_PING_DEFAULT_DELAY = 30 * 60 * 1000; + +// Ping types. +const PING_TYPE_MAIN = "main"; +const PING_TYPE_DELETION_REQUEST = "deletion-request"; +const PING_TYPE_UNINSTALL = "uninstall"; + +// Session ping reasons. +const REASON_GATHER_PAYLOAD = "gather-payload"; +const REASON_GATHER_SUBSESSION_PAYLOAD = "gather-subsession-payload"; + +XPCOMUtils.defineLazyServiceGetter( + this, + "Telemetry", + "@mozilla.org/base/telemetry;1", + "nsITelemetry" +); + +ChromeUtils.defineModuleGetter( + this, + "jwcrypto", + "resource://services-crypto/jwcrypto.jsm" +); + +XPCOMUtils.defineLazyModuleGetters(this, { + ClientID: "resource://gre/modules/ClientID.jsm", + CoveragePing: "resource://gre/modules/CoveragePing.jsm", + AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm", + TelemetryStorage: "resource://gre/modules/TelemetryStorage.jsm", + TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.jsm", + TelemetryArchive: "resource://gre/modules/TelemetryArchive.jsm", + TelemetrySession: "resource://gre/modules/TelemetrySession.jsm", + TelemetrySend: "resource://gre/modules/TelemetrySend.jsm", + TelemetryReportingPolicy: + "resource://gre/modules/TelemetryReportingPolicy.jsm", + TelemetryModules: "resource://gre/modules/ModulesPing.jsm", + TelemetryUntrustedModulesPing: + "resource://gre/modules/UntrustedModulesPing.jsm", + UpdatePing: "resource://gre/modules/UpdatePing.jsm", + TelemetryHealthPing: "resource://gre/modules/HealthPing.jsm", + TelemetryEventPing: "resource://gre/modules/EventPing.jsm", + EcosystemTelemetry: "resource://gre/modules/EcosystemTelemetry.jsm", + TelemetryPrioPing: "resource://gre/modules/PrioPing.jsm", + UninstallPing: "resource://gre/modules/UninstallPing.jsm", + OS: "resource://gre/modules/osfile.jsm", +}); + +/** + * This is a policy object used to override behavior for testing. + */ +var Policy = { + now: () => new Date(), + generatePingId: () => Utils.generateUUID(), + getCachedClientID: () => ClientID.getCachedClientID(), +}; + +var EXPORTED_SYMBOLS = ["TelemetryController"]; + +var TelemetryController = Object.freeze({ + /** + * Used only for testing purposes. + */ + testInitLogging() { + TelemetryControllerBase.configureLogging(); + }, + + /** + * Used only for testing purposes. + */ + testReset() { + return Impl.reset(); + }, + + /** + * Used only for testing purposes. + */ + testSetup() { + return Impl.setupTelemetry(true); + }, + + /** + * Used only for testing purposes. + */ + testShutdown() { + return Impl.shutdown(); + }, + + /** + * Used only for testing purposes. + */ + testPromiseJsProbeRegistration() { + return Promise.resolve(Impl._probeRegistrationPromise); + }, + + /** + * Register 'dynamic builtin' probes from the JSON definition files. + * This is needed to support adding new probes in developer builds + * without rebuilding the whole codebase. + * + * This is not meant to be used outside of local developer builds. + */ + testRegisterJsProbes() { + return Impl.registerJsProbes(); + }, + + /** + * Used only for testing purposes. + */ + testPromiseDeletionRequestPingSubmitted() { + return Promise.resolve(Impl._deletionRequestPingSubmittedPromise); + }, + + /** + * Send a notification. + */ + observe(aSubject, aTopic, aData) { + return Impl.observe(aSubject, aTopic, aData); + }, + + /** + * Submit ping payloads to Telemetry. This will assemble a complete ping, adding + * environment data, client id and some general info. + * Depending on configuration, the ping will be sent to the server (immediately or later) + * and archived locally. + * + * To identify the different pings and to be able to query them pings have a type. + * A type is a string identifier that should be unique to the type ping that is being submitted, + * it should only contain alphanumeric characters and '-' for separation, i.e. satisfy: + * /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} [aOptions] Options object. + * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client + * id, false otherwise. + * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the + * environment data. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * @returns {Promise} Test-only - a promise that resolves with the ping id once the ping is stored or sent. + */ + submitExternalPing(aType, aPayload, aOptions = {}) { + aOptions.addClientId = aOptions.addClientId || false; + aOptions.addEnvironment = aOptions.addEnvironment || false; + aOptions.usePingSender = aOptions.usePingSender || false; + + return Impl.submitExternalPing(aType, aPayload, aOptions); + }, + + /** + * Get the current session ping data as it would be sent out or stored. + * + * @param {bool} aSubsession Whether to get subsession data. Optional, defaults to false. + * @return {object} The current ping data if Telemetry is enabled, null otherwise. + */ + getCurrentPingData(aSubsession = false) { + return Impl.getCurrentPingData(aSubsession); + }, + + /** + * Save a ping to disk. + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} [aOptions] Options object. + * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client + * id, false otherwise. + * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the + * environment data. + * @param {Boolean} [aOptions.overwrite=false] true overwrites a ping with the same name, + * if found. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * + * @returns {Promise} A promise that resolves with the ping id when the ping is saved to + * disk. + */ + addPendingPing(aType, aPayload, aOptions = {}) { + let options = aOptions; + options.addClientId = aOptions.addClientId || false; + options.addEnvironment = aOptions.addEnvironment || false; + options.overwrite = aOptions.overwrite || false; + + return Impl.addPendingPing(aType, aPayload, options); + }, + + /** + * Check if we have an aborted-session ping from a previous session. + * If so, submit and then remove it. + * + * @return {Promise} Promise that is resolved when the ping is saved. + */ + checkAbortedSessionPing() { + return Impl.checkAbortedSessionPing(); + }, + + /** + * Save an aborted-session ping to disk without adding it to the pending pings. + * + * @param {Object} aPayload The ping payload data. + * @return {Promise} Promise that is resolved when the ping is saved. + */ + saveAbortedSessionPing(aPayload) { + return Impl.saveAbortedSessionPing(aPayload); + }, + + /** + * Remove the aborted-session ping if any exists. + * + * @return {Promise} Promise that is resolved when the ping was removed. + */ + removeAbortedSessionPing() { + return Impl.removeAbortedSessionPing(); + }, + + /** + * Create an uninstall ping and write it to disk, replacing any already present. + * This is stored independently from other pings, and only read by + * the Windows uninstaller. + * + * WINDOWS ONLY, does nothing and resolves immediately on other platforms. + * + * @return {Promise} Resolved when the ping has been saved. + */ + saveUninstallPing() { + return Impl.saveUninstallPing(); + }, + + /** + * Allows the sync ping to tell the controller that it is initializing, so + * should be included in the orderly shutdown process. + * + * @param {Function} aFnShutdown The function to call as telemetry shuts down. + + */ + registerSyncPingShutdown(afnShutdown) { + Impl.registerSyncPingShutdown(afnShutdown); + }, + + /** + * Allows waiting for TelemetryControllers delayed initialization to complete. + * The returned promise is guaranteed to resolve before TelemetryController is shutting down. + * @return {Promise} Resolved when delayed TelemetryController initialization completed. + */ + promiseInitialized() { + return Impl.promiseInitialized(); + }, +}); + +var Impl = { + _initialized: false, + _initStarted: false, // Whether we started setting up TelemetryController. + _shuttingDown: false, // Whether the browser is shutting down. + _shutDown: false, // Whether the browser has shut down. + _logger: null, + _prevValues: {}, + // The previous build ID, if this is the first run with a new build. + // Undefined if this is not the first run, or the previous build ID is unknown. + _previousBuildID: undefined, + _clientID: null, + // A task performing delayed initialization + _delayedInitTask: null, + // The deferred promise resolved when the initialization task completes. + _delayedInitTaskDeferred: null, + + // This is a public barrier Telemetry clients can use to add blockers to the shutdown + // of TelemetryController. + // After this barrier, clients can not submit Telemetry pings anymore. + _shutdownBarrier: new AsyncShutdown.Barrier( + "TelemetryController: Waiting for clients." + ), + // This is a private barrier blocked by pending async ping activity (sending & saving). + _connectionsBarrier: new AsyncShutdown.Barrier( + "TelemetryController: Waiting for pending ping activity" + ), + // This is true when running in the test infrastructure. + _testMode: false, + // The task performing the delayed sending of the "new-profile" ping. + _delayedNewPingTask: null, + // The promise used to wait for the JS probe registration (dynamic builtin). + _probeRegistrationPromise: null, + // The promise of any outstanding task sending the "deletion-request" ping. + _deletionRequestPingSubmittedPromise: null, + // A function to shutdown the sync/fxa ping, or null if that ping has not + // self-initialized. + _fnSyncPingShutdown: null, + + get _log() { + return TelemetryControllerBase.log; + }, + + /** + * Get the data for the "application" section of the ping. + */ + _getApplicationSection() { + // Querying architecture and update channel can throw. Make sure to recover and null + // those fields. + let arch = null; + try { + arch = Services.sysinfo.get("arch"); + } catch (e) { + this._log.trace( + "_getApplicationSection - Unable to get system architecture.", + e + ); + } + + let updateChannel = null; + try { + updateChannel = Utils.getUpdateChannel(); + } catch (e) { + this._log.trace( + "_getApplicationSection - Unable to get update channel.", + e + ); + } + + return { + architecture: arch, + buildId: Services.appinfo.appBuildID, + name: Services.appinfo.name, + version: Services.appinfo.version, + displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY, + vendor: Services.appinfo.vendor, + platformVersion: Services.appinfo.platformVersion, + xpcomAbi: Services.appinfo.XPCOMABI, + channel: updateChannel, + }; + }, + + /** + * Assemble a complete ping following the common ping format specification. + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} aOptions Options object. + * @param {Boolean} aOptions.addClientId true if the ping should contain the client + * id, false otherwise. + * @param {Boolean} aOptions.addEnvironment true if the ping should contain the + * environment data. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending. + * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key). + * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled. + * @param {String} [aOptions.studyName=null] the study name to use. + * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled. + * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled. + * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled. + * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise. + * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the + * pioneer id to the provided value. Only works if aOptions.addPioneerId=true. + * @returns {Object} An object that contains the assembled ping data. + */ + assemblePing: function assemblePing(aType, aPayload, aOptions = {}) { + this._log.trace( + "assemblePing - Type " + aType + ", aOptions " + JSON.stringify(aOptions) + ); + + // Clone the payload data so we don't race against unexpected changes in subobjects that are + // still referenced by other code. + // We can't trust all callers to do this properly on their own. + let payload = Cu.cloneInto(aPayload, {}); + + // Fill the common ping fields. + let pingData = { + type: aType, + id: Policy.generatePingId(), + creationDate: Policy.now().toISOString(), + version: PING_FORMAT_VERSION, + application: this._getApplicationSection(), + payload, + }; + + if (aOptions.addClientId || aOptions.overrideClientId) { + pingData.clientId = aOptions.overrideClientId || this._clientID; + } + + if (aOptions.addEnvironment) { + pingData.environment = + aOptions.overrideEnvironment || TelemetryEnvironment.currentEnvironment; + + // On Android store a flag if the client ID was reset from a canary ID. + if (AppConstants.platform == "android" && ClientID.wasCanaryClientID()) { + pingData.environment.profile.wasCanary = true; + } + } + + return pingData; + }, + + /** + * Track any pending ping send and save tasks through the promise passed here. + * This is needed to block shutdown on any outstanding ping activity. + */ + _trackPendingPingTask(aPromise) { + this._connectionsBarrier.client.addBlocker( + "Waiting for ping task", + aPromise + ); + }, + + /** + * Internal function to assemble a complete ping, adding environment data, client id + * and some general info. This waits on the client id to be loaded/generated if it's + * not yet available. Note that this function is synchronous unless we need to load + * the client id. + * Depending on configuration, the ping will be sent to the server (immediately or later) + * and archived locally. + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} [aOptions] Options object. + * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client + * id, false otherwise. + * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the + * environment data. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender. + * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending. + * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key). + * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled. + * @param {String} [aOptions.studyName=null] the study name to use. + * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled. + * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled. + * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled. + * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise. + * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the + * pioneer id to the provided value. Only works if aOptions.addPioneerId=true. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent. + */ + async _submitPingLogic(aType, aPayload, aOptions) { + // Make sure to have a clientId if we need one. This cover the case of submitting + // a ping early during startup, before Telemetry is initialized, if no client id was + // cached. + if (!this._clientID && aOptions.addClientId && !aOptions.overrideClientId) { + this._log.trace("_submitPingLogic - Waiting on client id"); + Telemetry.getHistogramById( + "TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID" + ).add(); + // We can safely call |getClientID| here and during initialization: we would still + // spawn and return one single loading task. + this._clientID = await ClientID.getClientID(); + } + + let pingData = this.assemblePing(aType, aPayload, aOptions); + this._log.trace("submitExternalPing - ping assembled, id: " + pingData.id); + + if (aOptions.useEncryption === true) { + try { + if (!aOptions.publicKey) { + throw new Error("Public key is required when using encryption."); + } + + if ( + !( + aOptions.schemaName && + aOptions.schemaNamespace && + aOptions.schemaVersion + ) + ) { + throw new Error( + "Schema name, namespace, and version are required when using encryption." + ); + } + + const payload = {}; + payload.encryptedData = await jwcrypto.generateJWE( + aOptions.publicKey, + new TextEncoder("utf-8").encode(JSON.stringify(aPayload)) + ); + + payload.schemaVersion = aOptions.schemaVersion; + payload.schemaName = aOptions.schemaName; + payload.schemaNamespace = aOptions.schemaNamespace; + + payload.encryptionKeyId = aOptions.encryptionKeyId; + + if (aOptions.addPioneerId === true) { + if (aOptions.overridePioneerId) { + // The caller provided a substitute id, let's use that + // instead of querying the pref. + payload.pioneerId = aOptions.overridePioneerId; + } else { + // This will throw if there is no pioneer ID set. + payload.pioneerId = Services.prefs.getStringPref( + "toolkit.telemetry.pioneerId" + ); + } + payload.studyName = aOptions.studyName; + } + + pingData.payload = payload; + } catch (e) { + this._log.error("_submitPingLogic - Unable to encrypt ping", e); + // Do not attempt to continue + throw e; + } + } + + // Always persist the pings if we are allowed to. We should not yield on any of the + // following operations to keep this function synchronous for the majority of the calls. + let archivePromise = TelemetryArchive.promiseArchivePing( + pingData + ).catch(e => + this._log.error( + "submitExternalPing - Failed to archive ping " + pingData.id, + e + ) + ); + let p = [archivePromise]; + + p.push( + TelemetrySend.submitPing(pingData, { + usePingSender: aOptions.usePingSender, + }) + ); + + return Promise.all(p).then(() => pingData.id); + }, + + /** + * Submit ping payloads to Telemetry. + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} [aOptions] Options object. + * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client + * id, false otherwise. + * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the + * environment data. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender. + * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending. + * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key). + * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled. + * @param {String} [aOptions.studyName=null] the study name to use. + * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled. + * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled. + * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled. + * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise. + * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the + * pioneer id to the provided value. Only works if aOptions.addPioneerId=true. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent. + */ + submitExternalPing: function send(aType, aPayload, aOptions) { + this._log.trace( + "submitExternalPing - type: " + + aType + + ", aOptions: " + + JSON.stringify(aOptions) + ); + + // Reject pings sent after shutdown. + if (this._shutDown) { + const errorMessage = + "submitExternalPing - Submission is not allowed after shutdown, discarding ping of type: " + + aType; + this._log.error(errorMessage); + return Promise.reject(new Error(errorMessage)); + } + + // Enforce the type string to only contain sane characters. + const typeUuid = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i; + if (!typeUuid.test(aType)) { + this._log.error("submitExternalPing - invalid ping type: " + aType); + let histogram = Telemetry.getKeyedHistogramById( + "TELEMETRY_INVALID_PING_TYPE_SUBMITTED" + ); + histogram.add(aType, 1); + return Promise.reject(new Error("Invalid type string submitted.")); + } + // Enforce that the payload is an object. + if ( + aPayload === null || + typeof aPayload !== "object" || + Array.isArray(aPayload) + ) { + this._log.error( + "submitExternalPing - invalid payload type: " + typeof aPayload + ); + let histogram = Telemetry.getHistogramById( + "TELEMETRY_INVALID_PAYLOAD_SUBMITTED" + ); + histogram.add(1); + return Promise.reject(new Error("Invalid payload type submitted.")); + } + + // We're trying to track down missing sync pings (bug 1663573), so record + // a temporary cross-checking counter. + if (aType == "sync" && aPayload.why == "shutdown") { + Telemetry.scalarSet("telemetry.sync_shutdown_ping_sent", true); + } + + let promise = this._submitPingLogic(aType, aPayload, aOptions); + this._trackPendingPingTask(promise); + return promise; + }, + + /** + * Save a ping to disk. + * + * @param {String} aType The type of the ping. + * @param {Object} aPayload The actual data payload for the ping. + * @param {Object} aOptions Options object. + * @param {Boolean} aOptions.addClientId true if the ping should contain the client id, + * false otherwise. + * @param {Boolean} aOptions.addEnvironment true if the ping should contain the + * environment data. + * @param {Boolean} aOptions.overwrite true overwrites a ping with the same name, if found. + * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data. + * @param {String} [aOptions.overrideClientId=undefined] if set, override the + * client id to the provided value. Implies aOptions.addClientId=true. + * + * @returns {Promise} A promise that resolves with the ping id when the ping is saved to + * disk. + */ + addPendingPing: function addPendingPing(aType, aPayload, aOptions) { + this._log.trace( + "addPendingPing - Type " + + aType + + ", aOptions " + + JSON.stringify(aOptions) + ); + + let pingData = this.assemblePing(aType, aPayload, aOptions); + + let savePromise = TelemetryStorage.savePendingPing(pingData); + let archivePromise = TelemetryArchive.promiseArchivePing(pingData).catch( + e => { + this._log.error( + "addPendingPing - Failed to archive ping " + pingData.id, + e + ); + } + ); + + // Wait for both the archiving and ping persistence to complete. + let promises = [savePromise, archivePromise]; + return Promise.all(promises).then(() => pingData.id); + }, + + /** + * Check whether we have an aborted-session ping. If so add it to the pending pings and archive it. + * + * @return {Promise} Promise that is resolved when the ping is submitted and archived. + */ + async checkAbortedSessionPing() { + let ping = await TelemetryStorage.loadAbortedSessionPing(); + this._log.trace( + "checkAbortedSessionPing - found aborted-session ping: " + !!ping + ); + if (!ping) { + return; + } + + try { + // Previous aborted-session might have been with a canary client ID. + // Don't send it. + if (ping.clientId != Utils.knownClientID) { + await TelemetryStorage.addPendingPing(ping); + await TelemetryArchive.promiseArchivePing(ping); + } + } catch (e) { + this._log.error( + "checkAbortedSessionPing - Unable to add the pending ping", + e + ); + } finally { + await TelemetryStorage.removeAbortedSessionPing(); + } + }, + + /** + * Save an aborted-session ping to disk without adding it to the pending pings. + * + * @param {Object} aPayload The ping payload data. + * @return {Promise} Promise that is resolved when the ping is saved. + */ + saveAbortedSessionPing(aPayload) { + this._log.trace("saveAbortedSessionPing"); + const options = { addClientId: true, addEnvironment: true }; + const pingData = this.assemblePing(PING_TYPE_MAIN, aPayload, options); + return TelemetryStorage.saveAbortedSessionPing(pingData); + }, + + removeAbortedSessionPing() { + return TelemetryStorage.removeAbortedSessionPing(); + }, + + async saveUninstallPing() { + if (AppConstants.platform != "win") { + return undefined; + } + + this._log.trace("saveUninstallPing"); + + let payload = {}; + try { + payload.otherInstalls = UninstallPing.getOtherInstallsCount(); + this._log.info( + "saveUninstallPing - otherInstalls", + payload.otherInstalls + ); + } catch (e) { + this._log.warn("saveUninstallPing - getOtherInstallCount failed", e); + } + const options = { addClientId: true, addEnvironment: true }; + const pingData = this.assemblePing(PING_TYPE_UNINSTALL, payload, options); + + return TelemetryStorage.saveUninstallPing(pingData); + }, + + /** + * This triggers basic telemetry initialization and schedules a full initialized for later + * for performance reasons. + * + * This delayed initialization means TelemetryController init can be in the following states: + * 1) setupTelemetry was never called + * or it was called and + * 2) _delayedInitTask was scheduled, but didn't run yet. + * 3) _delayedInitTask is currently running. + * 4) _delayedInitTask finished running and is nulled out. + * + * @return {Promise} Resolved when TelemetryController and TelemetrySession are fully + * initialized. This is only used in tests. + */ + setupTelemetry: function setupTelemetry(testing) { + this._initStarted = true; + this._shuttingDown = false; + this._shutDown = false; + this._testMode = testing; + + this._log.trace("setupTelemetry"); + + if (this._delayedInitTask) { + this._log.error("setupTelemetry - init task already running"); + return this._delayedInitTaskDeferred.promise; + } + + if (this._initialized && !this._testMode) { + this._log.error("setupTelemetry - already initialized"); + return Promise.resolve(); + } + + // Enable adding scalars in artifact builds and build faster modes. + // The function is async: we intentionally don't wait for it to complete + // as we don't want to delay startup. + this._probeRegistrationPromise = this.registerJsProbes(); + + // This will trigger displaying the datachoices infobar. + TelemetryReportingPolicy.setup(); + + if (!TelemetryControllerBase.enableTelemetryRecording()) { + this._log.config( + "setupChromeProcess - Telemetry recording is disabled, skipping Chrome process setup." + ); + return Promise.resolve(); + } + + this._attachObservers(); + + // Perform a lightweight, early initialization for the component, just registering + // a few observers and initializing the session. + TelemetrySession.earlyInit(this._testMode); + Services.telemetry.earlyInit(); + + // Annotate crash reports so that we get pings for startup crashes + TelemetrySend.earlyInit(); + + // For very short session durations, we may never load the client + // id from disk. + // We try to cache it in prefs to avoid this, even though this may + // lead to some stale client ids. + this._clientID = ClientID.getCachedClientID(); + + // Init the update ping telemetry as early as possible. This won't have + // an impact on startup. + UpdatePing.earlyInit(); + + // Delay full telemetry initialization to give the browser time to + // run various late initializers. Otherwise our gathered memory + // footprint and other numbers would be too optimistic. + this._delayedInitTaskDeferred = PromiseUtils.defer(); + this._delayedInitTask = new DeferredTask( + async () => { + try { + // TODO: This should probably happen after all the delayed init here. + this._initialized = true; + await TelemetryEnvironment.delayedInit(); + + // Load the ClientID. + this._clientID = await ClientID.getClientID(); + + // Fix-up a canary client ID if detected. + const uploadEnabled = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.FhrUploadEnabled, + false + ); + if (uploadEnabled && this._clientID == Utils.knownClientID) { + this._log.trace( + "Upload enabled, but got canary client ID. Resetting." + ); + await ClientID.removeClientIDs(); + this._clientID = await ClientID.getClientID(); + } else if (!uploadEnabled && this._clientID != Utils.knownClientID) { + this._log.trace( + "Upload disabled, but got a valid client ID. Setting canary client ID." + ); + await ClientID.setCanaryClientIDs(); + this._clientID = await ClientID.getClientID(); + } + + await TelemetrySend.setup(this._testMode); + + // Perform TelemetrySession delayed init. + await TelemetrySession.delayedInit(); + await Services.telemetry.delayedInit(); + + if ( + Services.prefs.getBoolPref( + TelemetryUtils.Preferences.NewProfilePingEnabled, + false + ) && + !TelemetrySession.newProfilePingSent + ) { + // Kick off the scheduling of the new-profile ping. + this.scheduleNewProfilePing(); + } + + // Purge the pings archive by removing outdated pings. We don't wait for + // this task to complete, but TelemetryStorage blocks on it during + // shutdown. + TelemetryStorage.runCleanPingArchiveTask(); + + // Now that FHR/healthreporter is gone, make sure to remove FHR's DB from + // the profile directory. This is a temporary measure that we should drop + // in the future. + TelemetryStorage.removeFHRDatabase(); + + // The init sequence is forced to run on shutdown for short sessions and + // we don't want to start TelemetryModules as the timer registration will fail. + if (!this._shuttingDown) { + // Report the modules loaded in the Firefox process. + TelemetryModules.start(); + + // Send coverage ping. + await CoveragePing.startup(); + + // Start the untrusted modules ping, which reports events where + // untrusted modules were loaded into the Firefox process. + if (AppConstants.platform == "win") { + TelemetryUntrustedModulesPing.start(); + } + } + + TelemetryEventPing.startup(); + EcosystemTelemetry.startup(); + TelemetryPrioPing.startup(); + + if (uploadEnabled) { + await this.saveUninstallPing().catch(e => + this._log.warn("_delayedInitTask - saveUninstallPing failed", e) + ); + } else { + await TelemetryStorage.removeUninstallPings().catch(e => + this._log.warn("_delayedInitTask - saveUninstallPing", e) + ); + } + + this._delayedInitTaskDeferred.resolve(); + } catch (e) { + this._delayedInitTaskDeferred.reject(e); + } finally { + this._delayedInitTask = null; + } + }, + this._testMode ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY, + this._testMode ? 0 : undefined + ); + + AsyncShutdown.sendTelemetry.addBlocker( + "TelemetryController: shutting down", + () => this.shutdown(), + () => this._getState() + ); + + this._delayedInitTask.arm(); + return this._delayedInitTaskDeferred.promise; + }, + + // Do proper shutdown waiting and cleanup. + async _cleanupOnShutdown() { + if (!this._initialized) { + return; + } + + this._detachObservers(); + + // Now do an orderly shutdown. + try { + if (this._delayedNewPingTask) { + await this._delayedNewPingTask.finalize(); + } + + UpdatePing.shutdown(); + + TelemetryEventPing.shutdown(); + EcosystemTelemetry.shutdown(); + await TelemetryPrioPing.shutdown(); + + // Shutdown the sync ping if it is initialized - this is likely, but not + // guaranteed, to submit a "shutdown" sync ping. + if (this._fnSyncPingShutdown) { + this._fnSyncPingShutdown(); + } + + // Stop the datachoices infobar display. + TelemetryReportingPolicy.shutdown(); + TelemetryEnvironment.shutdown(); + + // Stop any ping sending. + await TelemetrySend.shutdown(); + + // Send latest data. + await TelemetryHealthPing.shutdown(); + + await TelemetrySession.shutdown(); + await Services.telemetry.shutdown(); + + // First wait for clients processing shutdown. + await this._shutdownBarrier.wait(); + + // ... and wait for any outstanding async ping activity. + await this._connectionsBarrier.wait(); + + if (AppConstants.platform !== "android") { + // No PingSender on Android. + TelemetrySend.flushPingSenderBatch(); + } + + // Perform final shutdown operations. + await TelemetryStorage.shutdown(); + } finally { + // Reset state. + this._initialized = false; + this._initStarted = false; + this._shutDown = true; + } + }, + + shutdown() { + this._log.trace("shutdown"); + + this._shuttingDown = true; + + // We can be in one the following states here: + // 1) setupTelemetry was never called + // or it was called and + // 2) _delayedInitTask was scheduled, but didn't run yet. + // 3) _delayedInitTask is running now. + // 4) _delayedInitTask finished running already. + + // This handles 1). + if (!this._initStarted) { + this._shutDown = true; + return Promise.resolve(); + } + + // This handles 4). + if (!this._delayedInitTask) { + // We already ran the delayed initialization. + return this._cleanupOnShutdown(); + } + + // This handles 2) and 3). + return this._delayedInitTask + .finalize() + .then(() => this._cleanupOnShutdown()); + }, + + /** + * This observer drives telemetry. + */ + observe(aSubject, aTopic, aData) { + // The logger might still be not available at this point. + if (aTopic == "profile-after-change") { + // If we don't have a logger, we need to make sure |Log.repository.getLogger()| is + // called before |getLoggerWithMessagePrefix|. Otherwise logging won't work. + TelemetryControllerBase.configureLogging(); + } + + this._log.trace(`observe - ${aTopic} notified.`); + + switch (aTopic) { + case "profile-after-change": + // profile-after-change is only registered for chrome processes. + return this.setupTelemetry(); + case "nsPref:changed": + if (aData == TelemetryUtils.Preferences.FhrUploadEnabled) { + return this._onUploadPrefChange(); + } + } + return undefined; + }, + + /** + * Register the sync ping's shutdown handler. + */ + registerSyncPingShutdown(fnShutdown) { + if (this._fnSyncPingShutdown) { + throw new Error("The sync ping shutdown handler is already registered."); + } + this._fnSyncPingShutdown = fnShutdown; + }, + + /** + * Get an object describing the current state of this module for AsyncShutdown diagnostics. + */ + _getState() { + return { + initialized: this._initialized, + initStarted: this._initStarted, + haveDelayedInitTask: !!this._delayedInitTask, + shutdownBarrier: this._shutdownBarrier.state, + connectionsBarrier: this._connectionsBarrier.state, + sendModule: TelemetrySend.getShutdownState(), + haveDelayedNewProfileTask: !!this._delayedNewPingTask, + }; + }, + + /** + * Called whenever the FHR Upload preference changes (e.g. when user disables FHR from + * the preferences panel), this triggers sending the "deletion-request" ping. + */ + _onUploadPrefChange() { + const uploadEnabled = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.FhrUploadEnabled, + false + ); + if (uploadEnabled) { + this._log.trace( + "_onUploadPrefChange - upload was enabled again. Resetting client ID" + ); + + // Delete cached client ID immediately, so other usage is forced to refetch it. + this._clientID = null; + + // Generate a new client ID and make sure this module uses the new version + let p = (async () => { + await ClientID.removeClientIDs(); + let id = await ClientID.getClientID(); + this._clientID = id; + Telemetry.scalarSet("telemetry.data_upload_optin", true); + + await this.saveUninstallPing().catch(e => + this._log.warn("_onUploadPrefChange - saveUninstallPing failed", e) + ); + })(); + + this._shutdownBarrier.client.addBlocker( + "TelemetryController: resetting client ID after data upload was enabled", + p + ); + + return; + } + + let p = (async () => { + try { + // 1. Cancel the current pings. + // 2. Clear unpersisted pings + await TelemetrySend.clearCurrentPings(); + + // 3. Remove all pending pings + await TelemetryStorage.removeAppDataPings(); + await TelemetryStorage.runRemovePendingPingsTask(); + await TelemetryStorage.removeUninstallPings(); + } catch (e) { + this._log.error( + "_onUploadPrefChange - error clearing pending pings", + e + ); + } finally { + // 4. Reset session and subsession counter + TelemetrySession.resetSubsessionCounter(); + + // 5. Collect any additional identifiers we want to send in the + // deletion request. + const scalars = Telemetry.getSnapshotForScalars( + "deletion-request", + /* clear */ true + ); + + // 6. Set ClientID to a known value + let oldClientId = await ClientID.getClientID(); + await ClientID.setCanaryClientIDs(); + this._clientID = await ClientID.getClientID(); + + // 7. Send the deletion-request ping. + this._log.trace("_onUploadPrefChange - Sending deletion-request ping."); + this.submitExternalPing( + PING_TYPE_DELETION_REQUEST, + { scalars }, + { overrideClientId: oldClientId } + ); + this._deletionRequestPingSubmittedPromise = null; + } + })(); + + this._deletionRequestPingSubmittedPromise = p; + this._shutdownBarrier.client.addBlocker( + "TelemetryController: removing pending pings after data upload was disabled", + p + ); + + Services.obs.notifyObservers( + null, + TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC + ); + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + _attachObservers() { + if (TelemetryControllerBase.IS_UNIFIED_TELEMETRY) { + // Watch the FHR upload setting to trigger "deletion-request" pings. + Services.prefs.addObserver( + TelemetryUtils.Preferences.FhrUploadEnabled, + this, + true + ); + } + }, + + /** + * Remove the preference observer to avoid leaks. + */ + _detachObservers() { + if (TelemetryControllerBase.IS_UNIFIED_TELEMETRY) { + Services.prefs.removeObserver( + TelemetryUtils.Preferences.FhrUploadEnabled, + this + ); + } + }, + + /** + * Allows waiting for TelemetryControllers delayed initialization to complete. + * This will complete before TelemetryController is shutting down. + * @return {Promise} Resolved when delayed TelemetryController initialization completed. + */ + promiseInitialized() { + return this._delayedInitTaskDeferred.promise; + }, + + getCurrentPingData(aSubsession) { + this._log.trace("getCurrentPingData - subsession: " + aSubsession); + + // Telemetry is disabled, don't gather any data. + if (!Telemetry.canRecordBase) { + return null; + } + + const reason = aSubsession + ? REASON_GATHER_SUBSESSION_PAYLOAD + : REASON_GATHER_PAYLOAD; + const type = PING_TYPE_MAIN; + const payload = TelemetrySession.getPayload(reason); + const options = { addClientId: true, addEnvironment: true }; + const ping = this.assemblePing(type, payload, options); + + return ping; + }, + + async reset() { + this._clientID = null; + this._fnSyncPingShutdown = null; + this._detachObservers(); + + let sessionReset = TelemetrySession.testReset(); + + this._connectionsBarrier = new AsyncShutdown.Barrier( + "TelemetryController: Waiting for pending ping activity" + ); + this._shutdownBarrier = new AsyncShutdown.Barrier( + "TelemetryController: Waiting for clients." + ); + + // We need to kick of the controller setup first for tests that check the + // cached client id. + let controllerSetup = this.setupTelemetry(true); + + await sessionReset; + await TelemetrySend.reset(); + await TelemetryStorage.reset(); + await TelemetryEnvironment.testReset(); + + await controllerSetup; + }, + + /** + * Schedule sending the "new-profile" ping. + */ + scheduleNewProfilePing() { + this._log.trace("scheduleNewProfilePing"); + + const sendDelay = Services.prefs.getIntPref( + TelemetryUtils.Preferences.NewProfilePingDelay, + NEWPROFILE_PING_DEFAULT_DELAY + ); + + this._delayedNewPingTask = new DeferredTask(async () => { + try { + await this.sendNewProfilePing(); + } finally { + this._delayedNewPingTask = null; + } + }, sendDelay); + + this._delayedNewPingTask.arm(); + }, + + /** + * Generate and send the new-profile ping + */ + async sendNewProfilePing() { + this._log.trace( + "sendNewProfilePing - shutting down: " + this._shuttingDown + ); + + const scalars = Telemetry.getSnapshotForScalars( + "new-profile", + /* clear */ true + ); + + // Generate the payload. + const payload = { + reason: this._shuttingDown ? "shutdown" : "startup", + processes: { + parent: { + scalars: scalars.parent, + }, + }, + }; + + // Generate and send the "new-profile" ping. This uses the + // pingsender if we're shutting down. + let options = { + addClientId: true, + addEnvironment: true, + usePingSender: this._shuttingDown, + }; + // TODO: we need to be smarter about when to send the ping (and save the + // state to file). |requestIdleCallback| is currently only accessible + // through DOM. See bug 1361996. + await TelemetryController.submitExternalPing( + "new-profile", + payload, + options + ).then( + () => TelemetrySession.markNewProfilePingSent(), + e => + this._log.error( + "sendNewProfilePing - failed to submit new-profile ping", + e + ) + ); + }, + + /** + * Register 'dynamic builtin' probes from the JSON definition files. + * This is needed to support adding new probes in developer builds + * without rebuilding the whole codebase. + * + * This is not meant to be used outside of local developer builds. + */ + async registerJsProbes() { + // We don't support this outside of developer builds. + if (AppConstants.MOZILLA_OFFICIAL && !this._testMode) { + return; + } + + this._log.trace("registerJsProbes - registering builtin JS probes"); + + await this.registerScalarProbes(); + await this.registerEventProbes(); + }, + + _loadProbeDefinitions(filename) { + let probeFile = Services.dirsvc.get("GreD", Ci.nsIFile); + probeFile.append(filename); + if (!probeFile.exists()) { + this._log.trace( + `loadProbeDefinitions - no builtin JS probe file ${filename}` + ); + return null; + } + + return OS.File.read(probeFile.path, { encoding: "utf-8" }); + }, + + async registerScalarProbes() { + this._log.trace( + "registerScalarProbes - registering scalar builtin JS probes" + ); + + // Load the scalar probes JSON file. + const scalarProbeFilename = "ScalarArtifactDefinitions.json"; + let scalarJSProbes = {}; + try { + let fileContent = await this._loadProbeDefinitions(scalarProbeFilename); + scalarJSProbes = JSON.parse(fileContent, (property, value) => { + // Fixup the "kind" property: it's a string, and we need the constant + // coming from nsITelemetry. + if (property !== "kind" || typeof value != "string") { + return value; + } + + let newValue; + switch (value) { + case "nsITelemetry::SCALAR_TYPE_COUNT": + newValue = Telemetry.SCALAR_TYPE_COUNT; + break; + case "nsITelemetry::SCALAR_TYPE_BOOLEAN": + newValue = Telemetry.SCALAR_TYPE_BOOLEAN; + break; + case "nsITelemetry::SCALAR_TYPE_STRING": + newValue = Telemetry.SCALAR_TYPE_STRING; + break; + } + return newValue; + }); + } catch (ex) { + this._log.error( + `registerScalarProbes - there was an error loading ${scalarProbeFilename}`, + ex + ); + } + + // Register the builtin probes. + for (let category in scalarJSProbes) { + // Expire the expired scalars + for (let name in scalarJSProbes[category]) { + let def = scalarJSProbes[category][name]; + if ( + !def || + !def.expires || + def.expires == "never" || + def.expires == "default" + ) { + continue; + } + if ( + Services.vc.compare(AppConstants.MOZ_APP_VERSION, def.expires) >= 0 + ) { + def.expired = true; + } + } + Telemetry.registerBuiltinScalars(category, scalarJSProbes[category]); + } + }, + + async registerEventProbes() { + this._log.trace( + "registerEventProbes - registering builtin JS Event probes" + ); + + // Load the event probes JSON file. + const eventProbeFilename = "EventArtifactDefinitions.json"; + let eventJSProbes = {}; + try { + let fileContent = await this._loadProbeDefinitions(eventProbeFilename); + eventJSProbes = JSON.parse(fileContent); + } catch (ex) { + this._log.error( + `registerEventProbes - there was an error loading ${eventProbeFilename}`, + ex + ); + } + + // Register the builtin probes. + for (let category in eventJSProbes) { + for (let name in eventJSProbes[category]) { + let def = eventJSProbes[category][name]; + if ( + !def || + !def.expires || + def.expires == "never" || + def.expires == "default" + ) { + continue; + } + if ( + Services.vc.compare(AppConstants.MOZ_APP_VERSION, def.expires) >= 0 + ) { + def.expired = true; + } + } + Telemetry.registerBuiltinEvents(category, eventJSProbes[category]); + } + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryEnvironment.jsm b/toolkit/components/telemetry/app/TelemetryEnvironment.jsm new file mode 100644 index 0000000000..8e82ad49bb --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryEnvironment.jsm @@ -0,0 +1,2128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryEnvironment"]; + +const myScope = this; + +const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm"); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this); +const { ObjectUtils } = ChromeUtils.import( + "resource://gre/modules/ObjectUtils.jsm" +); +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +if (AppConstants.MOZ_GLEAN) { + Cu.importGlobalProperties(["Glean"]); +} + +const Utils = TelemetryUtils; + +const { AddonManager, AddonManagerPrivate } = ChromeUtils.import( + "resource://gre/modules/AddonManager.jsm" +); + +ChromeUtils.defineModuleGetter( + this, + "AttributionCode", + "resource:///modules/AttributionCode.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "ProfileAge", + "resource://gre/modules/ProfileAge.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "WindowsRegistry", + "resource://gre/modules/WindowsRegistry.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "UpdateUtils", + "resource://gre/modules/UpdateUtils.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "fxAccounts", + "resource://gre/modules/FxAccounts.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "WindowsVersionInfo", + "resource://gre/modules/components-utils/WindowsVersionInfo.jsm" +); + +// The maximum length of a string (e.g. description) in the addons section. +const MAX_ADDON_STRING_LENGTH = 100; +// The maximum length of a string value in the settings.attribution object. +const MAX_ATTRIBUTION_STRING_LENGTH = 100; +// The maximum lengths for the experiment id and branch in the experiments section. +const MAX_EXPERIMENT_ID_LENGTH = 100; +const MAX_EXPERIMENT_BRANCH_LENGTH = 100; +const MAX_EXPERIMENT_TYPE_LENGTH = 20; +const MAX_EXPERIMENT_ENROLLMENT_ID_LENGTH = 40; + +/** + * This is a policy object used to override behavior for testing. + */ +// eslint-disable-next-line no-unused-vars +var Policy = { + now: () => new Date(), + _intlLoaded: false, + _browserDelayedStartup() { + if (Policy._intlLoaded) { + return Promise.resolve(); + } + return new Promise(resolve => { + let startupTopic = "browser-delayed-startup-finished"; + Services.obs.addObserver(function observer(subject, topic) { + if (topic == startupTopic) { + Services.obs.removeObserver(observer, startupTopic); + resolve(); + } + }, startupTopic); + }); + }, +}; + +// This is used to buffer calls to setExperimentActive and friends, so that we +// don't prematurely initialize our environment if it is called early during +// startup. +var gActiveExperimentStartupBuffer = new Map(); + +var gGlobalEnvironment; +function getGlobal() { + if (!gGlobalEnvironment) { + gGlobalEnvironment = new EnvironmentCache(); + } + return gGlobalEnvironment; +} + +var TelemetryEnvironment = { + get currentEnvironment() { + return getGlobal().currentEnvironment; + }, + + onInitialized() { + return getGlobal().onInitialized(); + }, + + delayedInit() { + return getGlobal().delayedInit(); + }, + + registerChangeListener(name, listener) { + return getGlobal().registerChangeListener(name, listener); + }, + + unregisterChangeListener(name) { + return getGlobal().unregisterChangeListener(name); + }, + + /** + * Add an experiment annotation to the environment. + * If an annotation with the same id already exists, it will be overwritten. + * This triggers a new subsession, subject to throttling. + * + * @param {String} id The id of the active experiment. + * @param {String} branch The experiment branch. + * @param {Object} [options] Optional object with options. + * @param {String} [options.type=false] The specific experiment type. + * @param {String} [options.enrollmentId=undefined] The id of the enrollment. + */ + setExperimentActive(id, branch, options = {}) { + if (gGlobalEnvironment) { + gGlobalEnvironment.setExperimentActive(id, branch, options); + } else { + gActiveExperimentStartupBuffer.set(id, { branch, options }); + } + }, + + /** + * Remove an experiment annotation from the environment. + * If the annotation exists, a new subsession will triggered. + * + * @param {String} id The id of the active experiment. + */ + setExperimentInactive(id) { + if (gGlobalEnvironment) { + gGlobalEnvironment.setExperimentInactive(id); + } else { + gActiveExperimentStartupBuffer.delete(id); + } + }, + + /** + * Returns an object containing the data for the active experiments. + * + * The returned object is of the format: + * + * { + * "<experiment id>": { branch: "<branch>" }, + * // … + * } + */ + getActiveExperiments() { + if (gGlobalEnvironment) { + return gGlobalEnvironment.getActiveExperiments(); + } + + const result = {}; + for (const [id, { branch }] of gActiveExperimentStartupBuffer.entries()) { + result[id] = branch; + } + return result; + }, + + shutdown() { + return getGlobal().shutdown(); + }, + + // Policy to use when saving preferences. Exported for using them in tests. + // Reports "<user-set>" if there is a value set on the user branch + RECORD_PREF_STATE: 1, + + // Reports the value set on the user branch, if one is set + RECORD_PREF_VALUE: 2, + + // Reports the active value (set on either the user or default branch) + // for this pref, if one is set + RECORD_DEFAULTPREF_VALUE: 3, + + // Reports "<set>" if a value for this pref is defined on either the user + // or default branch + RECORD_DEFAULTPREF_STATE: 4, + + // Testing method + async testWatchPreferences(prefMap) { + return getGlobal()._watchPreferences(prefMap); + }, + + /** + * Intended for use in tests only. + * + * In multiple tests we need a way to shut and re-start telemetry together + * with TelemetryEnvironment. This is problematic due to the fact that + * TelemetryEnvironment is a singleton. We, therefore, need this helper + * method to be able to re-set TelemetryEnvironment. + */ + testReset() { + return getGlobal().reset(); + }, + + /** + * Intended for use in tests only. + */ + testCleanRestart() { + getGlobal().shutdown(); + gGlobalEnvironment = null; + gActiveExperimentStartupBuffer = new Map(); + return getGlobal(); + }, +}; + +const RECORD_PREF_STATE = TelemetryEnvironment.RECORD_PREF_STATE; +const RECORD_PREF_VALUE = TelemetryEnvironment.RECORD_PREF_VALUE; +const RECORD_DEFAULTPREF_VALUE = TelemetryEnvironment.RECORD_DEFAULTPREF_VALUE; +const RECORD_DEFAULTPREF_STATE = TelemetryEnvironment.RECORD_DEFAULTPREF_STATE; +const DEFAULT_ENVIRONMENT_PREFS = new Map([ + ["app.feedback.baseURL", { what: RECORD_PREF_VALUE }], + ["app.support.baseURL", { what: RECORD_PREF_VALUE }], + ["accessibility.browsewithcaret", { what: RECORD_PREF_VALUE }], + ["accessibility.force_disabled", { what: RECORD_PREF_VALUE }], + ["app.normandy.test-prefs.bool", { what: RECORD_PREF_VALUE }], + ["app.normandy.test-prefs.integer", { what: RECORD_PREF_VALUE }], + ["app.normandy.test-prefs.string", { what: RECORD_PREF_VALUE }], + ["app.shield.optoutstudies.enabled", { what: RECORD_PREF_VALUE }], + ["app.update.interval", { what: RECORD_PREF_VALUE }], + ["app.update.service.enabled", { what: RECORD_PREF_VALUE }], + ["app.update.silent", { what: RECORD_PREF_VALUE }], + ["browser.cache.disk.enable", { what: RECORD_PREF_VALUE }], + ["browser.cache.disk.capacity", { what: RECORD_PREF_VALUE }], + ["browser.cache.memory.enable", { what: RECORD_PREF_VALUE }], + ["browser.cache.offline.enable", { what: RECORD_PREF_VALUE }], + ["browser.formfill.enable", { what: RECORD_PREF_VALUE }], + ["browser.newtabpage.enabled", { what: RECORD_PREF_VALUE }], + ["browser.shell.checkDefaultBrowser", { what: RECORD_PREF_VALUE }], + ["browser.search.ignoredJAREngines", { what: RECORD_DEFAULTPREF_VALUE }], + ["browser.search.region", { what: RECORD_PREF_VALUE }], + ["browser.search.suggest.enabled", { what: RECORD_PREF_VALUE }], + ["browser.search.widget.inNavBar", { what: RECORD_DEFAULTPREF_VALUE }], + ["browser.startup.homepage", { what: RECORD_PREF_STATE }], + ["browser.startup.page", { what: RECORD_PREF_VALUE }], + ["browser.urlbar.suggest.searches", { what: RECORD_PREF_VALUE }], + ["devtools.chrome.enabled", { what: RECORD_PREF_VALUE }], + ["devtools.debugger.enabled", { what: RECORD_PREF_VALUE }], + ["devtools.debugger.remote-enabled", { what: RECORD_PREF_VALUE }], + ["dom.ipc.plugins.enabled", { what: RECORD_PREF_VALUE }], + ["dom.ipc.plugins.sandbox-level.flash", { what: RECORD_PREF_VALUE }], + ["dom.ipc.processCount", { what: RECORD_PREF_VALUE }], + ["dom.max_script_run_time", { what: RECORD_PREF_VALUE }], + ["extensions.autoDisableScopes", { what: RECORD_PREF_VALUE }], + ["extensions.enabledScopes", { what: RECORD_PREF_VALUE }], + ["extensions.blocklist.enabled", { what: RECORD_PREF_VALUE }], + ["extensions.formautofill.addresses.enabled", { what: RECORD_PREF_VALUE }], + [ + "extensions.formautofill.addresses.capture.enabled", + { what: RECORD_PREF_VALUE }, + ], + ["extensions.formautofill.creditCards.enabled", { what: RECORD_PREF_VALUE }], + [ + "extensions.formautofill.creditCards.available", + { what: RECORD_PREF_VALUE }, + ], + ["extensions.formautofill.creditCards.used", { what: RECORD_PREF_VALUE }], + ["extensions.strictCompatibility", { what: RECORD_PREF_VALUE }], + ["extensions.update.enabled", { what: RECORD_PREF_VALUE }], + ["extensions.update.url", { what: RECORD_PREF_VALUE }], + ["extensions.update.background.url", { what: RECORD_PREF_VALUE }], + ["extensions.screenshots.disabled", { what: RECORD_PREF_VALUE }], + ["general.config.filename", { what: RECORD_DEFAULTPREF_STATE }], + ["general.smoothScroll", { what: RECORD_PREF_VALUE }], + ["gfx.direct2d.disabled", { what: RECORD_PREF_VALUE }], + ["gfx.direct2d.force-enabled", { what: RECORD_PREF_VALUE }], + ["gfx.webrender.all", { what: RECORD_PREF_VALUE }], + ["gfx.webrender.all.qualified", { what: RECORD_PREF_VALUE }], + ["gfx.webrender.force-disabled", { what: RECORD_PREF_VALUE }], + ["layers.acceleration.disabled", { what: RECORD_PREF_VALUE }], + ["layers.acceleration.force-enabled", { what: RECORD_PREF_VALUE }], + ["layers.async-pan-zoom.enabled", { what: RECORD_PREF_VALUE }], + ["layers.async-video-oop.enabled", { what: RECORD_PREF_VALUE }], + ["layers.async-video.enabled", { what: RECORD_PREF_VALUE }], + ["layers.componentalpha.enabled", { what: RECORD_PREF_VALUE }], + ["layers.d3d11.disable-warp", { what: RECORD_PREF_VALUE }], + ["layers.d3d11.force-warp", { what: RECORD_PREF_VALUE }], + [ + "layers.offmainthreadcomposition.force-disabled", + { what: RECORD_PREF_VALUE }, + ], + ["layers.prefer-d3d9", { what: RECORD_PREF_VALUE }], + ["layers.prefer-opengl", { what: RECORD_PREF_VALUE }], + ["layout.css.devPixelsPerPx", { what: RECORD_PREF_VALUE }], + ["marionette.enabled", { what: RECORD_PREF_VALUE }], + ["network.proxy.autoconfig_url", { what: RECORD_PREF_STATE }], + ["network.proxy.http", { what: RECORD_PREF_STATE }], + ["network.proxy.ssl", { what: RECORD_PREF_STATE }], + ["network.trr.mode", { what: RECORD_PREF_VALUE }], + ["pdfjs.disabled", { what: RECORD_PREF_VALUE }], + ["places.history.enabled", { what: RECORD_PREF_VALUE }], + ["plugins.show_infobar", { what: RECORD_PREF_VALUE }], + ["privacy.fuzzyfox.enabled", { what: RECORD_PREF_VALUE }], + ["privacy.trackingprotection.enabled", { what: RECORD_PREF_VALUE }], + ["privacy.donottrackheader.enabled", { what: RECORD_PREF_VALUE }], + ["security.enterprise_roots.auto-enabled", { what: RECORD_PREF_VALUE }], + ["security.enterprise_roots.enabled", { what: RECORD_PREF_VALUE }], + ["security.pki.mitm_detected", { what: RECORD_PREF_VALUE }], + ["security.mixed_content.block_active_content", { what: RECORD_PREF_VALUE }], + ["security.mixed_content.block_display_content", { what: RECORD_PREF_VALUE }], + ["security.tls.version.enable-deprecated", { what: RECORD_PREF_VALUE }], + ["signon.management.page.breach-alerts.enabled", { what: RECORD_PREF_VALUE }], + ["signon.autofillForms", { what: RECORD_PREF_VALUE }], + ["signon.generation.enabled", { what: RECORD_PREF_VALUE }], + ["signon.rememberSignons", { what: RECORD_PREF_VALUE }], + ["toolkit.telemetry.pioneerId", { what: RECORD_PREF_STATE }], + ["widget.content.allow-gtk-dark-theme", { what: RECORD_DEFAULTPREF_VALUE }], + ["widget.content.gtk-theme-override", { what: RECORD_PREF_STATE }], + [ + "widget.content.gtk-high-contrast.enabled", + { what: RECORD_DEFAULTPREF_VALUE }, + ], + ["xpinstall.signatures.required", { what: RECORD_PREF_VALUE }], +]); + +const LOGGER_NAME = "Toolkit.Telemetry"; + +const PREF_BLOCKLIST_ENABLED = "extensions.blocklist.enabled"; +const PREF_DISTRIBUTION_ID = "distribution.id"; +const PREF_DISTRIBUTION_VERSION = "distribution.version"; +const PREF_DISTRIBUTOR = "app.distributor"; +const PREF_DISTRIBUTOR_CHANNEL = "app.distributor.channel"; +const PREF_APP_PARTNER_BRANCH = "app.partner."; +const PREF_PARTNER_ID = "mozilla.partner.id"; + +const COMPOSITOR_CREATED_TOPIC = "compositor:created"; +const COMPOSITOR_PROCESS_ABORTED_TOPIC = "compositor:process-aborted"; +const DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC = + "distribution-customization-complete"; +const GFX_FEATURES_READY_TOPIC = "gfx-features-ready"; +const SEARCH_ENGINE_MODIFIED_TOPIC = "browser-search-engine-modified"; +const SEARCH_SERVICE_TOPIC = "browser-search-service"; +const SESSIONSTORE_WINDOWS_RESTORED_TOPIC = "sessionstore-windows-restored"; +const PREF_CHANGED_TOPIC = "nsPref:changed"; +const BLOCKLIST_LOADED_TOPIC = "plugin-blocklist-loaded"; +const AUTO_UPDATE_PREF_CHANGE_TOPIC = "auto-update-config-change"; +const SERVICES_INFO_CHANGE_TOPIC = "sync-ui-state:update"; + +/** + * Enforces the parameter to a boolean value. + * @param aValue The input value. + * @return {Boolean|Object} If aValue is a boolean or a number, returns its truthfulness + * value. Otherwise, return null. + */ +function enforceBoolean(aValue) { + if (typeof aValue !== "number" && typeof aValue !== "boolean") { + return null; + } + return Boolean(aValue); +} + +/** + * Get the current browser locale. + * @return a string with the locale or null on failure. + */ +function getBrowserLocale() { + try { + return Services.locale.appLocaleAsBCP47; + } catch (e) { + return null; + } +} + +/** + * Get the current OS locale. + * @return a string with the OS locale or null on failure. + */ +function getSystemLocale() { + try { + return Cc["@mozilla.org/intl/ospreferences;1"].getService( + Ci.mozIOSPreferences + ).systemLocale; + } catch (e) { + return null; + } +} + +/** + * Get the current OS locales. + * @return an array of strings with the OS locales or null on failure. + */ +function getSystemLocales() { + try { + return Cc["@mozilla.org/intl/ospreferences;1"].getService( + Ci.mozIOSPreferences + ).systemLocales; + } catch (e) { + return null; + } +} + +/** + * Get the current OS regional preference locales. + * @return an array of strings with the OS regional preference locales or null on failure. + */ +function getRegionalPrefsLocales() { + try { + return Cc["@mozilla.org/intl/ospreferences;1"].getService( + Ci.mozIOSPreferences + ).regionalPrefsLocales; + } catch (e) { + return null; + } +} + +function getIntlSettings() { + return { + requestedLocales: Services.locale.requestedLocales, + availableLocales: Services.locale.availableLocales, + appLocales: Services.locale.appLocalesAsBCP47, + systemLocales: getSystemLocales(), + regionalPrefsLocales: getRegionalPrefsLocales(), + acceptLanguages: Services.prefs + .getComplexValue("intl.accept_languages", Ci.nsIPrefLocalizedString) + .data.split(",") + .map(str => str.trim()), + }; +} + +/** + * Safely get a sysinfo property and return its value. If the property is not + * available, return aDefault. + * + * @param aPropertyName the property name to get. + * @param aDefault the value to return if aPropertyName is not available. + * @return The property value, if available, or aDefault. + */ +function getSysinfoProperty(aPropertyName, aDefault) { + try { + // |getProperty| may throw if |aPropertyName| does not exist. + return Services.sysinfo.getProperty(aPropertyName); + } catch (e) {} + + return aDefault; +} + +/** + * Safely get a gfxInfo field and return its value. If the field is not available, return + * aDefault. + * + * @param aPropertyName the property name to get. + * @param aDefault the value to return if aPropertyName is not available. + * @return The property value, if available, or aDefault. + */ +function getGfxField(aPropertyName, aDefault) { + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo); + + try { + // Accessing the field may throw if |aPropertyName| does not exist. + let gfxProp = gfxInfo[aPropertyName]; + if (gfxProp !== undefined && gfxProp !== "") { + return gfxProp; + } + } catch (e) {} + + return aDefault; +} + +/** + * Returns a substring of the input string. + * + * @param {String} aString The input string. + * @param {Integer} aMaxLength The maximum length of the returned substring. If this is + * greater than the length of the input string, we return the whole input string. + * @return {String} The substring or null if the input string is null. + */ +function limitStringToLength(aString, aMaxLength) { + if (typeof aString !== "string") { + return null; + } + return aString.substring(0, aMaxLength); +} + +/** + * Force a value to be a string. + * Only if the value is null, null is returned instead. + */ +function forceToStringOrNull(aValue) { + if (aValue === null) { + return null; + } + + return String(aValue); +} + +/** + * Get the information about a graphic adapter. + * + * @param aSuffix A suffix to add to the properties names. + * @return An object containing the adapter properties. + */ +function getGfxAdapter(aSuffix = "") { + // Note that gfxInfo, and so getGfxField, might return "Unknown" for the RAM on failures, + // not null. + let memoryMB = parseInt(getGfxField("adapterRAM" + aSuffix, null), 10); + if (Number.isNaN(memoryMB)) { + memoryMB = null; + } + + return { + description: getGfxField("adapterDescription" + aSuffix, null), + vendorID: getGfxField("adapterVendorID" + aSuffix, null), + deviceID: getGfxField("adapterDeviceID" + aSuffix, null), + subsysID: getGfxField("adapterSubsysID" + aSuffix, null), + RAM: memoryMB, + driver: getGfxField("adapterDriver" + aSuffix, null), + driverVendor: getGfxField("adapterDriverVendor" + aSuffix, null), + driverVersion: getGfxField("adapterDriverVersion" + aSuffix, null), + driverDate: getGfxField("adapterDriverDate" + aSuffix, null), + }; +} + +/** + * Encapsulates the asynchronous magic interfacing with the addon manager. The builder + * is owned by a parent environment object and is an addon listener. + */ +function EnvironmentAddonBuilder(environment) { + this._environment = environment; + + // The pending task blocks addon manager shutdown. It can either be the initial load + // or a change load. + this._pendingTask = null; + + // Have we added an observer to listen for blocklist changes that still needs to be + // removed: + this._blocklistObserverAdded = false; + + // Set to true once initial load is complete and we're watching for changes. + this._loaded = false; + + // The state reported by the shutdown blocker if we hang shutdown. + this._shutdownState = "Initial"; +} +EnvironmentAddonBuilder.prototype = { + /** + * Get the initial set of addons. + * @returns Promise<void> when the initial load is complete. + */ + async init() { + AddonManager.beforeShutdown.addBlocker( + "EnvironmentAddonBuilder", + () => this._shutdownBlocker(), + { fetchState: () => this._shutdownState } + ); + + this._pendingTask = (async () => { + try { + this._shutdownState = "Awaiting _updateAddons"; + // Gather initial addons details + await this._updateAddons(true); + + if (!this._environment._addonsAreFull) { + // The addon database has not been loaded, wait for it to + // initialize and gather full data as soon as it does. + this._shutdownState = "Awaiting AddonManagerPrivate.databaseReady"; + await AddonManagerPrivate.databaseReady; + + // Now gather complete addons details. + this._shutdownState = "Awaiting second _updateAddons"; + await this._updateAddons(); + } + } catch (err) { + this._environment._log.error("init - Exception in _updateAddons", err); + } finally { + this._pendingTask = null; + this._shutdownState = "_pendingTask init complete. No longer blocking."; + } + })(); + + return this._pendingTask; + }, + + /** + * Register an addon listener and watch for changes. + */ + watchForChanges() { + this._loaded = true; + AddonManager.addAddonListener(this); + }, + + // AddonListener + onEnabled(addon) { + this._onAddonChange(addon); + }, + onDisabled(addon) { + this._onAddonChange(addon); + }, + onInstalled(addon) { + this._onAddonChange(addon); + }, + onUninstalling(addon) { + this._onAddonChange(addon); + }, + onUninstalled(addon) { + this._onAddonChange(addon); + }, + + _onAddonChange(addon) { + if (addon && addon.isBuiltin && !addon.isSystem) { + return; + } + this._environment._log.trace("_onAddonChange"); + this._checkForChanges("addons-changed"); + }, + + // nsIObserver + observe(aSubject, aTopic, aData) { + this._environment._log.trace("observe - Topic " + aTopic); + if (aTopic == BLOCKLIST_LOADED_TOPIC) { + Services.obs.removeObserver(this, BLOCKLIST_LOADED_TOPIC); + this._blocklistObserverAdded = false; + let plugins = this._getActivePlugins(); + let gmpPluginsPromise = this._getActiveGMPlugins(); + gmpPluginsPromise.then( + gmpPlugins => { + let { addons } = this._environment._currentEnvironment; + addons.activePlugins = plugins; + addons.activeGMPlugins = gmpPlugins; + }, + err => { + this._environment._log.error( + "blocklist observe: Error collecting plugins", + err + ); + } + ); + } + }, + + _checkForChanges(changeReason) { + if (this._pendingTask) { + this._environment._log.trace( + "_checkForChanges - task already pending, dropping change with reason " + + changeReason + ); + return; + } + + this._shutdownState = "_checkForChanges awaiting _updateAddons"; + this._pendingTask = this._updateAddons().then( + result => { + this._pendingTask = null; + this._shutdownState = "No longer blocking, _updateAddons resolved"; + if (result.changed) { + this._environment._onEnvironmentChange( + changeReason, + result.oldEnvironment + ); + } + }, + err => { + this._pendingTask = null; + this._shutdownState = "No longer blocking, _updateAddons rejected"; + this._environment._log.error( + "_checkForChanges: Error collecting addons", + err + ); + } + ); + }, + + _shutdownBlocker() { + if (this._loaded) { + AddonManager.removeAddonListener(this); + if (this._blocklistObserverAdded) { + Services.obs.removeObserver(this, BLOCKLIST_LOADED_TOPIC); + } + } + + // At startup, _pendingTask is set to a Promise that does not resolve + // until the addons database has been read so complete details about + // addons are available. Returning it here will cause it to block + // profileBeforeChange, guranteeing that full information will be + // available by the time profileBeforeChangeTelemetry is fired. + return this._pendingTask; + }, + + /** + * Collect the addon data for the environment. + * + * This should only be called from _pendingTask; otherwise we risk + * running this during addon manager shutdown. + * + * @param {boolean} [atStartup] + * True if this is the first check we're performing at startup. In that + * situation, we defer some more expensive initialization. + * + * @returns Promise<Object> This returns a Promise resolved with a status object with the following members: + * changed - Whether the environment changed. + * oldEnvironment - Only set if a change occured, contains the environment data before the change. + */ + async _updateAddons(atStartup) { + this._environment._log.trace("_updateAddons"); + + let addons = { + activeAddons: await this._getActiveAddons(), + theme: await this._getActiveTheme(), + activePlugins: this._getActivePlugins(atStartup), + activeGMPlugins: await this._getActiveGMPlugins(atStartup), + }; + + let result = { + changed: + !this._environment._currentEnvironment.addons || + !ObjectUtils.deepEqual( + addons.activeAddons, + this._environment._currentEnvironment.addons.activeAddons + ), + }; + + if (result.changed) { + this._environment._log.trace("_updateAddons: addons differ"); + result.oldEnvironment = Cu.cloneInto( + this._environment._currentEnvironment, + myScope + ); + } + this._environment._currentEnvironment.addons = addons; + + return result; + }, + + /** + * Get the addon data in object form. + * @return Promise<object> containing the addon data. + */ + async _getActiveAddons() { + // Request addons, asynchronously. + let { addons: allAddons, fullData } = await AddonManager.getActiveAddons([ + "extension", + "service", + ]); + + this._environment._addonsAreFull = fullData; + let activeAddons = {}; + for (let addon of allAddons) { + // Don't collect any information about the new built-in search webextensions + if (addon.isBuiltin && !addon.isSystem) { + continue; + } + // Weird addon data in the wild can lead to exceptions while collecting + // the data. + try { + // Make sure to have valid dates. + let updateDate = new Date(Math.max(0, addon.updateDate)); + + activeAddons[addon.id] = { + version: limitStringToLength(addon.version, MAX_ADDON_STRING_LENGTH), + scope: addon.scope, + type: addon.type, + updateDay: Utils.millisecondsToDays(updateDate.getTime()), + isSystem: addon.isSystem, + isWebExtension: addon.isWebExtension, + multiprocessCompatible: true, + }; + + // getActiveAddons() gives limited data during startup and full + // data after the addons database is loaded. + if (fullData) { + let installDate = new Date(Math.max(0, addon.installDate)); + Object.assign(activeAddons[addon.id], { + blocklisted: + addon.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED, + description: limitStringToLength( + addon.description, + MAX_ADDON_STRING_LENGTH + ), + name: limitStringToLength(addon.name, MAX_ADDON_STRING_LENGTH), + userDisabled: enforceBoolean(addon.userDisabled), + appDisabled: addon.appDisabled, + foreignInstall: enforceBoolean(addon.foreignInstall), + hasBinaryComponents: false, + installDay: Utils.millisecondsToDays(installDate.getTime()), + signedState: addon.signedState, + }); + } + } catch (ex) { + this._environment._log.error( + "_getActiveAddons - An addon was discarded due to an error", + ex + ); + continue; + } + } + + return activeAddons; + }, + + /** + * Get the currently active theme data in object form. + * @return Promise<object> containing the active theme data. + */ + async _getActiveTheme() { + // Request themes, asynchronously. + let { addons: themes } = await AddonManager.getActiveAddons(["theme"]); + + let activeTheme = {}; + // We only store information about the active theme. + let theme = themes.find(theme => theme.isActive); + if (theme) { + // Make sure to have valid dates. + let installDate = new Date(Math.max(0, theme.installDate)); + let updateDate = new Date(Math.max(0, theme.updateDate)); + + activeTheme = { + id: theme.id, + blocklisted: + theme.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED, + description: limitStringToLength( + theme.description, + MAX_ADDON_STRING_LENGTH + ), + name: limitStringToLength(theme.name, MAX_ADDON_STRING_LENGTH), + userDisabled: enforceBoolean(theme.userDisabled), + appDisabled: theme.appDisabled, + version: limitStringToLength(theme.version, MAX_ADDON_STRING_LENGTH), + scope: theme.scope, + foreignInstall: enforceBoolean(theme.foreignInstall), + hasBinaryComponents: false, + installDay: Utils.millisecondsToDays(installDate.getTime()), + updateDay: Utils.millisecondsToDays(updateDate.getTime()), + }; + } + + return activeTheme; + }, + + /** + * Get the plugins data in object form. + * + * @param {boolean} [atStartup] + * True if this is the first check we're performing at startup. In that + * situation, we defer some more expensive initialization. + * + * @return Object containing the plugins data. + */ + _getActivePlugins(atStartup) { + // If we haven't yet loaded the blocklist, pass back dummy data for now, + // and add an observer to update this data as soon as we get it. + if (atStartup || !Services.blocklist.isLoaded) { + if (!this._blocklistObserverAdded) { + Services.obs.addObserver(this, BLOCKLIST_LOADED_TOPIC); + this._blocklistObserverAdded = true; + } + return [ + { + name: "dummy", + version: "0.1", + description: "Blocklist unavailable", + blocklisted: false, + disabled: true, + clicktoplay: false, + mimeTypes: ["text/there.is.only.blocklist"], + updateDay: Utils.millisecondsToDays(Date.now()), + }, + ]; + } + let pluginTags = Cc["@mozilla.org/plugin/host;1"] + .getService(Ci.nsIPluginHost) + .getPluginTags(); + + let activePlugins = []; + for (let tag of pluginTags) { + // Skip plugins which are not active. + if (tag.disabled) { + continue; + } + + try { + // Make sure to have a valid date. + let updateDate = new Date(Math.max(0, tag.lastModifiedTime)); + + activePlugins.push({ + name: limitStringToLength(tag.name, MAX_ADDON_STRING_LENGTH), + version: limitStringToLength(tag.version, MAX_ADDON_STRING_LENGTH), + description: limitStringToLength( + tag.description, + MAX_ADDON_STRING_LENGTH + ), + blocklisted: tag.blocklisted, + disabled: tag.disabled, + clicktoplay: tag.clicktoplay, + mimeTypes: tag.getMimeTypes(), + updateDay: Utils.millisecondsToDays(updateDate.getTime()), + }); + } catch (ex) { + this._environment._log.error( + "_getActivePlugins - A plugin was discarded due to an error", + ex + ); + continue; + } + } + + return activePlugins; + }, + + /** + * Get the GMPlugins data in object form. + * + * @param {boolean} [atStartup] + * True if this is the first check we're performing at startup. In that + * situation, we defer some more expensive initialization. + * + * @return Object containing the GMPlugins data. + * + * This should only be called from _pendingTask; otherwise we risk + * running this during addon manager shutdown. + */ + async _getActiveGMPlugins(atStartup) { + // If we haven't yet loaded the blocklist, pass back dummy data for now, + // and add an observer to update this data as soon as we get it. + if (atStartup || !Services.blocklist.isLoaded) { + if (!this._blocklistObserverAdded) { + Services.obs.addObserver(this, BLOCKLIST_LOADED_TOPIC); + this._blocklistObserverAdded = true; + } + return { + "dummy-gmp": { + version: "0.1", + userDisabled: false, + applyBackgroundUpdates: 1, + }, + }; + } + // Request plugins, asynchronously. + let allPlugins = await AddonManager.getAddonsByTypes(["plugin"]); + + let activeGMPlugins = {}; + for (let plugin of allPlugins) { + // Only get info for active GMplugins. + if (!plugin.isGMPlugin || !plugin.isActive) { + continue; + } + + try { + activeGMPlugins[plugin.id] = { + version: plugin.version, + userDisabled: enforceBoolean(plugin.userDisabled), + applyBackgroundUpdates: plugin.applyBackgroundUpdates, + }; + } catch (ex) { + this._environment._log.error( + "_getActiveGMPlugins - A GMPlugin was discarded due to an error", + ex + ); + continue; + } + } + + return activeGMPlugins; + }, +}; + +function EnvironmentCache() { + this._log = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + "TelemetryEnvironment::" + ); + this._log.trace("constructor"); + + this._shutdown = false; + // Don't allow querying the search service too early to prevent + // impacting the startup performance. + this._canQuerySearch = false; + // To guard against slowing down startup, defer gathering heavy environment + // entries until the session is restored. + this._sessionWasRestored = false; + + // A map of listeners that will be called on environment changes. + this._changeListeners = new Map(); + + // A map of watched preferences which trigger an Environment change when + // modified. Every entry contains a recording policy (RECORD_PREF_*). + this._watchedPrefs = DEFAULT_ENVIRONMENT_PREFS; + + this._currentEnvironment = { + build: this._getBuild(), + partner: this._getPartner(), + system: this._getSystem(), + }; + + this._addObservers(); + + // Build the remaining asynchronous parts of the environment. Don't register change listeners + // until the initial environment has been built. + + let p = [this._updateSettings()]; + this._addonBuilder = new EnvironmentAddonBuilder(this); + p.push(this._addonBuilder.init()); + + this._currentEnvironment.profile = {}; + p.push(this._updateProfile()); + if (AppConstants.MOZ_BUILD_APP == "browser") { + p.push(this._loadAttributionAsync()); + } + p.push(this._loadAutoUpdateAsync()); + p.push(this._loadIntlData()); + + for (const [ + id, + { branch, options }, + ] of gActiveExperimentStartupBuffer.entries()) { + this.setExperimentActive(id, branch, options); + } + gActiveExperimentStartupBuffer = null; + + let setup = () => { + this._initTask = null; + this._startWatchingPrefs(); + this._addonBuilder.watchForChanges(); + this._updateGraphicsFeatures(); + return this.currentEnvironment; + }; + + this._initTask = Promise.all(p).then( + () => setup(), + err => { + // log errors but eat them for consumers + this._log.error("EnvironmentCache - error while initializing", err); + return setup(); + } + ); + + // Addons may contain partial or full data depending on whether the Addons DB + // has had a chance to load. Do we have full data yet? + this._addonsAreFull = false; +} +EnvironmentCache.prototype = { + /** + * The current environment data. The returned data is cloned to avoid + * unexpected sharing or mutation. + * @returns object + */ + get currentEnvironment() { + return Cu.cloneInto(this._currentEnvironment, myScope); + }, + + /** + * Wait for the current enviroment to be fully initialized. + * @returns Promise<object> + */ + onInitialized() { + if (this._initTask) { + return this._initTask; + } + return Promise.resolve(this.currentEnvironment); + }, + + /** + * This gets called when the delayed init completes. + */ + async delayedInit() { + this._processData = await Services.sysinfo.processInfo; + let processData = await Services.sysinfo.processInfo; + // Remove isWow64 and isWowARM64 from processData + // to strip it down to just CPU info + delete processData.isWow64; + delete processData.isWowARM64; + + let oldEnv = null; + if (!this._initTask) { + oldEnv = this.currentEnvironment; + } + + this._cpuData = this._getCPUData(); + // Augment the return value from the promises with cached values + this._cpuData = { ...processData, ...this._cpuData }; + + this._currentEnvironment.system.cpu = this._getCPUData(); + + if (AppConstants.platform == "win") { + this._hddData = await Services.sysinfo.diskInfo; + if (AppConstants.MOZ_GLEAN) { + Glean.fogValidation.profileDiskIsSsd.set( + this._hddData.profile.type == "SSD" + ); + } + let osData = await Services.sysinfo.osInfo; + + if (!this._initTask) { + // We've finished creating the initial env, so notify for the update + // This is all a bit awkward because `currentEnvironment` clones + // the object, which we need to pass to the notification, but we + // should only notify once we've updated the current environment... + // Ideally, _onEnvironmentChange should somehow deal with all this + // instead of all the consumers. + oldEnv = this.currentEnvironment; + } + + this._osData = this._getOSData(); + + // Augment the return values from the promises with cached values + this._osData = Object.assign(osData, this._osData); + + this._currentEnvironment.system.os = this._getOSData(); + this._currentEnvironment.system.hdd = this._getHDDData(); + + // Windows only values stored in processData + this._currentEnvironment.system.isWow64 = this._getProcessData().isWow64; + this._currentEnvironment.system.isWowARM64 = this._getProcessData().isWowARM64; + } + + if (!this._initTask) { + this._onEnvironmentChange("system-info", oldEnv); + } + }, + + /** + * Register a listener for environment changes. + * @param name The name of the listener. If a new listener is registered + * with the same name, the old listener will be replaced. + * @param listener function(reason, oldEnvironment) - Will receive a reason for + the change and the environment data before the change. + */ + registerChangeListener(name, listener) { + this._log.trace("registerChangeListener for " + name); + if (this._shutdown) { + this._log.warn("registerChangeListener - already shutdown"); + return; + } + this._changeListeners.set(name, listener); + }, + + /** + * Unregister from listening to environment changes. + * It's fine to call this on an unitialized TelemetryEnvironment. + * @param name The name of the listener to remove. + */ + unregisterChangeListener(name) { + this._log.trace("unregisterChangeListener for " + name); + if (this._shutdown) { + this._log.warn("registerChangeListener - already shutdown"); + return; + } + this._changeListeners.delete(name); + }, + + setExperimentActive(id, branch, options) { + this._log.trace(`setExperimentActive - id: ${id}, branch: ${branch}`); + // Make sure both the id and the branch have sane lengths. + const saneId = limitStringToLength(id, MAX_EXPERIMENT_ID_LENGTH); + const saneBranch = limitStringToLength( + branch, + MAX_EXPERIMENT_BRANCH_LENGTH + ); + if (!saneId || !saneBranch) { + this._log.error( + "setExperimentActive - the provided arguments are not strings." + ); + return; + } + + // Warn the user about any content truncation. + if (saneId.length != id.length || saneBranch.length != branch.length) { + this._log.warn( + "setExperimentActive - the experiment id or branch were truncated." + ); + } + + // Truncate the experiment type if present. + if (options.hasOwnProperty("type")) { + let type = limitStringToLength(options.type, MAX_EXPERIMENT_TYPE_LENGTH); + if (type.length != options.type.length) { + options.type = type; + this._log.warn( + "setExperimentActive - the experiment type was truncated." + ); + } + } + + // Truncate the enrollment id if present. + if (options.hasOwnProperty("enrollmentId")) { + let enrollmentId = limitStringToLength( + options.enrollmentId, + MAX_EXPERIMENT_ENROLLMENT_ID_LENGTH + ); + if (enrollmentId.length != options.enrollmentId.length) { + options.enrollmentId = enrollmentId; + this._log.warn( + "setExperimentActive - the enrollment id was truncated." + ); + } + } + + let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); + // Add the experiment annotation. + let experiments = this._currentEnvironment.experiments || {}; + experiments[saneId] = { branch: saneBranch }; + if (options.hasOwnProperty("type")) { + experiments[saneId].type = options.type; + } + if (options.hasOwnProperty("enrollmentId")) { + experiments[saneId].enrollmentId = options.enrollmentId; + } + this._currentEnvironment.experiments = experiments; + // Notify of the change. + this._onEnvironmentChange("experiment-annotation-changed", oldEnvironment); + }, + + setExperimentInactive(id) { + this._log.trace("setExperimentInactive"); + let experiments = this._currentEnvironment.experiments || {}; + if (id in experiments) { + // Only attempt to notify if a previous annotation was found and removed. + let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); + // Remove the experiment annotation. + delete this._currentEnvironment.experiments[id]; + // Notify of the change. + this._onEnvironmentChange( + "experiment-annotation-changed", + oldEnvironment + ); + } + }, + + getActiveExperiments() { + return Cu.cloneInto(this._currentEnvironment.experiments || {}, myScope); + }, + + shutdown() { + this._log.trace("shutdown"); + this._shutdown = true; + }, + + /** + * Only used in tests, set the preferences to watch. + * @param aPreferences A map of preferences names and their recording policy. + */ + async _watchPreferences(aPreferences) { + this._stopWatchingPrefs(); + this._watchedPrefs = aPreferences; + await this._updateSettings(); + this._startWatchingPrefs(); + }, + + /** + * Get an object containing the values for the watched preferences. Depending on the + * policy, the value for a preference or whether it was changed by user is reported. + * + * @return An object containing the preferences values. + */ + _getPrefData() { + let prefData = {}; + for (let [pref, policy] of this._watchedPrefs.entries()) { + let prefValue = this._getPrefValue(pref, policy.what); + + if (prefValue === undefined) { + continue; + } + + prefData[pref] = prefValue; + } + return prefData; + }, + + /** + * Get the value of a preference given the preference name and the policy. + * @param pref Name of the preference. + * @param what Policy of the preference. + * + * @returns The value we need to store for this preference. It can be undefined + * or null if the preference is invalid or has a value set by the user. + */ + _getPrefValue(pref, what) { + // Check the policy for the preference and decide if we need to store its value + // or whether it changed from the default value. + let prefType = Services.prefs.getPrefType(pref); + + if ( + what == TelemetryEnvironment.RECORD_DEFAULTPREF_VALUE || + what == TelemetryEnvironment.RECORD_DEFAULTPREF_STATE + ) { + // For default prefs, make sure they exist + if (prefType == Ci.nsIPrefBranch.PREF_INVALID) { + return undefined; + } + } else if (!Services.prefs.prefHasUserValue(pref)) { + // For user prefs, make sure they are set + return undefined; + } + + if (what == TelemetryEnvironment.RECORD_DEFAULTPREF_STATE) { + return "<set>"; + } else if (what == TelemetryEnvironment.RECORD_PREF_STATE) { + return "<user-set>"; + } else if (prefType == Ci.nsIPrefBranch.PREF_STRING) { + return Services.prefs.getStringPref(pref); + } else if (prefType == Ci.nsIPrefBranch.PREF_BOOL) { + return Services.prefs.getBoolPref(pref); + } else if (prefType == Ci.nsIPrefBranch.PREF_INT) { + return Services.prefs.getIntPref(pref); + } else if (prefType == Ci.nsIPrefBranch.PREF_INVALID) { + return null; + } + throw new Error( + `Unexpected preference type ("${prefType}") for "${pref}".` + ); + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + /** + * Start watching the preferences. + */ + _startWatchingPrefs() { + this._log.trace("_startWatchingPrefs - " + this._watchedPrefs); + + Services.prefs.addObserver("", this, true); + }, + + _onPrefChanged(aData) { + this._log.trace("_onPrefChanged"); + let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); + this._currentEnvironment.settings.userPrefs[aData] = this._getPrefValue( + aData, + this._watchedPrefs.get(aData).what + ); + this._onEnvironmentChange("pref-changed", oldEnvironment); + }, + + /** + * Do not receive any more change notifications for the preferences. + */ + _stopWatchingPrefs() { + this._log.trace("_stopWatchingPrefs"); + + Services.prefs.removeObserver("", this); + }, + + _addObservers() { + // Watch the search engine change and service topics. + Services.obs.addObserver(this, SESSIONSTORE_WINDOWS_RESTORED_TOPIC); + Services.obs.addObserver(this, COMPOSITOR_CREATED_TOPIC); + Services.obs.addObserver(this, COMPOSITOR_PROCESS_ABORTED_TOPIC); + Services.obs.addObserver(this, DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC); + Services.obs.addObserver(this, GFX_FEATURES_READY_TOPIC); + Services.obs.addObserver(this, SEARCH_ENGINE_MODIFIED_TOPIC); + Services.obs.addObserver(this, SEARCH_SERVICE_TOPIC); + Services.obs.addObserver(this, AUTO_UPDATE_PREF_CHANGE_TOPIC); + Services.obs.addObserver(this, SERVICES_INFO_CHANGE_TOPIC); + }, + + _removeObservers() { + Services.obs.removeObserver(this, SESSIONSTORE_WINDOWS_RESTORED_TOPIC); + Services.obs.removeObserver(this, COMPOSITOR_CREATED_TOPIC); + Services.obs.removeObserver(this, COMPOSITOR_PROCESS_ABORTED_TOPIC); + try { + Services.obs.removeObserver( + this, + DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC + ); + } catch (ex) {} + Services.obs.removeObserver(this, GFX_FEATURES_READY_TOPIC); + Services.obs.removeObserver(this, SEARCH_ENGINE_MODIFIED_TOPIC); + Services.obs.removeObserver(this, SEARCH_SERVICE_TOPIC); + Services.obs.removeObserver(this, AUTO_UPDATE_PREF_CHANGE_TOPIC); + Services.obs.removeObserver(this, SERVICES_INFO_CHANGE_TOPIC); + }, + + observe(aSubject, aTopic, aData) { + this._log.trace("observe - aTopic: " + aTopic + ", aData: " + aData); + switch (aTopic) { + case SEARCH_ENGINE_MODIFIED_TOPIC: + if ( + aData != "engine-default" && + aData != "engine-default-private" && + aData != "engine-changed" + ) { + return; + } + if ( + aData == "engine-changed" && + aSubject.QueryInterface(Ci.nsISearchEngine) && + Services.search.defaultEngine != aSubject + ) { + return; + } + // Record the new default search choice and send the change notification. + this._onSearchEngineChange(); + break; + case SEARCH_SERVICE_TOPIC: + if (aData != "init-complete") { + return; + } + // Now that the search engine init is complete, record the default search choice. + this._canQuerySearch = true; + this._updateSearchEngine(); + break; + case GFX_FEATURES_READY_TOPIC: + case COMPOSITOR_CREATED_TOPIC: + // Full graphics information is not available until we have created at + // least one off-main-thread-composited window. Thus we wait for the + // first compositor to be created and then query nsIGfxInfo again. + this._updateGraphicsFeatures(); + break; + case COMPOSITOR_PROCESS_ABORTED_TOPIC: + // Our compositor process has been killed for whatever reason, so refresh + // our reported graphics features and trigger an environment change. + this._onCompositorProcessAborted(); + break; + case DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC: + // Distribution customizations are applied after final-ui-startup. query + // partner prefs again when they are ready. + this._updatePartner(); + Services.obs.removeObserver(this, aTopic); + break; + case SESSIONSTORE_WINDOWS_RESTORED_TOPIC: + this._sessionWasRestored = true; + // Make sure to initialize the search service once we've done restoring + // the windows, so that we don't risk loosing search data. + Services.search.init(); + // The default browser check could take some time, so just call it after + // the session was restored. + this._updateDefaultBrowser(); + break; + case PREF_CHANGED_TOPIC: + let options = this._watchedPrefs.get(aData); + if (options && !options.requiresRestart) { + this._onPrefChanged(aData); + } + break; + case AUTO_UPDATE_PREF_CHANGE_TOPIC: + this._currentEnvironment.settings.update.autoDownload = aData == "true"; + break; + case SERVICES_INFO_CHANGE_TOPIC: + this._updateServicesInfo(); + break; + } + }, + + /** + * Update the default search engine value. + */ + async _updateSearchEngine() { + if (!this._canQuerySearch) { + this._log.trace("_updateSearchEngine - ignoring early call"); + return; + } + + this._log.trace( + "_updateSearchEngine - isInitialized: " + Services.search.isInitialized + ); + if (!Services.search.isInitialized) { + return; + } + + // Make sure we have a settings section. + this._currentEnvironment.settings = this._currentEnvironment.settings || {}; + + // Update the search engine entry in the current environment. + const defaultEngineInfo = await Services.search.getDefaultEngineInfo(); + this._currentEnvironment.settings.defaultSearchEngine = + defaultEngineInfo.defaultSearchEngine; + this._currentEnvironment.settings.defaultSearchEngineData = { + ...defaultEngineInfo.defaultSearchEngineData, + }; + if ("defaultPrivateSearchEngine" in defaultEngineInfo) { + this._currentEnvironment.settings.defaultPrivateSearchEngine = + defaultEngineInfo.defaultPrivateSearchEngine; + } + if ("defaultPrivateSearchEngineData" in defaultEngineInfo) { + this._currentEnvironment.settings.defaultPrivateSearchEngineData = { + ...defaultEngineInfo.defaultPrivateSearchEngineData, + }; + } + }, + + /** + * Update the default search engine value and trigger the environment change. + */ + async _onSearchEngineChange() { + this._log.trace("_onSearchEngineChange"); + + // Finally trigger the environment change notification. + let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); + await this._updateSearchEngine(); + this._onEnvironmentChange("search-engine-changed", oldEnvironment); + }, + + /** + * Refresh the Telemetry environment and trigger an environment change due to + * a change in compositor process (normally this will mean we've fallen back + * from out-of-process to in-process compositing). + */ + _onCompositorProcessAborted() { + this._log.trace("_onCompositorProcessAborted"); + + // Trigger the environment change notification. + let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope); + this._updateGraphicsFeatures(); + this._onEnvironmentChange("gfx-features-changed", oldEnvironment); + }, + + /** + * Update the graphics features object. + */ + _updateGraphicsFeatures() { + let gfxData = this._currentEnvironment.system.gfx; + try { + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo); + gfxData.features = gfxInfo.getFeatures(); + } catch (e) { + this._log.error("nsIGfxInfo.getFeatures() caught error", e); + } + }, + + /** + * Update the partner prefs. + */ + _updatePartner() { + this._currentEnvironment.partner = this._getPartner(); + }, + + /** + * Get the build data in object form. + * @return Object containing the build data. + */ + _getBuild() { + let buildData = { + applicationId: Services.appinfo.ID || null, + applicationName: Services.appinfo.name || null, + architecture: Services.sysinfo.get("arch"), + buildId: Services.appinfo.appBuildID || null, + version: Services.appinfo.version || null, + vendor: Services.appinfo.vendor || null, + displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY || null, + platformVersion: Services.appinfo.platformVersion || null, + xpcomAbi: Services.appinfo.XPCOMABI, + updaterAvailable: AppConstants.MOZ_UPDATER, + }; + + return buildData; + }, + + /** + * Determine if we're the default browser. + * @returns null on error, true if we are the default browser, or false otherwise. + */ + _isDefaultBrowser() { + let isDefault = (service, ...args) => { + try { + return !!service.isDefaultBrowser(...args); + } catch (ex) { + this._log.error( + "_isDefaultBrowser - Could not determine if default browser", + ex + ); + return null; + } + }; + + if (!("@mozilla.org/browser/shell-service;1" in Cc)) { + this._log.info( + "_isDefaultBrowser - Could not obtain browser shell service" + ); + return null; + } + + try { + let { ShellService } = ChromeUtils.import( + "resource:///modules/ShellService.jsm" + ); + // This uses the same set of flags used by the pref pane. + return isDefault(ShellService, false, true); + } catch (ex) { + this._log.error("_isDefaultBrowser - Could not obtain shell service JSM"); + } + + try { + let shellService = Cc["@mozilla.org/browser/shell-service;1"].getService( + Ci.nsIShellService + ); + // This uses the same set of flags used by the pref pane. + return isDefault(shellService, true); + } catch (ex) { + this._log.error("_isDefaultBrowser - Could not obtain shell service", ex); + return null; + } + }, + + _updateDefaultBrowser() { + if (AppConstants.platform === "android") { + return; + } + // Make sure to have a settings section. + this._currentEnvironment.settings = this._currentEnvironment.settings || {}; + this._currentEnvironment.settings.isDefaultBrowser = this + ._sessionWasRestored + ? this._isDefaultBrowser() + : null; + }, + + /** + * Update the cached settings data. + */ + async _updateSettings() { + let updateChannel = null; + try { + updateChannel = Utils.getUpdateChannel(); + } catch (e) {} + + this._currentEnvironment.settings = { + blocklistEnabled: Services.prefs.getBoolPref( + PREF_BLOCKLIST_ENABLED, + true + ), + e10sEnabled: Services.appinfo.browserTabsRemoteAutostart, + e10sMultiProcesses: Services.appinfo.maxWebProcessCount, + fissionEnabled: Services.appinfo.fissionAutostart, + telemetryEnabled: Utils.isTelemetryEnabled, + locale: getBrowserLocale(), + // We need to wait for browser-delayed-startup-finished to ensure that the locales + // have settled, once that's happened we can get the intl data directly. + intl: Policy._intlLoaded ? getIntlSettings() : {}, + update: { + channel: updateChannel, + enabled: !Services.policies || Services.policies.isAllowed("appUpdate"), + }, + userPrefs: this._getPrefData(), + sandbox: this._getSandboxData(), + }; + + // Services.appinfo.launcherProcessState is not available in all build + // configurations, in which case an exception may be thrown. + try { + this._currentEnvironment.settings.launcherProcessState = + Services.appinfo.launcherProcessState; + } catch (e) {} + + this._currentEnvironment.settings.addonCompatibilityCheckEnabled = + AddonManager.checkCompatibility; + + this._updateAttribution(); + this._updateDefaultBrowser(); + await this._updateSearchEngine(); + this._updateAutoDownload(); + }, + + _getSandboxData() { + let effectiveContentProcessLevel = null; + try { + let sandboxSettings = Cc[ + "@mozilla.org/sandbox/sandbox-settings;1" + ].getService(Ci.mozISandboxSettings); + effectiveContentProcessLevel = + sandboxSettings.effectiveContentSandboxLevel; + } catch (e) {} + return { + effectiveContentProcessLevel, + }; + }, + + /** + * Update the cached profile data. + * @returns Promise<> resolved when the I/O is complete. + */ + async _updateProfile() { + let profileAccessor = await ProfileAge(); + + let creationDate = await profileAccessor.created; + let resetDate = await profileAccessor.reset; + let firstUseDate = await profileAccessor.firstUse; + + this._currentEnvironment.profile.creationDate = Utils.millisecondsToDays( + creationDate + ); + if (resetDate) { + this._currentEnvironment.profile.resetDate = Utils.millisecondsToDays( + resetDate + ); + } + if (firstUseDate) { + this._currentEnvironment.profile.firstUseDate = Utils.millisecondsToDays( + firstUseDate + ); + } + }, + + /** + * Load the attribution data object and updates the environment. + * @returns Promise<> resolved when the I/O is complete. + */ + async _loadAttributionAsync() { + try { + await AttributionCode.getAttrDataAsync(); + } catch (e) { + // The AttributionCode.jsm module might not be always available + // (e.g. tests). Gracefully handle this. + return; + } + this._updateAttribution(); + }, + + /** + * Update the environment with the cached attribution data. + */ + _updateAttribution() { + let data = null; + try { + data = AttributionCode.getCachedAttributionData(); + } catch (e) { + // The AttributionCode.jsm module might not be always available + // (e.g. tests). Gracefully handle this. + } + + if (!data || !Object.keys(data).length) { + return; + } + + let attributionData = {}; + for (let key in data) { + attributionData[key] = limitStringToLength( + data[key], + MAX_ATTRIBUTION_STRING_LENGTH + ); + } + this._currentEnvironment.settings.attribution = attributionData; + }, + + /** + * Load the auto update pref and adds it to the environment + */ + async _loadAutoUpdateAsync() { + if (AppConstants.MOZ_UPDATER) { + this._updateAutoDownloadCache = await UpdateUtils.getAppUpdateAutoEnabled(); + } else { + this._updateAutoDownloadCache = false; + } + this._updateAutoDownload(); + }, + + /** + * Update the environment with the cached value for whether updates can auto- + * download. + */ + _updateAutoDownload() { + if (this._updateAutoDownloadCache === undefined) { + return; + } + this._currentEnvironment.settings.update.autoDownload = this._updateAutoDownloadCache; + }, + + /** + * Get i18n data about the system. + * @return A promise of completion. + */ + async _loadIntlData() { + // Wait for the startup topic. + await Policy._browserDelayedStartup(); + this._currentEnvironment.settings.intl = getIntlSettings(); + Policy._intlLoaded = true; + }, + // This exists as a separate function for testing. + async _getFxaSignedInUser() { + return fxAccounts.getSignedInUser(); + }, + + async _updateServicesInfo() { + let syncEnabled = false; + let accountEnabled = false; + let weaveService = Cc["@mozilla.org/weave/service;1"].getService() + .wrappedJSObject; + syncEnabled = weaveService && weaveService.enabled; + if (syncEnabled) { + // All sync users are account users, definitely. + accountEnabled = true; + } else { + // Not all account users are sync users. See if they're signed into FxA. + try { + let user = await this._getFxaSignedInUser(); + if (user) { + accountEnabled = true; + } + } catch (e) { + // We don't know. This might be a transient issue which will clear + // itself up later, but the information in telemetry is quite possibly stale + // (this is called from a change listener), so clear it out to avoid + // reporting data which might be wrong until we can figure it out. + delete this._currentEnvironment.services; + this._log.error("_updateServicesInfo() caught error", e); + return; + } + } + this._currentEnvironment.services = { + accountEnabled, + syncEnabled, + }; + }, + + /** + * Get the partner data in object form. + * @return Object containing the partner data. + */ + _getPartner() { + let partnerData = { + distributionId: Services.prefs.getStringPref(PREF_DISTRIBUTION_ID, null), + distributionVersion: Services.prefs.getStringPref( + PREF_DISTRIBUTION_VERSION, + null + ), + partnerId: Services.prefs.getStringPref(PREF_PARTNER_ID, null), + distributor: Services.prefs.getStringPref(PREF_DISTRIBUTOR, null), + distributorChannel: Services.prefs.getStringPref( + PREF_DISTRIBUTOR_CHANNEL, + null + ), + }; + + // Get the PREF_APP_PARTNER_BRANCH branch and append its children to partner data. + let partnerBranch = Services.prefs.getBranch(PREF_APP_PARTNER_BRANCH); + partnerData.partnerNames = partnerBranch.getChildList(""); + + return partnerData; + }, + + _cpuData: null, + /** + * Get the CPU information. + * @return Object containing the CPU information data. + */ + _getCPUData() { + if (this._cpuData) { + return this._cpuData; + } + + this._cpuData = {}; + + const CPU_EXTENSIONS = [ + "hasMMX", + "hasSSE", + "hasSSE2", + "hasSSE3", + "hasSSSE3", + "hasSSE4A", + "hasSSE4_1", + "hasSSE4_2", + "hasAVX", + "hasAVX2", + "hasAES", + "hasEDSP", + "hasARMv6", + "hasARMv7", + "hasNEON", + "hasUserCET", + ]; + + // Enumerate the available CPU extensions. + let availableExts = []; + for (let ext of CPU_EXTENSIONS) { + if (getSysinfoProperty(ext, false)) { + availableExts.push(ext); + } + } + + this._cpuData.extensions = availableExts; + + return this._cpuData; + }, + + _processData: null, + /** + * Get the process information. + * @return Object containing the process information data. + */ + _getProcessData() { + if (this._processData) { + return this._processData; + } + return {}; + }, + + /** + * Get the device information, if we are on a portable device. + * @return Object containing the device information data, or null if + * not a portable device. + */ + _getDeviceData() { + if (AppConstants.platform !== "android") { + return null; + } + + return { + model: getSysinfoProperty("device", null), + manufacturer: getSysinfoProperty("manufacturer", null), + hardware: getSysinfoProperty("hardware", null), + isTablet: getSysinfoProperty("tablet", null), + }; + }, + + _osData: null, + /** + * Get the OS information. + * @return Object containing the OS data. + */ + _getOSData() { + if (this._osData) { + return this._osData; + } + this._osData = { + name: forceToStringOrNull(getSysinfoProperty("name", null)), + version: forceToStringOrNull(getSysinfoProperty("version", null)), + locale: forceToStringOrNull(getSystemLocale()), + }; + + if (AppConstants.platform == "android") { + this._osData.kernelVersion = forceToStringOrNull( + getSysinfoProperty("kernel_version", null) + ); + } else if (AppConstants.platform === "win") { + // The path to the "UBR" key, queried to get additional version details on Windows. + const WINDOWS_UBR_KEY_PATH = + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; + + let versionInfo = WindowsVersionInfo.get({ throwOnError: false }); + this._osData.servicePackMajor = versionInfo.servicePackMajor; + this._osData.servicePackMinor = versionInfo.servicePackMinor; + this._osData.windowsBuildNumber = versionInfo.buildNumber; + // We only need the UBR if we're at or above Windows 10. + if ( + typeof this._osData.version === "string" && + Services.vc.compare(this._osData.version, "10") >= 0 + ) { + // Query the UBR key and only add it to the environment if it's available. + // |readRegKey| doesn't throw, but rather returns 'undefined' on error. + let ubr = WindowsRegistry.readRegKey( + Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE, + WINDOWS_UBR_KEY_PATH, + "UBR", + Ci.nsIWindowsRegKey.WOW64_64 + ); + this._osData.windowsUBR = ubr !== undefined ? ubr : null; + } + } + + return this._osData; + }, + + _hddData: null, + /** + * Get the HDD information. + * @return Object containing the HDD data. + */ + _getHDDData() { + if (this._hddData) { + return this._hddData; + } + let nullData = { model: null, revision: null, type: null }; + return { profile: nullData, binary: nullData, system: nullData }; + }, + + /** + * Get registered security product information. + * @return Object containing the security product data + */ + _getSecurityAppData() { + const maxStringLength = 256; + + const keys = [ + ["registeredAntiVirus", "antivirus"], + ["registeredAntiSpyware", "antispyware"], + ["registeredFirewall", "firewall"], + ]; + + let result = {}; + + for (let [inKey, outKey] of keys) { + let prop = getSysinfoProperty(inKey, null); + if (prop) { + prop = limitStringToLength(prop, maxStringLength).split(";"); + } + + result[outKey] = prop; + } + + return result; + }, + + /** + * Get the GFX information. + * @return Object containing the GFX data. + */ + _getGFXData() { + let gfxData = { + D2DEnabled: getGfxField("D2DEnabled", null), + DWriteEnabled: getGfxField("DWriteEnabled", null), + ContentBackend: getGfxField("ContentBackend", null), + Headless: getGfxField("isHeadless", null), + EmbeddedInFirefoxReality: getGfxField("EmbeddedInFirefoxReality", null), + // The following line is disabled due to main thread jank and will be enabled + // again as part of bug 1154500. + // DWriteVersion: getGfxField("DWriteVersion", null), + adapters: [], + monitors: [], + features: {}, + }; + + if (AppConstants.platform !== "android") { + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo); + try { + gfxData.monitors = gfxInfo.getMonitors(); + } catch (e) { + this._log.error("nsIGfxInfo.getMonitors() caught error", e); + } + } + + try { + let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo); + gfxData.features = gfxInfo.getFeatures(); + } catch (e) { + this._log.error("nsIGfxInfo.getFeatures() caught error", e); + } + + // GfxInfo does not yet expose a way to iterate through all the adapters. + gfxData.adapters.push(getGfxAdapter("")); + gfxData.adapters[0].GPUActive = true; + + // If we have a second adapter add it to the gfxData.adapters section. + let hasGPU2 = getGfxField("adapterDeviceID2", null) !== null; + if (!hasGPU2) { + this._log.trace("_getGFXData - Only one display adapter detected."); + return gfxData; + } + + this._log.trace("_getGFXData - Two display adapters detected."); + + gfxData.adapters.push(getGfxAdapter("2")); + gfxData.adapters[1].GPUActive = getGfxField("isGPU2Active", null); + + return gfxData; + }, + + /** + * Get the system data in object form. + * @return Object containing the system data. + */ + _getSystem() { + let memoryMB = getSysinfoProperty("memsize", null); + if (memoryMB) { + // Send RAM size in megabytes. Rounding because sysinfo doesn't + // always provide RAM in multiples of 1024. + memoryMB = Math.round(memoryMB / 1024 / 1024); + } + + let virtualMB = getSysinfoProperty("virtualmemsize", null); + if (virtualMB) { + // Send the total virtual memory size in megabytes. Rounding because + // sysinfo doesn't always provide RAM in multiples of 1024. + virtualMB = Math.round(virtualMB / 1024 / 1024); + } + + let data = { + memoryMB, + virtualMaxMB: virtualMB, + cpu: this._getCPUData(), + os: this._getOSData(), + hdd: this._getHDDData(), + gfx: this._getGFXData(), + appleModelId: getSysinfoProperty("appleModelId", null), + }; + + if (AppConstants.platform === "win") { + data = { ...this._getProcessData(), ...data }; + } else if (AppConstants.platform == "android") { + data.device = this._getDeviceData(); + } + + // Windows 8+ + if (AppConstants.isPlatformAndVersionAtLeast("win", "6.2")) { + data.sec = this._getSecurityAppData(); + } + + return data; + }, + + _onEnvironmentChange(what, oldEnvironment) { + this._log.trace("_onEnvironmentChange for " + what); + + // We are already skipping change events in _checkChanges if there is a pending change task running. + if (this._shutdown) { + this._log.trace("_onEnvironmentChange - Already shut down."); + return; + } + + if (ObjectUtils.deepEqual(this._currentEnvironment, oldEnvironment)) { + Services.telemetry.scalarAdd("telemetry.environment_didnt_change", 1); + } + + for (let [name, listener] of this._changeListeners) { + try { + this._log.debug("_onEnvironmentChange - calling " + name); + listener(what, oldEnvironment); + } catch (e) { + this._log.error( + "_onEnvironmentChange - listener " + name + " caught error", + e + ); + } + } + }, + + reset() { + this._shutdown = false; + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm b/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm new file mode 100644 index 0000000000..89f93de514 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm @@ -0,0 +1,584 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryReportingPolicy"]; + +ChromeUtils.import("resource://gre/modules/Log.jsm", this); +ChromeUtils.import("resource://gre/modules/Services.jsm", this); +ChromeUtils.import("resource://gre/modules/Timer.jsm", this); +ChromeUtils.import("resource://services-common/observers.js", this); +ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this); + +ChromeUtils.defineModuleGetter( + this, + "TelemetrySend", + "resource://gre/modules/TelemetrySend.jsm" +); + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetryReportingPolicy::"; + +// Oldest year to allow in date preferences. The FHR infobar was implemented in +// 2012 and no dates older than that should be encountered. +const OLDEST_ALLOWED_ACCEPTANCE_YEAR = 2012; + +const PREF_BRANCH = "datareporting.policy."; + +// The following preferences are deprecated and will be purged during the preferences +// migration process. +const DEPRECATED_FHR_PREFS = [ + PREF_BRANCH + "dataSubmissionPolicyAccepted", + PREF_BRANCH + "dataSubmissionPolicyBypassAcceptance", + PREF_BRANCH + "dataSubmissionPolicyResponseType", + PREF_BRANCH + "dataSubmissionPolicyResponseTime", +]; + +// How much time until we display the data choices notification bar, on the first run. +const NOTIFICATION_DELAY_FIRST_RUN_MSEC = 60 * 1000; // 60s +// Same as above, for the next runs. +const NOTIFICATION_DELAY_NEXT_RUNS_MSEC = 10 * 1000; // 10s + +/** + * This is a policy object used to override behavior within this module. + * Tests override properties on this object to allow for control of behavior + * that would otherwise be very hard to cover. + */ +var Policy = { + now: () => new Date(), + setShowInfobarTimeout: (callback, delayMs) => setTimeout(callback, delayMs), + clearShowInfobarTimeout: id => clearTimeout(id), +}; + +/** + * Represents a request to display data policy. + * + * Receivers of these instances are expected to call one or more of the on* + * functions when events occur. + * + * When one of these requests is received, the first thing a callee should do + * is present notification to the user of the data policy. When the notice + * is displayed to the user, the callee should call `onUserNotifyComplete`. + * + * If for whatever reason the callee could not display a notice, + * it should call `onUserNotifyFailed`. + * + * @param {Object} aLog The log object used to log the error in case of failures. + */ +function NotifyPolicyRequest(aLog) { + this._log = aLog; +} + +NotifyPolicyRequest.prototype = Object.freeze({ + /** + * Called when the user is notified of the policy. + */ + onUserNotifyComplete() { + return TelemetryReportingPolicyImpl._userNotified(); + }, + + /** + * Called when there was an error notifying the user about the policy. + * + * @param error + * (Error) Explains what went wrong. + */ + onUserNotifyFailed(error) { + this._log.error("onUserNotifyFailed - " + error); + }, +}); + +var TelemetryReportingPolicy = { + // The current policy version number. If the version number stored in the prefs + // is smaller than this, data upload will be disabled until the user is re-notified + // about the policy changes. + DEFAULT_DATAREPORTING_POLICY_VERSION: 1, + + /** + * Setup the policy. + */ + setup() { + return TelemetryReportingPolicyImpl.setup(); + }, + + /** + * Shutdown and clear the policy. + */ + shutdown() { + return TelemetryReportingPolicyImpl.shutdown(); + }, + + /** + * Check if we are allowed to upload data. In order to submit data both these conditions + * should be true: + * - The data submission preference should be true. + * - The datachoices infobar should have been displayed. + * + * @return {Boolean} True if we are allowed to upload data, false otherwise. + */ + canUpload() { + return TelemetryReportingPolicyImpl.canUpload(); + }, + + /** + * Check if this is the first time the browser ran. + */ + isFirstRun() { + return TelemetryReportingPolicyImpl.isFirstRun(); + }, + + /** + * Test only method, restarts the policy. + */ + reset() { + return TelemetryReportingPolicyImpl.reset(); + }, + + /** + * Test only method, used to check if user is notified of the policy in tests. + */ + testIsUserNotified() { + return TelemetryReportingPolicyImpl.isUserNotifiedOfCurrentPolicy; + }, + + /** + * Test only method, used to simulate the infobar being shown in xpcshell tests. + */ + testInfobarShown() { + return TelemetryReportingPolicyImpl._userNotified(); + }, + + /** + * Test only method, used to trigger an update of the "first run" state. + */ + testUpdateFirstRun() { + TelemetryReportingPolicyImpl._isFirstRun = undefined; + TelemetryReportingPolicyImpl.isFirstRun(); + }, +}; + +var TelemetryReportingPolicyImpl = { + _logger: null, + // Keep track of the notification status if user wasn't notified already. + _notificationInProgress: false, + // The timer used to show the datachoices notification at startup. + _startupNotificationTimerId: null, + // Keep track of the first session state, as the related preference + // is flipped right after the browser starts. + _isFirstRun: undefined, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + + return this._logger; + }, + + /** + * Get the date the policy was notified. + * @return {Object} A date object or null on errors. + */ + get dataSubmissionPolicyNotifiedDate() { + let prefString = Services.prefs.getStringPref( + TelemetryUtils.Preferences.AcceptedPolicyDate, + "0" + ); + let valueInteger = parseInt(prefString, 10); + + // Bail out if we didn't store any value yet. + if (valueInteger == 0) { + this._log.info( + "get dataSubmissionPolicyNotifiedDate - No date stored yet." + ); + return null; + } + + // If an invalid value is saved in the prefs, bail out too. + if (Number.isNaN(valueInteger)) { + this._log.error( + "get dataSubmissionPolicyNotifiedDate - Invalid date stored." + ); + return null; + } + + // Make sure the notification date is newer then the oldest allowed date. + let date = new Date(valueInteger); + if (date.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) { + this._log.error( + "get dataSubmissionPolicyNotifiedDate - The stored date is too old." + ); + return null; + } + + return date; + }, + + /** + * Set the date the policy was notified. + * @param {Object} aDate A valid date object. + */ + set dataSubmissionPolicyNotifiedDate(aDate) { + this._log.trace("set dataSubmissionPolicyNotifiedDate - aDate: " + aDate); + + if (!aDate || aDate.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) { + this._log.error( + "set dataSubmissionPolicyNotifiedDate - Invalid notification date." + ); + return; + } + + Services.prefs.setStringPref( + TelemetryUtils.Preferences.AcceptedPolicyDate, + aDate.getTime().toString() + ); + }, + + /** + * Whether submission of data is allowed. + * + * This is the master switch for remote server communication. If it is + * false, we never request upload or deletion. + */ + get dataSubmissionEnabled() { + // Default is true because we are opt-out. + return Services.prefs.getBoolPref( + TelemetryUtils.Preferences.DataSubmissionEnabled, + true + ); + }, + + get currentPolicyVersion() { + return Services.prefs.getIntPref( + TelemetryUtils.Preferences.CurrentPolicyVersion, + TelemetryReportingPolicy.DEFAULT_DATAREPORTING_POLICY_VERSION + ); + }, + + /** + * The minimum policy version which for dataSubmissionPolicyAccepted to + * to be valid. + */ + get minimumPolicyVersion() { + const minPolicyVersion = Services.prefs.getIntPref( + TelemetryUtils.Preferences.MinimumPolicyVersion, + 1 + ); + + // First check if the current channel has a specific minimum policy version. If not, + // use the general minimum policy version. + let channel = ""; + try { + channel = TelemetryUtils.getUpdateChannel(); + } catch (e) { + this._log.error( + "minimumPolicyVersion - Unable to retrieve the current channel." + ); + return minPolicyVersion; + } + const channelPref = + TelemetryUtils.Preferences.MinimumPolicyVersion + ".channel-" + channel; + return Services.prefs.getIntPref(channelPref, minPolicyVersion); + }, + + get dataSubmissionPolicyAcceptedVersion() { + return Services.prefs.getIntPref( + TelemetryUtils.Preferences.AcceptedPolicyVersion, + 0 + ); + }, + + set dataSubmissionPolicyAcceptedVersion(value) { + Services.prefs.setIntPref( + TelemetryUtils.Preferences.AcceptedPolicyVersion, + value + ); + }, + + /** + * Checks to see if the user has been notified about data submission + * @return {Bool} True if user has been notified and the notification is still valid, + * false otherwise. + */ + get isUserNotifiedOfCurrentPolicy() { + // If we don't have a sane notification date, the user was not notified yet. + if ( + !this.dataSubmissionPolicyNotifiedDate || + this.dataSubmissionPolicyNotifiedDate.getTime() <= 0 + ) { + return false; + } + + // The accepted policy version should not be less than the minimum policy version. + if (this.dataSubmissionPolicyAcceptedVersion < this.minimumPolicyVersion) { + return false; + } + + // Otherwise the user was already notified. + return true; + }, + + /** + * Test only method, restarts the policy. + */ + reset() { + this.shutdown(); + this._isFirstRun = undefined; + return this.setup(); + }, + + /** + * Setup the policy. + */ + setup() { + this._log.trace("setup"); + + // Migrate the data choices infobar, if needed. + this._migratePreferences(); + + // Add the event observers. + Services.obs.addObserver(this, "sessionstore-windows-restored"); + }, + + /** + * Clean up the reporting policy. + */ + shutdown() { + this._log.trace("shutdown"); + + this._detachObservers(); + + Policy.clearShowInfobarTimeout(this._startupNotificationTimerId); + }, + + /** + * Detach the observers that were attached during setup. + */ + _detachObservers() { + Services.obs.removeObserver(this, "sessionstore-windows-restored"); + }, + + /** + * Check if we are allowed to upload data. In order to submit data both these conditions + * should be true: + * - The data submission preference should be true. + * - The datachoices infobar should have been displayed. + * + * @return {Boolean} True if we are allowed to upload data, false otherwise. + */ + canUpload() { + // If data submission is disabled, there's no point in showing the infobar. Just + // forbid to upload. + if (!this.dataSubmissionEnabled) { + return false; + } + + // Submission is enabled. We enable upload if user is notified or we need to bypass + // the policy. + const bypassNotification = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.BypassNotification, + false + ); + return this.isUserNotifiedOfCurrentPolicy || bypassNotification; + }, + + isFirstRun() { + if (this._isFirstRun === undefined) { + this._isFirstRun = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.FirstRun, + true + ); + } + return this._isFirstRun; + }, + + /** + * Migrate the data policy preferences, if needed. + */ + _migratePreferences() { + // Current prefs are mostly the same than the old ones, except for some deprecated ones. + for (let pref of DEPRECATED_FHR_PREFS) { + Services.prefs.clearUserPref(pref); + } + }, + + /** + * Determine whether the user should be notified. + */ + _shouldNotify() { + if (!this.dataSubmissionEnabled) { + this._log.trace( + "_shouldNotify - Data submission disabled by the policy." + ); + return false; + } + + const bypassNotification = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.BypassNotification, + false + ); + if (this.isUserNotifiedOfCurrentPolicy || bypassNotification) { + this._log.trace( + "_shouldNotify - User already notified or bypassing the policy." + ); + return false; + } + + if (this._notificationInProgress) { + this._log.trace( + "_shouldNotify - User not notified, notification already in progress." + ); + return false; + } + + return true; + }, + + /** + * Show the data choices infobar if needed. + */ + _showInfobar() { + if (!this._shouldNotify()) { + return; + } + + this._log.trace("_showInfobar - User not notified, notifying now."); + this._notificationInProgress = true; + let request = new NotifyPolicyRequest(this._log); + Observers.notify("datareporting:notify-data-policy:request", request); + }, + + /** + * Called when the user is notified with the infobar or otherwise. + */ + _userNotified() { + this._log.trace("_userNotified"); + this._recordNotificationData(); + TelemetrySend.notifyCanUpload(); + }, + + /** + * Record date and the version of the accepted policy. + */ + _recordNotificationData() { + this._log.trace("_recordNotificationData"); + this.dataSubmissionPolicyNotifiedDate = Policy.now(); + this.dataSubmissionPolicyAcceptedVersion = this.currentPolicyVersion; + // The user was notified and the notification data saved: the notification + // is no longer in progress. + this._notificationInProgress = false; + }, + + /** + * Try to open the privacy policy in a background tab instead of showing the infobar. + */ + _openFirstRunPage() { + if (!this._shouldNotify()) { + return false; + } + + let firstRunPolicyURL = Services.prefs.getStringPref( + TelemetryUtils.Preferences.FirstRunURL, + "" + ); + if (!firstRunPolicyURL) { + return false; + } + firstRunPolicyURL = Services.urlFormatter.formatURL(firstRunPolicyURL); + + const { BrowserWindowTracker } = ChromeUtils.import( + "resource:///modules/BrowserWindowTracker.jsm" + ); + let win = BrowserWindowTracker.getTopWindow(); + + if (!win) { + this._log.info( + "Couldn't find browser window to open first-run page. Falling back to infobar." + ); + return false; + } + + // We'll consider the user notified once the privacy policy has been loaded + // in a background tab even if that tab hasn't been selected. + let tab; + let progressListener = {}; + progressListener.onStateChange = ( + aBrowser, + aWebProgress, + aRequest, + aStateFlags, + aStatus + ) => { + if ( + aWebProgress.isTopLevel && + tab && + tab.linkedBrowser == aBrowser && + aStateFlags & Ci.nsIWebProgressListener.STATE_STOP && + aStateFlags & Ci.nsIWebProgressListener.STATE_IS_NETWORK + ) { + let uri = aBrowser.documentURI; + if ( + uri && + !/^about:(blank|neterror|certerror|blocked)/.test(uri.spec) + ) { + this._userNotified(); + } else { + this._log.info( + "Failed to load first-run page. Falling back to infobar." + ); + this._showInfobar(); + } + removeListeners(); + } + }; + + let removeListeners = () => { + win.removeEventListener("unload", removeListeners); + win.gBrowser.removeTabsProgressListener(progressListener); + }; + + win.addEventListener("unload", removeListeners); + win.gBrowser.addTabsProgressListener(progressListener); + + tab = win.gBrowser.loadOneTab(firstRunPolicyURL, { + inBackground: true, + triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal(), + }); + return true; + }, + + observe(aSubject, aTopic, aData) { + if (aTopic != "sessionstore-windows-restored") { + return; + } + + if (this.isFirstRun()) { + // We're performing the first run, flip firstRun preference for subsequent runs. + Services.prefs.setBoolPref(TelemetryUtils.Preferences.FirstRun, false); + + try { + if (this._openFirstRunPage()) { + return; + } + } catch (e) { + this._log.error("Failed to open privacy policy tab: " + e); + } + } + + // Show the info bar. + const delay = this.isFirstRun() + ? NOTIFICATION_DELAY_FIRST_RUN_MSEC + : NOTIFICATION_DELAY_NEXT_RUNS_MSEC; + + this._startupNotificationTimerId = Policy.setShowInfobarTimeout( + // Calling |canUpload| eventually shows the infobar, if needed. + () => this._showInfobar(), + delay + ); + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryScheduler.jsm b/toolkit/components/telemetry/app/TelemetryScheduler.jsm new file mode 100644 index 0000000000..81f59083fa --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryScheduler.jsm @@ -0,0 +1,434 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryScheduler"]; + +const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm"); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); +const { TelemetrySession } = ChromeUtils.import( + "resource://gre/modules/TelemetrySession.jsm" +); +const { TelemetryUtils } = ChromeUtils.import( + "resource://gre/modules/TelemetryUtils.jsm" +); +const { XPCOMUtils } = ChromeUtils.import( + "resource://gre/modules/XPCOMUtils.jsm", + this +); +const { clearTimeout, setTimeout } = ChromeUtils.import( + "resource://gre/modules/Timer.jsm" +); +// Other pings +const { EcosystemTelemetry } = ChromeUtils.import( + "resource://gre/modules/EcosystemTelemetry.jsm" +); +const { TelemetryPrioPing } = ChromeUtils.import( + "resource://gre/modules/PrioPing.jsm" +); + +XPCOMUtils.defineLazyServiceGetters(this, { + idleService: ["@mozilla.org/widget/useridleservice;1", "nsIUserIdleService"], +}); + +const MIN_SUBSESSION_LENGTH_MS = + Services.prefs.getIntPref("toolkit.telemetry.minSubsessionLength", 5 * 60) * + 1000; + +const LOGGER_NAME = "Toolkit.Telemetry"; + +// Seconds of idle time before pinging. +// On idle-daily a gather-telemetry notification is fired, during it probes can +// start asynchronous tasks to gather data. +const IDLE_TIMEOUT_SECONDS = Services.prefs.getIntPref( + "toolkit.telemetry.idleTimeout", + 5 * 60 +); + +// Execute a scheduler tick every 5 minutes. +const SCHEDULER_TICK_INTERVAL_MS = + Services.prefs.getIntPref( + "toolkit.telemetry.scheduler.tickInterval", + 5 * 60 + ) * 1000; +// When user is idle, execute a scheduler tick every 60 minutes. +const SCHEDULER_TICK_IDLE_INTERVAL_MS = + Services.prefs.getIntPref( + "toolkit.telemetry.scheduler.idleTickInterval", + 60 * 60 + ) * 1000; + +// The maximum time (ms) until the tick should moved from the idle +// queue to the regular queue if it hasn't been executed yet. +const SCHEDULER_TICK_MAX_IDLE_DELAY_MS = 60 * 1000; + +// The frequency at which we persist session data to the disk to prevent data loss +// in case of aborted sessions (currently 5 minutes). +const ABORTED_SESSION_UPDATE_INTERVAL_MS = 5 * 60 * 1000; + +// The tolerance we have when checking if it's midnight (15 minutes). +const SCHEDULER_MIDNIGHT_TOLERANCE_MS = 15 * 60 * 1000; + +/** + * This is a policy object used to override behavior for testing. + */ +var Policy = { + now: () => new Date(), + setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs), + clearSchedulerTickTimeout: id => clearTimeout(id), + prioEncode: (batchID, prioParams) => PrioEncoder.encode(batchID, prioParams), +}; + +/** + * TelemetryScheduler contains a single timer driving all regularly-scheduled + * Telemetry related jobs. Having a single place with this logic simplifies + * reasoning about scheduling actions in a single place, making it easier to + * coordinate jobs and coalesce them. + */ +var TelemetryScheduler = { + // Tracks the main ping + _lastDailyPingTime: 0, + // Tracks the aborted session ping + _lastSessionCheckpointTime: 0, + // Tracks all other pings at regular intervals + _lastPeriodicPingTime: 0, + + _log: null, + + // The timer which drives the scheduler. + _schedulerTimer: null, + // The interval used by the scheduler timer. + _schedulerInterval: 0, + _shuttingDown: true, + _isUserIdle: false, + + /** + * Initialises the scheduler and schedules the first daily/aborted session pings. + */ + init() { + this._log = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + "TelemetryScheduler::" + ); + this._log.trace("init"); + this._shuttingDown = false; + this._isUserIdle = false; + + // Initialize the last daily ping and aborted session last due times to the current time. + // Otherwise, we might end up sending daily pings even if the subsession is not long enough. + let now = Policy.now(); + this._lastDailyPingTime = now.getTime(); + this._lastPeriodicPingTime = now.getTime(); + this._lastSessionCheckpointTime = now.getTime(); + this._rescheduleTimeout(); + + idleService.addIdleObserver(this, IDLE_TIMEOUT_SECONDS); + Services.obs.addObserver(this, "wake_notification"); + }, + + /** + * Stops the scheduler. + */ + shutdown() { + if (this._shuttingDown) { + if (this._log) { + this._log.error("shutdown - Already shut down"); + } else { + Cu.reportError("TelemetryScheduler.shutdown - Already shut down"); + } + return; + } + + this._log.trace("shutdown"); + if (this._schedulerTimer) { + Policy.clearSchedulerTickTimeout(this._schedulerTimer); + this._schedulerTimer = null; + } + + idleService.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS); + Services.obs.removeObserver(this, "wake_notification"); + + this._shuttingDown = true; + }, + + _clearTimeout() { + if (this._schedulerTimer) { + Policy.clearSchedulerTickTimeout(this._schedulerTimer); + } + }, + + /** + * Reschedules the tick timer. + */ + _rescheduleTimeout() { + this._log.trace("_rescheduleTimeout - isUserIdle: " + this._isUserIdle); + if (this._shuttingDown) { + this._log.warn("_rescheduleTimeout - already shutdown"); + return; + } + + this._clearTimeout(); + + const now = Policy.now(); + let timeout = SCHEDULER_TICK_INTERVAL_MS; + + // When the user is idle we want to fire the timer less often. + if (this._isUserIdle) { + timeout = SCHEDULER_TICK_IDLE_INTERVAL_MS; + // We need to make sure though that we don't miss sending pings around + // midnight when we use the longer idle intervals. + const nextMidnight = TelemetryUtils.getNextMidnight(now); + timeout = Math.min(timeout, nextMidnight.getTime() - now.getTime()); + } + + this._log.trace( + "_rescheduleTimeout - scheduling next tick for " + + new Date(now.getTime() + timeout) + ); + this._schedulerTimer = Policy.setSchedulerTickTimeout( + () => this._onSchedulerTick(), + timeout + ); + }, + + _sentPingToday(pingTime, nowDate) { + // This is today's date and also the previous midnight (0:00). + const todayDate = TelemetryUtils.truncateToDays(nowDate); + // We consider a ping sent for today if it occured after or at 00:00 today. + return pingTime >= todayDate.getTime(); + }, + + /** + * Checks if we can send a daily ping or not. + * @param {Object} nowDate A date object. + * @return {Boolean} True if we can send the daily ping, false otherwise. + */ + _isDailyPingDue(nowDate) { + // The daily ping is not due if we already sent one today. + if (this._sentPingToday(this._lastDailyPingTime, nowDate)) { + this._log.trace("_isDailyPingDue - already sent one today"); + return false; + } + + // Avoid overly short sessions. + const timeSinceLastDaily = nowDate.getTime() - this._lastDailyPingTime; + if (timeSinceLastDaily < MIN_SUBSESSION_LENGTH_MS) { + this._log.trace( + "_isDailyPingDue - delaying daily to keep minimum session length" + ); + return false; + } + + this._log.trace("_isDailyPingDue - is due"); + return true; + }, + + /** + * Checks if we can send a regular ping or not. + * @param {Object} nowDate A date object. + * @return {Boolean} True if we can send the regular pings, false otherwise. + */ + _isPeriodicPingDue(nowDate) { + // The periodic ping is not due if we already sent one today. + if (this._sentPingToday(this._lastPeriodicPingTime, nowDate)) { + this._log.trace("_isPeriodicPingDue - already sent one today"); + return false; + } + + this._log.trace("_isPeriodicPingDue - is due"); + return true; + }, + + /** + * An helper function to save an aborted-session ping. + * @param {Number} now The current time, in milliseconds. + * @param {Object} [competingPayload=null] If we are coalescing the daily and the + * aborted-session pings, this is the payload for the former. Note + * that the reason field of this payload will be changed. + * @return {Promise} A promise resolved when the ping is saved. + */ + _saveAbortedPing(now, competingPayload = null) { + this._lastSessionCheckpointTime = now; + return TelemetrySession.saveAbortedSessionPing(competingPayload).catch(e => + this._log.error("_saveAbortedPing - Failed", e) + ); + }, + + /** + * The notifications handler. + */ + observe(aSubject, aTopic, aData) { + this._log.trace("observe - aTopic: " + aTopic); + switch (aTopic) { + case "idle": + // If the user is idle, increase the tick interval. + this._isUserIdle = true; + return this._onSchedulerTick(); + case "active": + // User is back to work, restore the original tick interval. + this._isUserIdle = false; + return this._onSchedulerTick(true); + case "wake_notification": + // The machine woke up from sleep, trigger a tick to avoid sessions + // spanning more than a day. + // This is needed because sleep time does not count towards timeouts + // on Mac & Linux - see bug 1262386, bug 1204823 et al. + return this._onSchedulerTick(true); + } + return undefined; + }, + + /** + * Creates an object with a method `dispatch` that will call `dispatchFn` unless + * the method `cancel` is called beforehand. + * + * This is used to wrap main thread idle dispatch since it does not provide a + * cancel mechanism. + */ + _makeIdleDispatch(dispatchFn) { + this._log.trace("_makeIdleDispatch"); + let fn = dispatchFn; + let l = msg => this._log.trace(msg); // need to bind `this` + return { + cancel() { + fn = undefined; + }, + dispatch(resolve, reject) { + l("_makeIdleDispatch.dispatch - !!fn: " + !!fn); + if (!fn) { + return Promise.resolve().then(resolve, reject); + } + return fn(resolve, reject); + }, + }; + }, + + /** + * Performs a scheduler tick. This function manages Telemetry recurring operations. + * @param {Boolean} [dispatchOnIdle=false] If true, the tick is dispatched in the + * next idle cycle of the main thread. + * @return {Promise} A promise, only used when testing, resolved when the scheduled + * operation completes. + */ + _onSchedulerTick(dispatchOnIdle = false) { + this._log.trace("_onSchedulerTick - dispatchOnIdle: " + dispatchOnIdle); + // This call might not be triggered from a timeout. In that case we don't want to + // leave any previously scheduled timeouts pending. + this._clearTimeout(); + + if (this._idleDispatch) { + this._idleDispatch.cancel(); + } + + if (this._shuttingDown) { + this._log.warn("_onSchedulerTick - already shutdown."); + return Promise.reject(new Error("Already shutdown.")); + } + + let promise = Promise.resolve(); + try { + if (dispatchOnIdle) { + this._idleDispatch = this._makeIdleDispatch((resolve, reject) => { + this._log.trace( + "_onSchedulerTick - ildeDispatchToMainThread dispatch" + ); + return this._schedulerTickLogic().then(resolve, reject); + }); + promise = new Promise((resolve, reject) => + Services.tm.idleDispatchToMainThread(() => { + return this._idleDispatch + ? this._idleDispatch.dispatch(resolve, reject) + : Promise.resolve().then(resolve, reject); + }, SCHEDULER_TICK_MAX_IDLE_DELAY_MS) + ); + } else { + promise = this._schedulerTickLogic(); + } + } catch (e) { + this._log.error("_onSchedulerTick - There was an exception", e); + } finally { + this._rescheduleTimeout(); + } + + // This promise is returned to make testing easier. + return promise; + }, + + /** + * Implements the scheduler logic. + * @return {Promise} Resolved when the scheduled task completes. Only used in tests. + */ + _schedulerTickLogic() { + this._log.trace("_schedulerTickLogic"); + + let nowDate = Policy.now(); + let now = nowDate.getTime(); + + // Check if the daily ping is due. + const shouldSendDaily = this._isDailyPingDue(nowDate); + // Check if other regular pings are due. + const shouldSendPeriodic = this._isPeriodicPingDue(nowDate); + + if (shouldSendPeriodic) { + this._log.trace("_schedulerTickLogic - Periodic ping due."); + this._lastPeriodicPingTime = now; + // Send other pings. + EcosystemTelemetry.periodicPing(); + TelemetryPrioPing.periodicPing(); + } + + if (shouldSendDaily) { + this._log.trace("_schedulerTickLogic - Daily ping due."); + this._lastDailyPingTime = now; + return TelemetrySession.sendDailyPing(); + } + + // Check if the aborted-session ping is due. If a daily ping was saved above, it was + // already duplicated as an aborted-session ping. + const isAbortedPingDue = + now - this._lastSessionCheckpointTime >= + ABORTED_SESSION_UPDATE_INTERVAL_MS; + if (isAbortedPingDue) { + this._log.trace("_schedulerTickLogic - Aborted session ping due."); + return this._saveAbortedPing(now); + } + + // No ping is due. + this._log.trace("_schedulerTickLogic - No ping due."); + return Promise.resolve(); + }, + + /** + * Re-schedule the daily ping if some other equivalent ping was sent. + * + * This is only called from TelemetrySession when a main ping with reason 'environment-change' + * is sent. + * + * @param {Object} [payload] The payload of the ping that was sent, + * to be stored as an aborted-session ping. + */ + rescheduleDailyPing(payload) { + if (this._shuttingDown) { + this._log.error("rescheduleDailyPing - already shutdown"); + return; + } + + this._log.trace("rescheduleDailyPing"); + let now = Policy.now(); + + // We just generated an environment-changed ping, save it as an aborted session and + // update the schedules. + this._saveAbortedPing(now.getTime(), payload); + + // If we're close to midnight, skip today's daily ping and reschedule it for tomorrow. + let nearestMidnight = TelemetryUtils.getNearestMidnight( + now, + SCHEDULER_MIDNIGHT_TOLERANCE_MS + ); + if (nearestMidnight) { + this._lastDailyPingTime = now.getTime(); + } + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetrySend.jsm b/toolkit/components/telemetry/app/TelemetrySend.jsm new file mode 100644 index 0000000000..caf11f4406 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetrySend.jsm @@ -0,0 +1,1600 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This module is responsible for uploading pings to the server and persisting + * pings that can't be send now. + * Those pending pings are persisted on disk and sent at the next opportunity, + * newest first. + */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetrySend"]; + +ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this); +ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this); +const { ClientID } = ChromeUtils.import("resource://gre/modules/ClientID.jsm"); +ChromeUtils.import("resource://gre/modules/Log.jsm", this); +const { PromiseUtils } = ChromeUtils.import( + "resource://gre/modules/PromiseUtils.jsm" +); +ChromeUtils.import("resource://gre/modules/ServiceRequest.jsm", this); +ChromeUtils.import("resource://gre/modules/Services.jsm", this); +ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this); +ChromeUtils.import("resource://gre/modules/Timer.jsm", this); + +ChromeUtils.defineModuleGetter( + this, + "TelemetryStorage", + "resource://gre/modules/TelemetryStorage.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "TelemetryReportingPolicy", + "resource://gre/modules/TelemetryReportingPolicy.jsm" +); +ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm"); +XPCOMUtils.defineLazyServiceGetter( + this, + "Telemetry", + "@mozilla.org/base/telemetry;1", + "nsITelemetry" +); +ChromeUtils.defineModuleGetter( + this, + "TelemetryHealthPing", + "resource://gre/modules/HealthPing.jsm" +); + +const Utils = TelemetryUtils; + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetrySend::"; + +const TOPIC_IDLE_DAILY = "idle-daily"; +// The following topics are notified when Firefox is closing +// because the OS is shutting down. +const TOPIC_QUIT_APPLICATION_GRANTED = "quit-application-granted"; +const TOPIC_QUIT_APPLICATION_FORCED = "quit-application-forced"; +const PREF_CHANGED_TOPIC = "nsPref:changed"; +const TOPIC_PROFILE_CHANGE_NET_TEARDOWN = "profile-change-net-teardown"; + +// Whether the FHR/Telemetry unification features are enabled. +// Changing this pref requires a restart. +const IS_UNIFIED_TELEMETRY = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.Unified, + false +); + +const MS_IN_A_MINUTE = 60 * 1000; + +const PING_TYPE_DELETION_REQUEST = "deletion-request"; + +// We try to spread "midnight" pings out over this interval. +const MIDNIGHT_FUZZING_INTERVAL_MS = 60 * MS_IN_A_MINUTE; +// We delay sending "midnight" pings on this client by this interval. +const MIDNIGHT_FUZZING_DELAY_MS = Math.random() * MIDNIGHT_FUZZING_INTERVAL_MS; + +// Timeout after which we consider a ping submission failed. +const PING_SUBMIT_TIMEOUT_MS = 1.5 * MS_IN_A_MINUTE; + +// To keep resource usage in check, we limit ping sending to a maximum number +// of pings per minute. +const MAX_PING_SENDS_PER_MINUTE = 10; + +// If we have more pending pings then we can send right now, we schedule the next +// send for after SEND_TICK_DELAY. +const SEND_TICK_DELAY = 1 * MS_IN_A_MINUTE; +// If we had any ping send failures since the last ping, we use a backoff timeout +// for the next ping sends. We increase the delay exponentially up to a limit of +// SEND_MAXIMUM_BACKOFF_DELAY_MS. +// This exponential backoff will be reset by external ping submissions & idle-daily. +const SEND_MAXIMUM_BACKOFF_DELAY_MS = 120 * MS_IN_A_MINUTE; + +// Strings to map from XHR.errorCode to TELEMETRY_SEND_FAILURE_TYPE. +// Echoes XMLHttpRequestMainThread's ErrorType enum. +// Make sure that any additions done to XHR_ERROR_TYPE enum are also mirrored in +// TELEMETRY_SEND_FAILURE_TYPE and TELEMETRY_SEND_FAILURE_TYPE_PER_PING's labels. +const XHR_ERROR_TYPE = [ + "eOK", + "eRequest", + "eUnreachable", + "eChannelOpen", + "eRedirect", + "eTerminated", +]; + +/** + * This is a policy object used to override behavior within this module. + * Tests override properties on this object to allow for control of behavior + * that would otherwise be very hard to cover. + */ +var Policy = { + now: () => new Date(), + midnightPingFuzzingDelay: () => MIDNIGHT_FUZZING_DELAY_MS, + pingSubmissionTimeout: () => PING_SUBMIT_TIMEOUT_MS, + setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs), + clearSchedulerTickTimeout: id => clearTimeout(id), + gzipCompressString: data => gzipCompressString(data), +}; + +/** + * Determine if the ping has the new v4 ping format or the legacy v2 one or earlier. + */ +function isV4PingFormat(aPing) { + return ( + "id" in aPing && + "application" in aPing && + "version" in aPing && + aPing.version >= 2 + ); +} + +/** + * Check if the provided ping is a deletion-request ping. + * @param {Object} aPing The ping to check. + * @return {Boolean} True if the ping is a deletion-request ping, false otherwise. + */ +function isDeletionRequestPing(aPing) { + return isV4PingFormat(aPing) && aPing.type == PING_TYPE_DELETION_REQUEST; +} + +/** + * Save the provided ping as a pending ping. + * @param {Object} aPing The ping to save. + * @return {Promise} A promise resolved when the ping is saved. + */ +function savePing(aPing) { + return TelemetryStorage.savePendingPing(aPing); +} + +/** + * @return {String} This returns a string with the gzip compressed data. + */ +function gzipCompressString(string) { + let observer = { + buffer: "", + onStreamComplete(loader, context, status, length, result) { + // String.fromCharCode can only deal with 500,000 characters at + // a time, so chunk the result into parts of that size. + const chunkSize = 500000; + for (let offset = 0; offset < result.length; offset += chunkSize) { + this.buffer += String.fromCharCode.apply( + String, + result.slice(offset, offset + chunkSize) + ); + } + }, + }; + + let scs = Cc["@mozilla.org/streamConverters;1"].getService( + Ci.nsIStreamConverterService + ); + let listener = Cc["@mozilla.org/network/stream-loader;1"].createInstance( + Ci.nsIStreamLoader + ); + listener.init(observer); + let converter = scs.asyncConvertData("uncompressed", "gzip", listener, null); + let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance( + Ci.nsIStringInputStream + ); + stringStream.data = string; + converter.onStartRequest(null, null); + converter.onDataAvailable(null, stringStream, 0, string.length); + converter.onStopRequest(null, null, null); + return observer.buffer; +} + +var TelemetrySend = { + get pendingPingCount() { + return TelemetrySendImpl.pendingPingCount; + }, + + /** + * Partial setup that runs immediately at startup. This currently triggers + * the crash report annotations. + */ + earlyInit() { + TelemetrySendImpl.earlyInit(); + }, + + /** + * Initializes this module. + * + * @param {Boolean} testing Whether this is run in a test. This changes some behavior + * to enable proper testing. + * @return {Promise} Resolved when setup is finished. + */ + setup(testing = false) { + return TelemetrySendImpl.setup(testing); + }, + + /** + * Shutdown this module - this will cancel any pending ping tasks and wait for + * outstanding async activity like network and disk I/O. + * + * @return {Promise} Promise that is resolved when shutdown is finished. + */ + shutdown() { + return TelemetrySendImpl.shutdown(); + }, + + /** + * Flushes all pings to pingsender that were both + * 1. submitted after profile-change-net-teardown, and + * 2. wanting to be sent using pingsender. + */ + flushPingSenderBatch() { + TelemetrySendImpl.flushPingSenderBatch(); + }, + + /** + * Submit a ping for sending. This will: + * - send the ping right away if possible or + * - save the ping to disk and send it at the next opportunity + * + * @param {Object} ping The ping data to send, must be serializable to JSON. + * @param {Object} [aOptions] Options object. + * @param {Boolean} [options.usePingSender=false] if true, send the ping using the PingSender. + * @return {Promise} Test-only - a promise that is resolved when the ping is sent or saved. + */ + submitPing(ping, options = {}) { + options.usePingSender = options.usePingSender || false; + return TelemetrySendImpl.submitPing(ping, options); + }, + + /** + * Check if sending is disabled. If Telemetry is not allowed to upload, + * pings are not sent to the server. + * If trying to send a deletion-request ping, don't block it. + * + * @param {Object} [ping=null] A ping to be checked. + * @return {Boolean} True if pings can be send to the servers, false otherwise. + */ + sendingEnabled(ping = null) { + return TelemetrySendImpl.sendingEnabled(ping); + }, + + /** + * Notify that we can start submitting data to the servers. + */ + notifyCanUpload() { + return TelemetrySendImpl.notifyCanUpload(); + }, + + /** + * Only used in tests. Used to reset the module data to emulate a restart. + */ + reset() { + return TelemetrySendImpl.reset(); + }, + + /** + * Only used in tests. + */ + setServer(server) { + return TelemetrySendImpl.setServer(server); + }, + + /** + * Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests. + */ + clearCurrentPings() { + return TelemetrySendImpl.clearCurrentPings(); + }, + + /** + * Only used in tests to wait on outgoing pending pings. + */ + testWaitOnOutgoingPings() { + return TelemetrySendImpl.promisePendingPingActivity(); + }, + + /** + * Only used in tests to set whether it is too late in shutdown to send pings. + */ + testTooLateToSend(tooLate) { + TelemetrySendImpl._tooLateToSend = tooLate; + }, + + /** + * Test-only - this allows overriding behavior to enable ping sending in debug builds. + */ + setTestModeEnabled(testing) { + TelemetrySendImpl.setTestModeEnabled(testing); + }, + + /** + * This returns state info for this module for AsyncShutdown timeout diagnostics. + */ + getShutdownState() { + return TelemetrySendImpl.getShutdownState(); + }, + + /** + * Send a ping using the ping sender. + * This method will not wait for the ping to be sent, instead it will return + * as soon as the pingsender program has been launched. + * + * This method is currently exposed here only for testing purposes as it's + * only used internally. + * + * @param {Array}<Object> pings An array of objects holding url / path pairs + * for each ping to be sent. The URL represent the telemetry server the + * ping will be sent to and the path points to the ping data. The ping + * data files will be deleted if the pings have been submitted + * successfully. + * @param {callback} observer A function called with parameters + * (subject, topic, data) and a topic of "process-finished" or + * "process-failed" after pingsender completion. + * + * @throws NS_ERROR_FAILURE if we couldn't find or run the pingsender + * executable. + * @throws NS_ERROR_NOT_IMPLEMENTED on Android as the pingsender is not + * available. + */ + testRunPingSender(pings, observer) { + return TelemetrySendImpl.runPingSender(pings, observer); + }, +}; + +var CancellableTimeout = { + _deferred: null, + _timer: null, + + /** + * This waits until either the given timeout passed or the timeout was cancelled. + * + * @param {Number} timeoutMs The timeout in ms. + * @return {Promise<bool>} Promise that is resolved with false if the timeout was cancelled, + * false otherwise. + */ + promiseWaitOnTimeout(timeoutMs) { + if (!this._deferred) { + this._deferred = PromiseUtils.defer(); + this._timer = Policy.setSchedulerTickTimeout( + () => this._onTimeout(), + timeoutMs + ); + } + + return this._deferred.promise; + }, + + _onTimeout() { + if (this._deferred) { + this._deferred.resolve(false); + this._timer = null; + this._deferred = null; + } + }, + + cancelTimeout() { + if (this._deferred) { + Policy.clearSchedulerTickTimeout(this._timer); + this._deferred.resolve(true); + this._timer = null; + this._deferred = null; + } + }, +}; + +/** + * SendScheduler implements the timer & scheduling behavior for ping sends. + */ +var SendScheduler = { + // Whether any ping sends failed since the last tick. If yes, we start with our exponential + // backoff timeout. + _sendsFailed: false, + // The current retry delay after ping send failures. We use this for the exponential backoff, + // increasing this value everytime we had send failures since the last tick. + _backoffDelay: SEND_TICK_DELAY, + _shutdown: false, + _sendTask: null, + // A string that tracks the last seen send task state, null if it never ran. + _sendTaskState: null, + + _logger: null, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + "Scheduler::" + ); + } + + return this._logger; + }, + + shutdown() { + this._log.trace("shutdown"); + this._shutdown = true; + CancellableTimeout.cancelTimeout(); + return Promise.resolve(this._sendTask); + }, + + start() { + this._log.trace("start"); + this._sendsFailed = false; + this._backoffDelay = SEND_TICK_DELAY; + this._shutdown = false; + }, + + /** + * Only used for testing, resets the state to emulate a restart. + */ + reset() { + this._log.trace("reset"); + return this.shutdown().then(() => this.start()); + }, + + /** + * Notify the scheduler of a failure in sending out pings that warrants retrying. + * This will trigger the exponential backoff timer behavior on the next tick. + */ + notifySendsFailed() { + this._log.trace("notifySendsFailed"); + if (this._sendsFailed) { + return; + } + + this._sendsFailed = true; + this._log.trace("notifySendsFailed - had send failures"); + }, + + /** + * Returns whether ping submissions are currently throttled. + */ + isThrottled() { + const now = Policy.now(); + const nextPingSendTime = this._getNextPingSendTime(now); + return nextPingSendTime > now.getTime(); + }, + + waitOnSendTask() { + return Promise.resolve(this._sendTask); + }, + + triggerSendingPings(immediately) { + this._log.trace( + "triggerSendingPings - active send task: " + + !!this._sendTask + + ", immediately: " + + immediately + ); + + if (!this._sendTask) { + this._sendTask = this._doSendTask(); + let clear = () => (this._sendTask = null); + this._sendTask.then(clear, clear); + } else if (immediately) { + CancellableTimeout.cancelTimeout(); + } + + return this._sendTask; + }, + + async _doSendTask() { + this._sendTaskState = "send task started"; + this._backoffDelay = SEND_TICK_DELAY; + this._sendsFailed = false; + + const resetBackoffTimer = () => { + this._backoffDelay = SEND_TICK_DELAY; + }; + + for (;;) { + this._log.trace("_doSendTask iteration"); + this._sendTaskState = "start iteration"; + + if (this._shutdown) { + this._log.trace("_doSendTask - shutting down, bailing out"); + this._sendTaskState = "bail out - shutdown check"; + return; + } + + // Get a list of pending pings, sorted by last modified, descending. + // Filter out all the pings we can't send now. This addresses scenarios like "deletion-request" pings + // which can be sent even when upload is disabled. + let pending = TelemetryStorage.getPendingPingList(); + let current = TelemetrySendImpl.getUnpersistedPings(); + this._log.trace( + "_doSendTask - pending: " + + pending.length + + ", current: " + + current.length + ); + // Note that the two lists contain different kind of data. |pending| only holds ping + // info, while |current| holds actual ping data. + if (!TelemetrySendImpl.sendingEnabled()) { + // If sending is disabled, only handle deletion-request pings + pending = []; + current = current.filter(p => isDeletionRequestPing(p)); + } + this._log.trace( + "_doSendTask - can send - pending: " + + pending.length + + ", current: " + + current.length + ); + + // Bail out if there is nothing to send. + if (!pending.length && !current.length) { + this._log.trace("_doSendTask - no pending pings, bailing out"); + this._sendTaskState = "bail out - no pings to send"; + return; + } + + // If we are currently throttled (e.g. fuzzing to avoid midnight spikes), wait for the next send window. + const now = Policy.now(); + if (this.isThrottled()) { + const nextPingSendTime = this._getNextPingSendTime(now); + this._log.trace( + "_doSendTask - throttled, delaying ping send to " + + new Date(nextPingSendTime) + ); + this._sendTaskState = "wait for throttling to pass"; + + const delay = nextPingSendTime - now.getTime(); + const cancelled = await CancellableTimeout.promiseWaitOnTimeout(delay); + if (cancelled) { + this._log.trace( + "_doSendTask - throttling wait was cancelled, resetting backoff timer" + ); + resetBackoffTimer(); + } + + continue; + } + + let sending = pending.slice(0, MAX_PING_SENDS_PER_MINUTE); + pending = pending.slice(MAX_PING_SENDS_PER_MINUTE); + this._log.trace( + "_doSendTask - triggering sending of " + + sending.length + + " pings now" + + ", " + + pending.length + + " pings waiting" + ); + + this._sendsFailed = false; + const sendStartTime = Policy.now(); + this._sendTaskState = "wait on ping sends"; + await TelemetrySendImpl.sendPings( + current, + sending.map(p => p.id) + ); + if (this._shutdown || TelemetrySend.pendingPingCount == 0) { + this._log.trace( + "_doSendTask - bailing out after sending, shutdown: " + + this._shutdown + + ", pendingPingCount: " + + TelemetrySend.pendingPingCount + ); + this._sendTaskState = "bail out - shutdown & pending check after send"; + return; + } + + // Calculate the delay before sending the next batch of pings. + // We start with a delay that makes us send max. 1 batch per minute. + // If we had send failures in the last batch, we will override this with + // a backoff delay. + const timeSinceLastSend = Policy.now() - sendStartTime; + let nextSendDelay = Math.max(0, SEND_TICK_DELAY - timeSinceLastSend); + + if (!this._sendsFailed) { + this._log.trace( + "_doSendTask - had no send failures, resetting backoff timer" + ); + resetBackoffTimer(); + } else { + const newDelay = Math.min( + SEND_MAXIMUM_BACKOFF_DELAY_MS, + this._backoffDelay * 2 + ); + this._log.trace( + "_doSendTask - had send failures, backing off -" + + " old timeout: " + + this._backoffDelay + + ", new timeout: " + + newDelay + ); + this._backoffDelay = newDelay; + nextSendDelay = this._backoffDelay; + } + + this._log.trace( + "_doSendTask - waiting for next send opportunity, timeout is " + + nextSendDelay + ); + this._sendTaskState = "wait on next send opportunity"; + const cancelled = await CancellableTimeout.promiseWaitOnTimeout( + nextSendDelay + ); + if (cancelled) { + this._log.trace( + "_doSendTask - batch send wait was cancelled, resetting backoff timer" + ); + resetBackoffTimer(); + } + } + }, + + /** + * This helper calculates the next time that we can send pings at. + * Currently this mostly redistributes ping sends from midnight until one hour after + * to avoid submission spikes around local midnight for daily pings. + * + * @param now Date The current time. + * @return Number The next time (ms from UNIX epoch) when we can send pings. + */ + _getNextPingSendTime(now) { + // 1. First we check if the pref is set to skip any delay and send immediately. + // 2. Next we check if the time is between 0am and 1am. If it's not, we send + // immediately. + // 3. If we confirmed the time is indeed between 0am and 1am in step 1, we disallow + // sending before (midnight + fuzzing delay), which is a random time between 0am-1am + // (decided at startup). + + let disableFuzzingDelay = Services.prefs.getBoolPref( + TelemetryUtils.Preferences.DisableFuzzingDelay, + false + ); + if (disableFuzzingDelay) { + return now.getTime(); + } + + const midnight = Utils.truncateToDays(now); + // Don't delay pings if we are not within the fuzzing interval. + if (now.getTime() - midnight.getTime() > MIDNIGHT_FUZZING_INTERVAL_MS) { + return now.getTime(); + } + + // Delay ping send if we are within the midnight fuzzing range. + // We spread those ping sends out between |midnight| and |midnight + midnightPingFuzzingDelay|. + return midnight.getTime() + Policy.midnightPingFuzzingDelay(); + }, + + getShutdownState() { + return { + shutdown: this._shutdown, + hasSendTask: !!this._sendTask, + sendsFailed: this._sendsFailed, + sendTaskState: this._sendTaskState, + backoffDelay: this._backoffDelay, + }; + }, +}; + +var TelemetrySendImpl = { + _sendingEnabled: false, + // Tracks the shutdown state. + _shutdown: false, + _logger: null, + // This tracks all pending ping requests to the server. + _pendingPingRequests: new Map(), + // This tracks all the pending async ping activity. + _pendingPingActivity: new Set(), + // This is true when running in the test infrastructure. + _testMode: false, + // This holds pings that we currently try and haven't persisted yet. + _currentPings: new Map(), + // Used to skip spawning the pingsender if OS is shutting down. + _isOSShutdown: false, + // Has the network shut down, making it too late to send pings? + _tooLateToSend: false, + // Array of {url, path} awaiting flushPingSenderBatch(). + _pingSenderBatch: [], + + OBSERVER_TOPICS: [ + TOPIC_IDLE_DAILY, + TOPIC_QUIT_APPLICATION_GRANTED, + TOPIC_QUIT_APPLICATION_FORCED, + TOPIC_PROFILE_CHANGE_NET_TEARDOWN, + ], + + OBSERVED_PREFERENCES: [ + TelemetryUtils.Preferences.TelemetryEnabled, + TelemetryUtils.Preferences.FhrUploadEnabled, + ], + + // Whether sending pings has been overridden. + get _overrideOfficialCheck() { + return Services.prefs.getBoolPref( + TelemetryUtils.Preferences.OverrideOfficialCheck, + false + ); + }, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + + return this._logger; + }, + + get pendingPingRequests() { + return this._pendingPingRequests; + }, + + get pendingPingCount() { + return ( + TelemetryStorage.getPendingPingList().length + this._currentPings.size + ); + }, + + setTestModeEnabled(testing) { + this._testMode = testing; + }, + + earlyInit() { + this._annotateCrashReport(); + + // Install the observer to detect OS shutdown early enough, so + // that we catch this before the delayed setup happens. + Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_FORCED); + Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_GRANTED); + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + async setup(testing) { + this._log.trace("setup"); + + this._testMode = testing; + + Services.obs.addObserver(this, TOPIC_IDLE_DAILY); + Services.obs.addObserver(this, TOPIC_PROFILE_CHANGE_NET_TEARDOWN); + + this._server = Services.prefs.getStringPref( + TelemetryUtils.Preferences.Server, + undefined + ); + this._sendingEnabled = true; + + // Annotate crash reports so that crash pings are sent correctly and listen + // to pref changes to adjust the annotations accordingly. + for (let pref of this.OBSERVED_PREFERENCES) { + Services.prefs.addObserver(pref, this, true); + } + this._annotateCrashReport(); + + // Check the pending pings on disk now. + try { + await this._checkPendingPings(); + } catch (ex) { + this._log.error("setup - _checkPendingPings rejected", ex); + } + + // Enforce the pending pings storage quota. It could take a while so don't + // block on it. + TelemetryStorage.runEnforcePendingPingsQuotaTask(); + + // Start sending pings, but don't block on this. + SendScheduler.triggerSendingPings(true); + }, + + /** + * Triggers the crash report annotations depending on the current + * configuration. This communicates to the crash reporter if it can send a + * crash ping or not. This method can be called safely before setup() has + * been called. + */ + _annotateCrashReport() { + try { + const cr = Cc["@mozilla.org/toolkit/crash-reporter;1"]; + if (cr) { + const crs = cr.getService(Ci.nsICrashReporter); + + let clientId = ClientID.getCachedClientID(); + let server = + this._server || + Services.prefs.getStringPref( + TelemetryUtils.Preferences.Server, + undefined + ); + + if (!this.sendingEnabled() || !TelemetryReportingPolicy.canUpload()) { + // If we cannot send pings then clear the crash annotations + crs.removeCrashReportAnnotation("TelemetryClientId"); + crs.removeCrashReportAnnotation("TelemetryServerURL"); + } else { + crs.annotateCrashReport("TelemetryClientId", clientId); + crs.annotateCrashReport("TelemetryServerURL", server); + } + } + } catch (e) { + // Ignore errors when crash reporting is disabled + } + }, + + /** + * Discard old pings from the pending pings and detect overdue ones. + * @return {Boolean} True if we have overdue pings, false otherwise. + */ + async _checkPendingPings() { + // Scan the pending pings - that gives us a list sorted by last modified, descending. + let infos = await TelemetryStorage.loadPendingPingList(); + this._log.info("_checkPendingPings - pending ping count: " + infos.length); + if (!infos.length) { + this._log.trace("_checkPendingPings - no pending pings"); + return; + } + + const now = Policy.now(); + + // Submit the age of the pending pings. + for (let pingInfo of infos) { + const ageInDays = Utils.millisecondsToDays( + Math.abs(now.getTime() - pingInfo.lastModificationDate) + ); + Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_AGE").add(ageInDays); + } + }, + + async shutdown() { + this._shutdown = true; + + for (let pref of this.OBSERVED_PREFERENCES) { + // FIXME: When running tests this causes errors to be printed out if + // TelemetrySend.shutdown() is called twice in a row without calling + // TelemetrySend.setup() in-between. + Services.prefs.removeObserver(pref, this); + } + + for (let topic of this.OBSERVER_TOPICS) { + try { + Services.obs.removeObserver(this, topic); + } catch (ex) { + this._log.error( + "shutdown - failed to remove observer for " + topic, + ex + ); + } + } + + // We can't send anymore now. + this._sendingEnabled = false; + + // Cancel any outgoing requests. + await this._cancelOutgoingRequests(); + + // Stop any active send tasks. + await SendScheduler.shutdown(); + + // Wait for any outstanding async ping activity. + await this.promisePendingPingActivity(); + + // Save any outstanding pending pings to disk. + await this._persistCurrentPings(); + }, + + flushPingSenderBatch() { + if (this._pingSenderBatch.length === 0) { + return; + } + this._log.trace( + `flushPingSenderBatch - Sending ${this._pingSenderBatch.length} pings.` + ); + this.runPingSender(this._pingSenderBatch); + }, + + reset() { + this._log.trace("reset"); + + this._shutdown = false; + this._currentPings = new Map(); + this._tooLateToSend = false; + this._isOSShutdown = false; + this._sendingEnabled = true; + + const histograms = [ + "TELEMETRY_SUCCESS", + "TELEMETRY_SEND_SUCCESS", + "TELEMETRY_SEND_FAILURE", + "TELEMETRY_SEND_FAILURE_TYPE", + ]; + + histograms.forEach(h => Telemetry.getHistogramById(h).clear()); + + const keyedHistograms = ["TELEMETRY_SEND_FAILURE_TYPE_PER_PING"]; + + keyedHistograms.forEach(h => Telemetry.getKeyedHistogramById(h).clear()); + + return SendScheduler.reset(); + }, + + /** + * Notify that we can start submitting data to the servers. + */ + notifyCanUpload() { + if (!this._sendingEnabled) { + this._log.trace( + "notifyCanUpload - notifying before sending is enabled. Ignoring." + ); + return Promise.resolve(); + } + // Let the scheduler trigger sending pings if possible, also inform the + // crash reporter that it can send crash pings if appropriate. + SendScheduler.triggerSendingPings(true); + this._annotateCrashReport(); + + return this.promisePendingPingActivity(); + }, + + observe(subject, topic, data) { + let setOSShutdown = () => { + this._log.trace("setOSShutdown - in OS shutdown"); + this._isOSShutdown = true; + }; + + switch (topic) { + case TOPIC_IDLE_DAILY: + SendScheduler.triggerSendingPings(true); + break; + case TOPIC_QUIT_APPLICATION_FORCED: + setOSShutdown(); + break; + case TOPIC_QUIT_APPLICATION_GRANTED: + if (data == "syncShutdown") { + setOSShutdown(); + } + break; + case PREF_CHANGED_TOPIC: + if (this.OBSERVED_PREFERENCES.includes(data)) { + this._annotateCrashReport(); + } + break; + case TOPIC_PROFILE_CHANGE_NET_TEARDOWN: + this._tooLateToSend = true; + break; + } + }, + + /** + * Spawn the PingSender process that sends a ping. This function does + * not return an error or throw, it only logs an error. + * + * Even if the function doesn't fail, it doesn't mean that the ping was + * successfully sent, as we have no control over the spawned process. If it, + * succeeds, the ping is eventually removed from the disk to prevent duplicated + * submissions. + * + * @param {String} pingId The id of the ping to send. + * @param {String} submissionURL The complete Telemetry-compliant URL for the ping. + */ + _sendWithPingSender(pingId, submissionURL) { + this._log.trace( + "_sendWithPingSender - sending " + pingId + " to " + submissionURL + ); + try { + const pingPath = OS.Path.join(TelemetryStorage.pingDirectoryPath, pingId); + if (this._tooLateToSend) { + // We're in shutdown. Batch pings destined for pingsender. + this._log.trace("_sendWithPingSender - too late to send. Batching."); + this._pingSenderBatch.push({ url: submissionURL, path: pingPath }); + return; + } + this.runPingSender([{ url: submissionURL, path: pingPath }]); + } catch (e) { + this._log.error("_sendWithPingSender - failed to submit ping", e); + } + }, + + submitPing(ping, options) { + this._log.trace( + "submitPing - ping id: " + + ping.id + + ", options: " + + JSON.stringify(options) + ); + + if (!this.sendingEnabled(ping)) { + this._log.trace("submitPing - Telemetry is not allowed to send pings."); + return Promise.resolve(); + } + + // Send the ping using the PingSender, if requested and the user was + // notified of our policy. We don't support the pingsender on Android, + // so ignore this option on that platform (see bug 1335917). + // Moreover, if the OS is shutting down, we don't want to spawn the + // pingsender as it could unnecessarily slow down OS shutdown. + // Additionally, it could be be killed before it can complete its tasks, + // for example after successfully sending the ping but before removing + // the copy from the disk, resulting in receiving duplicate pings when + // Firefox restarts. + if ( + options.usePingSender && + !this._isOSShutdown && + TelemetryReportingPolicy.canUpload() && + AppConstants.platform != "android" + ) { + const url = this._buildSubmissionURL(ping); + // Serialize the ping to the disk and then spawn the PingSender. + return savePing(ping).then(() => this._sendWithPingSender(ping.id, url)); + } + + if (!this.canSendNow) { + // Sending is disabled or throttled, add this to the persisted pending pings. + this._log.trace( + "submitPing - can't send ping now, persisting to disk - " + + "canSendNow: " + + this.canSendNow + ); + return savePing(ping); + } + + // Let the scheduler trigger sending pings if possible. + // As a safety mechanism, this resets any currently active throttling. + this._log.trace("submitPing - can send pings, trying to send now"); + this._currentPings.set(ping.id, ping); + SendScheduler.triggerSendingPings(true); + return Promise.resolve(); + }, + + /** + * Only used in tests. + */ + setServer(server) { + this._log.trace("setServer", server); + this._server = server; + }, + + /** + * Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests. + */ + async clearCurrentPings() { + if (this._shutdown) { + this._log.trace("clearCurrentPings - in shutdown, bailing out"); + return; + } + + // Temporarily disable the scheduler. It must not try to reschedule ping sending + // while we're deleting them. + await SendScheduler.shutdown(); + + // Now that the ping activity has settled, abort outstanding ping requests. + this._cancelOutgoingRequests(); + + // Also, purge current pings. + this._currentPings.clear(); + + // We might have been interrupted and shutdown could have been started. + // We need to bail out in that case to avoid triggering send activity etc. + // at unexpected times. + if (this._shutdown) { + this._log.trace( + "clearCurrentPings - in shutdown, not spinning SendScheduler up again" + ); + return; + } + + // Enable the scheduler again and spin the send task. + SendScheduler.start(); + SendScheduler.triggerSendingPings(true); + }, + + _cancelOutgoingRequests() { + // Abort any pending ping XHRs. + for (let [id, request] of this._pendingPingRequests) { + this._log.trace( + "_cancelOutgoingRequests - aborting ping request for id " + id + ); + try { + request.abort(); + } catch (e) { + this._log.error( + "_cancelOutgoingRequests - failed to abort request for id " + id, + e + ); + } + } + this._pendingPingRequests.clear(); + }, + + sendPings(currentPings, persistedPingIds) { + let pingSends = []; + + // Prioritize health pings to enable low-latency monitoring. + currentPings = [ + ...currentPings.filter(ping => ping.type === "health"), + ...currentPings.filter(ping => ping.type !== "health"), + ]; + + for (let current of currentPings) { + let ping = current; + let p = (async () => { + try { + await this._doPing(ping, ping.id, false); + } catch (ex) { + this._log.info( + "sendPings - ping " + ping.id + " not sent, saving to disk", + ex + ); + await savePing(ping); + } finally { + this._currentPings.delete(ping.id); + } + })(); + + this._trackPendingPingTask(p); + pingSends.push(p); + } + + if (persistedPingIds.length) { + pingSends.push( + this._sendPersistedPings(persistedPingIds).catch(ex => { + this._log.info("sendPings - persisted pings not sent", ex); + }) + ); + } + + return Promise.all(pingSends); + }, + + /** + * Send the persisted pings to the server. + * + * @param {Array<string>} List of ping ids that should be sent. + * + * @return Promise A promise that is resolved when all pings finished sending or failed. + */ + async _sendPersistedPings(pingIds) { + this._log.trace("sendPersistedPings"); + + if (this.pendingPingCount < 1) { + this._log.trace("_sendPersistedPings - no pings to send"); + return; + } + + if (pingIds.length < 1) { + this._log.trace("sendPersistedPings - no pings to send"); + return; + } + + // We can send now. + // If there are any send failures, _doPing() sets up handlers that e.g. trigger backoff timer behavior. + this._log.trace( + "sendPersistedPings - sending " + pingIds.length + " pings" + ); + let pingSendPromises = []; + for (let pingId of pingIds) { + const id = pingId; + pingSendPromises.push( + TelemetryStorage.loadPendingPing(id) + .then(data => this._doPing(data, id, true)) + .catch(e => + this._log.error("sendPersistedPings - failed to send ping " + id, e) + ) + ); + } + + let promise = Promise.all(pingSendPromises); + this._trackPendingPingTask(promise); + await promise; + }, + + _onPingRequestFinished(success, startTime, id, isPersisted) { + this._log.trace( + "_onPingRequestFinished - success: " + + success + + ", persisted: " + + isPersisted + ); + + let sendId = success ? "TELEMETRY_SEND_SUCCESS" : "TELEMETRY_SEND_FAILURE"; + let hsend = Telemetry.getHistogramById(sendId); + let hsuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS"); + + hsend.add(Utils.monotonicNow() - startTime); + hsuccess.add(success); + + if (!success) { + // Let the scheduler know about send failures for triggering backoff timeouts. + SendScheduler.notifySendsFailed(); + } + + if (success && isPersisted) { + return TelemetryStorage.removePendingPing(id); + } + return Promise.resolve(); + }, + + _buildSubmissionURL(ping) { + const version = isV4PingFormat(ping) + ? AppConstants.TELEMETRY_PING_FORMAT_VERSION + : 1; + return this._server + this._getSubmissionPath(ping) + "?v=" + version; + }, + + _getSubmissionPath(ping) { + // The new ping format contains an "application" section, the old one doesn't. + let pathComponents; + if (isV4PingFormat(ping)) { + // We insert the Ping id in the URL to simplify server handling of duplicated + // pings. + let app = ping.application; + pathComponents = [ + ping.id, + ping.type, + app.name, + app.version, + app.channel, + app.buildId, + ]; + } else { + // This is a ping in the old format. + if (!("slug" in ping)) { + // That's odd, we don't have a slug. Generate one so that TelemetryStorage.jsm works. + ping.slug = Utils.generateUUID(); + } + + // Do we have enough info to build a submission URL? + let payload = "payload" in ping ? ping.payload : null; + if (payload && "info" in payload) { + let info = ping.payload.info; + pathComponents = [ + ping.slug, + info.reason, + info.appName, + info.appVersion, + info.appUpdateChannel, + info.appBuildID, + ]; + } else { + // Only use the UUID as the slug. + pathComponents = [ping.slug]; + } + } + + let slug = pathComponents.join("/"); + return "/submit/telemetry/" + slug; + }, + + _doPing(ping, id, isPersisted) { + if (!this.sendingEnabled(ping)) { + // We can't send the pings to the server, so don't try to. + this._log.trace("_doPing - Can't send ping " + ping.id); + return Promise.resolve(); + } + + if (this._tooLateToSend) { + // Too late to send now. Reject so we pend the ping to send it next time. + this._log.trace("_doPing - Too late to send ping " + ping.id); + Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE").add("eTooLate"); + Telemetry.getKeyedHistogramById( + "TELEMETRY_SEND_FAILURE_TYPE_PER_PING" + ).add(ping.type, "eTooLate"); + return Promise.reject(); + } + + this._log.trace( + "_doPing - server: " + + this._server + + ", persisted: " + + isPersisted + + ", id: " + + id + ); + + const url = this._buildSubmissionURL(ping); + + // Don't send cookies with these requests. + let request = new ServiceRequest({ mozAnon: true }); + request.mozBackgroundRequest = true; + request.timeout = Policy.pingSubmissionTimeout(); + + request.open("POST", url, true); + request.overrideMimeType("text/plain"); + request.setRequestHeader("Content-Type", "application/json; charset=UTF-8"); + request.setRequestHeader("Date", Policy.now().toUTCString()); + + this._pendingPingRequests.set(id, request); + + const monotonicStartTime = Utils.monotonicNow(); + let deferred = PromiseUtils.defer(); + + let onRequestFinished = (success, event) => { + let onCompletion = () => { + if (success) { + deferred.resolve(); + } else { + deferred.reject(event); + } + }; + + this._pendingPingRequests.delete(id); + this._onPingRequestFinished( + success, + monotonicStartTime, + id, + isPersisted + ).then( + () => onCompletion(), + error => { + this._log.error( + "_doPing - request success: " + success + ", error: " + error + ); + onCompletion(); + } + ); + }; + + let errorhandler = event => { + let failure = event.type; + if (failure === "error") { + failure = XHR_ERROR_TYPE[request.errorCode]; + } + + TelemetryHealthPing.recordSendFailure(failure); + + if (this.fallbackHttp) { + // only one attempt + this.fallbackHttp = false; + + request.channel.securityInfo + .QueryInterface(Ci.nsITransportSecurityInfo) + .QueryInterface(Ci.nsISerializable); + if (request.channel.securityInfo.errorCodeString.startsWith("SEC_")) { + // re-open the request with the HTTP version of the URL + let fallbackUrl = new URL(url); + fallbackUrl.protocol = "http:"; + // TODO encrypt payload + request.open("POST", fallbackUrl, true); + request.sendInputStream(this.payloadStream); + } + } + + Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE").add(failure); + Telemetry.getKeyedHistogramById( + "TELEMETRY_SEND_FAILURE_TYPE_PER_PING" + ).add(ping.type, failure); + + this._log.error( + "_doPing - error making request to " + url + ": " + failure + ); + onRequestFinished(false, event); + }; + request.onerror = errorhandler; + request.ontimeout = errorhandler; + request.onabort = errorhandler; + + request.onload = event => { + let status = request.status; + let statusClass = status - (status % 100); + let success = false; + + if (statusClass === 200) { + // We can treat all 2XX as success. + this._log.info("_doPing - successfully loaded, status: " + status); + success = true; + } else if (statusClass === 400) { + // 4XX means that something with the request was broken. + this._log.error( + "_doPing - error submitting to " + + url + + ", status: " + + status + + " - ping request broken?" + ); + Telemetry.getHistogramById( + "TELEMETRY_PING_EVICTED_FOR_SERVER_ERRORS" + ).add(); + // TODO: we should handle this better, but for now we should avoid resubmitting + // broken requests by pretending success. + success = true; + } else if (statusClass === 500) { + // 5XX means there was a server-side error and we should try again later. + this._log.error( + "_doPing - error submitting to " + + url + + ", status: " + + status + + " - server error, should retry later" + ); + } else { + // We received an unexpected status code. + this._log.error( + "_doPing - error submitting to " + + url + + ", status: " + + status + + ", type: " + + event.type + ); + } + + onRequestFinished(success, event); + }; + + // If that's a legacy ping format, just send its payload. + let networkPayload = isV4PingFormat(ping) ? ping : ping.payload; + request.setRequestHeader("Content-Encoding", "gzip"); + let converter = Cc[ + "@mozilla.org/intl/scriptableunicodeconverter" + ].createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + let startTime = Utils.monotonicNow(); + let utf8Payload = converter.ConvertFromUnicode( + JSON.stringify(networkPayload) + ); + utf8Payload += converter.Finish(); + Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add( + Utils.monotonicNow() - startTime + ); + + let payloadStream = Cc[ + "@mozilla.org/io/string-input-stream;1" + ].createInstance(Ci.nsIStringInputStream); + startTime = Utils.monotonicNow(); + payloadStream.data = Policy.gzipCompressString(utf8Payload); + + // Check the size and drop pings which are too big. + const compressedPingSizeBytes = payloadStream.data.length; + if (compressedPingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) { + this._log.error( + "_doPing - submitted ping exceeds the size limit, size: " + + compressedPingSizeBytes + ); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add(); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB").add( + Math.floor(compressedPingSizeBytes / 1024 / 1024) + ); + // We don't need to call |request.abort()| as it was not sent yet. + this._pendingPingRequests.delete(id); + + TelemetryHealthPing.recordDiscardedPing(ping.type); + return TelemetryStorage.removePendingPing(id); + } + + Telemetry.getHistogramById("TELEMETRY_COMPRESS").add( + Utils.monotonicNow() - startTime + ); + request.sendInputStream(payloadStream); + + this.payloadStream = payloadStream; + + return deferred.promise; + }, + + /** + * Check if sending is temporarily disabled. + * @return {Boolean} True if we can send pings to the server right now, false if + * sending is temporarily disabled. + */ + get canSendNow() { + // If the reporting policy was not accepted yet, don't send pings. + if (!TelemetryReportingPolicy.canUpload()) { + return false; + } + + return this._sendingEnabled; + }, + + /** + * Check if sending is disabled. If Telemetry is not allowed to upload, + * pings are not sent to the server. + * If trying to send a "deletion-request" ping, don't block it. + * If unified telemetry is off, don't send pings if Telemetry is disabled. + * + * @param {Object} [ping=null] A ping to be checked. + * @return {Boolean} True if pings can be send to the servers, false otherwise. + */ + sendingEnabled(ping = null) { + // We only send pings from official builds, but allow overriding this for tests. + if ( + !Telemetry.isOfficialTelemetry && + !this._testMode && + !this._overrideOfficialCheck + ) { + return false; + } + + // With unified Telemetry, the FHR upload setting controls whether we can send pings. + // The Telemetry pref enables sending extended data sets instead. + if (IS_UNIFIED_TELEMETRY) { + // "deletion-request" pings are sent once even if the upload is disabled. + if (ping && isDeletionRequestPing(ping)) { + return true; + } + return Services.prefs.getBoolPref( + TelemetryUtils.Preferences.FhrUploadEnabled, + false + ); + } + + // Without unified Telemetry, the Telemetry enabled pref controls ping sending. + return Utils.isTelemetryEnabled; + }, + + /** + * Track any pending ping send and save tasks through the promise passed here. + * This is needed to block shutdown on any outstanding ping activity. + */ + _trackPendingPingTask(promise) { + let clear = () => this._pendingPingActivity.delete(promise); + promise.then(clear, clear); + this._pendingPingActivity.add(promise); + }, + + /** + * Return a promise that allows to wait on pending pings. + * @return {Object<Promise>} A promise resolved when all the pending pings promises + * are resolved. + */ + promisePendingPingActivity() { + this._log.trace("promisePendingPingActivity - Waiting for ping task"); + let p = Array.from(this._pendingPingActivity, p => + p.catch(ex => { + this._log.error( + "promisePendingPingActivity - ping activity had an error", + ex + ); + }) + ); + p.push(SendScheduler.waitOnSendTask()); + return Promise.all(p); + }, + + async _persistCurrentPings() { + for (let [id, ping] of this._currentPings) { + try { + await savePing(ping); + this._log.trace("_persistCurrentPings - saved ping " + id); + } catch (ex) { + this._log.error("_persistCurrentPings - failed to save ping " + id, ex); + } finally { + this._currentPings.delete(id); + } + } + }, + + /** + * Returns the current pending, not yet persisted, pings, newest first. + */ + getUnpersistedPings() { + let current = [...this._currentPings.values()]; + current.reverse(); + return current; + }, + + getShutdownState() { + return { + sendingEnabled: this._sendingEnabled, + pendingPingRequestCount: this._pendingPingRequests.size, + pendingPingActivityCount: this._pendingPingActivity.size, + unpersistedPingCount: this._currentPings.size, + persistedPingCount: TelemetryStorage.getPendingPingList().length, + schedulerState: SendScheduler.getShutdownState(), + }; + }, + + runPingSender(pings, observer) { + if (AppConstants.platform === "android") { + throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED); + } + + const exeName = + AppConstants.platform === "win" ? "pingsender.exe" : "pingsender"; + + let exe = Services.dirsvc.get("GreBinD", Ci.nsIFile); + exe.append(exeName); + + let params = pings.flatMap(ping => [ping.url, ping.path]); + let process = Cc["@mozilla.org/process/util;1"].createInstance( + Ci.nsIProcess + ); + process.init(exe); + process.startHidden = true; + process.noShell = true; + process.runAsync(params, params.length, observer); + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryStorage.jsm b/toolkit/components/telemetry/app/TelemetryStorage.jsm new file mode 100644 index 0000000000..bc65282fed --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryStorage.jsm @@ -0,0 +1,2221 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryStorage"]; + +ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this); +const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm"); +ChromeUtils.import("resource://gre/modules/Services.jsm", this); +ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this); +ChromeUtils.import("resource://gre/modules/osfile.jsm", this); +ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this); +ChromeUtils.import("resource://gre/modules/Preferences.jsm", this); + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetryStorage::"; + +const Telemetry = Services.telemetry; +const Utils = TelemetryUtils; + +// Compute the path of the pings archive on the first use. +const DATAREPORTING_DIR = "datareporting"; +const PINGS_ARCHIVE_DIR = "archived"; +const ABORTED_SESSION_FILE_NAME = "aborted-session-ping"; +const SESSION_STATE_FILE_NAME = "session-state.json"; + +XPCOMUtils.defineLazyGetter(this, "gDataReportingDir", function() { + return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR); +}); +XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() { + return OS.Path.join(gDataReportingDir, PINGS_ARCHIVE_DIR); +}); +XPCOMUtils.defineLazyGetter(this, "gAbortedSessionFilePath", function() { + return OS.Path.join(gDataReportingDir, ABORTED_SESSION_FILE_NAME); +}); +ChromeUtils.defineModuleGetter( + this, + "CommonUtils", + "resource://services-common/utils.js" +); +ChromeUtils.defineModuleGetter( + this, + "TelemetryHealthPing", + "resource://gre/modules/HealthPing.jsm" +); +// Maxmimum time, in milliseconds, archive pings should be retained. +const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000; // 60 days + +// Maximum space the archive can take on disk (in Bytes). +const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB +// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes). +const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB +// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes). +const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB + +// The maximum size a pending/archived ping can take on disk. +const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB + +// This special value is submitted when the archive is outside of the quota. +const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300; + +// This special value is submitted when the pending pings is outside of the quota, as +// we don't know the size of the pings above the quota. +const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping + * from the disk fails. + */ +function PingReadError( + message = "Error reading the ping file", + becauseNoSuchFile = false +) { + Error.call(this, message); + let error = new Error(); + this.name = "PingReadError"; + this.message = message; + this.stack = error.stack; + this.becauseNoSuchFile = becauseNoSuchFile; +} +PingReadError.prototype = Object.create(Error.prototype); +PingReadError.prototype.constructor = PingReadError; + +/** + * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON + * content fails. + */ +function PingParseError(message = "Error parsing ping content") { + Error.call(this, message); + let error = new Error(); + this.name = "PingParseError"; + this.message = message; + this.stack = error.stack; +} +PingParseError.prototype = Object.create(Error.prototype); +PingParseError.prototype.constructor = PingParseError; + +/** + * This is a policy object used to override behavior for testing. + */ +var Policy = { + now: () => new Date(), + getArchiveQuota: () => ARCHIVE_QUOTA_BYTES, + getPendingPingsQuota: () => + AppConstants.platform == "android" + ? PENDING_PINGS_QUOTA_BYTES_MOBILE + : PENDING_PINGS_QUOTA_BYTES_DESKTOP, + /** + * @param {string} id The ID of the ping that will be written into the file. Can be "*" to + * make a pattern to find all pings for this installation. + * @return + * { + * directory: <nsIFile>, // Directory to save pings + * file: <string>, // File name for this ping (or pattern for all pings) + * } + */ + getUninstallPingPath: id => { + // UpdRootD is e.g. C:\ProgramData\Mozilla\updates\<PATH HASH> + const updateDirectory = Services.dirsvc.get("UpdRootD", Ci.nsIFile); + const installPathHash = updateDirectory.leafName; + + return { + // e.g. C:\ProgramData\Mozilla + directory: updateDirectory.parent.parent.clone(), + file: `uninstall_ping_${installPathHash}_${id}.json`, + }; + }, +}; + +/** + * Wait for all promises in iterable to resolve or reject. This function + * always resolves its promise with undefined, and never rejects. + */ +function waitForAll(it) { + let dummy = () => {}; + let promises = Array.from(it, p => p.catch(dummy)); + return Promise.all(promises); +} + +/** + * Permanently intern the given string. This is mainly used for the ping.type + * strings that can be excessively duplicated in the _archivedPings map. Do not + * pass large or temporary strings to this function. + */ +function internString(str) { + return Symbol.keyFor(Symbol.for(str)); +} + +var TelemetryStorage = { + get pingDirectoryPath() { + return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings"); + }, + + /** + * The maximum size a ping can have, in bytes. + */ + get MAXIMUM_PING_SIZE() { + return PING_FILE_MAXIMUM_SIZE_BYTES; + }, + /** + * Shutdown & block on any outstanding async activity in this module. + * + * @return {Promise} Promise that is resolved when shutdown is complete. + */ + shutdown() { + return TelemetryStorageImpl.shutdown(); + }, + + /** + * Save an archived ping to disk. + * + * @param {object} ping The ping data to archive. + * @return {promise} Promise that is resolved when the ping is successfully archived. + */ + saveArchivedPing(ping) { + return TelemetryStorageImpl.saveArchivedPing(ping); + }, + + /** + * Load an archived ping from disk. + * + * @param {string} id The pings id. + * @return {promise<object>} Promise that is resolved with the ping data. + */ + loadArchivedPing(id) { + return TelemetryStorageImpl.loadArchivedPing(id); + }, + + /** + * Get a list of info on the archived pings. + * This will scan the archive directory and grab basic data about the existing + * pings out of their filename. + * + * @return {promise<sequence<object>>} + */ + loadArchivedPingList() { + return TelemetryStorageImpl.loadArchivedPingList(); + }, + + /** + * Clean the pings archive by removing old pings. + * This will scan the archive directory. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runCleanPingArchiveTask() { + return TelemetryStorageImpl.runCleanPingArchiveTask(); + }, + + /** + * Run the task to enforce the pending pings quota. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runEnforcePendingPingsQuotaTask() { + return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask(); + }, + + /** + * Run the task to remove all the pending pings + * + * @return {Promise} Resolved when the pings are removed. + */ + runRemovePendingPingsTask() { + return TelemetryStorageImpl.runRemovePendingPingsTask(); + }, + + /** + * Remove all pings that are stored in the userApplicationDataDir + * under the "Pending Pings" sub-directory. + */ + removeAppDataPings() { + return TelemetryStorageImpl.removeAppDataPings(); + }, + + /** + * Reset the storage state in tests. + */ + reset() { + return TelemetryStorageImpl.reset(); + }, + + /** + * Test method that allows waiting on the archive clean task to finish. + */ + testCleanupTaskPromise() { + return TelemetryStorageImpl._cleanArchiveTask || Promise.resolve(); + }, + + /** + * Test method that allows waiting on the pending pings quota task to finish. + */ + testPendingQuotaTaskPromise() { + return ( + TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve() + ); + }, + + /** + * Save a pending - outgoing - ping to disk and track it. + * + * @param {Object} ping The ping data. + * @return {Promise} Resolved when the ping was saved. + */ + savePendingPing(ping) { + return TelemetryStorageImpl.savePendingPing(ping); + }, + + /** + * Saves session data to disk. + * @param {Object} sessionData The session data. + * @return {Promise} Resolved when the data was saved. + */ + saveSessionData(sessionData) { + return TelemetryStorageImpl.saveSessionData(sessionData); + }, + + /** + * Loads session data from a session data file. + * @return {Promise<object>} Resolved with the session data in object form. + */ + loadSessionData() { + return TelemetryStorageImpl.loadSessionData(); + }, + + /** + * Load a pending ping from disk by id. + * + * @param {String} id The pings id. + * @return {Promise} Resolved with the loaded ping data. + */ + loadPendingPing(id) { + return TelemetryStorageImpl.loadPendingPing(id); + }, + + /** + * Remove a pending ping from disk by id. + * + * @param {String} id The pings id. + * @return {Promise} Resolved when the ping was removed. + */ + removePendingPing(id) { + return TelemetryStorageImpl.removePendingPing(id); + }, + + /** + * Returns a list of the currently pending pings in the format: + * { + * id: <string>, // The pings UUID. + * lastModificationDate: <number>, // Timestamp of the pings last modification. + * } + * This populates the list by scanning the disk. + * + * @return {Promise<sequence>} Resolved with the ping list. + */ + loadPendingPingList() { + return TelemetryStorageImpl.loadPendingPingList(); + }, + + /** + * Returns a list of the currently pending pings in the format: + * { + * id: <string>, // The pings UUID. + * lastModificationDate: <number>, // Timestamp of the pings last modification. + * } + * This does not scan pending pings on disk. + * + * @return {sequence} The current pending ping list. + */ + getPendingPingList() { + return TelemetryStorageImpl.getPendingPingList(); + }, + + /** + * Save an aborted-session ping to disk. This goes to a special location so + * it is not picked up as a pending ping. + * + * @param {object} ping The ping data to save. + * @return {promise} Promise that is resolved when the ping is successfully saved. + */ + saveAbortedSessionPing(ping) { + return TelemetryStorageImpl.saveAbortedSessionPing(ping); + }, + + /** + * Load the aborted-session ping from disk if present. + * + * @return {promise<object>} Promise that is resolved with the ping data if found. + * Otherwise returns null. + */ + loadAbortedSessionPing() { + return TelemetryStorageImpl.loadAbortedSessionPing(); + }, + + /** + * Remove the aborted-session ping if present. + * + * @return {promise} Promise that is resolved once the ping is removed. + */ + removeAbortedSessionPing() { + return TelemetryStorageImpl.removeAbortedSessionPing(); + }, + + /** + * Save an uninstall ping to disk, removing any old ones from this + * installation first. + * This is stored independently from other pings, and only read by + * the Windows uninstaller. + * + * WINDOWS ONLY, does nothing and resolves immediately on other platforms. + * + * @return {promise} Promise that is resolved when the ping has been saved. + */ + saveUninstallPing(ping) { + return TelemetryStorageImpl.saveUninstallPing(ping); + }, + + /** + * Remove all uninstall pings from this installation. + * + * WINDOWS ONLY, does nothing and resolves immediately on other platforms. + * + * @return {promise} Promise that is resolved when the pings have been removed. + */ + removeUninstallPings() { + return TelemetryStorageImpl.removeUninstallPings(); + }, + + /** + * Save a single ping to a file. + * + * @param {object} ping The content of the ping to save. + * @param {string} file The destination file. + * @param {bool} overwrite If |true|, the file will be overwritten if it exists, + * if |false| the file will not be overwritten and no error will be reported if + * the file exists. + * @returns {promise} + */ + savePingToFile(ping, file, overwrite) { + return TelemetryStorageImpl.savePingToFile(ping, file, overwrite); + }, + + /** + * Save a ping to its file. + * + * @param {object} ping The content of the ping to save. + * @param {bool} overwrite If |true|, the file will be overwritten + * if it exists. + * @returns {promise} + */ + savePing(ping, overwrite) { + return TelemetryStorageImpl.savePing(ping, overwrite); + }, + + /** + * Add a ping to the saved pings directory so that it gets saved + * and sent along with other pings. + * + * @param {Object} pingData The ping object. + * @return {Promise} A promise resolved when the ping is saved to the pings directory. + */ + addPendingPing(pingData) { + return TelemetryStorageImpl.addPendingPing(pingData); + }, + + /** + * Remove the file for a ping + * + * @param {object} ping The ping. + * @returns {promise} + */ + cleanupPingFile(ping) { + return TelemetryStorageImpl.cleanupPingFile(ping); + }, + + /** + * Loads a ping file. + * @param {String} aFilePath The path of the ping file. + * @return {Promise<Object>} A promise resolved with the ping content or rejected if the + * ping contains invalid data. + */ + async loadPingFile(aFilePath) { + return TelemetryStorageImpl.loadPingFile(aFilePath); + }, + + /** + * Remove FHR database files. This is temporary and will be dropped in + * the future. + * @return {Promise} Resolved when the database files are deleted. + */ + removeFHRDatabase() { + return TelemetryStorageImpl.removeFHRDatabase(); + }, + + /** + * Only used in tests, builds an archived ping path from the ping metadata. + * @param {String} aPingId The ping id. + * @param {Object} aDate The ping creation date. + * @param {String} aType The ping type. + * @return {String} The full path to the archived ping. + */ + _testGetArchivedPingPath(aPingId, aDate, aType) { + return getArchivedPingPath(aPingId, aDate, aType); + }, + + /** + * Only used in tests, this helper extracts ping metadata from a given filename. + * + * @param fileName {String} The filename. + * @return {Object} Null if the filename didn't match the expected form. + * Otherwise an object with the extracted data in the form: + * { timestamp: <number>, + * id: <string>, + * type: <string> } + */ + _testGetArchivedPingDataFromFileName(aFileName) { + return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName); + }, + + /** + * Only used in tests, this helper allows cleaning up the pending ping storage. + */ + testClearPendingPings() { + return TelemetryStorageImpl.runRemovePendingPingsTask(); + }, +}; + +/** + * This object allows the serialisation of asynchronous tasks. This is particularly + * useful to serialise write access to the disk in order to prevent race conditions + * to corrupt the data being written. + * We are using this to synchronize saving to the file that TelemetrySession persists + * its state in. + */ +function SaveSerializer() { + this._queuedOperations = []; + this._queuedInProgress = false; + this._log = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); +} + +SaveSerializer.prototype = { + /** + * Enqueues an operation to a list to serialise their execution in order to prevent race + * conditions. Useful to serialise access to disk. + * + * @param {Function} aFunction The task function to enqueue. It must return a promise. + * @return {Promise} A promise resolved when the enqueued task completes. + */ + enqueueTask(aFunction) { + let promise = new Promise((resolve, reject) => + this._queuedOperations.push([aFunction, resolve, reject]) + ); + + if (this._queuedOperations.length == 1) { + this._popAndPerformQueuedOperation(); + } + return promise; + }, + + /** + * Make sure to flush all the pending operations. + * @return {Promise} A promise resolved when all the pending operations have completed. + */ + flushTasks() { + let dummyTask = () => new Promise(resolve => resolve()); + return this.enqueueTask(dummyTask); + }, + + /** + * Pop a task from the queue, executes it and continue to the next one. + * This function recursively pops all the tasks. + */ + _popAndPerformQueuedOperation() { + if (!this._queuedOperations.length || this._queuedInProgress) { + return; + } + + this._log.trace( + "_popAndPerformQueuedOperation - Performing queued operation." + ); + let [func, resolve, reject] = this._queuedOperations.shift(); + let promise; + + try { + this._queuedInProgress = true; + promise = func(); + } catch (ex) { + this._log.warn( + "_popAndPerformQueuedOperation - Queued operation threw during execution. ", + ex + ); + this._queuedInProgress = false; + reject(ex); + this._popAndPerformQueuedOperation(); + return; + } + + if (!promise || typeof promise.then != "function") { + let msg = "Queued operation did not return a promise: " + func; + this._log.warn("_popAndPerformQueuedOperation - " + msg); + + this._queuedInProgress = false; + reject(new Error(msg)); + this._popAndPerformQueuedOperation(); + return; + } + + promise.then( + result => { + this._queuedInProgress = false; + resolve(result); + this._popAndPerformQueuedOperation(); + }, + error => { + this._log.warn( + "_popAndPerformQueuedOperation - Failure when performing queued operation.", + error + ); + this._queuedInProgress = false; + reject(error); + this._popAndPerformQueuedOperation(); + } + ); + }, +}; + +var TelemetryStorageImpl = { + _logger: null, + // Used to serialize aborted session ping writes to disk. + _abortedSessionSerializer: new SaveSerializer(), + // Used to serialize session state writes to disk. + _stateSaveSerializer: new SaveSerializer(), + + // Tracks the archived pings in a Map of (id -> {timestampCreated, type}). + // We use this to cache info on archived pings to avoid scanning the disk more than once. + _archivedPings: new Map(), + // A set of promises for pings currently being archived + _activelyArchiving: new Set(), + // Track the archive loading task to prevent multiple tasks from being executed. + _scanArchiveTask: null, + // Track the archive cleanup task. + _cleanArchiveTask: null, + // Whether we already scanned the archived pings on disk. + _scannedArchiveDirectory: false, + + // Track the pending ping removal task. + _removePendingPingsTask: null, + + // This tracks all the pending async ping save activity. + _activePendingPingSaves: new Set(), + + // Tracks the pending pings in a Map of (id -> {timestampCreated, type}). + // We use this to cache info on pending pings to avoid scanning the disk more than once. + _pendingPings: new Map(), + + // Track the pending pings enforce quota task. + _enforcePendingPingsQuotaTask: null, + + // Track the shutdown process to bail out of the clean up task quickly. + _shutdown: false, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix( + LOGGER_NAME, + LOGGER_PREFIX + ); + } + + return this._logger; + }, + + /** + * Shutdown & block on any outstanding async activity in this module. + * + * @return {Promise} Promise that is resolved when shutdown is complete. + */ + async shutdown() { + this._shutdown = true; + + // If the following tasks are still running, block on them. They will bail out as soon + // as possible. + await this._abortedSessionSerializer.flushTasks().catch(ex => { + this._log.error("shutdown - failed to flush aborted-session writes", ex); + }); + + if (this._cleanArchiveTask) { + await this._cleanArchiveTask.catch(ex => { + this._log.error("shutdown - the archive cleaning task failed", ex); + }); + } + + if (this._enforcePendingPingsQuotaTask) { + await this._enforcePendingPingsQuotaTask.catch(ex => { + this._log.error("shutdown - the pending pings quota task failed", ex); + }); + } + + if (this._removePendingPingsTask) { + await this._removePendingPingsTask.catch(ex => { + this._log.error("shutdown - the pending pings removal task failed", ex); + }); + } + + // Wait on pending pings still being saved. While OS.File should have shutdown + // blockers in place, we a) have seen weird errors being reported that might + // indicate a bad shutdown path and b) might have completion handlers hanging + // off the save operations that don't expect to be late in shutdown. + await this.promisePendingPingSaves(); + }, + + /** + * Save an archived ping to disk. + * + * @param {object} ping The ping data to archive. + * @return {promise} Promise that is resolved when the ping is successfully archived. + */ + saveArchivedPing(ping) { + let promise = this._saveArchivedPingTask(ping); + this._activelyArchiving.add(promise); + promise.then( + r => { + this._activelyArchiving.delete(promise); + }, + e => { + this._activelyArchiving.delete(promise); + } + ); + return promise; + }, + + async _saveArchivedPingTask(ping) { + const creationDate = new Date(ping.creationDate); + if (this._archivedPings.has(ping.id)) { + const data = this._archivedPings.get(ping.id); + if (data.timestampCreated > creationDate.getTime()) { + this._log.error( + "saveArchivedPing - trying to overwrite newer ping with the same id" + ); + return Promise.reject( + new Error("trying to overwrite newer ping with the same id") + ); + } + this._log.warn( + "saveArchivedPing - overwriting older ping with the same id" + ); + } + + // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4'). + const filePath = + getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4"; + await OS.File.makeDir(OS.Path.dirname(filePath), { + ignoreExisting: true, + from: OS.Constants.Path.profileDir, + }); + await this.savePingToFile( + ping, + filePath, + /* overwrite*/ true, + /* compressed*/ true + ); + + this._archivedPings.set(ping.id, { + timestampCreated: creationDate.getTime(), + type: internString(ping.type), + }); + + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add(); + return undefined; + }, + + /** + * Load an archived ping from disk. + * + * @param {string} id The pings id. + * @return {promise<object>} Promise that is resolved with the ping data. + */ + async loadArchivedPing(id) { + const data = this._archivedPings.get(id); + if (!data) { + this._log.trace("loadArchivedPing - no ping with id: " + id); + return Promise.reject( + new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id) + ); + } + + const path = getArchivedPingPath( + id, + new Date(data.timestampCreated), + data.type + ); + const pathCompressed = path + "lz4"; + + // Purge pings which are too big. + let checkSize = async function(path) { + const fileSize = (await OS.File.stat(path)).size; + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + Telemetry.getHistogramById( + "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB" + ).add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById( + "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED" + ).add(); + await OS.File.remove(path, { ignoreAbsent: true }); + throw new Error( + "loadArchivedPing - exceeded the maximum ping size: " + fileSize + ); + } + }; + + let ping; + try { + // Try to load a compressed version of the archived ping first. + this._log.trace( + "loadArchivedPing - loading ping from: " + pathCompressed + ); + await checkSize(pathCompressed); + ping = await this.loadPingFile(pathCompressed, /* compressed*/ true); + } catch (ex) { + if (!ex.becauseNoSuchFile) { + throw ex; + } + // If that fails, look for the uncompressed version. + this._log.trace( + "loadArchivedPing - compressed ping not found, loading: " + path + ); + await checkSize(path); + ping = await this.loadPingFile(path, /* compressed*/ false); + } + + return ping; + }, + + /** + * Saves session data to disk. + */ + saveSessionData(sessionData) { + return this._stateSaveSerializer.enqueueTask(() => + this._saveSessionData(sessionData) + ); + }, + + async _saveSessionData(sessionData) { + let dataDir = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR); + await OS.File.makeDir(dataDir); + + let filePath = OS.Path.join(gDataReportingDir, SESSION_STATE_FILE_NAME); + try { + await CommonUtils.writeJSON(sessionData, filePath); + } catch (e) { + this._log.error( + "_saveSessionData - Failed to write session data to " + filePath, + e + ); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1); + } + }, + + /** + * Loads session data from the session data file. + * @return {Promise<Object>} A promise resolved with an object on success, + * with null otherwise. + */ + loadSessionData() { + return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData()); + }, + + async _loadSessionData() { + const dataFile = OS.Path.join( + OS.Constants.Path.profileDir, + DATAREPORTING_DIR, + SESSION_STATE_FILE_NAME + ); + let content; + try { + content = await OS.File.read(dataFile, { encoding: "utf-8" }); + } catch (ex) { + this._log.info("_loadSessionData - can not load session data file", ex); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1); + return null; + } + + let data; + try { + data = JSON.parse(content); + } catch (ex) { + this._log.error("_loadSessionData - failed to parse session data", ex); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1); + return null; + } + + return data; + }, + + /** + * Remove an archived ping from disk. + * + * @param {string} id The pings id. + * @param {number} timestampCreated The pings creation timestamp. + * @param {string} type The pings type. + * @return {promise<object>} Promise that is resolved when the pings is removed. + */ + async _removeArchivedPing(id, timestampCreated, type) { + this._log.trace( + "_removeArchivedPing - id: " + + id + + ", timestampCreated: " + + timestampCreated + + ", type: " + + type + ); + const path = getArchivedPingPath(id, new Date(timestampCreated), type); + const pathCompressed = path + "lz4"; + + this._log.trace("_removeArchivedPing - removing ping from: " + path); + await OS.File.remove(path, { ignoreAbsent: true }); + await OS.File.remove(pathCompressed, { ignoreAbsent: true }); + // Remove the ping from the cache. + this._archivedPings.delete(id); + }, + + /** + * Clean the pings archive by removing old pings. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runCleanPingArchiveTask() { + // If there's an archive cleaning task already running, return it. + if (this._cleanArchiveTask) { + return this._cleanArchiveTask; + } + + // Make sure to clear |_cleanArchiveTask| once done. + let clear = () => (this._cleanArchiveTask = null); + // Since there's no archive cleaning task running, start it. + this._cleanArchiveTask = this._cleanArchive().then(clear, clear); + return this._cleanArchiveTask; + }, + + /** + * Removes pings which are too old from the pings archive. + * @return {Promise} Resolved when the ping age check is complete. + */ + async _purgeOldPings() { + this._log.trace("_purgeOldPings"); + + const nowDate = Policy.now(); + const startTimeStamp = nowDate.getTime(); + let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath); + let subdirs = (await dirIterator.nextBatch()).filter(e => e.isDir); + dirIterator.close(); + + // Keep track of the newest removed month to update the cache, if needed. + let newestRemovedMonthTimestamp = null; + let evictedDirsCount = 0; + let maxDirAgeInMonths = 0; + + // Walk through the monthly subdirs of the form <YYYY-MM>/ + for (let dir of subdirs) { + if (this._shutdown) { + this._log.trace( + "_purgeOldPings - Terminating the clean up task due to shutdown" + ); + return; + } + + if (!isValidArchiveDir(dir.name)) { + this._log.warn( + "_purgeOldPings - skipping invalidly named subdirectory " + dir.path + ); + continue; + } + + const archiveDate = getDateFromArchiveDir(dir.name); + if (!archiveDate) { + this._log.warn( + "_purgeOldPings - skipping invalid subdirectory date " + dir.path + ); + continue; + } + + // If this archive directory is older than allowed, remove it. + if ( + startTimeStamp - archiveDate.getTime() > + MAX_ARCHIVED_PINGS_RETENTION_MS + ) { + try { + await OS.File.removeDir(dir.path); + evictedDirsCount++; + + // Update the newest removed month. + newestRemovedMonthTimestamp = Math.max( + archiveDate, + newestRemovedMonthTimestamp + ); + } catch (ex) { + this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex); + } + } else { + // We're not removing this directory, so record the age for the oldest directory. + const dirAgeInMonths = Utils.getElapsedTimeInMonths( + archiveDate, + nowDate + ); + maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths); + } + } + + // Trigger scanning of the archived pings. + await this.loadArchivedPingList(); + + // Refresh the cache: we could still skip this, but it's cheap enough to keep it + // to avoid introducing task dependencies. + if (newestRemovedMonthTimestamp) { + // Scan the archive cache for pings older than the newest directory pruned above. + for (let [id, info] of this._archivedPings) { + const timestampCreated = new Date(info.timestampCreated); + if (timestampCreated.getTime() > newestRemovedMonthTimestamp) { + continue; + } + // Remove outdated pings from the cache. + this._archivedPings.delete(id); + } + } + + const endTimeStamp = Policy.now().getTime(); + + // Save the time it takes to evict old directories and the eviction count. + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").add( + evictedDirsCount + ); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS").add( + Math.ceil(endTimeStamp - startTimeStamp) + ); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE").add( + maxDirAgeInMonths + ); + }, + + /** + * Enforce a disk quota for the pings archive. + * @return {Promise} Resolved when the quota check is complete. + */ + async _enforceArchiveQuota() { + this._log.trace("_enforceArchiveQuota"); + let startTimeStamp = Policy.now().getTime(); + + // Build an ordered list, from newer to older, of archived pings. + let pingList = Array.from(this._archivedPings, p => ({ + id: p[0], + timestampCreated: p[1].timestampCreated, + type: p[1].type, + })); + + pingList.sort((a, b) => b.timestampCreated - a.timestampCreated); + + // If our archive is too big, we should reduce it to reach 90% of the quota. + const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9; + // The index of the last ping to keep. Pings older than this one will be deleted if + // the archive exceeds the quota. + let lastPingIndexToKeep = null; + let archiveSizeInBytes = 0; + + // Find the disk size of the archive. + for (let i = 0; i < pingList.length; i++) { + if (this._shutdown) { + this._log.trace( + "_enforceArchiveQuota - Terminating the clean up task due to shutdown" + ); + return; + } + + let ping = pingList[i]; + + // Get the size for this ping. + const fileSize = await getArchivedPingSize( + ping.id, + new Date(ping.timestampCreated), + ping.type + ); + if (!fileSize) { + this._log.warn( + "_enforceArchiveQuota - Unable to find the size of ping " + ping.id + ); + continue; + } + + // Enforce a maximum file size limit on archived pings. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error( + "_enforceArchiveQuota - removing file exceeding size limit, size: " + + fileSize + ); + // We just remove the ping from the disk, we don't bother removing it from pingList + // since it won't contribute to the quota. + await this._removeArchivedPing( + ping.id, + ping.timestampCreated, + ping.type + ).catch(e => + this._log.error( + "_enforceArchiveQuota - failed to remove archived ping" + ping.id + ) + ); + Telemetry.getHistogramById( + "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB" + ).add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById( + "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED" + ).add(); + continue; + } + + archiveSizeInBytes += fileSize; + + if (archiveSizeInBytes < SAFE_QUOTA) { + // We save the index of the last ping which is ok to keep in order to speed up ping + // pruning. + lastPingIndexToKeep = i; + } else if (archiveSizeInBytes > Policy.getArchiveQuota()) { + // Ouch, our ping archive is too big. Bail out and start pruning! + break; + } + } + + // Save the time it takes to check if the archive is over-quota. + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS").add( + Math.round(Policy.now().getTime() - startTimeStamp) + ); + + let submitProbes = (sizeInMB, evictedPings, elapsedMs) => { + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add( + evictedPings + ); + Telemetry.getHistogramById( + "TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS" + ).add(elapsedMs); + }; + + // Check if we're using too much space. If not, submit the archive size and bail out. + if (archiveSizeInBytes < Policy.getArchiveQuota()) { + submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0); + return; + } + + this._log.info( + "_enforceArchiveQuota - archive size: " + + archiveSizeInBytes + + "bytes" + + ", safety quota: " + + SAFE_QUOTA + + "bytes" + ); + + startTimeStamp = Policy.now().getTime(); + let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1); + + // Remove all the pings older than the last one which we are safe to keep. + for (let ping of pingsToPurge) { + if (this._shutdown) { + this._log.trace( + "_enforceArchiveQuota - Terminating the clean up task due to shutdown" + ); + return; + } + + // This list is guaranteed to be in order, so remove the pings at its + // beginning (oldest). + await this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type); + } + + const endTimeStamp = Policy.now().getTime(); + submitProbes( + ARCHIVE_SIZE_PROBE_SPECIAL_VALUE, + pingsToPurge.length, + Math.ceil(endTimeStamp - startTimeStamp) + ); + }, + + async _cleanArchive() { + this._log.trace("cleanArchiveTask"); + + if (!(await OS.File.exists(gPingsArchivePath))) { + return; + } + + // Remove pings older than allowed. + try { + await this._purgeOldPings(); + } catch (ex) { + this._log.error( + "_cleanArchive - There was an error removing old directories", + ex + ); + } + + // Make sure we respect the archive disk quota. + await this._enforceArchiveQuota(); + }, + + /** + * Run the task to enforce the pending pings quota. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + async runEnforcePendingPingsQuotaTask() { + // If there's a cleaning task already running, return it. + if (this._enforcePendingPingsQuotaTask) { + return this._enforcePendingPingsQuotaTask; + } + + // Since there's no quota enforcing task running, start it. + try { + this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota(); + await this._enforcePendingPingsQuotaTask; + } finally { + this._enforcePendingPingsQuotaTask = null; + } + return undefined; + }, + + /** + * Enforce a disk quota for the pending pings. + * @return {Promise} Resolved when the quota check is complete. + */ + async _enforcePendingPingsQuota() { + this._log.trace("_enforcePendingPingsQuota"); + let startTimeStamp = Policy.now().getTime(); + + // Build an ordered list, from newer to older, of pending pings. + let pingList = Array.from(this._pendingPings, p => ({ + id: p[0], + lastModificationDate: p[1].lastModificationDate, + })); + + pingList.sort((a, b) => b.lastModificationDate - a.lastModificationDate); + + // If our pending pings directory is too big, we should reduce it to reach 90% of the quota. + const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9; + // The index of the last ping to keep. Pings older than this one will be deleted if + // the pending pings directory size exceeds the quota. + let lastPingIndexToKeep = null; + let pendingPingsSizeInBytes = 0; + + // Find the disk size of the pending pings directory. + for (let i = 0; i < pingList.length; i++) { + if (this._shutdown) { + this._log.trace( + "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown" + ); + return; + } + + let ping = pingList[i]; + + // Get the size for this ping. + const fileSize = await getPendingPingSize(ping.id); + if (!fileSize) { + this._log.warn( + "_enforcePendingPingsQuota - Unable to find the size of ping " + + ping.id + ); + continue; + } + + pendingPingsSizeInBytes += fileSize; + if (pendingPingsSizeInBytes < SAFE_QUOTA) { + // We save the index of the last ping which is ok to keep in order to speed up ping + // pruning. + lastPingIndexToKeep = i; + } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) { + // Ouch, our pending pings directory size is too big. Bail out and start pruning! + break; + } + } + + // Save the time it takes to check if the pending pings are over-quota. + Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS").add( + Math.round(Policy.now().getTime() - startTimeStamp) + ); + + let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => { + Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add( + sizeInMB + ); + Telemetry.getHistogramById( + "TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA" + ).add(evictedPings); + Telemetry.getHistogramById( + "TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS" + ).add(elapsedMs); + }; + + // Check if we're using too much space. If not, bail out. + if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) { + recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0); + return; + } + + this._log.info( + "_enforcePendingPingsQuota - size: " + + pendingPingsSizeInBytes + + "bytes" + + ", safety quota: " + + SAFE_QUOTA + + "bytes" + ); + + startTimeStamp = Policy.now().getTime(); + let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1); + + // Remove all the pings older than the last one which we are safe to keep. + for (let ping of pingsToPurge) { + if (this._shutdown) { + this._log.trace( + "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown" + ); + return; + } + + // This list is guaranteed to be in order, so remove the pings at its + // beginning (oldest). + await this.removePendingPing(ping.id); + } + + const endTimeStamp = Policy.now().getTime(); + // We don't know the size of the pending pings directory if we are above the quota, + // since we stop scanning once we reach the quota. We use a special value to show + // this condition. + recordHistograms( + PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE, + pingsToPurge.length, + Math.ceil(endTimeStamp - startTimeStamp) + ); + }, + + /** + * Reset the storage state in tests. + */ + reset() { + this._shutdown = false; + this._scannedArchiveDirectory = false; + this._archivedPings = new Map(); + this._scannedPendingDirectory = false; + this._pendingPings = new Map(); + }, + + /** + * Get a list of info on the archived pings. + * This will scan the archive directory and grab basic data about the existing + * pings out of their filename. + * + * @return {promise<sequence<object>>} + */ + async loadArchivedPingList() { + // If there's an archive loading task already running, return it. + if (this._scanArchiveTask) { + return this._scanArchiveTask; + } + + await waitForAll(this._activelyArchiving); + + if (this._scannedArchiveDirectory) { + this._log.trace( + "loadArchivedPingList - Archive already scanned, hitting cache." + ); + return this._archivedPings; + } + + // Since there's no archive loading task running, start it. + let result; + try { + this._scanArchiveTask = this._scanArchive(); + result = await this._scanArchiveTask; + } finally { + this._scanArchiveTask = null; + } + return result; + }, + + async _scanArchive() { + this._log.trace("_scanArchive"); + + let submitProbes = (pingCount, dirCount) => { + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").add( + pingCount + ); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT").add( + dirCount + ); + }; + + if (!(await OS.File.exists(gPingsArchivePath))) { + submitProbes(0, 0); + return new Map(); + } + + let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath); + let subdirs = (await dirIterator.nextBatch()) + .filter(e => e.isDir) + .filter(e => isValidArchiveDir(e.name)); + dirIterator.close(); + + // Walk through the monthly subdirs of the form <YYYY-MM>/ + for (let dir of subdirs) { + this._log.trace("_scanArchive - checking in subdir: " + dir.path); + let pingIterator = new OS.File.DirectoryIterator(dir.path); + let pings = (await pingIterator.nextBatch()).filter(e => !e.isDir); + pingIterator.close(); + + // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]". + for (let p of pings) { + // data may be null if the filename doesn't match the above format. + let data = this._getArchivedPingDataFromFileName(p.name); + if (!data) { + continue; + } + + // In case of conflicts, overwrite only with newer pings. + if (this._archivedPings.has(data.id)) { + const overwrite = + data.timestamp > this._archivedPings.get(data.id).timestampCreated; + this._log.warn( + "_scanArchive - have seen this id before: " + + data.id + + ", overwrite: " + + overwrite + ); + if (!overwrite) { + continue; + } + + await this._removeArchivedPing( + data.id, + data.timestampCreated, + data.type + ).catch(e => + this._log.warn("_scanArchive - failed to remove ping", e) + ); + } + + this._archivedPings.set(data.id, { + timestampCreated: data.timestamp, + type: internString(data.type), + }); + } + } + + // Mark the archive as scanned, so we no longer hit the disk. + this._scannedArchiveDirectory = true; + // Update the ping and directories count histograms. + submitProbes(this._archivedPings.size, subdirs.length); + return this._archivedPings; + }, + + /** + * Save a single ping to a file. + * + * @param {object} ping The content of the ping to save. + * @param {string} file The destination file. + * @param {bool} overwrite If |true|, the file will be overwritten if it exists, + * if |false| the file will not be overwritten and no error will be reported if + * the file exists. + * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no + * compression will be used. + * @returns {promise} + */ + async savePingToFile(ping, filePath, overwrite, compress = false) { + try { + this._log.trace("savePingToFile - path: " + filePath); + let pingString = JSON.stringify(ping); + let options = { tmpPath: filePath + ".tmp", noOverwrite: !overwrite }; + if (compress) { + options.compression = "lz4"; + } + await OS.File.writeAtomic(filePath, pingString, options); + } catch (e) { + if (!e.becauseExists) { + throw e; + } + } + }, + + /** + * Save a ping to its file. + * + * @param {object} ping The content of the ping to save. + * @param {bool} overwrite If |true|, the file will be overwritten + * if it exists. + * @returns {promise} + */ + async savePing(ping, overwrite) { + await getPingDirectory(); + let file = pingFilePath(ping); + await this.savePingToFile(ping, file, overwrite); + return file; + }, + + /** + * Add a ping to the saved pings directory so that it gets saved + * and sent along with other pings. + * Note: that the original ping file will not be modified. + * + * @param {Object} ping The ping object. + * @return {Promise} A promise resolved when the ping is saved to the pings directory. + */ + addPendingPing(ping) { + return this.savePendingPing(ping); + }, + + /** + * Remove the file for a ping + * + * @param {object} ping The ping. + * @returns {promise} + */ + cleanupPingFile(ping) { + return OS.File.remove(pingFilePath(ping)); + }, + + savePendingPing(ping) { + let p = this.savePing(ping, true).then(path => { + this._pendingPings.set(ping.id, { + path, + lastModificationDate: Policy.now().getTime(), + }); + this._log.trace("savePendingPing - saved ping with id " + ping.id); + }); + this._trackPendingPingSaveTask(p); + return p; + }, + + async loadPendingPing(id) { + this._log.trace("loadPendingPing - id: " + id); + let info = this._pendingPings.get(id); + if (!info) { + this._log.trace("loadPendingPing - unknown id " + id); + throw new Error( + "TelemetryStorage.loadPendingPing - no ping with id " + id + ); + } + + // Try to get the dimension of the ping. If that fails, update the histograms. + let fileSize = 0; + try { + fileSize = (await OS.File.stat(info.path)).size; + } catch (e) { + if (!(e instanceof OS.File.Error) || !e.becauseNoSuchFile) { + throw e; + } + // Fall through and let |loadPingFile| report the error. + } + + // Purge pings which are too big. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + await this.removePendingPing(id); + Telemetry.getHistogramById( + "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB" + ).add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add(); + + // Currently we don't have the ping type available without loading the ping from disk. + // Bug 1384903 will fix that. + TelemetryHealthPing.recordDiscardedPing("<unknown>"); + throw new Error( + "loadPendingPing - exceeded the maximum ping size: " + fileSize + ); + } + + // Try to load the ping file. Update the related histograms on failure. + let ping; + try { + ping = await this.loadPingFile(info.path, false); + } catch (e) { + // If we failed to load the ping, check what happened and update the histogram. + if (e instanceof PingReadError) { + Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add(); + } else if (e instanceof PingParseError) { + Telemetry.getHistogramById( + "TELEMETRY_PENDING_LOAD_FAILURE_PARSE" + ).add(); + } + + // Remove the ping from the cache, so we don't try to load it again. + this._pendingPings.delete(id); + // Then propagate the rejection. + throw e; + } + + return ping; + }, + + removePendingPing(id) { + let info = this._pendingPings.get(id); + if (!info) { + this._log.trace("removePendingPing - unknown id " + id); + return Promise.resolve(); + } + + this._log.trace( + "removePendingPing - deleting ping with id: " + + id + + ", path: " + + info.path + ); + this._pendingPings.delete(id); + return OS.File.remove(info.path).catch(ex => + this._log.error("removePendingPing - failed to remove ping", ex) + ); + }, + + /** + * Track any pending ping save tasks through the promise passed here. + * This is needed to block on any outstanding ping save activity. + * + * @param {Object<Promise>} The save promise to track. + */ + _trackPendingPingSaveTask(promise) { + let clear = () => this._activePendingPingSaves.delete(promise); + promise.then(clear, clear); + this._activePendingPingSaves.add(promise); + }, + + /** + * Return a promise that allows to wait on pending pings being saved. + * @return {Object<Promise>} A promise resolved when all the pending pings save promises + * are resolved. + */ + promisePendingPingSaves() { + // Make sure to wait for all the promises, even if they reject. We don't need to log + // the failures here, as they are already logged elsewhere. + return waitForAll(this._activePendingPingSaves); + }, + + /** + * Run the task to remove all the pending pings + * + * @return {Promise} Resolved when the pings are removed. + */ + async runRemovePendingPingsTask() { + // If we already have a pending pings removal task active, return that. + if (this._removePendingPingsTask) { + return this._removePendingPingsTask; + } + + // Start the task to remove all pending pings. Also make sure to clear the task once done. + try { + this._removePendingPingsTask = this.removePendingPings(); + await this._removePendingPingsTask; + } finally { + this._removePendingPingsTask = null; + } + return undefined; + }, + + async removePendingPings() { + this._log.trace("removePendingPings - removing all pending pings"); + + // Wait on pending pings still being saved, so so we don't miss removing them. + await this.promisePendingPingSaves(); + + // Individually remove existing pings, so we don't interfere with operations expecting + // the pending pings directory to exist. + const directory = TelemetryStorage.pingDirectoryPath; + let iter = new OS.File.DirectoryIterator(directory); + + try { + if (!(await iter.exists())) { + this._log.trace( + "removePendingPings - the pending pings directory doesn't exist" + ); + return; + } + + let files = (await iter.nextBatch()).filter(e => !e.isDir); + for (let file of files) { + try { + await OS.File.remove(file.path); + } catch (ex) { + this._log.error( + "removePendingPings - failed to remove file " + file.path, + ex + ); + continue; + } + } + } finally { + await iter.close(); + } + }, + + /** + * Iterate through all pings in the userApplicationDataDir under the "Pending Pings" sub-directory + * and yield each file. + */ + async *_iterateAppDataPings() { + this._log.trace("_iterateAppDataPings"); + + // The test suites might not create and define the "UAppData" directory. + // We account for that here instead of manually going through each test using + // telemetry to manually create the directory and define the constant. + if (!OS.Constants.Path.userApplicationDataDir) { + this._log.trace( + "_iterateAppDataPings - userApplicationDataDir is not defined. Is this a test?" + ); + return; + } + + const appDataPendingPings = OS.Path.join( + OS.Constants.Path.userApplicationDataDir, + "Pending Pings" + ); + + // Iterate through the pending ping files. + let iter = new OS.File.DirectoryIterator(appDataPendingPings); + try { + // Check if appDataPendingPings exists and bail out if it doesn't. + if (!(await iter.exists())) { + this._log.trace( + "_iterateAppDataPings - the AppData pending pings directory doesn't exist." + ); + return; + } + + let files = (await iter.nextBatch()).filter(e => !e.isDir); + for (let file of files) { + yield file; + } + } finally { + await iter.close(); + } + }, + + /** + * Remove all pings that are stored in the userApplicationDataDir + * under the "Pending Pings" sub-directory. + */ + async removeAppDataPings() { + this._log.trace("removeAppDataPings"); + + for await (const file of this._iterateAppDataPings()) { + try { + await OS.File.remove(file.path); + } catch (ex) { + this._log.error( + "removeAppDataPings - failed to remove file " + file.path, + ex + ); + } + } + }, + + /** + * Migrate pings that are stored in the userApplicationDataDir + * under the "Pending Pings" sub-directory. + */ + async _migrateAppDataPings() { + this._log.trace("_migrateAppDataPings"); + + for await (const file of this._iterateAppDataPings()) { + try { + // Load the ping data from the original file. + const pingData = await this.loadPingFile(file.path); + + // Save it among the pending pings in the user profile, overwrite on + // ping id collision. + await TelemetryStorage.savePing(pingData, true); + } catch (ex) { + this._log.error( + "_migrateAppDataPings - failed to load or migrate file. Removing. " + + file.path, + ex + ); + } + + try { + // Finally remove the file. + await OS.File.remove(file.path); + } catch (ex) { + this._log.error( + "_migrateAppDataPings - failed to remove file " + file.path, + ex + ); + } + } + }, + + loadPendingPingList() { + // If we already have a pending scanning task active, return that. + if (this._scanPendingPingsTask) { + return this._scanPendingPingsTask; + } + + if (this._scannedPendingDirectory) { + this._log.trace( + "loadPendingPingList - Pending already scanned, hitting cache." + ); + return Promise.resolve(this._buildPingList()); + } + + // Since there's no pending pings scan task running, start it. + // Also make sure to clear the task once done. + this._scanPendingPingsTask = this._scanPendingPings().then( + pings => { + this._scanPendingPingsTask = null; + return pings; + }, + ex => { + this._scanPendingPingsTask = null; + throw ex; + } + ); + return this._scanPendingPingsTask; + }, + + getPendingPingList() { + return this._buildPingList(); + }, + + async _scanPendingPings() { + this._log.trace("_scanPendingPings"); + + // Before pruning the pending pings, migrate over the ones from the user + // application data directory (mainly crash pings that failed to be sent). + await this._migrateAppDataPings(); + + let directory = TelemetryStorage.pingDirectoryPath; + let iter = new OS.File.DirectoryIterator(directory); + let exists = await iter.exists(); + + try { + if (!exists) { + return []; + } + + let files = (await iter.nextBatch()).filter(e => !e.isDir); + + for (let file of files) { + if (this._shutdown) { + return []; + } + + let info; + try { + info = await OS.File.stat(file.path); + } catch (ex) { + this._log.error( + "_scanPendingPings - failed to stat file " + file.path, + ex + ); + continue; + } + + // Enforce a maximum file size limit on pending pings. + if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error( + "_scanPendingPings - removing file exceeding size limit " + + file.path + ); + try { + await OS.File.remove(file.path); + } catch (ex) { + this._log.error( + "_scanPendingPings - failed to remove file " + file.path, + ex + ); + } finally { + Telemetry.getHistogramById( + "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB" + ).add(Math.floor(info.size / 1024 / 1024)); + Telemetry.getHistogramById( + "TELEMETRY_PING_SIZE_EXCEEDED_PENDING" + ).add(); + + // Currently we don't have the ping type available without loading the ping from disk. + // Bug 1384903 will fix that. + TelemetryHealthPing.recordDiscardedPing("<unknown>"); + } + continue; + } + + let id = OS.Path.basename(file.path); + if (!UUID_REGEX.test(id)) { + this._log.trace("_scanPendingPings - filename is not a UUID: " + id); + id = Utils.generateUUID(); + } + + this._pendingPings.set(id, { + path: file.path, + lastModificationDate: info.lastModificationDate.getTime(), + }); + } + } finally { + await iter.close(); + } + + this._scannedPendingDirectory = true; + return this._buildPingList(); + }, + + _buildPingList() { + const list = Array.from(this._pendingPings, p => ({ + id: p[0], + lastModificationDate: p[1].lastModificationDate, + })); + + list.sort((a, b) => b.lastModificationDate - a.lastModificationDate); + return list; + }, + + /** + * Loads a ping file. + * @param {String} aFilePath The path of the ping file. + * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4. + * @return {Promise<Object>} A promise resolved with the ping content or rejected if the + * ping contains invalid data. + * @throws {PingReadError} There was an error while reading the ping file from the disk. + * @throws {PingParseError} There was an error while parsing the JSON content of the ping file. + */ + async loadPingFile(aFilePath, aCompressed = false) { + let options = {}; + if (aCompressed) { + options.compression = "lz4"; + } + + let array; + try { + array = await OS.File.read(aFilePath, options); + } catch (e) { + this._log.trace("loadPingfile - unreadable ping " + aFilePath, e); + throw new PingReadError(e.message, e.becauseNoSuchFile); + } + + let decoder = new TextDecoder(); + let string = decoder.decode(array); + let ping; + try { + ping = JSON.parse(string); + } catch (e) { + this._log.trace("loadPingfile - unparseable ping " + aFilePath, e); + await OS.File.remove(aFilePath).catch(ex => { + this._log.error( + "loadPingFile - failed removing unparseable ping file", + ex + ); + }); + throw new PingParseError(e.message); + } + + return ping; + }, + + /** + * Archived pings are saved with file names of the form: + * "<timestamp>.<uuid>.<type>.[json|jsonlz4]" + * This helper extracts that data from a given filename. + * + * @param fileName {String} The filename. + * @return {Object} Null if the filename didn't match the expected form. + * Otherwise an object with the extracted data in the form: + * { timestamp: <number>, + * id: <string>, + * type: <string> } + */ + _getArchivedPingDataFromFileName(fileName) { + // Extract the parts. + let parts = fileName.split("."); + if (parts.length != 4) { + this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts"); + return null; + } + + let [timestamp, uuid, type, extension] = parts; + if (extension != "json" && extension != "jsonlz4") { + this._log.trace( + "_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension" + ); + return null; + } + + // Check for a valid timestamp. + timestamp = parseInt(timestamp); + if (Number.isNaN(timestamp)) { + this._log.trace( + "_getArchivedPingDataFromFileName - should have a valid timestamp" + ); + return null; + } + + // Check for a valid UUID. + if (!UUID_REGEX.test(uuid)) { + this._log.trace( + "_getArchivedPingDataFromFileName - should have a valid id" + ); + return null; + } + + // Check for a valid type string. + const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i; + if (!typeRegex.test(type)) { + this._log.trace( + "_getArchivedPingDataFromFileName - should have a valid type" + ); + return null; + } + + return { + timestamp, + id: uuid, + type, + }; + }, + + async saveAbortedSessionPing(ping) { + this._log.trace( + "saveAbortedSessionPing - ping path: " + gAbortedSessionFilePath + ); + await OS.File.makeDir(gDataReportingDir, { ignoreExisting: true }); + + return this._abortedSessionSerializer.enqueueTask(() => + this.savePingToFile(ping, gAbortedSessionFilePath, true) + ); + }, + + async loadAbortedSessionPing() { + let ping = null; + try { + ping = await this.loadPingFile(gAbortedSessionFilePath); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("loadAbortedSessionPing - no such file"); + } else { + this._log.error("loadAbortedSessionPing - error loading ping", ex); + } + } + return ping; + }, + + removeAbortedSessionPing() { + return this._abortedSessionSerializer.enqueueTask(async () => { + try { + await OS.File.remove(gAbortedSessionFilePath, { ignoreAbsent: false }); + this._log.trace("removeAbortedSessionPing - success"); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("removeAbortedSessionPing - no such file"); + } else { + this._log.error("removeAbortedSessionPing - error removing ping", ex); + } + } + }); + }, + + async saveUninstallPing(ping) { + if (AppConstants.platform != "win") { + return; + } + + // Remove any old pings from this install first. + await this.removeUninstallPings(); + + let { directory: pingFile, file } = Policy.getUninstallPingPath(ping.id); + pingFile.append(file); + + await this.savePingToFile(ping, pingFile.path, /* overwrite */ true); + }, + + async removeUninstallPings() { + if (AppConstants.platform != "win") { + return; + } + + const { directory, file } = Policy.getUninstallPingPath("*"); + + const iteratorOptions = { winPattern: file }; + const iterator = new OS.File.DirectoryIterator( + directory.path, + iteratorOptions + ); + + await iterator.forEach(async entry => { + this._log.trace("removeUninstallPings - removing", entry.path); + try { + await OS.File.remove(entry.path); + this._log.trace("removeUninstallPings - success"); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("removeUninstallPings - no such file"); + } else { + this._log.error("removeUninstallPings - error removing ping", ex); + } + } + }); + iterator.close(); + }, + + /** + * Remove FHR database files. This is temporary and will be dropped in + * the future. + * @return {Promise} Resolved when the database files are deleted. + */ + async removeFHRDatabase() { + this._log.trace("removeFHRDatabase"); + + // Let's try to remove the FHR DB with the default filename first. + const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite"; + + // Even if it's uncommon, there may be 2 additional files: - a "write ahead log" + // (-wal) file and a "shared memory file" (-shm). We need to remove them as well. + let FILES_TO_REMOVE = [ + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME), + OS.Path.join( + OS.Constants.Path.profileDir, + FHR_DB_DEFAULT_FILENAME + "-wal" + ), + OS.Path.join( + OS.Constants.Path.profileDir, + FHR_DB_DEFAULT_FILENAME + "-shm" + ), + ]; + + // FHR could have used either the default DB file name or a custom one + // through this preference. + const FHR_DB_CUSTOM_FILENAME = Preferences.get( + "datareporting.healthreport.dbName", + undefined + ); + if (FHR_DB_CUSTOM_FILENAME) { + FILES_TO_REMOVE.push( + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME), + OS.Path.join( + OS.Constants.Path.profileDir, + FHR_DB_CUSTOM_FILENAME + "-wal" + ), + OS.Path.join( + OS.Constants.Path.profileDir, + FHR_DB_CUSTOM_FILENAME + "-shm" + ) + ); + } + + for (let f of FILES_TO_REMOVE) { + await OS.File.remove(f, { ignoreAbsent: true }).catch(e => + this._log.error("removeFHRDatabase - failed to remove " + f, e) + ); + } + }, +}; + +// Utility functions + +function pingFilePath(ping) { + // Support legacy ping formats, who don't have an "id" field, but a "slug" field. + let pingIdentifier = ping.slug ? ping.slug : ping.id; + return OS.Path.join(TelemetryStorage.pingDirectoryPath, pingIdentifier); +} + +function getPingDirectory() { + return (async function() { + let directory = TelemetryStorage.pingDirectoryPath; + + if (!(await OS.File.exists(directory))) { + await OS.File.makeDir(directory, { unixMode: OS.Constants.S_IRWXU }); + } + + return directory; + })(); +} + +/** + * Build the path to the archived ping. + * @param {String} aPingId The ping id. + * @param {Object} aDate The ping creation date. + * @param {String} aType The ping type. + * @return {String} The full path to the archived ping. + */ +function getArchivedPingPath(aPingId, aDate, aType) { + // Get the ping creation date and generate the archive directory to hold it. Note + // that getMonth returns a 0-based month, so we need to add an offset. + let month = String(aDate.getMonth() + 1); + let archivedPingDir = OS.Path.join( + gPingsArchivePath, + aDate.getFullYear() + "-" + month.padStart(2, "0") + ); + // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json + let fileName = [aDate.getTime(), aPingId, aType, "json"].join("."); + return OS.Path.join(archivedPingDir, fileName); +} + +/** + * Get the size of the ping file on the disk. + * @return {Integer} The file size, in bytes, of the ping file or 0 on errors. + */ +var getArchivedPingSize = async function(aPingId, aDate, aType) { + const path = getArchivedPingPath(aPingId, aDate, aType); + let filePaths = [path + "lz4", path]; + + for (let path of filePaths) { + try { + return (await OS.File.stat(path)).size; + } catch (e) {} + } + + // That's odd, this ping doesn't seem to exist. + return 0; +}; + +/** + * Get the size of the pending ping file on the disk. + * @return {Integer} The file size, in bytes, of the ping file or 0 on errors. + */ +var getPendingPingSize = async function(aPingId) { + const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, aPingId); + try { + return (await OS.File.stat(path)).size; + } catch (e) {} + + // That's odd, this ping doesn't seem to exist. + return 0; +}; + +/** + * Check if a directory name is in the "YYYY-MM" format. + * @param {String} aDirName The name of the pings archive directory. + * @return {Boolean} True if the directory name is in the right format, false otherwise. + */ +function isValidArchiveDir(aDirName) { + const dirRegEx = /^[0-9]{4}-[0-9]{2}$/; + return dirRegEx.test(aDirName); +} + +/** + * Gets a date object from an archive directory name. + * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM + * format. + * @return {Object} A Date object or null if the dir name is not valid. + */ +function getDateFromArchiveDir(aDirName) { + let [year, month] = aDirName.split("-"); + year = parseInt(year); + month = parseInt(month); + // Make sure to have sane numbers. + if ( + !Number.isFinite(month) || + !Number.isFinite(year) || + month < 1 || + month > 12 + ) { + return null; + } + return new Date(year, month - 1, 1, 0, 0, 0); +} diff --git a/toolkit/components/telemetry/app/TelemetryTimestamps.jsm b/toolkit/components/telemetry/app/TelemetryTimestamps.jsm new file mode 100644 index 0000000000..b0cc72a725 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryTimestamps.jsm @@ -0,0 +1,55 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var EXPORTED_SYMBOLS = ["TelemetryTimestamps"]; + +/** + * This module's purpose is to collect timestamps for important + * application-specific events. + * + * The TelemetryController component attaches the timestamps stored by this module to + * the telemetry submission, substracting the process lifetime so that the times + * are relative to process startup. The overall goal is to produce a basic + * timeline of the startup process. + */ +var timeStamps = {}; + +var TelemetryTimestamps = { + /** + * Adds a timestamp to the list. The addition of TimeStamps that already have + * a value stored is ignored. + * + * @param name must be a unique, generally "camelCase" descriptor of what the + * timestamp represents. e.g.: "delayedStartupStarted" + * @param value is a timeStamp in milliseconds since the epoch. If omitted, + * defaults to Date.now(). + */ + add: function TT_add(name, value) { + // Default to "now" if not specified + if (value == null) { + value = Date.now(); + } + + if (isNaN(value)) { + throw new Error("Value must be a timestamp"); + } + + // If there's an existing value, just ignore the new value. + if (timeStamps.hasOwnProperty(name)) { + return; + } + + timeStamps[name] = value; + }, + + /** + * Returns a JS object containing all of the timeStamps as properties (can be + * easily serialized to JSON). Used by TelemetryController to retrieve the data + * to attach to the telemetry submission. + */ + get: function TT_get() { + // Return a copy of the object. + return Cu.cloneInto(timeStamps, {}); + }, +}; diff --git a/toolkit/components/telemetry/app/TelemetryUtils.jsm b/toolkit/components/telemetry/app/TelemetryUtils.jsm new file mode 100644 index 0000000000..5788a3d269 --- /dev/null +++ b/toolkit/components/telemetry/app/TelemetryUtils.jsm @@ -0,0 +1,301 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["TelemetryUtils"]; + +ChromeUtils.import("resource://gre/modules/Services.jsm", this); +const { TelemetryControllerBase } = ChromeUtils.import( + "resource://gre/modules/TelemetryControllerBase.jsm" +); +ChromeUtils.defineModuleGetter( + this, + "UpdateUtils", + "resource://gre/modules/UpdateUtils.jsm" +); + +const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; + +const IS_CONTENT_PROCESS = (function() { + // We cannot use Services.appinfo here because in telemetry xpcshell tests, + // appinfo is initially unavailable, and becomes available only later on. + // eslint-disable-next-line mozilla/use-services + let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime); + return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT; +})(); + +var TelemetryUtils = { + /** + * When telemetry is disabled, identifying information (such as client ID) + * should be removed. A topic event is emitted with a subject that matches + * this constant. When this happens, other systems that store identifying + * information about the client should delete that data. Please ask the + * Firefox Telemetry Team before relying on this topic. + * + * Here is an example of listening for that event: + * + * const { TelemetryUtils } = ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm"); + * + * class YourClass { + * constructor() { + * Services.obs.addObserver(this, TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC); + * } + * + * observe(subject, topic, data) { + * if (topic == TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC) { + * // Telemetry was disabled + * // subject and data are both unused + * } + * } + * } + */ + TELEMETRY_UPLOAD_DISABLED_TOPIC: "telemetry.upload.disabled", + + Preferences: Object.freeze({ + ...TelemetryControllerBase.Preferences, + + // General Preferences + ArchiveEnabled: "toolkit.telemetry.archive.enabled", + CachedClientId: "toolkit.telemetry.cachedClientID", + DisableFuzzingDelay: "toolkit.telemetry.testing.disableFuzzingDelay", + FirstRun: "toolkit.telemetry.reportingpolicy.firstRun", + FirstShutdownPingEnabled: "toolkit.telemetry.firstShutdownPing.enabled", + HealthPingEnabled: "toolkit.telemetry.healthping.enabled", + IPCBatchTimeout: "toolkit.telemetry.ipcBatchTimeout", + OverrideOfficialCheck: "toolkit.telemetry.send.overrideOfficialCheck", + OverrideUpdateChannel: "toolkit.telemetry.overrideUpdateChannel", + Server: "toolkit.telemetry.server", + ShutdownPingSender: "toolkit.telemetry.shutdownPingSender.enabled", + ShutdownPingSenderFirstSession: + "toolkit.telemetry.shutdownPingSender.enabledFirstSession", + TelemetryEnabled: "toolkit.telemetry.enabled", + UntrustedModulesPingFrequency: + "toolkit.telemetry.untrustedModulesPing.frequency", + UpdatePing: "toolkit.telemetry.updatePing.enabled", + NewProfilePingEnabled: "toolkit.telemetry.newProfilePing.enabled", + NewProfilePingDelay: "toolkit.telemetry.newProfilePing.delay", + PreviousBuildID: "toolkit.telemetry.previousBuildID", + + // Event Ping Preferences + EventPingMinimumFrequency: "toolkit.telemetry.eventping.minimumFrequency", + EventPingMaximumFrequency: "toolkit.telemetry.eventping.maximumFrequency", + + // Ecosystem Telemetry Preferences + EcosystemTelemetryEnabled: "toolkit.telemetry.ecosystemtelemetry.enabled", + EcosystemTelemetryAllowForNonProductionFxA: + "toolkit.telemetry.ecosystemtelemetry.allowForNonProductionFxA", + + // Prio Ping Preferences + PrioPingEnabled: "toolkit.telemetry.prioping.enabled", + PrioPingDataLimit: "toolkit.telemetry.prioping.dataLimit", + + // Data reporting Preferences + AcceptedPolicyDate: "datareporting.policy.dataSubmissionPolicyNotifiedTime", + AcceptedPolicyVersion: + "datareporting.policy.dataSubmissionPolicyAcceptedVersion", + BypassNotification: + "datareporting.policy.dataSubmissionPolicyBypassNotification", + CurrentPolicyVersion: "datareporting.policy.currentPolicyVersion", + DataSubmissionEnabled: "datareporting.policy.dataSubmissionEnabled", + FhrUploadEnabled: "datareporting.healthreport.uploadEnabled", + MinimumPolicyVersion: "datareporting.policy.minimumPolicyVersion", + FirstRunURL: "datareporting.policy.firstRunURL", + }), + + /** + * A fixed valid client ID used when Telemetry upload is disabled. + */ + get knownClientID() { + return "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"; + }, + + /** + * True if this is a content process. + */ + get isContentProcess() { + return IS_CONTENT_PROCESS; + }, + + /** + * Returns the state of the Telemetry enabled preference, making sure + * it correctly evaluates to a boolean type. + */ + get isTelemetryEnabled() { + return TelemetryControllerBase.isTelemetryEnabled; + }, + + /** + * Turn a millisecond timestamp into a day timestamp. + * + * @param aMsec A number of milliseconds since Unix epoch. + * @return The number of whole days since Unix epoch. + */ + millisecondsToDays(aMsec) { + return Math.floor(aMsec / MILLISECONDS_PER_DAY); + }, + + /** + * Takes a date and returns it truncated to a date with daily precision. + */ + truncateToDays(date) { + return new Date( + date.getFullYear(), + date.getMonth(), + date.getDate(), + 0, + 0, + 0, + 0 + ); + }, + + /** + * Takes a date and returns it truncated to a date with hourly precision. + */ + truncateToHours(date) { + return new Date( + date.getFullYear(), + date.getMonth(), + date.getDate(), + date.getHours(), + 0, + 0, + 0 + ); + }, + + /** + * Check if the difference between the times is within the provided tolerance. + * @param {Number} t1 A time in milliseconds. + * @param {Number} t2 A time in milliseconds. + * @param {Number} tolerance The tolerance, in milliseconds. + * @return {Boolean} True if the absolute time difference is within the tolerance, false + * otherwise. + */ + areTimesClose(t1, t2, tolerance) { + return Math.abs(t1 - t2) <= tolerance; + }, + + /** + * Get the next midnight for a date. + * @param {Object} date The date object to check. + * @return {Object} The Date object representing the next midnight. + */ + getNextMidnight(date) { + let nextMidnight = new Date(this.truncateToDays(date)); + nextMidnight.setDate(nextMidnight.getDate() + 1); + return nextMidnight; + }, + + /** + * Get the midnight which is closer to the provided date. + * @param {Object} date The date object to check. + * @param {Number} tolerance The tolerance within we find the closest midnight. + * @return {Object} The Date object representing the closes midnight, or null if midnight + * is not within the midnight tolerance. + */ + getNearestMidnight(date, tolerance) { + let lastMidnight = this.truncateToDays(date); + if (this.areTimesClose(date.getTime(), lastMidnight.getTime(), tolerance)) { + return lastMidnight; + } + + const nextMidnightDate = this.getNextMidnight(date); + if ( + this.areTimesClose(date.getTime(), nextMidnightDate.getTime(), tolerance) + ) { + return nextMidnightDate; + } + return null; + }, + + generateUUID() { + let str = Cc["@mozilla.org/uuid-generator;1"] + .getService(Ci.nsIUUIDGenerator) + .generateUUID() + .toString(); + // strip {} + return str.substring(1, str.length - 1); + }, + + /** + * Find how many months passed between two dates. + * @param {Object} aStartDate The starting date. + * @param {Object} aEndDate The ending date. + * @return {Integer} The number of months between the two dates. + */ + getElapsedTimeInMonths(aStartDate, aEndDate) { + return ( + aEndDate.getMonth() - + aStartDate.getMonth() + + 12 * (aEndDate.getFullYear() - aStartDate.getFullYear()) + ); + }, + + /** + * Date.toISOString() gives us UTC times, this gives us local times in + * the ISO date format. See http://www.w3.org/TR/NOTE-datetime + * @param {Object} date The input date. + * @return {String} The local time ISO string. + */ + toLocalTimeISOString(date) { + function padNumber(number, length) { + return number.toString().padStart(length, "0"); + } + + let sign = n => (n >= 0 ? "+" : "-"); + // getTimezoneOffset counter-intuitively returns -60 for UTC+1. + let tzOffset = -date.getTimezoneOffset(); + + // YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00) + return ( + padNumber(date.getFullYear(), 4) + + "-" + + padNumber(date.getMonth() + 1, 2) + + "-" + + padNumber(date.getDate(), 2) + + "T" + + padNumber(date.getHours(), 2) + + ":" + + padNumber(date.getMinutes(), 2) + + ":" + + padNumber(date.getSeconds(), 2) + + "." + + date.getMilliseconds() + + sign(tzOffset) + + padNumber(Math.floor(Math.abs(tzOffset / 60)), 2) + + ":" + + padNumber(Math.abs(tzOffset % 60), 2) + ); + }, + + /** + * @returns {number} The monotonic time since the process start + * or (non-monotonic) Date value if this fails back. + */ + monotonicNow() { + return Services.telemetry.msSinceProcessStart(); + }, + + /** + * @returns {string} The name of the update channel to report + * in telemetry. + * By default, this is the same as the name of the channel that + * the browser uses to download its updates. However in certain + * situations, a single update channel provides multiple (distinct) + * build types, that need to be distinguishable on Telemetry. + */ + getUpdateChannel() { + let overrideChannel = Services.prefs.getCharPref( + this.Preferences.OverrideUpdateChannel, + undefined + ); + if (overrideChannel) { + return overrideChannel; + } + + return UpdateUtils.getUpdateChannel(false); + }, +}; |