summaryrefslogtreecommitdiffstats
path: root/toolkit/components/telemetry/app
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
commit2aa4a82499d4becd2284cdb482213d541b8804dd (patch)
treeb80bf8bf13c3766139fbacc530efd0dd9d54394c /toolkit/components/telemetry/app
parentInitial commit. (diff)
downloadfirefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz
firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'toolkit/components/telemetry/app')
-rw-r--r--toolkit/components/telemetry/app/ClientID.jsm498
-rw-r--r--toolkit/components/telemetry/app/TelemetryArchive.jsm123
-rw-r--r--toolkit/components/telemetry/app/TelemetryController.jsm45
-rw-r--r--toolkit/components/telemetry/app/TelemetryControllerBase.jsm143
-rw-r--r--toolkit/components/telemetry/app/TelemetryControllerContent.jsm108
-rw-r--r--toolkit/components/telemetry/app/TelemetryControllerParent.jsm1450
-rw-r--r--toolkit/components/telemetry/app/TelemetryEnvironment.jsm2128
-rw-r--r--toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm584
-rw-r--r--toolkit/components/telemetry/app/TelemetryScheduler.jsm434
-rw-r--r--toolkit/components/telemetry/app/TelemetrySend.jsm1600
-rw-r--r--toolkit/components/telemetry/app/TelemetryStorage.jsm2221
-rw-r--r--toolkit/components/telemetry/app/TelemetryTimestamps.jsm55
-rw-r--r--toolkit/components/telemetry/app/TelemetryUtils.jsm301
13 files changed, 9690 insertions, 0 deletions
diff --git a/toolkit/components/telemetry/app/ClientID.jsm b/toolkit/components/telemetry/app/ClientID.jsm
new file mode 100644
index 0000000000..43d95ab9b8
--- /dev/null
+++ b/toolkit/components/telemetry/app/ClientID.jsm
@@ -0,0 +1,498 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["ClientID"];
+
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+const { AppConstants } = ChromeUtils.import(
+ "resource://gre/modules/AppConstants.jsm"
+);
+if (AppConstants.MOZ_GLEAN) {
+ Cu.importGlobalProperties(["Glean"]);
+}
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "ClientID::";
+// Must match ID in TelemetryUtils
+const CANARY_CLIENT_ID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0";
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "CommonUtils",
+ "resource://services-common/utils.js"
+);
+
+XPCOMUtils.defineLazyGetter(this, "CryptoHash", () => {
+ return Components.Constructor(
+ "@mozilla.org/security/hash;1",
+ "nsICryptoHash",
+ "initWithString"
+ );
+});
+
+XPCOMUtils.defineLazyGetter(this, "gDatareportingPath", () => {
+ return PathUtils.join(
+ Services.dirsvc.get("ProfD", Ci.nsIFile).path,
+ "datareporting"
+ );
+});
+
+XPCOMUtils.defineLazyGetter(this, "gStateFilePath", () => {
+ return PathUtils.join(gDatareportingPath, "state.json");
+});
+
+const PREF_CACHED_CLIENTID = "toolkit.telemetry.cachedClientID";
+
+const SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID =
+ "deletion.request.ecosystem_client_id";
+
+/**
+ * Checks if client ID has a valid format.
+ *
+ * @param {String} id A string containing the client ID.
+ * @return {Boolean} True when the client ID has valid format, or False
+ * otherwise.
+ */
+function isValidClientID(id) {
+ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+ return UUID_REGEX.test(id);
+}
+
+var ClientID = Object.freeze({
+ /**
+ * This returns a promise resolving to the the stable client ID we use for
+ * data reporting (FHR & Telemetry).
+ *
+ * WARNING: This functionality is duplicated for Android (see GeckoProfile.getClientId
+ * for more). There are Java tests (TestGeckoProfile) to ensure the functionality is
+ * consistent and Gecko tests to come (bug 1249156). However, THIS IS NOT FOOLPROOF.
+ * Be careful when changing this code and, in particular, the underlying file format.
+ *
+ * @return {Promise<string>} The stable client ID.
+ */
+ getClientID() {
+ return ClientIDImpl.getClientID();
+ },
+
+ /**
+ * Returns a promise resolving to the ecosystem client ID, used in ecosystem
+ * pings as a stable identifier for this profile.
+ *
+ * @return {Promise<string>} The ecosystem client ID.
+ */
+ getEcosystemClientID() {
+ return ClientIDImpl.getEcosystemClientID();
+ },
+
+ /**
+ * This returns true if the client ID prior to the last client ID reset was a canary client ID.
+ * Android only. Always returns null on Desktop.
+ */
+ wasCanaryClientID() {
+ if (AppConstants.platform == "android") {
+ return ClientIDImpl.wasCanaryClientID();
+ }
+
+ return null;
+ },
+
+ /**
+ * Get the client id synchronously without hitting the disk.
+ * This returns:
+ * - the current on-disk client id if it was already loaded
+ * - the client id that we cached into preferences (if any)
+ * - null otherwise
+ */
+ getCachedClientID() {
+ return ClientIDImpl.getCachedClientID();
+ },
+
+ /**
+ * Gets the in-memory cached ecosystem client ID if it was already loaded;
+ * `null` otherwise.
+ */
+ getCachedEcosystemClientID() {
+ return ClientIDImpl.getCachedEcosystemClientID();
+ },
+
+ async getClientIdHash() {
+ return ClientIDImpl.getClientIdHash();
+ },
+
+ /**
+ * Sets the main and ecosystem client IDs to the canary (known) client ID,
+ * writing them to disk and updating the cached versions.
+ *
+ * Use `removeClientIDs` followed by `get{Ecosystem}ClientID` to clear the
+ * existing IDs and generate new, random ones if required.
+ *
+ * @return {Promise<void>}
+ */
+ setCanaryClientIDs() {
+ return ClientIDImpl.setCanaryClientIDs();
+ },
+
+ /**
+ * Sets the ecosystem client IDs to a new random value while leaving other IDs
+ * unchanged, writing the result to disk and updating the cached identifier.
+ * This can be used when a user signs out, to avoid linking telemetry between
+ * different accounts.
+ *
+ * Use `removeClientIDs` followed by `get{Ecosystem}ClientID` to reset *all* the
+ * identifiers rather than just the ecosystem client id.
+ *
+ * @return {Promise<void>} Resolves when the change has been saved to disk.
+ */
+ resetEcosystemClientID() {
+ return ClientIDImpl.resetEcosystemClientID();
+ },
+
+ /**
+ * Clears the main and ecosystem client IDs asynchronously, removing them
+ * from disk. Use `getClientID()` and `getEcosystemClientID()` to generate
+ * fresh IDs after calling this method.
+ *
+ * Should only be used if a reset is explicitly requested by the user.
+ *
+ * @return {Promise<void>}
+ */
+ removeClientIDs() {
+ return ClientIDImpl.removeClientIDs();
+ },
+
+ /**
+ * Only used for testing. Invalidates the cached client IDs so that they're
+ * read again from file, but doesn't remove the existing IDs from disk.
+ */
+ _reset() {
+ return ClientIDImpl._reset();
+ },
+});
+
+var ClientIDImpl = {
+ _clientID: null,
+ _clientIDHash: null,
+ _ecosystemClientID: null,
+ _loadClientIdsTask: null,
+ _saveClientIdsTask: null,
+ _removeClientIdsTask: null,
+ _logger: null,
+ _wasCanary: null,
+
+ _loadClientIDs() {
+ if (this._loadClientIdsTask) {
+ return this._loadClientIdsTask;
+ }
+
+ this._loadClientIdsTask = this._doLoadClientIDs();
+ let clear = () => (this._loadClientIdsTask = null);
+ this._loadClientIdsTask.then(clear, clear);
+ return this._loadClientIdsTask;
+ },
+
+ /**
+ * Load the client IDs (Telemetry Client ID and Ecosystem Client ID) from the
+ * DataReporting Service state file. If either ID is missing, we generate a
+ * new one.
+ */
+ async _doLoadClientIDs() {
+ this._log.trace(`_doLoadClientIDs`);
+ // If there's a removal in progress, let's wait for it
+ await this._removeClientIdsTask;
+
+ // Try to load the client id from the DRS state file.
+ let hasCurrentClientID = false;
+ let hasCurrentEcosystemClientID = false;
+ try {
+ let state = await CommonUtils.readJSON(gStateFilePath);
+ if (AppConstants.platform == "android" && state && "wasCanary" in state) {
+ this._wasCanary = state.wasCanary;
+ }
+ if (state) {
+ hasCurrentClientID = this.updateClientID(state.clientID);
+ hasCurrentEcosystemClientID = this.updateEcosystemClientID(
+ state.ecosystemClientID
+ );
+ if (hasCurrentClientID && hasCurrentEcosystemClientID) {
+ this._log.trace(`_doLoadClientIDs: Client IDs loaded from state.`);
+ return {
+ clientID: this._clientID,
+ ecosystemClientID: this._ecosystemClientID,
+ };
+ }
+ }
+ } catch (e) {
+ // fall through to next option
+ }
+
+ // We're missing one or both IDs from the DRS state file, generate new ones.
+ if (!hasCurrentClientID) {
+ this.updateClientID(CommonUtils.generateUUID());
+ }
+ if (!hasCurrentEcosystemClientID) {
+ this.updateEcosystemClientID(CommonUtils.generateUUID());
+ }
+ this._saveClientIdsTask = this._saveClientIDs();
+
+ // Wait on persisting the id. Otherwise failure to save the ID would result in
+ // the client creating and subsequently sending multiple IDs to the server.
+ // This would appear as multiple clients submitting similar data, which would
+ // result in orphaning.
+ await this._saveClientIdsTask;
+
+ this._log.trace("_doLoadClientIDs: New client IDs loaded and persisted.");
+ return {
+ clientID: this._clientID,
+ ecosystemClientID: this._ecosystemClientID,
+ };
+ },
+
+ /**
+ * Save the client ID to the client ID file.
+ *
+ * @return {Promise} A promise resolved when the client ID is saved to disk.
+ */
+ async _saveClientIDs() {
+ this._log.trace(`_saveClientIDs`);
+ let obj = {
+ clientID: this._clientID,
+ ecosystemClientID: this._ecosystemClientID,
+ };
+ // We detected a canary client ID when resetting, storing this as a flag
+ if (AppConstants.platform == "android" && this._wasCanary) {
+ obj.wasCanary = true;
+ }
+ try {
+ await IOUtils.makeDirectory(gDatareportingPath);
+ } catch (ex) {
+ if (ex.name != "NotAllowedError") {
+ throw ex;
+ }
+ }
+ await CommonUtils.writeJSON(obj, gStateFilePath);
+ this._saveClientIdsTask = null;
+ },
+
+ /**
+ * This returns a promise resolving to the the stable client ID we use for
+ * data reporting (FHR & Telemetry).
+ *
+ * @return {Promise<string>} The stable client ID.
+ */
+ async getClientID() {
+ if (!this._clientID) {
+ let { clientID } = await this._loadClientIDs();
+ return clientID;
+ }
+
+ return Promise.resolve(this._clientID);
+ },
+
+ async getEcosystemClientID() {
+ if (!this._ecosystemClientID) {
+ let { ecosystemClientID } = await this._loadClientIDs();
+ return ecosystemClientID;
+ }
+
+ return Promise.resolve(this._ecosystemClientID);
+ },
+
+ /**
+ * This returns true if the client ID prior to the last client ID reset was a canary client ID.
+ * Android only. Always returns null on Desktop.
+ */
+ wasCanaryClientID() {
+ return this._wasCanary;
+ },
+
+ /**
+ * Get the client id synchronously without hitting the disk.
+ * This returns:
+ * - the current on-disk client id if it was already loaded
+ * - the client id that we cached into preferences (if any)
+ * - null otherwise
+ */
+ getCachedClientID() {
+ if (this._clientID) {
+ // Already loaded the client id from disk.
+ return this._clientID;
+ }
+
+ // If the client id cache contains a value of the wrong type,
+ // reset the pref. We need to do this before |getStringPref| since
+ // it will just return |null| in that case and we won't be able
+ // to distinguish between the missing pref and wrong type cases.
+ if (
+ Services.prefs.prefHasUserValue(PREF_CACHED_CLIENTID) &&
+ Services.prefs.getPrefType(PREF_CACHED_CLIENTID) !=
+ Ci.nsIPrefBranch.PREF_STRING
+ ) {
+ this._log.error(
+ "getCachedClientID - invalid client id type in preferences, resetting"
+ );
+ Services.prefs.clearUserPref(PREF_CACHED_CLIENTID);
+ }
+
+ // Not yet loaded, return the cached client id if we have one.
+ let id = Services.prefs.getStringPref(PREF_CACHED_CLIENTID, null);
+ if (id === null) {
+ return null;
+ }
+ if (!isValidClientID(id)) {
+ this._log.error(
+ "getCachedClientID - invalid client id in preferences, resetting",
+ id
+ );
+ Services.prefs.clearUserPref(PREF_CACHED_CLIENTID);
+ return null;
+ }
+ return id;
+ },
+
+ getCachedEcosystemClientID() {
+ return this._ecosystemClientID;
+ },
+
+ async getClientIdHash() {
+ if (!this._clientIDHash) {
+ let byteArr = new TextEncoder().encode(await this.getClientID());
+ let hash = new CryptoHash("sha256");
+ hash.update(byteArr, byteArr.length);
+ this._clientIDHash = CommonUtils.bytesAsHex(hash.finish(false));
+ }
+ return this._clientIDHash;
+ },
+
+ /*
+ * Resets the provider. This is for testing only.
+ */
+ async _reset() {
+ await this._loadClientIdsTask;
+ await this._saveClientIdsTask;
+ this._clientID = null;
+ this._clientIDHash = null;
+ this._ecosystemClientID = null;
+ },
+
+ async setCanaryClientIDs() {
+ this._log.trace("setCanaryClientIDs");
+ this.updateClientID(CANARY_CLIENT_ID);
+ this.updateEcosystemClientID(CANARY_CLIENT_ID);
+
+ this._saveClientIdsTask = this._saveClientIDs();
+ await this._saveClientIdsTask;
+ return this._clientID;
+ },
+
+ async resetEcosystemClientID() {
+ this._log.trace("resetEcosystemClientID");
+ this.updateEcosystemClientID(CommonUtils.generateUUID());
+ this._saveClientIdsTask = this._saveClientIDs();
+ await this._saveClientIdsTask;
+ return this._ecosystemClientID;
+ },
+
+ async _doRemoveClientIDs() {
+ this._log.trace("_doRemoveClientIDs");
+
+ // Reset the cached main and ecosystem client IDs.
+ this._clientID = null;
+ this._clientIDHash = null;
+ this._ecosystemClientID = null;
+
+ // Clear the client id from the preference cache.
+ Services.prefs.clearUserPref(PREF_CACHED_CLIENTID);
+
+ // Clear the old ecosystem client ID from the deletion request scalar store.
+ Services.telemetry.scalarSet(
+ SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID,
+ ""
+ );
+
+ // If there is a save in progress, wait for it to complete.
+ await this._saveClientIdsTask;
+
+ // Remove the client id from disk
+ await IOUtils.remove(gStateFilePath);
+ },
+
+ async removeClientIDs() {
+ this._log.trace("removeClientIDs");
+ let oldClientId = this._clientID;
+
+ // Wait for the removal.
+ // Asynchronous calls to getClientID will also be blocked on this.
+ this._removeClientIdsTask = this._doRemoveClientIDs();
+ let clear = () => (this._removeClientIdsTask = null);
+ this._removeClientIdsTask.then(clear, clear);
+
+ await this._removeClientIdsTask;
+
+ // On Android we detect resets after a canary client ID.
+ if (AppConstants.platform == "android") {
+ this._wasCanary = oldClientId == CANARY_CLIENT_ID;
+ }
+ },
+
+ /**
+ * Sets the client id to the given value and updates the value cached in
+ * preferences only if the given id is a valid.
+ *
+ * @param {String} id A string containing the client ID.
+ * @return {Boolean} True when the client ID has valid format, or False
+ * otherwise.
+ */
+ updateClientID(id) {
+ if (!isValidClientID(id)) {
+ this._log.error("updateClientID - invalid client ID", id);
+ return false;
+ }
+
+ this._clientID = id;
+
+ if (AppConstants.MOZ_GLEAN) {
+ Glean.fogValidation.legacyTelemetryClientId.set(this._clientID);
+ }
+
+ this._clientIDHash = null;
+ Services.prefs.setStringPref(PREF_CACHED_CLIENTID, this._clientID);
+ return true;
+ },
+
+ updateEcosystemClientID(id) {
+ if (!isValidClientID(id)) {
+ this._log.error(
+ "updateEcosystemClientID - invalid ecosystem client ID",
+ id
+ );
+ return false;
+ }
+ this._ecosystemClientID = id;
+ Services.telemetry.scalarSet(
+ SCALAR_DELETION_REQUEST_ECOSYSTEM_CLIENT_ID,
+ id
+ );
+ return true;
+ },
+
+ /**
+ * A helper for getting access to telemetry logger.
+ */
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+
+ return this._logger;
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryArchive.jsm b/toolkit/components/telemetry/app/TelemetryArchive.jsm
new file mode 100644
index 0000000000..bbbc6fce71
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryArchive.jsm
@@ -0,0 +1,123 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryArchive"];
+
+ChromeUtils.import("resource://gre/modules/Log.jsm", this);
+ChromeUtils.import("resource://gre/modules/Preferences.jsm", this);
+ChromeUtils.import("resource://gre/modules/osfile.jsm", this);
+ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "TelemetryArchive::";
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetryStorage",
+ "resource://gre/modules/TelemetryStorage.jsm"
+);
+
+var TelemetryArchive = {
+ /**
+ * Get a list of the archived pings, sorted by the creation date.
+ * Note that scanning the archived pings on disk is delayed on startup,
+ * use promizeInitialized() to access this after scanning.
+ *
+ * @return {Promise<sequence<Object>>}
+ * A list of the archived ping info in the form:
+ * { id: <string>,
+ * timestampCreated: <number>,
+ * type: <string> }
+ */
+ promiseArchivedPingList() {
+ return TelemetryArchiveImpl.promiseArchivedPingList();
+ },
+
+ /**
+ * Load an archived ping from disk by id, asynchronously.
+ *
+ * @param id {String} The pings UUID.
+ * @return {Promise<PingData>} A promise resolved with the pings data on success.
+ */
+ promiseArchivedPingById(id) {
+ return TelemetryArchiveImpl.promiseArchivedPingById(id);
+ },
+
+ /**
+ * Archive a ping and persist it to disk.
+ *
+ * @param {object} ping The ping data to archive.
+ * @return {promise} Promise that is resolved when the ping is successfully archived.
+ */
+ promiseArchivePing(ping) {
+ return TelemetryArchiveImpl.promiseArchivePing(ping);
+ },
+};
+
+/**
+ * Checks if pings can be archived. Some products (e.g. Thunderbird) might not want
+ * to do that.
+ * @return {Boolean} True if pings should be archived, false otherwise.
+ */
+function shouldArchivePings() {
+ return Preferences.get(TelemetryUtils.Preferences.ArchiveEnabled, false);
+}
+
+var TelemetryArchiveImpl = {
+ _logger: null,
+
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+
+ return this._logger;
+ },
+
+ promiseArchivePing(ping) {
+ if (!shouldArchivePings()) {
+ this._log.trace("promiseArchivePing - archiving is disabled");
+ return Promise.resolve();
+ }
+
+ for (let field of ["creationDate", "id", "type"]) {
+ if (!(field in ping)) {
+ this._log.warn("promiseArchivePing - missing field " + field);
+ return Promise.reject(new Error("missing field " + field));
+ }
+ }
+
+ return TelemetryStorage.saveArchivedPing(ping);
+ },
+
+ _buildArchivedPingList(archivedPingsMap) {
+ let list = Array.from(archivedPingsMap, p => ({
+ id: p[0],
+ timestampCreated: p[1].timestampCreated,
+ type: p[1].type,
+ }));
+
+ list.sort((a, b) => a.timestampCreated - b.timestampCreated);
+
+ return list;
+ },
+
+ promiseArchivedPingList() {
+ this._log.trace("promiseArchivedPingList");
+
+ return TelemetryStorage.loadArchivedPingList().then(loadedInfo => {
+ return this._buildArchivedPingList(loadedInfo);
+ });
+ },
+
+ promiseArchivedPingById(id) {
+ this._log.trace("promiseArchivedPingById - id: " + id);
+ return TelemetryStorage.loadArchivedPing(id);
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryController.jsm b/toolkit/components/telemetry/app/TelemetryController.jsm
new file mode 100644
index 0000000000..7d0dcd3b76
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryController.jsm
@@ -0,0 +1,45 @@
+/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * This module chooses the correct telemetry controller module to load
+ * based on the process:
+ *
+ * - TelemetryControllerParent is loaded only in the parent process, and
+ * contains code specific to the parent.
+ * - TelemetryControllerContent is loaded only in content processes, and
+ * contains code specific to them.
+ *
+ * Both the parent and the content modules load TelemetryControllerBase,
+ * which contains code which is common to all processes.
+ *
+ * This division is important for content process memory usage and
+ * startup time. The parent-specific code occupies tens of KB of memory
+ * which, multiplied by the number of content processes we have, adds up
+ * fast.
+ */
+
+var EXPORTED_SYMBOLS = ["TelemetryController"];
+
+// We can't use Services.appinfo here because tests stub out the appinfo
+// service, and if we touch Services.appinfo now, the built-in version
+// will be cached in place of the stub.
+const isParentProcess =
+ // eslint-disable-next-line mozilla/use-services
+ Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime).processType ===
+ Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT;
+
+var TelemetryController;
+if (isParentProcess) {
+ ({ TelemetryController } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryControllerParent.jsm"
+ ));
+} else {
+ ({ TelemetryController } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryControllerContent.jsm"
+ ));
+}
diff --git a/toolkit/components/telemetry/app/TelemetryControllerBase.jsm b/toolkit/components/telemetry/app/TelemetryControllerBase.jsm
new file mode 100644
index 0000000000..4f17fac5fe
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryControllerBase.jsm
@@ -0,0 +1,143 @@
+/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryControllerBase"];
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "AppConstants",
+ "resource://gre/modules/AppConstants.jsm"
+);
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "TelemetryController::";
+
+const PREF_BRANCH_LOG = "toolkit.telemetry.log.";
+const PREF_LOG_LEVEL = "toolkit.telemetry.log.level";
+const PREF_LOG_DUMP = "toolkit.telemetry.log.dump";
+
+const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled";
+
+const Preferences = Object.freeze({
+ OverridePreRelease: "toolkit.telemetry.testing.overridePreRelease",
+ Unified: "toolkit.telemetry.unified",
+});
+
+/**
+ * Setup Telemetry logging. This function also gets called when loggin related
+ * preferences change.
+ */
+var gLogger = null;
+var gPrefixLogger = null;
+var gLogAppenderDump = null;
+
+var TelemetryControllerBase = Object.freeze({
+ // Whether the FHR/Telemetry unification features are enabled.
+ // Changing this pref requires a restart.
+ IS_UNIFIED_TELEMETRY: Services.prefs.getBoolPref(Preferences.Unified, false),
+
+ Preferences,
+
+ /**
+ * Returns the state of the Telemetry enabled preference, making sure
+ * it correctly evaluates to a boolean type.
+ */
+ get isTelemetryEnabled() {
+ return Services.prefs.getBoolPref(PREF_TELEMETRY_ENABLED, false) === true;
+ },
+
+ get log() {
+ if (!gPrefixLogger) {
+ gPrefixLogger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+ return gPrefixLogger;
+ },
+
+ configureLogging() {
+ if (!gLogger) {
+ gLogger = Log.repository.getLogger(LOGGER_NAME);
+
+ // Log messages need to go to the browser console.
+ let consoleAppender = new Log.ConsoleAppender(new Log.BasicFormatter());
+ gLogger.addAppender(consoleAppender);
+
+ Services.prefs.addObserver(PREF_BRANCH_LOG, this.configureLogging);
+ }
+
+ // Make sure the logger keeps up with the logging level preference.
+ gLogger.level =
+ Log.Level[Services.prefs.getStringPref(PREF_LOG_LEVEL, "Warn")];
+
+ // If enabled in the preferences, add a dump appender.
+ let logDumping = Services.prefs.getBoolPref(PREF_LOG_DUMP, false);
+ if (logDumping != !!gLogAppenderDump) {
+ if (logDumping) {
+ gLogAppenderDump = new Log.DumpAppender(new Log.BasicFormatter());
+ gLogger.addAppender(gLogAppenderDump);
+ } else {
+ gLogger.removeAppender(gLogAppenderDump);
+ gLogAppenderDump = null;
+ }
+ }
+ },
+
+ /**
+ * Set the Telemetry core recording flag for Unified Telemetry.
+ */
+ setTelemetryRecordingFlags() {
+ // Enable extended Telemetry on pre-release channels and disable it
+ // on Release/ESR.
+ let prereleaseChannels = ["nightly", "aurora", "beta"];
+ if (!AppConstants.MOZILLA_OFFICIAL) {
+ // Turn extended telemetry for local developer builds.
+ prereleaseChannels.push("default");
+ }
+ const isPrereleaseChannel = prereleaseChannels.includes(
+ AppConstants.MOZ_UPDATE_CHANNEL
+ );
+ const isReleaseCandidateOnBeta =
+ AppConstants.MOZ_UPDATE_CHANNEL === "release" &&
+ Services.prefs.getCharPref("app.update.channel", null) === "beta";
+ Services.telemetry.canRecordBase = true;
+ Services.telemetry.canRecordExtended =
+ isPrereleaseChannel ||
+ isReleaseCandidateOnBeta ||
+ Services.prefs.getBoolPref(this.Preferences.OverridePreRelease, false);
+ },
+
+ /**
+ * Perform telemetry initialization for either chrome or content process.
+ * @return {Boolean} True if Telemetry is allowed to record at least base (FHR) data,
+ * false otherwise.
+ */
+ enableTelemetryRecording: function enableTelemetryRecording() {
+ // Configure base Telemetry recording.
+ // Unified Telemetry makes it opt-out. If extended Telemetry is enabled, base recording
+ // is always on as well.
+ if (this.IS_UNIFIED_TELEMETRY) {
+ this.setTelemetryRecordingFlags();
+ } else {
+ // We're not on unified Telemetry, stick to the old behaviour for
+ // supporting Fennec.
+ Services.telemetry.canRecordBase = Services.telemetry.canRecordExtended = this.isTelemetryEnabled;
+ }
+
+ this.log.config(
+ "enableTelemetryRecording - canRecordBase:" +
+ Services.telemetry.canRecordBase +
+ ", canRecordExtended: " +
+ Services.telemetry.canRecordExtended
+ );
+
+ return Services.telemetry.canRecordBase;
+ },
+});
diff --git a/toolkit/components/telemetry/app/TelemetryControllerContent.jsm b/toolkit/components/telemetry/app/TelemetryControllerContent.jsm
new file mode 100644
index 0000000000..7cc36deaa5
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryControllerContent.jsm
@@ -0,0 +1,108 @@
+/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const { DeferredTask } = ChromeUtils.import(
+ "resource://gre/modules/DeferredTask.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { TelemetryControllerBase } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryControllerBase.jsm"
+);
+
+// Delay before intializing telemetry (ms)
+const TELEMETRY_DELAY =
+ Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000;
+// Delay before initializing telemetry if we're testing (ms)
+const TELEMETRY_TEST_DELAY = 1;
+
+var EXPORTED_SYMBOLS = ["TelemetryController", "getTelemetryController"];
+
+var TelemetryController = Object.freeze({
+ /**
+ * Used only for testing purposes.
+ */
+ testInitLogging() {
+ TelemetryControllerBase.configureLogging();
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testSetupContent() {
+ return Impl.setupContentTelemetry(true);
+ },
+
+ /**
+ * Send a notification.
+ */
+ observe(aSubject, aTopic, aData) {
+ return Impl.observe(aSubject, aTopic, aData);
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
+});
+
+var Impl = {
+ // This is true when running in the test infrastructure.
+ _testMode: false,
+
+ get _log() {
+ return TelemetryControllerBase.log;
+ },
+
+ /**
+ * This triggers basic telemetry initialization for content processes.
+ * @param {Boolean} [testing=false] True if we are in test mode, false otherwise.
+ */
+ setupContentTelemetry(testing = false) {
+ this._testMode = testing;
+
+ // The thumbnail service also runs in a content process, even with e10s off.
+ // We need to check if e10s is on so we don't submit child payloads for it.
+ // We still need xpcshell child tests to work, so we skip this if test mode is enabled.
+ if (testing || Services.appinfo.browserTabsRemoteAutostart) {
+ // We call |enableTelemetryRecording| here to make sure that Telemetry.canRecord* flags
+ // are in sync between chrome and content processes.
+ if (!TelemetryControllerBase.enableTelemetryRecording()) {
+ this._log.trace(
+ "setupContentTelemetry - Content process recording disabled."
+ );
+ return;
+ }
+ }
+ Services.telemetry.earlyInit();
+
+ // FIXME: This is a terrible abuse of DeferredTask.
+ let delayedTask = new DeferredTask(
+ () => {
+ Services.telemetry.delayedInit();
+ },
+ testing ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY,
+ testing ? 0 : undefined
+ );
+
+ delayedTask.arm();
+ },
+
+ /**
+ * This observer drives telemetry.
+ */
+ observe(aSubject, aTopic, aData) {
+ if (aTopic == "app-startup") {
+ TelemetryControllerBase.configureLogging();
+
+ this._log.trace(`observe - ${aTopic} notified.`);
+
+ this.setupContentTelemetry();
+ }
+ },
+};
+
+// Used by service registration, which requires a callable function.
+function getTelemetryController() {
+ return TelemetryController;
+}
diff --git a/toolkit/components/telemetry/app/TelemetryControllerParent.jsm b/toolkit/components/telemetry/app/TelemetryControllerParent.jsm
new file mode 100644
index 0000000000..832b9e0b28
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryControllerParent.jsm
@@ -0,0 +1,1450 @@
+/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const { AppConstants } = ChromeUtils.import(
+ "resource://gre/modules/AppConstants.jsm"
+);
+const { PromiseUtils } = ChromeUtils.import(
+ "resource://gre/modules/PromiseUtils.jsm"
+);
+const { DeferredTask } = ChromeUtils.import(
+ "resource://gre/modules/DeferredTask.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { TelemetryUtils } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryUtils.jsm"
+);
+const { TelemetryControllerBase } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryControllerBase.jsm"
+);
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+const Utils = TelemetryUtils;
+
+const PING_FORMAT_VERSION = 4;
+
+// Delay before intializing telemetry (ms)
+const TELEMETRY_DELAY =
+ Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000;
+// Delay before initializing telemetry if we're testing (ms)
+const TELEMETRY_TEST_DELAY = 1;
+
+// How long to wait (ms) before sending the new profile ping on the first
+// run of a new profile.
+const NEWPROFILE_PING_DEFAULT_DELAY = 30 * 60 * 1000;
+
+// Ping types.
+const PING_TYPE_MAIN = "main";
+const PING_TYPE_DELETION_REQUEST = "deletion-request";
+const PING_TYPE_UNINSTALL = "uninstall";
+
+// Session ping reasons.
+const REASON_GATHER_PAYLOAD = "gather-payload";
+const REASON_GATHER_SUBSESSION_PAYLOAD = "gather-subsession-payload";
+
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "Telemetry",
+ "@mozilla.org/base/telemetry;1",
+ "nsITelemetry"
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "jwcrypto",
+ "resource://services-crypto/jwcrypto.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ ClientID: "resource://gre/modules/ClientID.jsm",
+ CoveragePing: "resource://gre/modules/CoveragePing.jsm",
+ AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm",
+ TelemetryStorage: "resource://gre/modules/TelemetryStorage.jsm",
+ TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.jsm",
+ TelemetryArchive: "resource://gre/modules/TelemetryArchive.jsm",
+ TelemetrySession: "resource://gre/modules/TelemetrySession.jsm",
+ TelemetrySend: "resource://gre/modules/TelemetrySend.jsm",
+ TelemetryReportingPolicy:
+ "resource://gre/modules/TelemetryReportingPolicy.jsm",
+ TelemetryModules: "resource://gre/modules/ModulesPing.jsm",
+ TelemetryUntrustedModulesPing:
+ "resource://gre/modules/UntrustedModulesPing.jsm",
+ UpdatePing: "resource://gre/modules/UpdatePing.jsm",
+ TelemetryHealthPing: "resource://gre/modules/HealthPing.jsm",
+ TelemetryEventPing: "resource://gre/modules/EventPing.jsm",
+ EcosystemTelemetry: "resource://gre/modules/EcosystemTelemetry.jsm",
+ TelemetryPrioPing: "resource://gre/modules/PrioPing.jsm",
+ UninstallPing: "resource://gre/modules/UninstallPing.jsm",
+ OS: "resource://gre/modules/osfile.jsm",
+});
+
+/**
+ * This is a policy object used to override behavior for testing.
+ */
+var Policy = {
+ now: () => new Date(),
+ generatePingId: () => Utils.generateUUID(),
+ getCachedClientID: () => ClientID.getCachedClientID(),
+};
+
+var EXPORTED_SYMBOLS = ["TelemetryController"];
+
+var TelemetryController = Object.freeze({
+ /**
+ * Used only for testing purposes.
+ */
+ testInitLogging() {
+ TelemetryControllerBase.configureLogging();
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testReset() {
+ return Impl.reset();
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testSetup() {
+ return Impl.setupTelemetry(true);
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testShutdown() {
+ return Impl.shutdown();
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testPromiseJsProbeRegistration() {
+ return Promise.resolve(Impl._probeRegistrationPromise);
+ },
+
+ /**
+ * Register 'dynamic builtin' probes from the JSON definition files.
+ * This is needed to support adding new probes in developer builds
+ * without rebuilding the whole codebase.
+ *
+ * This is not meant to be used outside of local developer builds.
+ */
+ testRegisterJsProbes() {
+ return Impl.registerJsProbes();
+ },
+
+ /**
+ * Used only for testing purposes.
+ */
+ testPromiseDeletionRequestPingSubmitted() {
+ return Promise.resolve(Impl._deletionRequestPingSubmittedPromise);
+ },
+
+ /**
+ * Send a notification.
+ */
+ observe(aSubject, aTopic, aData) {
+ return Impl.observe(aSubject, aTopic, aData);
+ },
+
+ /**
+ * Submit ping payloads to Telemetry. This will assemble a complete ping, adding
+ * environment data, client id and some general info.
+ * Depending on configuration, the ping will be sent to the server (immediately or later)
+ * and archived locally.
+ *
+ * To identify the different pings and to be able to query them pings have a type.
+ * A type is a string identifier that should be unique to the type ping that is being submitted,
+ * it should only contain alphanumeric characters and '-' for separation, i.e. satisfy:
+ * /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} [aOptions] Options object.
+ * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
+ * id, false otherwise.
+ * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
+ * environment data.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ * @returns {Promise} Test-only - a promise that resolves with the ping id once the ping is stored or sent.
+ */
+ submitExternalPing(aType, aPayload, aOptions = {}) {
+ aOptions.addClientId = aOptions.addClientId || false;
+ aOptions.addEnvironment = aOptions.addEnvironment || false;
+ aOptions.usePingSender = aOptions.usePingSender || false;
+
+ return Impl.submitExternalPing(aType, aPayload, aOptions);
+ },
+
+ /**
+ * Get the current session ping data as it would be sent out or stored.
+ *
+ * @param {bool} aSubsession Whether to get subsession data. Optional, defaults to false.
+ * @return {object} The current ping data if Telemetry is enabled, null otherwise.
+ */
+ getCurrentPingData(aSubsession = false) {
+ return Impl.getCurrentPingData(aSubsession);
+ },
+
+ /**
+ * Save a ping to disk.
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} [aOptions] Options object.
+ * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
+ * id, false otherwise.
+ * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
+ * environment data.
+ * @param {Boolean} [aOptions.overwrite=false] true overwrites a ping with the same name,
+ * if found.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ *
+ * @returns {Promise} A promise that resolves with the ping id when the ping is saved to
+ * disk.
+ */
+ addPendingPing(aType, aPayload, aOptions = {}) {
+ let options = aOptions;
+ options.addClientId = aOptions.addClientId || false;
+ options.addEnvironment = aOptions.addEnvironment || false;
+ options.overwrite = aOptions.overwrite || false;
+
+ return Impl.addPendingPing(aType, aPayload, options);
+ },
+
+ /**
+ * Check if we have an aborted-session ping from a previous session.
+ * If so, submit and then remove it.
+ *
+ * @return {Promise} Promise that is resolved when the ping is saved.
+ */
+ checkAbortedSessionPing() {
+ return Impl.checkAbortedSessionPing();
+ },
+
+ /**
+ * Save an aborted-session ping to disk without adding it to the pending pings.
+ *
+ * @param {Object} aPayload The ping payload data.
+ * @return {Promise} Promise that is resolved when the ping is saved.
+ */
+ saveAbortedSessionPing(aPayload) {
+ return Impl.saveAbortedSessionPing(aPayload);
+ },
+
+ /**
+ * Remove the aborted-session ping if any exists.
+ *
+ * @return {Promise} Promise that is resolved when the ping was removed.
+ */
+ removeAbortedSessionPing() {
+ return Impl.removeAbortedSessionPing();
+ },
+
+ /**
+ * Create an uninstall ping and write it to disk, replacing any already present.
+ * This is stored independently from other pings, and only read by
+ * the Windows uninstaller.
+ *
+ * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+ *
+ * @return {Promise} Resolved when the ping has been saved.
+ */
+ saveUninstallPing() {
+ return Impl.saveUninstallPing();
+ },
+
+ /**
+ * Allows the sync ping to tell the controller that it is initializing, so
+ * should be included in the orderly shutdown process.
+ *
+ * @param {Function} aFnShutdown The function to call as telemetry shuts down.
+
+ */
+ registerSyncPingShutdown(afnShutdown) {
+ Impl.registerSyncPingShutdown(afnShutdown);
+ },
+
+ /**
+ * Allows waiting for TelemetryControllers delayed initialization to complete.
+ * The returned promise is guaranteed to resolve before TelemetryController is shutting down.
+ * @return {Promise} Resolved when delayed TelemetryController initialization completed.
+ */
+ promiseInitialized() {
+ return Impl.promiseInitialized();
+ },
+});
+
+var Impl = {
+ _initialized: false,
+ _initStarted: false, // Whether we started setting up TelemetryController.
+ _shuttingDown: false, // Whether the browser is shutting down.
+ _shutDown: false, // Whether the browser has shut down.
+ _logger: null,
+ _prevValues: {},
+ // The previous build ID, if this is the first run with a new build.
+ // Undefined if this is not the first run, or the previous build ID is unknown.
+ _previousBuildID: undefined,
+ _clientID: null,
+ // A task performing delayed initialization
+ _delayedInitTask: null,
+ // The deferred promise resolved when the initialization task completes.
+ _delayedInitTaskDeferred: null,
+
+ // This is a public barrier Telemetry clients can use to add blockers to the shutdown
+ // of TelemetryController.
+ // After this barrier, clients can not submit Telemetry pings anymore.
+ _shutdownBarrier: new AsyncShutdown.Barrier(
+ "TelemetryController: Waiting for clients."
+ ),
+ // This is a private barrier blocked by pending async ping activity (sending & saving).
+ _connectionsBarrier: new AsyncShutdown.Barrier(
+ "TelemetryController: Waiting for pending ping activity"
+ ),
+ // This is true when running in the test infrastructure.
+ _testMode: false,
+ // The task performing the delayed sending of the "new-profile" ping.
+ _delayedNewPingTask: null,
+ // The promise used to wait for the JS probe registration (dynamic builtin).
+ _probeRegistrationPromise: null,
+ // The promise of any outstanding task sending the "deletion-request" ping.
+ _deletionRequestPingSubmittedPromise: null,
+ // A function to shutdown the sync/fxa ping, or null if that ping has not
+ // self-initialized.
+ _fnSyncPingShutdown: null,
+
+ get _log() {
+ return TelemetryControllerBase.log;
+ },
+
+ /**
+ * Get the data for the "application" section of the ping.
+ */
+ _getApplicationSection() {
+ // Querying architecture and update channel can throw. Make sure to recover and null
+ // those fields.
+ let arch = null;
+ try {
+ arch = Services.sysinfo.get("arch");
+ } catch (e) {
+ this._log.trace(
+ "_getApplicationSection - Unable to get system architecture.",
+ e
+ );
+ }
+
+ let updateChannel = null;
+ try {
+ updateChannel = Utils.getUpdateChannel();
+ } catch (e) {
+ this._log.trace(
+ "_getApplicationSection - Unable to get update channel.",
+ e
+ );
+ }
+
+ return {
+ architecture: arch,
+ buildId: Services.appinfo.appBuildID,
+ name: Services.appinfo.name,
+ version: Services.appinfo.version,
+ displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY,
+ vendor: Services.appinfo.vendor,
+ platformVersion: Services.appinfo.platformVersion,
+ xpcomAbi: Services.appinfo.XPCOMABI,
+ channel: updateChannel,
+ };
+ },
+
+ /**
+ * Assemble a complete ping following the common ping format specification.
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} aOptions Options object.
+ * @param {Boolean} aOptions.addClientId true if the ping should contain the client
+ * id, false otherwise.
+ * @param {Boolean} aOptions.addEnvironment true if the ping should contain the
+ * environment data.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending.
+ * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key).
+ * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled.
+ * @param {String} [aOptions.studyName=null] the study name to use.
+ * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled.
+ * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled.
+ * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled.
+ * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise.
+ * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the
+ * pioneer id to the provided value. Only works if aOptions.addPioneerId=true.
+ * @returns {Object} An object that contains the assembled ping data.
+ */
+ assemblePing: function assemblePing(aType, aPayload, aOptions = {}) {
+ this._log.trace(
+ "assemblePing - Type " + aType + ", aOptions " + JSON.stringify(aOptions)
+ );
+
+ // Clone the payload data so we don't race against unexpected changes in subobjects that are
+ // still referenced by other code.
+ // We can't trust all callers to do this properly on their own.
+ let payload = Cu.cloneInto(aPayload, {});
+
+ // Fill the common ping fields.
+ let pingData = {
+ type: aType,
+ id: Policy.generatePingId(),
+ creationDate: Policy.now().toISOString(),
+ version: PING_FORMAT_VERSION,
+ application: this._getApplicationSection(),
+ payload,
+ };
+
+ if (aOptions.addClientId || aOptions.overrideClientId) {
+ pingData.clientId = aOptions.overrideClientId || this._clientID;
+ }
+
+ if (aOptions.addEnvironment) {
+ pingData.environment =
+ aOptions.overrideEnvironment || TelemetryEnvironment.currentEnvironment;
+
+ // On Android store a flag if the client ID was reset from a canary ID.
+ if (AppConstants.platform == "android" && ClientID.wasCanaryClientID()) {
+ pingData.environment.profile.wasCanary = true;
+ }
+ }
+
+ return pingData;
+ },
+
+ /**
+ * Track any pending ping send and save tasks through the promise passed here.
+ * This is needed to block shutdown on any outstanding ping activity.
+ */
+ _trackPendingPingTask(aPromise) {
+ this._connectionsBarrier.client.addBlocker(
+ "Waiting for ping task",
+ aPromise
+ );
+ },
+
+ /**
+ * Internal function to assemble a complete ping, adding environment data, client id
+ * and some general info. This waits on the client id to be loaded/generated if it's
+ * not yet available. Note that this function is synchronous unless we need to load
+ * the client id.
+ * Depending on configuration, the ping will be sent to the server (immediately or later)
+ * and archived locally.
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} [aOptions] Options object.
+ * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
+ * id, false otherwise.
+ * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
+ * environment data.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender.
+ * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending.
+ * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key).
+ * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled.
+ * @param {String} [aOptions.studyName=null] the study name to use.
+ * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled.
+ * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled.
+ * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled.
+ * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise.
+ * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the
+ * pioneer id to the provided value. Only works if aOptions.addPioneerId=true.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ * @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent.
+ */
+ async _submitPingLogic(aType, aPayload, aOptions) {
+ // Make sure to have a clientId if we need one. This cover the case of submitting
+ // a ping early during startup, before Telemetry is initialized, if no client id was
+ // cached.
+ if (!this._clientID && aOptions.addClientId && !aOptions.overrideClientId) {
+ this._log.trace("_submitPingLogic - Waiting on client id");
+ Telemetry.getHistogramById(
+ "TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID"
+ ).add();
+ // We can safely call |getClientID| here and during initialization: we would still
+ // spawn and return one single loading task.
+ this._clientID = await ClientID.getClientID();
+ }
+
+ let pingData = this.assemblePing(aType, aPayload, aOptions);
+ this._log.trace("submitExternalPing - ping assembled, id: " + pingData.id);
+
+ if (aOptions.useEncryption === true) {
+ try {
+ if (!aOptions.publicKey) {
+ throw new Error("Public key is required when using encryption.");
+ }
+
+ if (
+ !(
+ aOptions.schemaName &&
+ aOptions.schemaNamespace &&
+ aOptions.schemaVersion
+ )
+ ) {
+ throw new Error(
+ "Schema name, namespace, and version are required when using encryption."
+ );
+ }
+
+ const payload = {};
+ payload.encryptedData = await jwcrypto.generateJWE(
+ aOptions.publicKey,
+ new TextEncoder("utf-8").encode(JSON.stringify(aPayload))
+ );
+
+ payload.schemaVersion = aOptions.schemaVersion;
+ payload.schemaName = aOptions.schemaName;
+ payload.schemaNamespace = aOptions.schemaNamespace;
+
+ payload.encryptionKeyId = aOptions.encryptionKeyId;
+
+ if (aOptions.addPioneerId === true) {
+ if (aOptions.overridePioneerId) {
+ // The caller provided a substitute id, let's use that
+ // instead of querying the pref.
+ payload.pioneerId = aOptions.overridePioneerId;
+ } else {
+ // This will throw if there is no pioneer ID set.
+ payload.pioneerId = Services.prefs.getStringPref(
+ "toolkit.telemetry.pioneerId"
+ );
+ }
+ payload.studyName = aOptions.studyName;
+ }
+
+ pingData.payload = payload;
+ } catch (e) {
+ this._log.error("_submitPingLogic - Unable to encrypt ping", e);
+ // Do not attempt to continue
+ throw e;
+ }
+ }
+
+ // Always persist the pings if we are allowed to. We should not yield on any of the
+ // following operations to keep this function synchronous for the majority of the calls.
+ let archivePromise = TelemetryArchive.promiseArchivePing(
+ pingData
+ ).catch(e =>
+ this._log.error(
+ "submitExternalPing - Failed to archive ping " + pingData.id,
+ e
+ )
+ );
+ let p = [archivePromise];
+
+ p.push(
+ TelemetrySend.submitPing(pingData, {
+ usePingSender: aOptions.usePingSender,
+ })
+ );
+
+ return Promise.all(p).then(() => pingData.id);
+ },
+
+ /**
+ * Submit ping payloads to Telemetry.
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} [aOptions] Options object.
+ * @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
+ * id, false otherwise.
+ * @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
+ * environment data.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {Boolean} [aOptions.usePingSender=false] if true, send the ping using the PingSender.
+ * @param {Boolean} [aOptions.useEncryption=false] if true, encrypt data client-side before sending.
+ * @param {Object} [aOptions.publicKey=null] the public key to use if encryption is enabled (JSON Web Key).
+ * @param {String} [aOptions.encryptionKeyId=null] the public key ID to use if encryption is enabled.
+ * @param {String} [aOptions.studyName=null] the study name to use.
+ * @param {String} [aOptions.schemaName=null] the schema name to use if encryption is enabled.
+ * @param {String} [aOptions.schemaNamespace=null] the schema namespace to use if encryption is enabled.
+ * @param {String} [aOptions.schemaVersion=null] the schema version to use if encryption is enabled.
+ * @param {Boolean} [aOptions.addPioneerId=false] true if the ping should contain the Pioneer id, false otherwise.
+ * @param {Boolean} [aOptions.overridePioneerId=undefined] if set, override the
+ * pioneer id to the provided value. Only works if aOptions.addPioneerId=true.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ * @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent.
+ */
+ submitExternalPing: function send(aType, aPayload, aOptions) {
+ this._log.trace(
+ "submitExternalPing - type: " +
+ aType +
+ ", aOptions: " +
+ JSON.stringify(aOptions)
+ );
+
+ // Reject pings sent after shutdown.
+ if (this._shutDown) {
+ const errorMessage =
+ "submitExternalPing - Submission is not allowed after shutdown, discarding ping of type: " +
+ aType;
+ this._log.error(errorMessage);
+ return Promise.reject(new Error(errorMessage));
+ }
+
+ // Enforce the type string to only contain sane characters.
+ const typeUuid = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
+ if (!typeUuid.test(aType)) {
+ this._log.error("submitExternalPing - invalid ping type: " + aType);
+ let histogram = Telemetry.getKeyedHistogramById(
+ "TELEMETRY_INVALID_PING_TYPE_SUBMITTED"
+ );
+ histogram.add(aType, 1);
+ return Promise.reject(new Error("Invalid type string submitted."));
+ }
+ // Enforce that the payload is an object.
+ if (
+ aPayload === null ||
+ typeof aPayload !== "object" ||
+ Array.isArray(aPayload)
+ ) {
+ this._log.error(
+ "submitExternalPing - invalid payload type: " + typeof aPayload
+ );
+ let histogram = Telemetry.getHistogramById(
+ "TELEMETRY_INVALID_PAYLOAD_SUBMITTED"
+ );
+ histogram.add(1);
+ return Promise.reject(new Error("Invalid payload type submitted."));
+ }
+
+ // We're trying to track down missing sync pings (bug 1663573), so record
+ // a temporary cross-checking counter.
+ if (aType == "sync" && aPayload.why == "shutdown") {
+ Telemetry.scalarSet("telemetry.sync_shutdown_ping_sent", true);
+ }
+
+ let promise = this._submitPingLogic(aType, aPayload, aOptions);
+ this._trackPendingPingTask(promise);
+ return promise;
+ },
+
+ /**
+ * Save a ping to disk.
+ *
+ * @param {String} aType The type of the ping.
+ * @param {Object} aPayload The actual data payload for the ping.
+ * @param {Object} aOptions Options object.
+ * @param {Boolean} aOptions.addClientId true if the ping should contain the client id,
+ * false otherwise.
+ * @param {Boolean} aOptions.addEnvironment true if the ping should contain the
+ * environment data.
+ * @param {Boolean} aOptions.overwrite true overwrites a ping with the same name, if found.
+ * @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
+ * @param {String} [aOptions.overrideClientId=undefined] if set, override the
+ * client id to the provided value. Implies aOptions.addClientId=true.
+ *
+ * @returns {Promise} A promise that resolves with the ping id when the ping is saved to
+ * disk.
+ */
+ addPendingPing: function addPendingPing(aType, aPayload, aOptions) {
+ this._log.trace(
+ "addPendingPing - Type " +
+ aType +
+ ", aOptions " +
+ JSON.stringify(aOptions)
+ );
+
+ let pingData = this.assemblePing(aType, aPayload, aOptions);
+
+ let savePromise = TelemetryStorage.savePendingPing(pingData);
+ let archivePromise = TelemetryArchive.promiseArchivePing(pingData).catch(
+ e => {
+ this._log.error(
+ "addPendingPing - Failed to archive ping " + pingData.id,
+ e
+ );
+ }
+ );
+
+ // Wait for both the archiving and ping persistence to complete.
+ let promises = [savePromise, archivePromise];
+ return Promise.all(promises).then(() => pingData.id);
+ },
+
+ /**
+ * Check whether we have an aborted-session ping. If so add it to the pending pings and archive it.
+ *
+ * @return {Promise} Promise that is resolved when the ping is submitted and archived.
+ */
+ async checkAbortedSessionPing() {
+ let ping = await TelemetryStorage.loadAbortedSessionPing();
+ this._log.trace(
+ "checkAbortedSessionPing - found aborted-session ping: " + !!ping
+ );
+ if (!ping) {
+ return;
+ }
+
+ try {
+ // Previous aborted-session might have been with a canary client ID.
+ // Don't send it.
+ if (ping.clientId != Utils.knownClientID) {
+ await TelemetryStorage.addPendingPing(ping);
+ await TelemetryArchive.promiseArchivePing(ping);
+ }
+ } catch (e) {
+ this._log.error(
+ "checkAbortedSessionPing - Unable to add the pending ping",
+ e
+ );
+ } finally {
+ await TelemetryStorage.removeAbortedSessionPing();
+ }
+ },
+
+ /**
+ * Save an aborted-session ping to disk without adding it to the pending pings.
+ *
+ * @param {Object} aPayload The ping payload data.
+ * @return {Promise} Promise that is resolved when the ping is saved.
+ */
+ saveAbortedSessionPing(aPayload) {
+ this._log.trace("saveAbortedSessionPing");
+ const options = { addClientId: true, addEnvironment: true };
+ const pingData = this.assemblePing(PING_TYPE_MAIN, aPayload, options);
+ return TelemetryStorage.saveAbortedSessionPing(pingData);
+ },
+
+ removeAbortedSessionPing() {
+ return TelemetryStorage.removeAbortedSessionPing();
+ },
+
+ async saveUninstallPing() {
+ if (AppConstants.platform != "win") {
+ return undefined;
+ }
+
+ this._log.trace("saveUninstallPing");
+
+ let payload = {};
+ try {
+ payload.otherInstalls = UninstallPing.getOtherInstallsCount();
+ this._log.info(
+ "saveUninstallPing - otherInstalls",
+ payload.otherInstalls
+ );
+ } catch (e) {
+ this._log.warn("saveUninstallPing - getOtherInstallCount failed", e);
+ }
+ const options = { addClientId: true, addEnvironment: true };
+ const pingData = this.assemblePing(PING_TYPE_UNINSTALL, payload, options);
+
+ return TelemetryStorage.saveUninstallPing(pingData);
+ },
+
+ /**
+ * This triggers basic telemetry initialization and schedules a full initialized for later
+ * for performance reasons.
+ *
+ * This delayed initialization means TelemetryController init can be in the following states:
+ * 1) setupTelemetry was never called
+ * or it was called and
+ * 2) _delayedInitTask was scheduled, but didn't run yet.
+ * 3) _delayedInitTask is currently running.
+ * 4) _delayedInitTask finished running and is nulled out.
+ *
+ * @return {Promise} Resolved when TelemetryController and TelemetrySession are fully
+ * initialized. This is only used in tests.
+ */
+ setupTelemetry: function setupTelemetry(testing) {
+ this._initStarted = true;
+ this._shuttingDown = false;
+ this._shutDown = false;
+ this._testMode = testing;
+
+ this._log.trace("setupTelemetry");
+
+ if (this._delayedInitTask) {
+ this._log.error("setupTelemetry - init task already running");
+ return this._delayedInitTaskDeferred.promise;
+ }
+
+ if (this._initialized && !this._testMode) {
+ this._log.error("setupTelemetry - already initialized");
+ return Promise.resolve();
+ }
+
+ // Enable adding scalars in artifact builds and build faster modes.
+ // The function is async: we intentionally don't wait for it to complete
+ // as we don't want to delay startup.
+ this._probeRegistrationPromise = this.registerJsProbes();
+
+ // This will trigger displaying the datachoices infobar.
+ TelemetryReportingPolicy.setup();
+
+ if (!TelemetryControllerBase.enableTelemetryRecording()) {
+ this._log.config(
+ "setupChromeProcess - Telemetry recording is disabled, skipping Chrome process setup."
+ );
+ return Promise.resolve();
+ }
+
+ this._attachObservers();
+
+ // Perform a lightweight, early initialization for the component, just registering
+ // a few observers and initializing the session.
+ TelemetrySession.earlyInit(this._testMode);
+ Services.telemetry.earlyInit();
+
+ // Annotate crash reports so that we get pings for startup crashes
+ TelemetrySend.earlyInit();
+
+ // For very short session durations, we may never load the client
+ // id from disk.
+ // We try to cache it in prefs to avoid this, even though this may
+ // lead to some stale client ids.
+ this._clientID = ClientID.getCachedClientID();
+
+ // Init the update ping telemetry as early as possible. This won't have
+ // an impact on startup.
+ UpdatePing.earlyInit();
+
+ // Delay full telemetry initialization to give the browser time to
+ // run various late initializers. Otherwise our gathered memory
+ // footprint and other numbers would be too optimistic.
+ this._delayedInitTaskDeferred = PromiseUtils.defer();
+ this._delayedInitTask = new DeferredTask(
+ async () => {
+ try {
+ // TODO: This should probably happen after all the delayed init here.
+ this._initialized = true;
+ await TelemetryEnvironment.delayedInit();
+
+ // Load the ClientID.
+ this._clientID = await ClientID.getClientID();
+
+ // Fix-up a canary client ID if detected.
+ const uploadEnabled = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ false
+ );
+ if (uploadEnabled && this._clientID == Utils.knownClientID) {
+ this._log.trace(
+ "Upload enabled, but got canary client ID. Resetting."
+ );
+ await ClientID.removeClientIDs();
+ this._clientID = await ClientID.getClientID();
+ } else if (!uploadEnabled && this._clientID != Utils.knownClientID) {
+ this._log.trace(
+ "Upload disabled, but got a valid client ID. Setting canary client ID."
+ );
+ await ClientID.setCanaryClientIDs();
+ this._clientID = await ClientID.getClientID();
+ }
+
+ await TelemetrySend.setup(this._testMode);
+
+ // Perform TelemetrySession delayed init.
+ await TelemetrySession.delayedInit();
+ await Services.telemetry.delayedInit();
+
+ if (
+ Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.NewProfilePingEnabled,
+ false
+ ) &&
+ !TelemetrySession.newProfilePingSent
+ ) {
+ // Kick off the scheduling of the new-profile ping.
+ this.scheduleNewProfilePing();
+ }
+
+ // Purge the pings archive by removing outdated pings. We don't wait for
+ // this task to complete, but TelemetryStorage blocks on it during
+ // shutdown.
+ TelemetryStorage.runCleanPingArchiveTask();
+
+ // Now that FHR/healthreporter is gone, make sure to remove FHR's DB from
+ // the profile directory. This is a temporary measure that we should drop
+ // in the future.
+ TelemetryStorage.removeFHRDatabase();
+
+ // The init sequence is forced to run on shutdown for short sessions and
+ // we don't want to start TelemetryModules as the timer registration will fail.
+ if (!this._shuttingDown) {
+ // Report the modules loaded in the Firefox process.
+ TelemetryModules.start();
+
+ // Send coverage ping.
+ await CoveragePing.startup();
+
+ // Start the untrusted modules ping, which reports events where
+ // untrusted modules were loaded into the Firefox process.
+ if (AppConstants.platform == "win") {
+ TelemetryUntrustedModulesPing.start();
+ }
+ }
+
+ TelemetryEventPing.startup();
+ EcosystemTelemetry.startup();
+ TelemetryPrioPing.startup();
+
+ if (uploadEnabled) {
+ await this.saveUninstallPing().catch(e =>
+ this._log.warn("_delayedInitTask - saveUninstallPing failed", e)
+ );
+ } else {
+ await TelemetryStorage.removeUninstallPings().catch(e =>
+ this._log.warn("_delayedInitTask - saveUninstallPing", e)
+ );
+ }
+
+ this._delayedInitTaskDeferred.resolve();
+ } catch (e) {
+ this._delayedInitTaskDeferred.reject(e);
+ } finally {
+ this._delayedInitTask = null;
+ }
+ },
+ this._testMode ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY,
+ this._testMode ? 0 : undefined
+ );
+
+ AsyncShutdown.sendTelemetry.addBlocker(
+ "TelemetryController: shutting down",
+ () => this.shutdown(),
+ () => this._getState()
+ );
+
+ this._delayedInitTask.arm();
+ return this._delayedInitTaskDeferred.promise;
+ },
+
+ // Do proper shutdown waiting and cleanup.
+ async _cleanupOnShutdown() {
+ if (!this._initialized) {
+ return;
+ }
+
+ this._detachObservers();
+
+ // Now do an orderly shutdown.
+ try {
+ if (this._delayedNewPingTask) {
+ await this._delayedNewPingTask.finalize();
+ }
+
+ UpdatePing.shutdown();
+
+ TelemetryEventPing.shutdown();
+ EcosystemTelemetry.shutdown();
+ await TelemetryPrioPing.shutdown();
+
+ // Shutdown the sync ping if it is initialized - this is likely, but not
+ // guaranteed, to submit a "shutdown" sync ping.
+ if (this._fnSyncPingShutdown) {
+ this._fnSyncPingShutdown();
+ }
+
+ // Stop the datachoices infobar display.
+ TelemetryReportingPolicy.shutdown();
+ TelemetryEnvironment.shutdown();
+
+ // Stop any ping sending.
+ await TelemetrySend.shutdown();
+
+ // Send latest data.
+ await TelemetryHealthPing.shutdown();
+
+ await TelemetrySession.shutdown();
+ await Services.telemetry.shutdown();
+
+ // First wait for clients processing shutdown.
+ await this._shutdownBarrier.wait();
+
+ // ... and wait for any outstanding async ping activity.
+ await this._connectionsBarrier.wait();
+
+ if (AppConstants.platform !== "android") {
+ // No PingSender on Android.
+ TelemetrySend.flushPingSenderBatch();
+ }
+
+ // Perform final shutdown operations.
+ await TelemetryStorage.shutdown();
+ } finally {
+ // Reset state.
+ this._initialized = false;
+ this._initStarted = false;
+ this._shutDown = true;
+ }
+ },
+
+ shutdown() {
+ this._log.trace("shutdown");
+
+ this._shuttingDown = true;
+
+ // We can be in one the following states here:
+ // 1) setupTelemetry was never called
+ // or it was called and
+ // 2) _delayedInitTask was scheduled, but didn't run yet.
+ // 3) _delayedInitTask is running now.
+ // 4) _delayedInitTask finished running already.
+
+ // This handles 1).
+ if (!this._initStarted) {
+ this._shutDown = true;
+ return Promise.resolve();
+ }
+
+ // This handles 4).
+ if (!this._delayedInitTask) {
+ // We already ran the delayed initialization.
+ return this._cleanupOnShutdown();
+ }
+
+ // This handles 2) and 3).
+ return this._delayedInitTask
+ .finalize()
+ .then(() => this._cleanupOnShutdown());
+ },
+
+ /**
+ * This observer drives telemetry.
+ */
+ observe(aSubject, aTopic, aData) {
+ // The logger might still be not available at this point.
+ if (aTopic == "profile-after-change") {
+ // If we don't have a logger, we need to make sure |Log.repository.getLogger()| is
+ // called before |getLoggerWithMessagePrefix|. Otherwise logging won't work.
+ TelemetryControllerBase.configureLogging();
+ }
+
+ this._log.trace(`observe - ${aTopic} notified.`);
+
+ switch (aTopic) {
+ case "profile-after-change":
+ // profile-after-change is only registered for chrome processes.
+ return this.setupTelemetry();
+ case "nsPref:changed":
+ if (aData == TelemetryUtils.Preferences.FhrUploadEnabled) {
+ return this._onUploadPrefChange();
+ }
+ }
+ return undefined;
+ },
+
+ /**
+ * Register the sync ping's shutdown handler.
+ */
+ registerSyncPingShutdown(fnShutdown) {
+ if (this._fnSyncPingShutdown) {
+ throw new Error("The sync ping shutdown handler is already registered.");
+ }
+ this._fnSyncPingShutdown = fnShutdown;
+ },
+
+ /**
+ * Get an object describing the current state of this module for AsyncShutdown diagnostics.
+ */
+ _getState() {
+ return {
+ initialized: this._initialized,
+ initStarted: this._initStarted,
+ haveDelayedInitTask: !!this._delayedInitTask,
+ shutdownBarrier: this._shutdownBarrier.state,
+ connectionsBarrier: this._connectionsBarrier.state,
+ sendModule: TelemetrySend.getShutdownState(),
+ haveDelayedNewProfileTask: !!this._delayedNewPingTask,
+ };
+ },
+
+ /**
+ * Called whenever the FHR Upload preference changes (e.g. when user disables FHR from
+ * the preferences panel), this triggers sending the "deletion-request" ping.
+ */
+ _onUploadPrefChange() {
+ const uploadEnabled = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ false
+ );
+ if (uploadEnabled) {
+ this._log.trace(
+ "_onUploadPrefChange - upload was enabled again. Resetting client ID"
+ );
+
+ // Delete cached client ID immediately, so other usage is forced to refetch it.
+ this._clientID = null;
+
+ // Generate a new client ID and make sure this module uses the new version
+ let p = (async () => {
+ await ClientID.removeClientIDs();
+ let id = await ClientID.getClientID();
+ this._clientID = id;
+ Telemetry.scalarSet("telemetry.data_upload_optin", true);
+
+ await this.saveUninstallPing().catch(e =>
+ this._log.warn("_onUploadPrefChange - saveUninstallPing failed", e)
+ );
+ })();
+
+ this._shutdownBarrier.client.addBlocker(
+ "TelemetryController: resetting client ID after data upload was enabled",
+ p
+ );
+
+ return;
+ }
+
+ let p = (async () => {
+ try {
+ // 1. Cancel the current pings.
+ // 2. Clear unpersisted pings
+ await TelemetrySend.clearCurrentPings();
+
+ // 3. Remove all pending pings
+ await TelemetryStorage.removeAppDataPings();
+ await TelemetryStorage.runRemovePendingPingsTask();
+ await TelemetryStorage.removeUninstallPings();
+ } catch (e) {
+ this._log.error(
+ "_onUploadPrefChange - error clearing pending pings",
+ e
+ );
+ } finally {
+ // 4. Reset session and subsession counter
+ TelemetrySession.resetSubsessionCounter();
+
+ // 5. Collect any additional identifiers we want to send in the
+ // deletion request.
+ const scalars = Telemetry.getSnapshotForScalars(
+ "deletion-request",
+ /* clear */ true
+ );
+
+ // 6. Set ClientID to a known value
+ let oldClientId = await ClientID.getClientID();
+ await ClientID.setCanaryClientIDs();
+ this._clientID = await ClientID.getClientID();
+
+ // 7. Send the deletion-request ping.
+ this._log.trace("_onUploadPrefChange - Sending deletion-request ping.");
+ this.submitExternalPing(
+ PING_TYPE_DELETION_REQUEST,
+ { scalars },
+ { overrideClientId: oldClientId }
+ );
+ this._deletionRequestPingSubmittedPromise = null;
+ }
+ })();
+
+ this._deletionRequestPingSubmittedPromise = p;
+ this._shutdownBarrier.client.addBlocker(
+ "TelemetryController: removing pending pings after data upload was disabled",
+ p
+ );
+
+ Services.obs.notifyObservers(
+ null,
+ TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC
+ );
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
+
+ _attachObservers() {
+ if (TelemetryControllerBase.IS_UNIFIED_TELEMETRY) {
+ // Watch the FHR upload setting to trigger "deletion-request" pings.
+ Services.prefs.addObserver(
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ this,
+ true
+ );
+ }
+ },
+
+ /**
+ * Remove the preference observer to avoid leaks.
+ */
+ _detachObservers() {
+ if (TelemetryControllerBase.IS_UNIFIED_TELEMETRY) {
+ Services.prefs.removeObserver(
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ this
+ );
+ }
+ },
+
+ /**
+ * Allows waiting for TelemetryControllers delayed initialization to complete.
+ * This will complete before TelemetryController is shutting down.
+ * @return {Promise} Resolved when delayed TelemetryController initialization completed.
+ */
+ promiseInitialized() {
+ return this._delayedInitTaskDeferred.promise;
+ },
+
+ getCurrentPingData(aSubsession) {
+ this._log.trace("getCurrentPingData - subsession: " + aSubsession);
+
+ // Telemetry is disabled, don't gather any data.
+ if (!Telemetry.canRecordBase) {
+ return null;
+ }
+
+ const reason = aSubsession
+ ? REASON_GATHER_SUBSESSION_PAYLOAD
+ : REASON_GATHER_PAYLOAD;
+ const type = PING_TYPE_MAIN;
+ const payload = TelemetrySession.getPayload(reason);
+ const options = { addClientId: true, addEnvironment: true };
+ const ping = this.assemblePing(type, payload, options);
+
+ return ping;
+ },
+
+ async reset() {
+ this._clientID = null;
+ this._fnSyncPingShutdown = null;
+ this._detachObservers();
+
+ let sessionReset = TelemetrySession.testReset();
+
+ this._connectionsBarrier = new AsyncShutdown.Barrier(
+ "TelemetryController: Waiting for pending ping activity"
+ );
+ this._shutdownBarrier = new AsyncShutdown.Barrier(
+ "TelemetryController: Waiting for clients."
+ );
+
+ // We need to kick of the controller setup first for tests that check the
+ // cached client id.
+ let controllerSetup = this.setupTelemetry(true);
+
+ await sessionReset;
+ await TelemetrySend.reset();
+ await TelemetryStorage.reset();
+ await TelemetryEnvironment.testReset();
+
+ await controllerSetup;
+ },
+
+ /**
+ * Schedule sending the "new-profile" ping.
+ */
+ scheduleNewProfilePing() {
+ this._log.trace("scheduleNewProfilePing");
+
+ const sendDelay = Services.prefs.getIntPref(
+ TelemetryUtils.Preferences.NewProfilePingDelay,
+ NEWPROFILE_PING_DEFAULT_DELAY
+ );
+
+ this._delayedNewPingTask = new DeferredTask(async () => {
+ try {
+ await this.sendNewProfilePing();
+ } finally {
+ this._delayedNewPingTask = null;
+ }
+ }, sendDelay);
+
+ this._delayedNewPingTask.arm();
+ },
+
+ /**
+ * Generate and send the new-profile ping
+ */
+ async sendNewProfilePing() {
+ this._log.trace(
+ "sendNewProfilePing - shutting down: " + this._shuttingDown
+ );
+
+ const scalars = Telemetry.getSnapshotForScalars(
+ "new-profile",
+ /* clear */ true
+ );
+
+ // Generate the payload.
+ const payload = {
+ reason: this._shuttingDown ? "shutdown" : "startup",
+ processes: {
+ parent: {
+ scalars: scalars.parent,
+ },
+ },
+ };
+
+ // Generate and send the "new-profile" ping. This uses the
+ // pingsender if we're shutting down.
+ let options = {
+ addClientId: true,
+ addEnvironment: true,
+ usePingSender: this._shuttingDown,
+ };
+ // TODO: we need to be smarter about when to send the ping (and save the
+ // state to file). |requestIdleCallback| is currently only accessible
+ // through DOM. See bug 1361996.
+ await TelemetryController.submitExternalPing(
+ "new-profile",
+ payload,
+ options
+ ).then(
+ () => TelemetrySession.markNewProfilePingSent(),
+ e =>
+ this._log.error(
+ "sendNewProfilePing - failed to submit new-profile ping",
+ e
+ )
+ );
+ },
+
+ /**
+ * Register 'dynamic builtin' probes from the JSON definition files.
+ * This is needed to support adding new probes in developer builds
+ * without rebuilding the whole codebase.
+ *
+ * This is not meant to be used outside of local developer builds.
+ */
+ async registerJsProbes() {
+ // We don't support this outside of developer builds.
+ if (AppConstants.MOZILLA_OFFICIAL && !this._testMode) {
+ return;
+ }
+
+ this._log.trace("registerJsProbes - registering builtin JS probes");
+
+ await this.registerScalarProbes();
+ await this.registerEventProbes();
+ },
+
+ _loadProbeDefinitions(filename) {
+ let probeFile = Services.dirsvc.get("GreD", Ci.nsIFile);
+ probeFile.append(filename);
+ if (!probeFile.exists()) {
+ this._log.trace(
+ `loadProbeDefinitions - no builtin JS probe file ${filename}`
+ );
+ return null;
+ }
+
+ return OS.File.read(probeFile.path, { encoding: "utf-8" });
+ },
+
+ async registerScalarProbes() {
+ this._log.trace(
+ "registerScalarProbes - registering scalar builtin JS probes"
+ );
+
+ // Load the scalar probes JSON file.
+ const scalarProbeFilename = "ScalarArtifactDefinitions.json";
+ let scalarJSProbes = {};
+ try {
+ let fileContent = await this._loadProbeDefinitions(scalarProbeFilename);
+ scalarJSProbes = JSON.parse(fileContent, (property, value) => {
+ // Fixup the "kind" property: it's a string, and we need the constant
+ // coming from nsITelemetry.
+ if (property !== "kind" || typeof value != "string") {
+ return value;
+ }
+
+ let newValue;
+ switch (value) {
+ case "nsITelemetry::SCALAR_TYPE_COUNT":
+ newValue = Telemetry.SCALAR_TYPE_COUNT;
+ break;
+ case "nsITelemetry::SCALAR_TYPE_BOOLEAN":
+ newValue = Telemetry.SCALAR_TYPE_BOOLEAN;
+ break;
+ case "nsITelemetry::SCALAR_TYPE_STRING":
+ newValue = Telemetry.SCALAR_TYPE_STRING;
+ break;
+ }
+ return newValue;
+ });
+ } catch (ex) {
+ this._log.error(
+ `registerScalarProbes - there was an error loading ${scalarProbeFilename}`,
+ ex
+ );
+ }
+
+ // Register the builtin probes.
+ for (let category in scalarJSProbes) {
+ // Expire the expired scalars
+ for (let name in scalarJSProbes[category]) {
+ let def = scalarJSProbes[category][name];
+ if (
+ !def ||
+ !def.expires ||
+ def.expires == "never" ||
+ def.expires == "default"
+ ) {
+ continue;
+ }
+ if (
+ Services.vc.compare(AppConstants.MOZ_APP_VERSION, def.expires) >= 0
+ ) {
+ def.expired = true;
+ }
+ }
+ Telemetry.registerBuiltinScalars(category, scalarJSProbes[category]);
+ }
+ },
+
+ async registerEventProbes() {
+ this._log.trace(
+ "registerEventProbes - registering builtin JS Event probes"
+ );
+
+ // Load the event probes JSON file.
+ const eventProbeFilename = "EventArtifactDefinitions.json";
+ let eventJSProbes = {};
+ try {
+ let fileContent = await this._loadProbeDefinitions(eventProbeFilename);
+ eventJSProbes = JSON.parse(fileContent);
+ } catch (ex) {
+ this._log.error(
+ `registerEventProbes - there was an error loading ${eventProbeFilename}`,
+ ex
+ );
+ }
+
+ // Register the builtin probes.
+ for (let category in eventJSProbes) {
+ for (let name in eventJSProbes[category]) {
+ let def = eventJSProbes[category][name];
+ if (
+ !def ||
+ !def.expires ||
+ def.expires == "never" ||
+ def.expires == "default"
+ ) {
+ continue;
+ }
+ if (
+ Services.vc.compare(AppConstants.MOZ_APP_VERSION, def.expires) >= 0
+ ) {
+ def.expired = true;
+ }
+ }
+ Telemetry.registerBuiltinEvents(category, eventJSProbes[category]);
+ }
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryEnvironment.jsm b/toolkit/components/telemetry/app/TelemetryEnvironment.jsm
new file mode 100644
index 0000000000..8e82ad49bb
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryEnvironment.jsm
@@ -0,0 +1,2128 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryEnvironment"];
+
+const myScope = this;
+
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
+const { ObjectUtils } = ChromeUtils.import(
+ "resource://gre/modules/ObjectUtils.jsm"
+);
+const { AppConstants } = ChromeUtils.import(
+ "resource://gre/modules/AppConstants.jsm"
+);
+if (AppConstants.MOZ_GLEAN) {
+ Cu.importGlobalProperties(["Glean"]);
+}
+
+const Utils = TelemetryUtils;
+
+const { AddonManager, AddonManagerPrivate } = ChromeUtils.import(
+ "resource://gre/modules/AddonManager.jsm"
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "AttributionCode",
+ "resource:///modules/AttributionCode.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "ProfileAge",
+ "resource://gre/modules/ProfileAge.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "WindowsRegistry",
+ "resource://gre/modules/WindowsRegistry.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "UpdateUtils",
+ "resource://gre/modules/UpdateUtils.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "fxAccounts",
+ "resource://gre/modules/FxAccounts.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "WindowsVersionInfo",
+ "resource://gre/modules/components-utils/WindowsVersionInfo.jsm"
+);
+
+// The maximum length of a string (e.g. description) in the addons section.
+const MAX_ADDON_STRING_LENGTH = 100;
+// The maximum length of a string value in the settings.attribution object.
+const MAX_ATTRIBUTION_STRING_LENGTH = 100;
+// The maximum lengths for the experiment id and branch in the experiments section.
+const MAX_EXPERIMENT_ID_LENGTH = 100;
+const MAX_EXPERIMENT_BRANCH_LENGTH = 100;
+const MAX_EXPERIMENT_TYPE_LENGTH = 20;
+const MAX_EXPERIMENT_ENROLLMENT_ID_LENGTH = 40;
+
+/**
+ * This is a policy object used to override behavior for testing.
+ */
+// eslint-disable-next-line no-unused-vars
+var Policy = {
+ now: () => new Date(),
+ _intlLoaded: false,
+ _browserDelayedStartup() {
+ if (Policy._intlLoaded) {
+ return Promise.resolve();
+ }
+ return new Promise(resolve => {
+ let startupTopic = "browser-delayed-startup-finished";
+ Services.obs.addObserver(function observer(subject, topic) {
+ if (topic == startupTopic) {
+ Services.obs.removeObserver(observer, startupTopic);
+ resolve();
+ }
+ }, startupTopic);
+ });
+ },
+};
+
+// This is used to buffer calls to setExperimentActive and friends, so that we
+// don't prematurely initialize our environment if it is called early during
+// startup.
+var gActiveExperimentStartupBuffer = new Map();
+
+var gGlobalEnvironment;
+function getGlobal() {
+ if (!gGlobalEnvironment) {
+ gGlobalEnvironment = new EnvironmentCache();
+ }
+ return gGlobalEnvironment;
+}
+
+var TelemetryEnvironment = {
+ get currentEnvironment() {
+ return getGlobal().currentEnvironment;
+ },
+
+ onInitialized() {
+ return getGlobal().onInitialized();
+ },
+
+ delayedInit() {
+ return getGlobal().delayedInit();
+ },
+
+ registerChangeListener(name, listener) {
+ return getGlobal().registerChangeListener(name, listener);
+ },
+
+ unregisterChangeListener(name) {
+ return getGlobal().unregisterChangeListener(name);
+ },
+
+ /**
+ * Add an experiment annotation to the environment.
+ * If an annotation with the same id already exists, it will be overwritten.
+ * This triggers a new subsession, subject to throttling.
+ *
+ * @param {String} id The id of the active experiment.
+ * @param {String} branch The experiment branch.
+ * @param {Object} [options] Optional object with options.
+ * @param {String} [options.type=false] The specific experiment type.
+ * @param {String} [options.enrollmentId=undefined] The id of the enrollment.
+ */
+ setExperimentActive(id, branch, options = {}) {
+ if (gGlobalEnvironment) {
+ gGlobalEnvironment.setExperimentActive(id, branch, options);
+ } else {
+ gActiveExperimentStartupBuffer.set(id, { branch, options });
+ }
+ },
+
+ /**
+ * Remove an experiment annotation from the environment.
+ * If the annotation exists, a new subsession will triggered.
+ *
+ * @param {String} id The id of the active experiment.
+ */
+ setExperimentInactive(id) {
+ if (gGlobalEnvironment) {
+ gGlobalEnvironment.setExperimentInactive(id);
+ } else {
+ gActiveExperimentStartupBuffer.delete(id);
+ }
+ },
+
+ /**
+ * Returns an object containing the data for the active experiments.
+ *
+ * The returned object is of the format:
+ *
+ * {
+ * "<experiment id>": { branch: "<branch>" },
+ * // …
+ * }
+ */
+ getActiveExperiments() {
+ if (gGlobalEnvironment) {
+ return gGlobalEnvironment.getActiveExperiments();
+ }
+
+ const result = {};
+ for (const [id, { branch }] of gActiveExperimentStartupBuffer.entries()) {
+ result[id] = branch;
+ }
+ return result;
+ },
+
+ shutdown() {
+ return getGlobal().shutdown();
+ },
+
+ // Policy to use when saving preferences. Exported for using them in tests.
+ // Reports "<user-set>" if there is a value set on the user branch
+ RECORD_PREF_STATE: 1,
+
+ // Reports the value set on the user branch, if one is set
+ RECORD_PREF_VALUE: 2,
+
+ // Reports the active value (set on either the user or default branch)
+ // for this pref, if one is set
+ RECORD_DEFAULTPREF_VALUE: 3,
+
+ // Reports "<set>" if a value for this pref is defined on either the user
+ // or default branch
+ RECORD_DEFAULTPREF_STATE: 4,
+
+ // Testing method
+ async testWatchPreferences(prefMap) {
+ return getGlobal()._watchPreferences(prefMap);
+ },
+
+ /**
+ * Intended for use in tests only.
+ *
+ * In multiple tests we need a way to shut and re-start telemetry together
+ * with TelemetryEnvironment. This is problematic due to the fact that
+ * TelemetryEnvironment is a singleton. We, therefore, need this helper
+ * method to be able to re-set TelemetryEnvironment.
+ */
+ testReset() {
+ return getGlobal().reset();
+ },
+
+ /**
+ * Intended for use in tests only.
+ */
+ testCleanRestart() {
+ getGlobal().shutdown();
+ gGlobalEnvironment = null;
+ gActiveExperimentStartupBuffer = new Map();
+ return getGlobal();
+ },
+};
+
+const RECORD_PREF_STATE = TelemetryEnvironment.RECORD_PREF_STATE;
+const RECORD_PREF_VALUE = TelemetryEnvironment.RECORD_PREF_VALUE;
+const RECORD_DEFAULTPREF_VALUE = TelemetryEnvironment.RECORD_DEFAULTPREF_VALUE;
+const RECORD_DEFAULTPREF_STATE = TelemetryEnvironment.RECORD_DEFAULTPREF_STATE;
+const DEFAULT_ENVIRONMENT_PREFS = new Map([
+ ["app.feedback.baseURL", { what: RECORD_PREF_VALUE }],
+ ["app.support.baseURL", { what: RECORD_PREF_VALUE }],
+ ["accessibility.browsewithcaret", { what: RECORD_PREF_VALUE }],
+ ["accessibility.force_disabled", { what: RECORD_PREF_VALUE }],
+ ["app.normandy.test-prefs.bool", { what: RECORD_PREF_VALUE }],
+ ["app.normandy.test-prefs.integer", { what: RECORD_PREF_VALUE }],
+ ["app.normandy.test-prefs.string", { what: RECORD_PREF_VALUE }],
+ ["app.shield.optoutstudies.enabled", { what: RECORD_PREF_VALUE }],
+ ["app.update.interval", { what: RECORD_PREF_VALUE }],
+ ["app.update.service.enabled", { what: RECORD_PREF_VALUE }],
+ ["app.update.silent", { what: RECORD_PREF_VALUE }],
+ ["browser.cache.disk.enable", { what: RECORD_PREF_VALUE }],
+ ["browser.cache.disk.capacity", { what: RECORD_PREF_VALUE }],
+ ["browser.cache.memory.enable", { what: RECORD_PREF_VALUE }],
+ ["browser.cache.offline.enable", { what: RECORD_PREF_VALUE }],
+ ["browser.formfill.enable", { what: RECORD_PREF_VALUE }],
+ ["browser.newtabpage.enabled", { what: RECORD_PREF_VALUE }],
+ ["browser.shell.checkDefaultBrowser", { what: RECORD_PREF_VALUE }],
+ ["browser.search.ignoredJAREngines", { what: RECORD_DEFAULTPREF_VALUE }],
+ ["browser.search.region", { what: RECORD_PREF_VALUE }],
+ ["browser.search.suggest.enabled", { what: RECORD_PREF_VALUE }],
+ ["browser.search.widget.inNavBar", { what: RECORD_DEFAULTPREF_VALUE }],
+ ["browser.startup.homepage", { what: RECORD_PREF_STATE }],
+ ["browser.startup.page", { what: RECORD_PREF_VALUE }],
+ ["browser.urlbar.suggest.searches", { what: RECORD_PREF_VALUE }],
+ ["devtools.chrome.enabled", { what: RECORD_PREF_VALUE }],
+ ["devtools.debugger.enabled", { what: RECORD_PREF_VALUE }],
+ ["devtools.debugger.remote-enabled", { what: RECORD_PREF_VALUE }],
+ ["dom.ipc.plugins.enabled", { what: RECORD_PREF_VALUE }],
+ ["dom.ipc.plugins.sandbox-level.flash", { what: RECORD_PREF_VALUE }],
+ ["dom.ipc.processCount", { what: RECORD_PREF_VALUE }],
+ ["dom.max_script_run_time", { what: RECORD_PREF_VALUE }],
+ ["extensions.autoDisableScopes", { what: RECORD_PREF_VALUE }],
+ ["extensions.enabledScopes", { what: RECORD_PREF_VALUE }],
+ ["extensions.blocklist.enabled", { what: RECORD_PREF_VALUE }],
+ ["extensions.formautofill.addresses.enabled", { what: RECORD_PREF_VALUE }],
+ [
+ "extensions.formautofill.addresses.capture.enabled",
+ { what: RECORD_PREF_VALUE },
+ ],
+ ["extensions.formautofill.creditCards.enabled", { what: RECORD_PREF_VALUE }],
+ [
+ "extensions.formautofill.creditCards.available",
+ { what: RECORD_PREF_VALUE },
+ ],
+ ["extensions.formautofill.creditCards.used", { what: RECORD_PREF_VALUE }],
+ ["extensions.strictCompatibility", { what: RECORD_PREF_VALUE }],
+ ["extensions.update.enabled", { what: RECORD_PREF_VALUE }],
+ ["extensions.update.url", { what: RECORD_PREF_VALUE }],
+ ["extensions.update.background.url", { what: RECORD_PREF_VALUE }],
+ ["extensions.screenshots.disabled", { what: RECORD_PREF_VALUE }],
+ ["general.config.filename", { what: RECORD_DEFAULTPREF_STATE }],
+ ["general.smoothScroll", { what: RECORD_PREF_VALUE }],
+ ["gfx.direct2d.disabled", { what: RECORD_PREF_VALUE }],
+ ["gfx.direct2d.force-enabled", { what: RECORD_PREF_VALUE }],
+ ["gfx.webrender.all", { what: RECORD_PREF_VALUE }],
+ ["gfx.webrender.all.qualified", { what: RECORD_PREF_VALUE }],
+ ["gfx.webrender.force-disabled", { what: RECORD_PREF_VALUE }],
+ ["layers.acceleration.disabled", { what: RECORD_PREF_VALUE }],
+ ["layers.acceleration.force-enabled", { what: RECORD_PREF_VALUE }],
+ ["layers.async-pan-zoom.enabled", { what: RECORD_PREF_VALUE }],
+ ["layers.async-video-oop.enabled", { what: RECORD_PREF_VALUE }],
+ ["layers.async-video.enabled", { what: RECORD_PREF_VALUE }],
+ ["layers.componentalpha.enabled", { what: RECORD_PREF_VALUE }],
+ ["layers.d3d11.disable-warp", { what: RECORD_PREF_VALUE }],
+ ["layers.d3d11.force-warp", { what: RECORD_PREF_VALUE }],
+ [
+ "layers.offmainthreadcomposition.force-disabled",
+ { what: RECORD_PREF_VALUE },
+ ],
+ ["layers.prefer-d3d9", { what: RECORD_PREF_VALUE }],
+ ["layers.prefer-opengl", { what: RECORD_PREF_VALUE }],
+ ["layout.css.devPixelsPerPx", { what: RECORD_PREF_VALUE }],
+ ["marionette.enabled", { what: RECORD_PREF_VALUE }],
+ ["network.proxy.autoconfig_url", { what: RECORD_PREF_STATE }],
+ ["network.proxy.http", { what: RECORD_PREF_STATE }],
+ ["network.proxy.ssl", { what: RECORD_PREF_STATE }],
+ ["network.trr.mode", { what: RECORD_PREF_VALUE }],
+ ["pdfjs.disabled", { what: RECORD_PREF_VALUE }],
+ ["places.history.enabled", { what: RECORD_PREF_VALUE }],
+ ["plugins.show_infobar", { what: RECORD_PREF_VALUE }],
+ ["privacy.fuzzyfox.enabled", { what: RECORD_PREF_VALUE }],
+ ["privacy.trackingprotection.enabled", { what: RECORD_PREF_VALUE }],
+ ["privacy.donottrackheader.enabled", { what: RECORD_PREF_VALUE }],
+ ["security.enterprise_roots.auto-enabled", { what: RECORD_PREF_VALUE }],
+ ["security.enterprise_roots.enabled", { what: RECORD_PREF_VALUE }],
+ ["security.pki.mitm_detected", { what: RECORD_PREF_VALUE }],
+ ["security.mixed_content.block_active_content", { what: RECORD_PREF_VALUE }],
+ ["security.mixed_content.block_display_content", { what: RECORD_PREF_VALUE }],
+ ["security.tls.version.enable-deprecated", { what: RECORD_PREF_VALUE }],
+ ["signon.management.page.breach-alerts.enabled", { what: RECORD_PREF_VALUE }],
+ ["signon.autofillForms", { what: RECORD_PREF_VALUE }],
+ ["signon.generation.enabled", { what: RECORD_PREF_VALUE }],
+ ["signon.rememberSignons", { what: RECORD_PREF_VALUE }],
+ ["toolkit.telemetry.pioneerId", { what: RECORD_PREF_STATE }],
+ ["widget.content.allow-gtk-dark-theme", { what: RECORD_DEFAULTPREF_VALUE }],
+ ["widget.content.gtk-theme-override", { what: RECORD_PREF_STATE }],
+ [
+ "widget.content.gtk-high-contrast.enabled",
+ { what: RECORD_DEFAULTPREF_VALUE },
+ ],
+ ["xpinstall.signatures.required", { what: RECORD_PREF_VALUE }],
+]);
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+
+const PREF_BLOCKLIST_ENABLED = "extensions.blocklist.enabled";
+const PREF_DISTRIBUTION_ID = "distribution.id";
+const PREF_DISTRIBUTION_VERSION = "distribution.version";
+const PREF_DISTRIBUTOR = "app.distributor";
+const PREF_DISTRIBUTOR_CHANNEL = "app.distributor.channel";
+const PREF_APP_PARTNER_BRANCH = "app.partner.";
+const PREF_PARTNER_ID = "mozilla.partner.id";
+
+const COMPOSITOR_CREATED_TOPIC = "compositor:created";
+const COMPOSITOR_PROCESS_ABORTED_TOPIC = "compositor:process-aborted";
+const DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC =
+ "distribution-customization-complete";
+const GFX_FEATURES_READY_TOPIC = "gfx-features-ready";
+const SEARCH_ENGINE_MODIFIED_TOPIC = "browser-search-engine-modified";
+const SEARCH_SERVICE_TOPIC = "browser-search-service";
+const SESSIONSTORE_WINDOWS_RESTORED_TOPIC = "sessionstore-windows-restored";
+const PREF_CHANGED_TOPIC = "nsPref:changed";
+const BLOCKLIST_LOADED_TOPIC = "plugin-blocklist-loaded";
+const AUTO_UPDATE_PREF_CHANGE_TOPIC = "auto-update-config-change";
+const SERVICES_INFO_CHANGE_TOPIC = "sync-ui-state:update";
+
+/**
+ * Enforces the parameter to a boolean value.
+ * @param aValue The input value.
+ * @return {Boolean|Object} If aValue is a boolean or a number, returns its truthfulness
+ * value. Otherwise, return null.
+ */
+function enforceBoolean(aValue) {
+ if (typeof aValue !== "number" && typeof aValue !== "boolean") {
+ return null;
+ }
+ return Boolean(aValue);
+}
+
+/**
+ * Get the current browser locale.
+ * @return a string with the locale or null on failure.
+ */
+function getBrowserLocale() {
+ try {
+ return Services.locale.appLocaleAsBCP47;
+ } catch (e) {
+ return null;
+ }
+}
+
+/**
+ * Get the current OS locale.
+ * @return a string with the OS locale or null on failure.
+ */
+function getSystemLocale() {
+ try {
+ return Cc["@mozilla.org/intl/ospreferences;1"].getService(
+ Ci.mozIOSPreferences
+ ).systemLocale;
+ } catch (e) {
+ return null;
+ }
+}
+
+/**
+ * Get the current OS locales.
+ * @return an array of strings with the OS locales or null on failure.
+ */
+function getSystemLocales() {
+ try {
+ return Cc["@mozilla.org/intl/ospreferences;1"].getService(
+ Ci.mozIOSPreferences
+ ).systemLocales;
+ } catch (e) {
+ return null;
+ }
+}
+
+/**
+ * Get the current OS regional preference locales.
+ * @return an array of strings with the OS regional preference locales or null on failure.
+ */
+function getRegionalPrefsLocales() {
+ try {
+ return Cc["@mozilla.org/intl/ospreferences;1"].getService(
+ Ci.mozIOSPreferences
+ ).regionalPrefsLocales;
+ } catch (e) {
+ return null;
+ }
+}
+
+function getIntlSettings() {
+ return {
+ requestedLocales: Services.locale.requestedLocales,
+ availableLocales: Services.locale.availableLocales,
+ appLocales: Services.locale.appLocalesAsBCP47,
+ systemLocales: getSystemLocales(),
+ regionalPrefsLocales: getRegionalPrefsLocales(),
+ acceptLanguages: Services.prefs
+ .getComplexValue("intl.accept_languages", Ci.nsIPrefLocalizedString)
+ .data.split(",")
+ .map(str => str.trim()),
+ };
+}
+
+/**
+ * Safely get a sysinfo property and return its value. If the property is not
+ * available, return aDefault.
+ *
+ * @param aPropertyName the property name to get.
+ * @param aDefault the value to return if aPropertyName is not available.
+ * @return The property value, if available, or aDefault.
+ */
+function getSysinfoProperty(aPropertyName, aDefault) {
+ try {
+ // |getProperty| may throw if |aPropertyName| does not exist.
+ return Services.sysinfo.getProperty(aPropertyName);
+ } catch (e) {}
+
+ return aDefault;
+}
+
+/**
+ * Safely get a gfxInfo field and return its value. If the field is not available, return
+ * aDefault.
+ *
+ * @param aPropertyName the property name to get.
+ * @param aDefault the value to return if aPropertyName is not available.
+ * @return The property value, if available, or aDefault.
+ */
+function getGfxField(aPropertyName, aDefault) {
+ let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo);
+
+ try {
+ // Accessing the field may throw if |aPropertyName| does not exist.
+ let gfxProp = gfxInfo[aPropertyName];
+ if (gfxProp !== undefined && gfxProp !== "") {
+ return gfxProp;
+ }
+ } catch (e) {}
+
+ return aDefault;
+}
+
+/**
+ * Returns a substring of the input string.
+ *
+ * @param {String} aString The input string.
+ * @param {Integer} aMaxLength The maximum length of the returned substring. If this is
+ * greater than the length of the input string, we return the whole input string.
+ * @return {String} The substring or null if the input string is null.
+ */
+function limitStringToLength(aString, aMaxLength) {
+ if (typeof aString !== "string") {
+ return null;
+ }
+ return aString.substring(0, aMaxLength);
+}
+
+/**
+ * Force a value to be a string.
+ * Only if the value is null, null is returned instead.
+ */
+function forceToStringOrNull(aValue) {
+ if (aValue === null) {
+ return null;
+ }
+
+ return String(aValue);
+}
+
+/**
+ * Get the information about a graphic adapter.
+ *
+ * @param aSuffix A suffix to add to the properties names.
+ * @return An object containing the adapter properties.
+ */
+function getGfxAdapter(aSuffix = "") {
+ // Note that gfxInfo, and so getGfxField, might return "Unknown" for the RAM on failures,
+ // not null.
+ let memoryMB = parseInt(getGfxField("adapterRAM" + aSuffix, null), 10);
+ if (Number.isNaN(memoryMB)) {
+ memoryMB = null;
+ }
+
+ return {
+ description: getGfxField("adapterDescription" + aSuffix, null),
+ vendorID: getGfxField("adapterVendorID" + aSuffix, null),
+ deviceID: getGfxField("adapterDeviceID" + aSuffix, null),
+ subsysID: getGfxField("adapterSubsysID" + aSuffix, null),
+ RAM: memoryMB,
+ driver: getGfxField("adapterDriver" + aSuffix, null),
+ driverVendor: getGfxField("adapterDriverVendor" + aSuffix, null),
+ driverVersion: getGfxField("adapterDriverVersion" + aSuffix, null),
+ driverDate: getGfxField("adapterDriverDate" + aSuffix, null),
+ };
+}
+
+/**
+ * Encapsulates the asynchronous magic interfacing with the addon manager. The builder
+ * is owned by a parent environment object and is an addon listener.
+ */
+function EnvironmentAddonBuilder(environment) {
+ this._environment = environment;
+
+ // The pending task blocks addon manager shutdown. It can either be the initial load
+ // or a change load.
+ this._pendingTask = null;
+
+ // Have we added an observer to listen for blocklist changes that still needs to be
+ // removed:
+ this._blocklistObserverAdded = false;
+
+ // Set to true once initial load is complete and we're watching for changes.
+ this._loaded = false;
+
+ // The state reported by the shutdown blocker if we hang shutdown.
+ this._shutdownState = "Initial";
+}
+EnvironmentAddonBuilder.prototype = {
+ /**
+ * Get the initial set of addons.
+ * @returns Promise<void> when the initial load is complete.
+ */
+ async init() {
+ AddonManager.beforeShutdown.addBlocker(
+ "EnvironmentAddonBuilder",
+ () => this._shutdownBlocker(),
+ { fetchState: () => this._shutdownState }
+ );
+
+ this._pendingTask = (async () => {
+ try {
+ this._shutdownState = "Awaiting _updateAddons";
+ // Gather initial addons details
+ await this._updateAddons(true);
+
+ if (!this._environment._addonsAreFull) {
+ // The addon database has not been loaded, wait for it to
+ // initialize and gather full data as soon as it does.
+ this._shutdownState = "Awaiting AddonManagerPrivate.databaseReady";
+ await AddonManagerPrivate.databaseReady;
+
+ // Now gather complete addons details.
+ this._shutdownState = "Awaiting second _updateAddons";
+ await this._updateAddons();
+ }
+ } catch (err) {
+ this._environment._log.error("init - Exception in _updateAddons", err);
+ } finally {
+ this._pendingTask = null;
+ this._shutdownState = "_pendingTask init complete. No longer blocking.";
+ }
+ })();
+
+ return this._pendingTask;
+ },
+
+ /**
+ * Register an addon listener and watch for changes.
+ */
+ watchForChanges() {
+ this._loaded = true;
+ AddonManager.addAddonListener(this);
+ },
+
+ // AddonListener
+ onEnabled(addon) {
+ this._onAddonChange(addon);
+ },
+ onDisabled(addon) {
+ this._onAddonChange(addon);
+ },
+ onInstalled(addon) {
+ this._onAddonChange(addon);
+ },
+ onUninstalling(addon) {
+ this._onAddonChange(addon);
+ },
+ onUninstalled(addon) {
+ this._onAddonChange(addon);
+ },
+
+ _onAddonChange(addon) {
+ if (addon && addon.isBuiltin && !addon.isSystem) {
+ return;
+ }
+ this._environment._log.trace("_onAddonChange");
+ this._checkForChanges("addons-changed");
+ },
+
+ // nsIObserver
+ observe(aSubject, aTopic, aData) {
+ this._environment._log.trace("observe - Topic " + aTopic);
+ if (aTopic == BLOCKLIST_LOADED_TOPIC) {
+ Services.obs.removeObserver(this, BLOCKLIST_LOADED_TOPIC);
+ this._blocklistObserverAdded = false;
+ let plugins = this._getActivePlugins();
+ let gmpPluginsPromise = this._getActiveGMPlugins();
+ gmpPluginsPromise.then(
+ gmpPlugins => {
+ let { addons } = this._environment._currentEnvironment;
+ addons.activePlugins = plugins;
+ addons.activeGMPlugins = gmpPlugins;
+ },
+ err => {
+ this._environment._log.error(
+ "blocklist observe: Error collecting plugins",
+ err
+ );
+ }
+ );
+ }
+ },
+
+ _checkForChanges(changeReason) {
+ if (this._pendingTask) {
+ this._environment._log.trace(
+ "_checkForChanges - task already pending, dropping change with reason " +
+ changeReason
+ );
+ return;
+ }
+
+ this._shutdownState = "_checkForChanges awaiting _updateAddons";
+ this._pendingTask = this._updateAddons().then(
+ result => {
+ this._pendingTask = null;
+ this._shutdownState = "No longer blocking, _updateAddons resolved";
+ if (result.changed) {
+ this._environment._onEnvironmentChange(
+ changeReason,
+ result.oldEnvironment
+ );
+ }
+ },
+ err => {
+ this._pendingTask = null;
+ this._shutdownState = "No longer blocking, _updateAddons rejected";
+ this._environment._log.error(
+ "_checkForChanges: Error collecting addons",
+ err
+ );
+ }
+ );
+ },
+
+ _shutdownBlocker() {
+ if (this._loaded) {
+ AddonManager.removeAddonListener(this);
+ if (this._blocklistObserverAdded) {
+ Services.obs.removeObserver(this, BLOCKLIST_LOADED_TOPIC);
+ }
+ }
+
+ // At startup, _pendingTask is set to a Promise that does not resolve
+ // until the addons database has been read so complete details about
+ // addons are available. Returning it here will cause it to block
+ // profileBeforeChange, guranteeing that full information will be
+ // available by the time profileBeforeChangeTelemetry is fired.
+ return this._pendingTask;
+ },
+
+ /**
+ * Collect the addon data for the environment.
+ *
+ * This should only be called from _pendingTask; otherwise we risk
+ * running this during addon manager shutdown.
+ *
+ * @param {boolean} [atStartup]
+ * True if this is the first check we're performing at startup. In that
+ * situation, we defer some more expensive initialization.
+ *
+ * @returns Promise<Object> This returns a Promise resolved with a status object with the following members:
+ * changed - Whether the environment changed.
+ * oldEnvironment - Only set if a change occured, contains the environment data before the change.
+ */
+ async _updateAddons(atStartup) {
+ this._environment._log.trace("_updateAddons");
+
+ let addons = {
+ activeAddons: await this._getActiveAddons(),
+ theme: await this._getActiveTheme(),
+ activePlugins: this._getActivePlugins(atStartup),
+ activeGMPlugins: await this._getActiveGMPlugins(atStartup),
+ };
+
+ let result = {
+ changed:
+ !this._environment._currentEnvironment.addons ||
+ !ObjectUtils.deepEqual(
+ addons.activeAddons,
+ this._environment._currentEnvironment.addons.activeAddons
+ ),
+ };
+
+ if (result.changed) {
+ this._environment._log.trace("_updateAddons: addons differ");
+ result.oldEnvironment = Cu.cloneInto(
+ this._environment._currentEnvironment,
+ myScope
+ );
+ }
+ this._environment._currentEnvironment.addons = addons;
+
+ return result;
+ },
+
+ /**
+ * Get the addon data in object form.
+ * @return Promise<object> containing the addon data.
+ */
+ async _getActiveAddons() {
+ // Request addons, asynchronously.
+ let { addons: allAddons, fullData } = await AddonManager.getActiveAddons([
+ "extension",
+ "service",
+ ]);
+
+ this._environment._addonsAreFull = fullData;
+ let activeAddons = {};
+ for (let addon of allAddons) {
+ // Don't collect any information about the new built-in search webextensions
+ if (addon.isBuiltin && !addon.isSystem) {
+ continue;
+ }
+ // Weird addon data in the wild can lead to exceptions while collecting
+ // the data.
+ try {
+ // Make sure to have valid dates.
+ let updateDate = new Date(Math.max(0, addon.updateDate));
+
+ activeAddons[addon.id] = {
+ version: limitStringToLength(addon.version, MAX_ADDON_STRING_LENGTH),
+ scope: addon.scope,
+ type: addon.type,
+ updateDay: Utils.millisecondsToDays(updateDate.getTime()),
+ isSystem: addon.isSystem,
+ isWebExtension: addon.isWebExtension,
+ multiprocessCompatible: true,
+ };
+
+ // getActiveAddons() gives limited data during startup and full
+ // data after the addons database is loaded.
+ if (fullData) {
+ let installDate = new Date(Math.max(0, addon.installDate));
+ Object.assign(activeAddons[addon.id], {
+ blocklisted:
+ addon.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
+ description: limitStringToLength(
+ addon.description,
+ MAX_ADDON_STRING_LENGTH
+ ),
+ name: limitStringToLength(addon.name, MAX_ADDON_STRING_LENGTH),
+ userDisabled: enforceBoolean(addon.userDisabled),
+ appDisabled: addon.appDisabled,
+ foreignInstall: enforceBoolean(addon.foreignInstall),
+ hasBinaryComponents: false,
+ installDay: Utils.millisecondsToDays(installDate.getTime()),
+ signedState: addon.signedState,
+ });
+ }
+ } catch (ex) {
+ this._environment._log.error(
+ "_getActiveAddons - An addon was discarded due to an error",
+ ex
+ );
+ continue;
+ }
+ }
+
+ return activeAddons;
+ },
+
+ /**
+ * Get the currently active theme data in object form.
+ * @return Promise<object> containing the active theme data.
+ */
+ async _getActiveTheme() {
+ // Request themes, asynchronously.
+ let { addons: themes } = await AddonManager.getActiveAddons(["theme"]);
+
+ let activeTheme = {};
+ // We only store information about the active theme.
+ let theme = themes.find(theme => theme.isActive);
+ if (theme) {
+ // Make sure to have valid dates.
+ let installDate = new Date(Math.max(0, theme.installDate));
+ let updateDate = new Date(Math.max(0, theme.updateDate));
+
+ activeTheme = {
+ id: theme.id,
+ blocklisted:
+ theme.blocklistState !== Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
+ description: limitStringToLength(
+ theme.description,
+ MAX_ADDON_STRING_LENGTH
+ ),
+ name: limitStringToLength(theme.name, MAX_ADDON_STRING_LENGTH),
+ userDisabled: enforceBoolean(theme.userDisabled),
+ appDisabled: theme.appDisabled,
+ version: limitStringToLength(theme.version, MAX_ADDON_STRING_LENGTH),
+ scope: theme.scope,
+ foreignInstall: enforceBoolean(theme.foreignInstall),
+ hasBinaryComponents: false,
+ installDay: Utils.millisecondsToDays(installDate.getTime()),
+ updateDay: Utils.millisecondsToDays(updateDate.getTime()),
+ };
+ }
+
+ return activeTheme;
+ },
+
+ /**
+ * Get the plugins data in object form.
+ *
+ * @param {boolean} [atStartup]
+ * True if this is the first check we're performing at startup. In that
+ * situation, we defer some more expensive initialization.
+ *
+ * @return Object containing the plugins data.
+ */
+ _getActivePlugins(atStartup) {
+ // If we haven't yet loaded the blocklist, pass back dummy data for now,
+ // and add an observer to update this data as soon as we get it.
+ if (atStartup || !Services.blocklist.isLoaded) {
+ if (!this._blocklistObserverAdded) {
+ Services.obs.addObserver(this, BLOCKLIST_LOADED_TOPIC);
+ this._blocklistObserverAdded = true;
+ }
+ return [
+ {
+ name: "dummy",
+ version: "0.1",
+ description: "Blocklist unavailable",
+ blocklisted: false,
+ disabled: true,
+ clicktoplay: false,
+ mimeTypes: ["text/there.is.only.blocklist"],
+ updateDay: Utils.millisecondsToDays(Date.now()),
+ },
+ ];
+ }
+ let pluginTags = Cc["@mozilla.org/plugin/host;1"]
+ .getService(Ci.nsIPluginHost)
+ .getPluginTags();
+
+ let activePlugins = [];
+ for (let tag of pluginTags) {
+ // Skip plugins which are not active.
+ if (tag.disabled) {
+ continue;
+ }
+
+ try {
+ // Make sure to have a valid date.
+ let updateDate = new Date(Math.max(0, tag.lastModifiedTime));
+
+ activePlugins.push({
+ name: limitStringToLength(tag.name, MAX_ADDON_STRING_LENGTH),
+ version: limitStringToLength(tag.version, MAX_ADDON_STRING_LENGTH),
+ description: limitStringToLength(
+ tag.description,
+ MAX_ADDON_STRING_LENGTH
+ ),
+ blocklisted: tag.blocklisted,
+ disabled: tag.disabled,
+ clicktoplay: tag.clicktoplay,
+ mimeTypes: tag.getMimeTypes(),
+ updateDay: Utils.millisecondsToDays(updateDate.getTime()),
+ });
+ } catch (ex) {
+ this._environment._log.error(
+ "_getActivePlugins - A plugin was discarded due to an error",
+ ex
+ );
+ continue;
+ }
+ }
+
+ return activePlugins;
+ },
+
+ /**
+ * Get the GMPlugins data in object form.
+ *
+ * @param {boolean} [atStartup]
+ * True if this is the first check we're performing at startup. In that
+ * situation, we defer some more expensive initialization.
+ *
+ * @return Object containing the GMPlugins data.
+ *
+ * This should only be called from _pendingTask; otherwise we risk
+ * running this during addon manager shutdown.
+ */
+ async _getActiveGMPlugins(atStartup) {
+ // If we haven't yet loaded the blocklist, pass back dummy data for now,
+ // and add an observer to update this data as soon as we get it.
+ if (atStartup || !Services.blocklist.isLoaded) {
+ if (!this._blocklistObserverAdded) {
+ Services.obs.addObserver(this, BLOCKLIST_LOADED_TOPIC);
+ this._blocklistObserverAdded = true;
+ }
+ return {
+ "dummy-gmp": {
+ version: "0.1",
+ userDisabled: false,
+ applyBackgroundUpdates: 1,
+ },
+ };
+ }
+ // Request plugins, asynchronously.
+ let allPlugins = await AddonManager.getAddonsByTypes(["plugin"]);
+
+ let activeGMPlugins = {};
+ for (let plugin of allPlugins) {
+ // Only get info for active GMplugins.
+ if (!plugin.isGMPlugin || !plugin.isActive) {
+ continue;
+ }
+
+ try {
+ activeGMPlugins[plugin.id] = {
+ version: plugin.version,
+ userDisabled: enforceBoolean(plugin.userDisabled),
+ applyBackgroundUpdates: plugin.applyBackgroundUpdates,
+ };
+ } catch (ex) {
+ this._environment._log.error(
+ "_getActiveGMPlugins - A GMPlugin was discarded due to an error",
+ ex
+ );
+ continue;
+ }
+ }
+
+ return activeGMPlugins;
+ },
+};
+
+function EnvironmentCache() {
+ this._log = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ "TelemetryEnvironment::"
+ );
+ this._log.trace("constructor");
+
+ this._shutdown = false;
+ // Don't allow querying the search service too early to prevent
+ // impacting the startup performance.
+ this._canQuerySearch = false;
+ // To guard against slowing down startup, defer gathering heavy environment
+ // entries until the session is restored.
+ this._sessionWasRestored = false;
+
+ // A map of listeners that will be called on environment changes.
+ this._changeListeners = new Map();
+
+ // A map of watched preferences which trigger an Environment change when
+ // modified. Every entry contains a recording policy (RECORD_PREF_*).
+ this._watchedPrefs = DEFAULT_ENVIRONMENT_PREFS;
+
+ this._currentEnvironment = {
+ build: this._getBuild(),
+ partner: this._getPartner(),
+ system: this._getSystem(),
+ };
+
+ this._addObservers();
+
+ // Build the remaining asynchronous parts of the environment. Don't register change listeners
+ // until the initial environment has been built.
+
+ let p = [this._updateSettings()];
+ this._addonBuilder = new EnvironmentAddonBuilder(this);
+ p.push(this._addonBuilder.init());
+
+ this._currentEnvironment.profile = {};
+ p.push(this._updateProfile());
+ if (AppConstants.MOZ_BUILD_APP == "browser") {
+ p.push(this._loadAttributionAsync());
+ }
+ p.push(this._loadAutoUpdateAsync());
+ p.push(this._loadIntlData());
+
+ for (const [
+ id,
+ { branch, options },
+ ] of gActiveExperimentStartupBuffer.entries()) {
+ this.setExperimentActive(id, branch, options);
+ }
+ gActiveExperimentStartupBuffer = null;
+
+ let setup = () => {
+ this._initTask = null;
+ this._startWatchingPrefs();
+ this._addonBuilder.watchForChanges();
+ this._updateGraphicsFeatures();
+ return this.currentEnvironment;
+ };
+
+ this._initTask = Promise.all(p).then(
+ () => setup(),
+ err => {
+ // log errors but eat them for consumers
+ this._log.error("EnvironmentCache - error while initializing", err);
+ return setup();
+ }
+ );
+
+ // Addons may contain partial or full data depending on whether the Addons DB
+ // has had a chance to load. Do we have full data yet?
+ this._addonsAreFull = false;
+}
+EnvironmentCache.prototype = {
+ /**
+ * The current environment data. The returned data is cloned to avoid
+ * unexpected sharing or mutation.
+ * @returns object
+ */
+ get currentEnvironment() {
+ return Cu.cloneInto(this._currentEnvironment, myScope);
+ },
+
+ /**
+ * Wait for the current enviroment to be fully initialized.
+ * @returns Promise<object>
+ */
+ onInitialized() {
+ if (this._initTask) {
+ return this._initTask;
+ }
+ return Promise.resolve(this.currentEnvironment);
+ },
+
+ /**
+ * This gets called when the delayed init completes.
+ */
+ async delayedInit() {
+ this._processData = await Services.sysinfo.processInfo;
+ let processData = await Services.sysinfo.processInfo;
+ // Remove isWow64 and isWowARM64 from processData
+ // to strip it down to just CPU info
+ delete processData.isWow64;
+ delete processData.isWowARM64;
+
+ let oldEnv = null;
+ if (!this._initTask) {
+ oldEnv = this.currentEnvironment;
+ }
+
+ this._cpuData = this._getCPUData();
+ // Augment the return value from the promises with cached values
+ this._cpuData = { ...processData, ...this._cpuData };
+
+ this._currentEnvironment.system.cpu = this._getCPUData();
+
+ if (AppConstants.platform == "win") {
+ this._hddData = await Services.sysinfo.diskInfo;
+ if (AppConstants.MOZ_GLEAN) {
+ Glean.fogValidation.profileDiskIsSsd.set(
+ this._hddData.profile.type == "SSD"
+ );
+ }
+ let osData = await Services.sysinfo.osInfo;
+
+ if (!this._initTask) {
+ // We've finished creating the initial env, so notify for the update
+ // This is all a bit awkward because `currentEnvironment` clones
+ // the object, which we need to pass to the notification, but we
+ // should only notify once we've updated the current environment...
+ // Ideally, _onEnvironmentChange should somehow deal with all this
+ // instead of all the consumers.
+ oldEnv = this.currentEnvironment;
+ }
+
+ this._osData = this._getOSData();
+
+ // Augment the return values from the promises with cached values
+ this._osData = Object.assign(osData, this._osData);
+
+ this._currentEnvironment.system.os = this._getOSData();
+ this._currentEnvironment.system.hdd = this._getHDDData();
+
+ // Windows only values stored in processData
+ this._currentEnvironment.system.isWow64 = this._getProcessData().isWow64;
+ this._currentEnvironment.system.isWowARM64 = this._getProcessData().isWowARM64;
+ }
+
+ if (!this._initTask) {
+ this._onEnvironmentChange("system-info", oldEnv);
+ }
+ },
+
+ /**
+ * Register a listener for environment changes.
+ * @param name The name of the listener. If a new listener is registered
+ * with the same name, the old listener will be replaced.
+ * @param listener function(reason, oldEnvironment) - Will receive a reason for
+ the change and the environment data before the change.
+ */
+ registerChangeListener(name, listener) {
+ this._log.trace("registerChangeListener for " + name);
+ if (this._shutdown) {
+ this._log.warn("registerChangeListener - already shutdown");
+ return;
+ }
+ this._changeListeners.set(name, listener);
+ },
+
+ /**
+ * Unregister from listening to environment changes.
+ * It's fine to call this on an unitialized TelemetryEnvironment.
+ * @param name The name of the listener to remove.
+ */
+ unregisterChangeListener(name) {
+ this._log.trace("unregisterChangeListener for " + name);
+ if (this._shutdown) {
+ this._log.warn("registerChangeListener - already shutdown");
+ return;
+ }
+ this._changeListeners.delete(name);
+ },
+
+ setExperimentActive(id, branch, options) {
+ this._log.trace(`setExperimentActive - id: ${id}, branch: ${branch}`);
+ // Make sure both the id and the branch have sane lengths.
+ const saneId = limitStringToLength(id, MAX_EXPERIMENT_ID_LENGTH);
+ const saneBranch = limitStringToLength(
+ branch,
+ MAX_EXPERIMENT_BRANCH_LENGTH
+ );
+ if (!saneId || !saneBranch) {
+ this._log.error(
+ "setExperimentActive - the provided arguments are not strings."
+ );
+ return;
+ }
+
+ // Warn the user about any content truncation.
+ if (saneId.length != id.length || saneBranch.length != branch.length) {
+ this._log.warn(
+ "setExperimentActive - the experiment id or branch were truncated."
+ );
+ }
+
+ // Truncate the experiment type if present.
+ if (options.hasOwnProperty("type")) {
+ let type = limitStringToLength(options.type, MAX_EXPERIMENT_TYPE_LENGTH);
+ if (type.length != options.type.length) {
+ options.type = type;
+ this._log.warn(
+ "setExperimentActive - the experiment type was truncated."
+ );
+ }
+ }
+
+ // Truncate the enrollment id if present.
+ if (options.hasOwnProperty("enrollmentId")) {
+ let enrollmentId = limitStringToLength(
+ options.enrollmentId,
+ MAX_EXPERIMENT_ENROLLMENT_ID_LENGTH
+ );
+ if (enrollmentId.length != options.enrollmentId.length) {
+ options.enrollmentId = enrollmentId;
+ this._log.warn(
+ "setExperimentActive - the enrollment id was truncated."
+ );
+ }
+ }
+
+ let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope);
+ // Add the experiment annotation.
+ let experiments = this._currentEnvironment.experiments || {};
+ experiments[saneId] = { branch: saneBranch };
+ if (options.hasOwnProperty("type")) {
+ experiments[saneId].type = options.type;
+ }
+ if (options.hasOwnProperty("enrollmentId")) {
+ experiments[saneId].enrollmentId = options.enrollmentId;
+ }
+ this._currentEnvironment.experiments = experiments;
+ // Notify of the change.
+ this._onEnvironmentChange("experiment-annotation-changed", oldEnvironment);
+ },
+
+ setExperimentInactive(id) {
+ this._log.trace("setExperimentInactive");
+ let experiments = this._currentEnvironment.experiments || {};
+ if (id in experiments) {
+ // Only attempt to notify if a previous annotation was found and removed.
+ let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope);
+ // Remove the experiment annotation.
+ delete this._currentEnvironment.experiments[id];
+ // Notify of the change.
+ this._onEnvironmentChange(
+ "experiment-annotation-changed",
+ oldEnvironment
+ );
+ }
+ },
+
+ getActiveExperiments() {
+ return Cu.cloneInto(this._currentEnvironment.experiments || {}, myScope);
+ },
+
+ shutdown() {
+ this._log.trace("shutdown");
+ this._shutdown = true;
+ },
+
+ /**
+ * Only used in tests, set the preferences to watch.
+ * @param aPreferences A map of preferences names and their recording policy.
+ */
+ async _watchPreferences(aPreferences) {
+ this._stopWatchingPrefs();
+ this._watchedPrefs = aPreferences;
+ await this._updateSettings();
+ this._startWatchingPrefs();
+ },
+
+ /**
+ * Get an object containing the values for the watched preferences. Depending on the
+ * policy, the value for a preference or whether it was changed by user is reported.
+ *
+ * @return An object containing the preferences values.
+ */
+ _getPrefData() {
+ let prefData = {};
+ for (let [pref, policy] of this._watchedPrefs.entries()) {
+ let prefValue = this._getPrefValue(pref, policy.what);
+
+ if (prefValue === undefined) {
+ continue;
+ }
+
+ prefData[pref] = prefValue;
+ }
+ return prefData;
+ },
+
+ /**
+ * Get the value of a preference given the preference name and the policy.
+ * @param pref Name of the preference.
+ * @param what Policy of the preference.
+ *
+ * @returns The value we need to store for this preference. It can be undefined
+ * or null if the preference is invalid or has a value set by the user.
+ */
+ _getPrefValue(pref, what) {
+ // Check the policy for the preference and decide if we need to store its value
+ // or whether it changed from the default value.
+ let prefType = Services.prefs.getPrefType(pref);
+
+ if (
+ what == TelemetryEnvironment.RECORD_DEFAULTPREF_VALUE ||
+ what == TelemetryEnvironment.RECORD_DEFAULTPREF_STATE
+ ) {
+ // For default prefs, make sure they exist
+ if (prefType == Ci.nsIPrefBranch.PREF_INVALID) {
+ return undefined;
+ }
+ } else if (!Services.prefs.prefHasUserValue(pref)) {
+ // For user prefs, make sure they are set
+ return undefined;
+ }
+
+ if (what == TelemetryEnvironment.RECORD_DEFAULTPREF_STATE) {
+ return "<set>";
+ } else if (what == TelemetryEnvironment.RECORD_PREF_STATE) {
+ return "<user-set>";
+ } else if (prefType == Ci.nsIPrefBranch.PREF_STRING) {
+ return Services.prefs.getStringPref(pref);
+ } else if (prefType == Ci.nsIPrefBranch.PREF_BOOL) {
+ return Services.prefs.getBoolPref(pref);
+ } else if (prefType == Ci.nsIPrefBranch.PREF_INT) {
+ return Services.prefs.getIntPref(pref);
+ } else if (prefType == Ci.nsIPrefBranch.PREF_INVALID) {
+ return null;
+ }
+ throw new Error(
+ `Unexpected preference type ("${prefType}") for "${pref}".`
+ );
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
+
+ /**
+ * Start watching the preferences.
+ */
+ _startWatchingPrefs() {
+ this._log.trace("_startWatchingPrefs - " + this._watchedPrefs);
+
+ Services.prefs.addObserver("", this, true);
+ },
+
+ _onPrefChanged(aData) {
+ this._log.trace("_onPrefChanged");
+ let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope);
+ this._currentEnvironment.settings.userPrefs[aData] = this._getPrefValue(
+ aData,
+ this._watchedPrefs.get(aData).what
+ );
+ this._onEnvironmentChange("pref-changed", oldEnvironment);
+ },
+
+ /**
+ * Do not receive any more change notifications for the preferences.
+ */
+ _stopWatchingPrefs() {
+ this._log.trace("_stopWatchingPrefs");
+
+ Services.prefs.removeObserver("", this);
+ },
+
+ _addObservers() {
+ // Watch the search engine change and service topics.
+ Services.obs.addObserver(this, SESSIONSTORE_WINDOWS_RESTORED_TOPIC);
+ Services.obs.addObserver(this, COMPOSITOR_CREATED_TOPIC);
+ Services.obs.addObserver(this, COMPOSITOR_PROCESS_ABORTED_TOPIC);
+ Services.obs.addObserver(this, DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC);
+ Services.obs.addObserver(this, GFX_FEATURES_READY_TOPIC);
+ Services.obs.addObserver(this, SEARCH_ENGINE_MODIFIED_TOPIC);
+ Services.obs.addObserver(this, SEARCH_SERVICE_TOPIC);
+ Services.obs.addObserver(this, AUTO_UPDATE_PREF_CHANGE_TOPIC);
+ Services.obs.addObserver(this, SERVICES_INFO_CHANGE_TOPIC);
+ },
+
+ _removeObservers() {
+ Services.obs.removeObserver(this, SESSIONSTORE_WINDOWS_RESTORED_TOPIC);
+ Services.obs.removeObserver(this, COMPOSITOR_CREATED_TOPIC);
+ Services.obs.removeObserver(this, COMPOSITOR_PROCESS_ABORTED_TOPIC);
+ try {
+ Services.obs.removeObserver(
+ this,
+ DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC
+ );
+ } catch (ex) {}
+ Services.obs.removeObserver(this, GFX_FEATURES_READY_TOPIC);
+ Services.obs.removeObserver(this, SEARCH_ENGINE_MODIFIED_TOPIC);
+ Services.obs.removeObserver(this, SEARCH_SERVICE_TOPIC);
+ Services.obs.removeObserver(this, AUTO_UPDATE_PREF_CHANGE_TOPIC);
+ Services.obs.removeObserver(this, SERVICES_INFO_CHANGE_TOPIC);
+ },
+
+ observe(aSubject, aTopic, aData) {
+ this._log.trace("observe - aTopic: " + aTopic + ", aData: " + aData);
+ switch (aTopic) {
+ case SEARCH_ENGINE_MODIFIED_TOPIC:
+ if (
+ aData != "engine-default" &&
+ aData != "engine-default-private" &&
+ aData != "engine-changed"
+ ) {
+ return;
+ }
+ if (
+ aData == "engine-changed" &&
+ aSubject.QueryInterface(Ci.nsISearchEngine) &&
+ Services.search.defaultEngine != aSubject
+ ) {
+ return;
+ }
+ // Record the new default search choice and send the change notification.
+ this._onSearchEngineChange();
+ break;
+ case SEARCH_SERVICE_TOPIC:
+ if (aData != "init-complete") {
+ return;
+ }
+ // Now that the search engine init is complete, record the default search choice.
+ this._canQuerySearch = true;
+ this._updateSearchEngine();
+ break;
+ case GFX_FEATURES_READY_TOPIC:
+ case COMPOSITOR_CREATED_TOPIC:
+ // Full graphics information is not available until we have created at
+ // least one off-main-thread-composited window. Thus we wait for the
+ // first compositor to be created and then query nsIGfxInfo again.
+ this._updateGraphicsFeatures();
+ break;
+ case COMPOSITOR_PROCESS_ABORTED_TOPIC:
+ // Our compositor process has been killed for whatever reason, so refresh
+ // our reported graphics features and trigger an environment change.
+ this._onCompositorProcessAborted();
+ break;
+ case DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC:
+ // Distribution customizations are applied after final-ui-startup. query
+ // partner prefs again when they are ready.
+ this._updatePartner();
+ Services.obs.removeObserver(this, aTopic);
+ break;
+ case SESSIONSTORE_WINDOWS_RESTORED_TOPIC:
+ this._sessionWasRestored = true;
+ // Make sure to initialize the search service once we've done restoring
+ // the windows, so that we don't risk loosing search data.
+ Services.search.init();
+ // The default browser check could take some time, so just call it after
+ // the session was restored.
+ this._updateDefaultBrowser();
+ break;
+ case PREF_CHANGED_TOPIC:
+ let options = this._watchedPrefs.get(aData);
+ if (options && !options.requiresRestart) {
+ this._onPrefChanged(aData);
+ }
+ break;
+ case AUTO_UPDATE_PREF_CHANGE_TOPIC:
+ this._currentEnvironment.settings.update.autoDownload = aData == "true";
+ break;
+ case SERVICES_INFO_CHANGE_TOPIC:
+ this._updateServicesInfo();
+ break;
+ }
+ },
+
+ /**
+ * Update the default search engine value.
+ */
+ async _updateSearchEngine() {
+ if (!this._canQuerySearch) {
+ this._log.trace("_updateSearchEngine - ignoring early call");
+ return;
+ }
+
+ this._log.trace(
+ "_updateSearchEngine - isInitialized: " + Services.search.isInitialized
+ );
+ if (!Services.search.isInitialized) {
+ return;
+ }
+
+ // Make sure we have a settings section.
+ this._currentEnvironment.settings = this._currentEnvironment.settings || {};
+
+ // Update the search engine entry in the current environment.
+ const defaultEngineInfo = await Services.search.getDefaultEngineInfo();
+ this._currentEnvironment.settings.defaultSearchEngine =
+ defaultEngineInfo.defaultSearchEngine;
+ this._currentEnvironment.settings.defaultSearchEngineData = {
+ ...defaultEngineInfo.defaultSearchEngineData,
+ };
+ if ("defaultPrivateSearchEngine" in defaultEngineInfo) {
+ this._currentEnvironment.settings.defaultPrivateSearchEngine =
+ defaultEngineInfo.defaultPrivateSearchEngine;
+ }
+ if ("defaultPrivateSearchEngineData" in defaultEngineInfo) {
+ this._currentEnvironment.settings.defaultPrivateSearchEngineData = {
+ ...defaultEngineInfo.defaultPrivateSearchEngineData,
+ };
+ }
+ },
+
+ /**
+ * Update the default search engine value and trigger the environment change.
+ */
+ async _onSearchEngineChange() {
+ this._log.trace("_onSearchEngineChange");
+
+ // Finally trigger the environment change notification.
+ let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope);
+ await this._updateSearchEngine();
+ this._onEnvironmentChange("search-engine-changed", oldEnvironment);
+ },
+
+ /**
+ * Refresh the Telemetry environment and trigger an environment change due to
+ * a change in compositor process (normally this will mean we've fallen back
+ * from out-of-process to in-process compositing).
+ */
+ _onCompositorProcessAborted() {
+ this._log.trace("_onCompositorProcessAborted");
+
+ // Trigger the environment change notification.
+ let oldEnvironment = Cu.cloneInto(this._currentEnvironment, myScope);
+ this._updateGraphicsFeatures();
+ this._onEnvironmentChange("gfx-features-changed", oldEnvironment);
+ },
+
+ /**
+ * Update the graphics features object.
+ */
+ _updateGraphicsFeatures() {
+ let gfxData = this._currentEnvironment.system.gfx;
+ try {
+ let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo);
+ gfxData.features = gfxInfo.getFeatures();
+ } catch (e) {
+ this._log.error("nsIGfxInfo.getFeatures() caught error", e);
+ }
+ },
+
+ /**
+ * Update the partner prefs.
+ */
+ _updatePartner() {
+ this._currentEnvironment.partner = this._getPartner();
+ },
+
+ /**
+ * Get the build data in object form.
+ * @return Object containing the build data.
+ */
+ _getBuild() {
+ let buildData = {
+ applicationId: Services.appinfo.ID || null,
+ applicationName: Services.appinfo.name || null,
+ architecture: Services.sysinfo.get("arch"),
+ buildId: Services.appinfo.appBuildID || null,
+ version: Services.appinfo.version || null,
+ vendor: Services.appinfo.vendor || null,
+ displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY || null,
+ platformVersion: Services.appinfo.platformVersion || null,
+ xpcomAbi: Services.appinfo.XPCOMABI,
+ updaterAvailable: AppConstants.MOZ_UPDATER,
+ };
+
+ return buildData;
+ },
+
+ /**
+ * Determine if we're the default browser.
+ * @returns null on error, true if we are the default browser, or false otherwise.
+ */
+ _isDefaultBrowser() {
+ let isDefault = (service, ...args) => {
+ try {
+ return !!service.isDefaultBrowser(...args);
+ } catch (ex) {
+ this._log.error(
+ "_isDefaultBrowser - Could not determine if default browser",
+ ex
+ );
+ return null;
+ }
+ };
+
+ if (!("@mozilla.org/browser/shell-service;1" in Cc)) {
+ this._log.info(
+ "_isDefaultBrowser - Could not obtain browser shell service"
+ );
+ return null;
+ }
+
+ try {
+ let { ShellService } = ChromeUtils.import(
+ "resource:///modules/ShellService.jsm"
+ );
+ // This uses the same set of flags used by the pref pane.
+ return isDefault(ShellService, false, true);
+ } catch (ex) {
+ this._log.error("_isDefaultBrowser - Could not obtain shell service JSM");
+ }
+
+ try {
+ let shellService = Cc["@mozilla.org/browser/shell-service;1"].getService(
+ Ci.nsIShellService
+ );
+ // This uses the same set of flags used by the pref pane.
+ return isDefault(shellService, true);
+ } catch (ex) {
+ this._log.error("_isDefaultBrowser - Could not obtain shell service", ex);
+ return null;
+ }
+ },
+
+ _updateDefaultBrowser() {
+ if (AppConstants.platform === "android") {
+ return;
+ }
+ // Make sure to have a settings section.
+ this._currentEnvironment.settings = this._currentEnvironment.settings || {};
+ this._currentEnvironment.settings.isDefaultBrowser = this
+ ._sessionWasRestored
+ ? this._isDefaultBrowser()
+ : null;
+ },
+
+ /**
+ * Update the cached settings data.
+ */
+ async _updateSettings() {
+ let updateChannel = null;
+ try {
+ updateChannel = Utils.getUpdateChannel();
+ } catch (e) {}
+
+ this._currentEnvironment.settings = {
+ blocklistEnabled: Services.prefs.getBoolPref(
+ PREF_BLOCKLIST_ENABLED,
+ true
+ ),
+ e10sEnabled: Services.appinfo.browserTabsRemoteAutostart,
+ e10sMultiProcesses: Services.appinfo.maxWebProcessCount,
+ fissionEnabled: Services.appinfo.fissionAutostart,
+ telemetryEnabled: Utils.isTelemetryEnabled,
+ locale: getBrowserLocale(),
+ // We need to wait for browser-delayed-startup-finished to ensure that the locales
+ // have settled, once that's happened we can get the intl data directly.
+ intl: Policy._intlLoaded ? getIntlSettings() : {},
+ update: {
+ channel: updateChannel,
+ enabled: !Services.policies || Services.policies.isAllowed("appUpdate"),
+ },
+ userPrefs: this._getPrefData(),
+ sandbox: this._getSandboxData(),
+ };
+
+ // Services.appinfo.launcherProcessState is not available in all build
+ // configurations, in which case an exception may be thrown.
+ try {
+ this._currentEnvironment.settings.launcherProcessState =
+ Services.appinfo.launcherProcessState;
+ } catch (e) {}
+
+ this._currentEnvironment.settings.addonCompatibilityCheckEnabled =
+ AddonManager.checkCompatibility;
+
+ this._updateAttribution();
+ this._updateDefaultBrowser();
+ await this._updateSearchEngine();
+ this._updateAutoDownload();
+ },
+
+ _getSandboxData() {
+ let effectiveContentProcessLevel = null;
+ try {
+ let sandboxSettings = Cc[
+ "@mozilla.org/sandbox/sandbox-settings;1"
+ ].getService(Ci.mozISandboxSettings);
+ effectiveContentProcessLevel =
+ sandboxSettings.effectiveContentSandboxLevel;
+ } catch (e) {}
+ return {
+ effectiveContentProcessLevel,
+ };
+ },
+
+ /**
+ * Update the cached profile data.
+ * @returns Promise<> resolved when the I/O is complete.
+ */
+ async _updateProfile() {
+ let profileAccessor = await ProfileAge();
+
+ let creationDate = await profileAccessor.created;
+ let resetDate = await profileAccessor.reset;
+ let firstUseDate = await profileAccessor.firstUse;
+
+ this._currentEnvironment.profile.creationDate = Utils.millisecondsToDays(
+ creationDate
+ );
+ if (resetDate) {
+ this._currentEnvironment.profile.resetDate = Utils.millisecondsToDays(
+ resetDate
+ );
+ }
+ if (firstUseDate) {
+ this._currentEnvironment.profile.firstUseDate = Utils.millisecondsToDays(
+ firstUseDate
+ );
+ }
+ },
+
+ /**
+ * Load the attribution data object and updates the environment.
+ * @returns Promise<> resolved when the I/O is complete.
+ */
+ async _loadAttributionAsync() {
+ try {
+ await AttributionCode.getAttrDataAsync();
+ } catch (e) {
+ // The AttributionCode.jsm module might not be always available
+ // (e.g. tests). Gracefully handle this.
+ return;
+ }
+ this._updateAttribution();
+ },
+
+ /**
+ * Update the environment with the cached attribution data.
+ */
+ _updateAttribution() {
+ let data = null;
+ try {
+ data = AttributionCode.getCachedAttributionData();
+ } catch (e) {
+ // The AttributionCode.jsm module might not be always available
+ // (e.g. tests). Gracefully handle this.
+ }
+
+ if (!data || !Object.keys(data).length) {
+ return;
+ }
+
+ let attributionData = {};
+ for (let key in data) {
+ attributionData[key] = limitStringToLength(
+ data[key],
+ MAX_ATTRIBUTION_STRING_LENGTH
+ );
+ }
+ this._currentEnvironment.settings.attribution = attributionData;
+ },
+
+ /**
+ * Load the auto update pref and adds it to the environment
+ */
+ async _loadAutoUpdateAsync() {
+ if (AppConstants.MOZ_UPDATER) {
+ this._updateAutoDownloadCache = await UpdateUtils.getAppUpdateAutoEnabled();
+ } else {
+ this._updateAutoDownloadCache = false;
+ }
+ this._updateAutoDownload();
+ },
+
+ /**
+ * Update the environment with the cached value for whether updates can auto-
+ * download.
+ */
+ _updateAutoDownload() {
+ if (this._updateAutoDownloadCache === undefined) {
+ return;
+ }
+ this._currentEnvironment.settings.update.autoDownload = this._updateAutoDownloadCache;
+ },
+
+ /**
+ * Get i18n data about the system.
+ * @return A promise of completion.
+ */
+ async _loadIntlData() {
+ // Wait for the startup topic.
+ await Policy._browserDelayedStartup();
+ this._currentEnvironment.settings.intl = getIntlSettings();
+ Policy._intlLoaded = true;
+ },
+ // This exists as a separate function for testing.
+ async _getFxaSignedInUser() {
+ return fxAccounts.getSignedInUser();
+ },
+
+ async _updateServicesInfo() {
+ let syncEnabled = false;
+ let accountEnabled = false;
+ let weaveService = Cc["@mozilla.org/weave/service;1"].getService()
+ .wrappedJSObject;
+ syncEnabled = weaveService && weaveService.enabled;
+ if (syncEnabled) {
+ // All sync users are account users, definitely.
+ accountEnabled = true;
+ } else {
+ // Not all account users are sync users. See if they're signed into FxA.
+ try {
+ let user = await this._getFxaSignedInUser();
+ if (user) {
+ accountEnabled = true;
+ }
+ } catch (e) {
+ // We don't know. This might be a transient issue which will clear
+ // itself up later, but the information in telemetry is quite possibly stale
+ // (this is called from a change listener), so clear it out to avoid
+ // reporting data which might be wrong until we can figure it out.
+ delete this._currentEnvironment.services;
+ this._log.error("_updateServicesInfo() caught error", e);
+ return;
+ }
+ }
+ this._currentEnvironment.services = {
+ accountEnabled,
+ syncEnabled,
+ };
+ },
+
+ /**
+ * Get the partner data in object form.
+ * @return Object containing the partner data.
+ */
+ _getPartner() {
+ let partnerData = {
+ distributionId: Services.prefs.getStringPref(PREF_DISTRIBUTION_ID, null),
+ distributionVersion: Services.prefs.getStringPref(
+ PREF_DISTRIBUTION_VERSION,
+ null
+ ),
+ partnerId: Services.prefs.getStringPref(PREF_PARTNER_ID, null),
+ distributor: Services.prefs.getStringPref(PREF_DISTRIBUTOR, null),
+ distributorChannel: Services.prefs.getStringPref(
+ PREF_DISTRIBUTOR_CHANNEL,
+ null
+ ),
+ };
+
+ // Get the PREF_APP_PARTNER_BRANCH branch and append its children to partner data.
+ let partnerBranch = Services.prefs.getBranch(PREF_APP_PARTNER_BRANCH);
+ partnerData.partnerNames = partnerBranch.getChildList("");
+
+ return partnerData;
+ },
+
+ _cpuData: null,
+ /**
+ * Get the CPU information.
+ * @return Object containing the CPU information data.
+ */
+ _getCPUData() {
+ if (this._cpuData) {
+ return this._cpuData;
+ }
+
+ this._cpuData = {};
+
+ const CPU_EXTENSIONS = [
+ "hasMMX",
+ "hasSSE",
+ "hasSSE2",
+ "hasSSE3",
+ "hasSSSE3",
+ "hasSSE4A",
+ "hasSSE4_1",
+ "hasSSE4_2",
+ "hasAVX",
+ "hasAVX2",
+ "hasAES",
+ "hasEDSP",
+ "hasARMv6",
+ "hasARMv7",
+ "hasNEON",
+ "hasUserCET",
+ ];
+
+ // Enumerate the available CPU extensions.
+ let availableExts = [];
+ for (let ext of CPU_EXTENSIONS) {
+ if (getSysinfoProperty(ext, false)) {
+ availableExts.push(ext);
+ }
+ }
+
+ this._cpuData.extensions = availableExts;
+
+ return this._cpuData;
+ },
+
+ _processData: null,
+ /**
+ * Get the process information.
+ * @return Object containing the process information data.
+ */
+ _getProcessData() {
+ if (this._processData) {
+ return this._processData;
+ }
+ return {};
+ },
+
+ /**
+ * Get the device information, if we are on a portable device.
+ * @return Object containing the device information data, or null if
+ * not a portable device.
+ */
+ _getDeviceData() {
+ if (AppConstants.platform !== "android") {
+ return null;
+ }
+
+ return {
+ model: getSysinfoProperty("device", null),
+ manufacturer: getSysinfoProperty("manufacturer", null),
+ hardware: getSysinfoProperty("hardware", null),
+ isTablet: getSysinfoProperty("tablet", null),
+ };
+ },
+
+ _osData: null,
+ /**
+ * Get the OS information.
+ * @return Object containing the OS data.
+ */
+ _getOSData() {
+ if (this._osData) {
+ return this._osData;
+ }
+ this._osData = {
+ name: forceToStringOrNull(getSysinfoProperty("name", null)),
+ version: forceToStringOrNull(getSysinfoProperty("version", null)),
+ locale: forceToStringOrNull(getSystemLocale()),
+ };
+
+ if (AppConstants.platform == "android") {
+ this._osData.kernelVersion = forceToStringOrNull(
+ getSysinfoProperty("kernel_version", null)
+ );
+ } else if (AppConstants.platform === "win") {
+ // The path to the "UBR" key, queried to get additional version details on Windows.
+ const WINDOWS_UBR_KEY_PATH =
+ "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion";
+
+ let versionInfo = WindowsVersionInfo.get({ throwOnError: false });
+ this._osData.servicePackMajor = versionInfo.servicePackMajor;
+ this._osData.servicePackMinor = versionInfo.servicePackMinor;
+ this._osData.windowsBuildNumber = versionInfo.buildNumber;
+ // We only need the UBR if we're at or above Windows 10.
+ if (
+ typeof this._osData.version === "string" &&
+ Services.vc.compare(this._osData.version, "10") >= 0
+ ) {
+ // Query the UBR key and only add it to the environment if it's available.
+ // |readRegKey| doesn't throw, but rather returns 'undefined' on error.
+ let ubr = WindowsRegistry.readRegKey(
+ Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
+ WINDOWS_UBR_KEY_PATH,
+ "UBR",
+ Ci.nsIWindowsRegKey.WOW64_64
+ );
+ this._osData.windowsUBR = ubr !== undefined ? ubr : null;
+ }
+ }
+
+ return this._osData;
+ },
+
+ _hddData: null,
+ /**
+ * Get the HDD information.
+ * @return Object containing the HDD data.
+ */
+ _getHDDData() {
+ if (this._hddData) {
+ return this._hddData;
+ }
+ let nullData = { model: null, revision: null, type: null };
+ return { profile: nullData, binary: nullData, system: nullData };
+ },
+
+ /**
+ * Get registered security product information.
+ * @return Object containing the security product data
+ */
+ _getSecurityAppData() {
+ const maxStringLength = 256;
+
+ const keys = [
+ ["registeredAntiVirus", "antivirus"],
+ ["registeredAntiSpyware", "antispyware"],
+ ["registeredFirewall", "firewall"],
+ ];
+
+ let result = {};
+
+ for (let [inKey, outKey] of keys) {
+ let prop = getSysinfoProperty(inKey, null);
+ if (prop) {
+ prop = limitStringToLength(prop, maxStringLength).split(";");
+ }
+
+ result[outKey] = prop;
+ }
+
+ return result;
+ },
+
+ /**
+ * Get the GFX information.
+ * @return Object containing the GFX data.
+ */
+ _getGFXData() {
+ let gfxData = {
+ D2DEnabled: getGfxField("D2DEnabled", null),
+ DWriteEnabled: getGfxField("DWriteEnabled", null),
+ ContentBackend: getGfxField("ContentBackend", null),
+ Headless: getGfxField("isHeadless", null),
+ EmbeddedInFirefoxReality: getGfxField("EmbeddedInFirefoxReality", null),
+ // The following line is disabled due to main thread jank and will be enabled
+ // again as part of bug 1154500.
+ // DWriteVersion: getGfxField("DWriteVersion", null),
+ adapters: [],
+ monitors: [],
+ features: {},
+ };
+
+ if (AppConstants.platform !== "android") {
+ let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo);
+ try {
+ gfxData.monitors = gfxInfo.getMonitors();
+ } catch (e) {
+ this._log.error("nsIGfxInfo.getMonitors() caught error", e);
+ }
+ }
+
+ try {
+ let gfxInfo = Cc["@mozilla.org/gfx/info;1"].getService(Ci.nsIGfxInfo);
+ gfxData.features = gfxInfo.getFeatures();
+ } catch (e) {
+ this._log.error("nsIGfxInfo.getFeatures() caught error", e);
+ }
+
+ // GfxInfo does not yet expose a way to iterate through all the adapters.
+ gfxData.adapters.push(getGfxAdapter(""));
+ gfxData.adapters[0].GPUActive = true;
+
+ // If we have a second adapter add it to the gfxData.adapters section.
+ let hasGPU2 = getGfxField("adapterDeviceID2", null) !== null;
+ if (!hasGPU2) {
+ this._log.trace("_getGFXData - Only one display adapter detected.");
+ return gfxData;
+ }
+
+ this._log.trace("_getGFXData - Two display adapters detected.");
+
+ gfxData.adapters.push(getGfxAdapter("2"));
+ gfxData.adapters[1].GPUActive = getGfxField("isGPU2Active", null);
+
+ return gfxData;
+ },
+
+ /**
+ * Get the system data in object form.
+ * @return Object containing the system data.
+ */
+ _getSystem() {
+ let memoryMB = getSysinfoProperty("memsize", null);
+ if (memoryMB) {
+ // Send RAM size in megabytes. Rounding because sysinfo doesn't
+ // always provide RAM in multiples of 1024.
+ memoryMB = Math.round(memoryMB / 1024 / 1024);
+ }
+
+ let virtualMB = getSysinfoProperty("virtualmemsize", null);
+ if (virtualMB) {
+ // Send the total virtual memory size in megabytes. Rounding because
+ // sysinfo doesn't always provide RAM in multiples of 1024.
+ virtualMB = Math.round(virtualMB / 1024 / 1024);
+ }
+
+ let data = {
+ memoryMB,
+ virtualMaxMB: virtualMB,
+ cpu: this._getCPUData(),
+ os: this._getOSData(),
+ hdd: this._getHDDData(),
+ gfx: this._getGFXData(),
+ appleModelId: getSysinfoProperty("appleModelId", null),
+ };
+
+ if (AppConstants.platform === "win") {
+ data = { ...this._getProcessData(), ...data };
+ } else if (AppConstants.platform == "android") {
+ data.device = this._getDeviceData();
+ }
+
+ // Windows 8+
+ if (AppConstants.isPlatformAndVersionAtLeast("win", "6.2")) {
+ data.sec = this._getSecurityAppData();
+ }
+
+ return data;
+ },
+
+ _onEnvironmentChange(what, oldEnvironment) {
+ this._log.trace("_onEnvironmentChange for " + what);
+
+ // We are already skipping change events in _checkChanges if there is a pending change task running.
+ if (this._shutdown) {
+ this._log.trace("_onEnvironmentChange - Already shut down.");
+ return;
+ }
+
+ if (ObjectUtils.deepEqual(this._currentEnvironment, oldEnvironment)) {
+ Services.telemetry.scalarAdd("telemetry.environment_didnt_change", 1);
+ }
+
+ for (let [name, listener] of this._changeListeners) {
+ try {
+ this._log.debug("_onEnvironmentChange - calling " + name);
+ listener(what, oldEnvironment);
+ } catch (e) {
+ this._log.error(
+ "_onEnvironmentChange - listener " + name + " caught error",
+ e
+ );
+ }
+ }
+ },
+
+ reset() {
+ this._shutdown = false;
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm b/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm
new file mode 100644
index 0000000000..89f93de514
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm
@@ -0,0 +1,584 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryReportingPolicy"];
+
+ChromeUtils.import("resource://gre/modules/Log.jsm", this);
+ChromeUtils.import("resource://gre/modules/Services.jsm", this);
+ChromeUtils.import("resource://gre/modules/Timer.jsm", this);
+ChromeUtils.import("resource://services-common/observers.js", this);
+ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetrySend",
+ "resource://gre/modules/TelemetrySend.jsm"
+);
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "TelemetryReportingPolicy::";
+
+// Oldest year to allow in date preferences. The FHR infobar was implemented in
+// 2012 and no dates older than that should be encountered.
+const OLDEST_ALLOWED_ACCEPTANCE_YEAR = 2012;
+
+const PREF_BRANCH = "datareporting.policy.";
+
+// The following preferences are deprecated and will be purged during the preferences
+// migration process.
+const DEPRECATED_FHR_PREFS = [
+ PREF_BRANCH + "dataSubmissionPolicyAccepted",
+ PREF_BRANCH + "dataSubmissionPolicyBypassAcceptance",
+ PREF_BRANCH + "dataSubmissionPolicyResponseType",
+ PREF_BRANCH + "dataSubmissionPolicyResponseTime",
+];
+
+// How much time until we display the data choices notification bar, on the first run.
+const NOTIFICATION_DELAY_FIRST_RUN_MSEC = 60 * 1000; // 60s
+// Same as above, for the next runs.
+const NOTIFICATION_DELAY_NEXT_RUNS_MSEC = 10 * 1000; // 10s
+
+/**
+ * This is a policy object used to override behavior within this module.
+ * Tests override properties on this object to allow for control of behavior
+ * that would otherwise be very hard to cover.
+ */
+var Policy = {
+ now: () => new Date(),
+ setShowInfobarTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+ clearShowInfobarTimeout: id => clearTimeout(id),
+};
+
+/**
+ * Represents a request to display data policy.
+ *
+ * Receivers of these instances are expected to call one or more of the on*
+ * functions when events occur.
+ *
+ * When one of these requests is received, the first thing a callee should do
+ * is present notification to the user of the data policy. When the notice
+ * is displayed to the user, the callee should call `onUserNotifyComplete`.
+ *
+ * If for whatever reason the callee could not display a notice,
+ * it should call `onUserNotifyFailed`.
+ *
+ * @param {Object} aLog The log object used to log the error in case of failures.
+ */
+function NotifyPolicyRequest(aLog) {
+ this._log = aLog;
+}
+
+NotifyPolicyRequest.prototype = Object.freeze({
+ /**
+ * Called when the user is notified of the policy.
+ */
+ onUserNotifyComplete() {
+ return TelemetryReportingPolicyImpl._userNotified();
+ },
+
+ /**
+ * Called when there was an error notifying the user about the policy.
+ *
+ * @param error
+ * (Error) Explains what went wrong.
+ */
+ onUserNotifyFailed(error) {
+ this._log.error("onUserNotifyFailed - " + error);
+ },
+});
+
+var TelemetryReportingPolicy = {
+ // The current policy version number. If the version number stored in the prefs
+ // is smaller than this, data upload will be disabled until the user is re-notified
+ // about the policy changes.
+ DEFAULT_DATAREPORTING_POLICY_VERSION: 1,
+
+ /**
+ * Setup the policy.
+ */
+ setup() {
+ return TelemetryReportingPolicyImpl.setup();
+ },
+
+ /**
+ * Shutdown and clear the policy.
+ */
+ shutdown() {
+ return TelemetryReportingPolicyImpl.shutdown();
+ },
+
+ /**
+ * Check if we are allowed to upload data. In order to submit data both these conditions
+ * should be true:
+ * - The data submission preference should be true.
+ * - The datachoices infobar should have been displayed.
+ *
+ * @return {Boolean} True if we are allowed to upload data, false otherwise.
+ */
+ canUpload() {
+ return TelemetryReportingPolicyImpl.canUpload();
+ },
+
+ /**
+ * Check if this is the first time the browser ran.
+ */
+ isFirstRun() {
+ return TelemetryReportingPolicyImpl.isFirstRun();
+ },
+
+ /**
+ * Test only method, restarts the policy.
+ */
+ reset() {
+ return TelemetryReportingPolicyImpl.reset();
+ },
+
+ /**
+ * Test only method, used to check if user is notified of the policy in tests.
+ */
+ testIsUserNotified() {
+ return TelemetryReportingPolicyImpl.isUserNotifiedOfCurrentPolicy;
+ },
+
+ /**
+ * Test only method, used to simulate the infobar being shown in xpcshell tests.
+ */
+ testInfobarShown() {
+ return TelemetryReportingPolicyImpl._userNotified();
+ },
+
+ /**
+ * Test only method, used to trigger an update of the "first run" state.
+ */
+ testUpdateFirstRun() {
+ TelemetryReportingPolicyImpl._isFirstRun = undefined;
+ TelemetryReportingPolicyImpl.isFirstRun();
+ },
+};
+
+var TelemetryReportingPolicyImpl = {
+ _logger: null,
+ // Keep track of the notification status if user wasn't notified already.
+ _notificationInProgress: false,
+ // The timer used to show the datachoices notification at startup.
+ _startupNotificationTimerId: null,
+ // Keep track of the first session state, as the related preference
+ // is flipped right after the browser starts.
+ _isFirstRun: undefined,
+
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+
+ return this._logger;
+ },
+
+ /**
+ * Get the date the policy was notified.
+ * @return {Object} A date object or null on errors.
+ */
+ get dataSubmissionPolicyNotifiedDate() {
+ let prefString = Services.prefs.getStringPref(
+ TelemetryUtils.Preferences.AcceptedPolicyDate,
+ "0"
+ );
+ let valueInteger = parseInt(prefString, 10);
+
+ // Bail out if we didn't store any value yet.
+ if (valueInteger == 0) {
+ this._log.info(
+ "get dataSubmissionPolicyNotifiedDate - No date stored yet."
+ );
+ return null;
+ }
+
+ // If an invalid value is saved in the prefs, bail out too.
+ if (Number.isNaN(valueInteger)) {
+ this._log.error(
+ "get dataSubmissionPolicyNotifiedDate - Invalid date stored."
+ );
+ return null;
+ }
+
+ // Make sure the notification date is newer then the oldest allowed date.
+ let date = new Date(valueInteger);
+ if (date.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) {
+ this._log.error(
+ "get dataSubmissionPolicyNotifiedDate - The stored date is too old."
+ );
+ return null;
+ }
+
+ return date;
+ },
+
+ /**
+ * Set the date the policy was notified.
+ * @param {Object} aDate A valid date object.
+ */
+ set dataSubmissionPolicyNotifiedDate(aDate) {
+ this._log.trace("set dataSubmissionPolicyNotifiedDate - aDate: " + aDate);
+
+ if (!aDate || aDate.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) {
+ this._log.error(
+ "set dataSubmissionPolicyNotifiedDate - Invalid notification date."
+ );
+ return;
+ }
+
+ Services.prefs.setStringPref(
+ TelemetryUtils.Preferences.AcceptedPolicyDate,
+ aDate.getTime().toString()
+ );
+ },
+
+ /**
+ * Whether submission of data is allowed.
+ *
+ * This is the master switch for remote server communication. If it is
+ * false, we never request upload or deletion.
+ */
+ get dataSubmissionEnabled() {
+ // Default is true because we are opt-out.
+ return Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.DataSubmissionEnabled,
+ true
+ );
+ },
+
+ get currentPolicyVersion() {
+ return Services.prefs.getIntPref(
+ TelemetryUtils.Preferences.CurrentPolicyVersion,
+ TelemetryReportingPolicy.DEFAULT_DATAREPORTING_POLICY_VERSION
+ );
+ },
+
+ /**
+ * The minimum policy version which for dataSubmissionPolicyAccepted to
+ * to be valid.
+ */
+ get minimumPolicyVersion() {
+ const minPolicyVersion = Services.prefs.getIntPref(
+ TelemetryUtils.Preferences.MinimumPolicyVersion,
+ 1
+ );
+
+ // First check if the current channel has a specific minimum policy version. If not,
+ // use the general minimum policy version.
+ let channel = "";
+ try {
+ channel = TelemetryUtils.getUpdateChannel();
+ } catch (e) {
+ this._log.error(
+ "minimumPolicyVersion - Unable to retrieve the current channel."
+ );
+ return minPolicyVersion;
+ }
+ const channelPref =
+ TelemetryUtils.Preferences.MinimumPolicyVersion + ".channel-" + channel;
+ return Services.prefs.getIntPref(channelPref, minPolicyVersion);
+ },
+
+ get dataSubmissionPolicyAcceptedVersion() {
+ return Services.prefs.getIntPref(
+ TelemetryUtils.Preferences.AcceptedPolicyVersion,
+ 0
+ );
+ },
+
+ set dataSubmissionPolicyAcceptedVersion(value) {
+ Services.prefs.setIntPref(
+ TelemetryUtils.Preferences.AcceptedPolicyVersion,
+ value
+ );
+ },
+
+ /**
+ * Checks to see if the user has been notified about data submission
+ * @return {Bool} True if user has been notified and the notification is still valid,
+ * false otherwise.
+ */
+ get isUserNotifiedOfCurrentPolicy() {
+ // If we don't have a sane notification date, the user was not notified yet.
+ if (
+ !this.dataSubmissionPolicyNotifiedDate ||
+ this.dataSubmissionPolicyNotifiedDate.getTime() <= 0
+ ) {
+ return false;
+ }
+
+ // The accepted policy version should not be less than the minimum policy version.
+ if (this.dataSubmissionPolicyAcceptedVersion < this.minimumPolicyVersion) {
+ return false;
+ }
+
+ // Otherwise the user was already notified.
+ return true;
+ },
+
+ /**
+ * Test only method, restarts the policy.
+ */
+ reset() {
+ this.shutdown();
+ this._isFirstRun = undefined;
+ return this.setup();
+ },
+
+ /**
+ * Setup the policy.
+ */
+ setup() {
+ this._log.trace("setup");
+
+ // Migrate the data choices infobar, if needed.
+ this._migratePreferences();
+
+ // Add the event observers.
+ Services.obs.addObserver(this, "sessionstore-windows-restored");
+ },
+
+ /**
+ * Clean up the reporting policy.
+ */
+ shutdown() {
+ this._log.trace("shutdown");
+
+ this._detachObservers();
+
+ Policy.clearShowInfobarTimeout(this._startupNotificationTimerId);
+ },
+
+ /**
+ * Detach the observers that were attached during setup.
+ */
+ _detachObservers() {
+ Services.obs.removeObserver(this, "sessionstore-windows-restored");
+ },
+
+ /**
+ * Check if we are allowed to upload data. In order to submit data both these conditions
+ * should be true:
+ * - The data submission preference should be true.
+ * - The datachoices infobar should have been displayed.
+ *
+ * @return {Boolean} True if we are allowed to upload data, false otherwise.
+ */
+ canUpload() {
+ // If data submission is disabled, there's no point in showing the infobar. Just
+ // forbid to upload.
+ if (!this.dataSubmissionEnabled) {
+ return false;
+ }
+
+ // Submission is enabled. We enable upload if user is notified or we need to bypass
+ // the policy.
+ const bypassNotification = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.BypassNotification,
+ false
+ );
+ return this.isUserNotifiedOfCurrentPolicy || bypassNotification;
+ },
+
+ isFirstRun() {
+ if (this._isFirstRun === undefined) {
+ this._isFirstRun = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.FirstRun,
+ true
+ );
+ }
+ return this._isFirstRun;
+ },
+
+ /**
+ * Migrate the data policy preferences, if needed.
+ */
+ _migratePreferences() {
+ // Current prefs are mostly the same than the old ones, except for some deprecated ones.
+ for (let pref of DEPRECATED_FHR_PREFS) {
+ Services.prefs.clearUserPref(pref);
+ }
+ },
+
+ /**
+ * Determine whether the user should be notified.
+ */
+ _shouldNotify() {
+ if (!this.dataSubmissionEnabled) {
+ this._log.trace(
+ "_shouldNotify - Data submission disabled by the policy."
+ );
+ return false;
+ }
+
+ const bypassNotification = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.BypassNotification,
+ false
+ );
+ if (this.isUserNotifiedOfCurrentPolicy || bypassNotification) {
+ this._log.trace(
+ "_shouldNotify - User already notified or bypassing the policy."
+ );
+ return false;
+ }
+
+ if (this._notificationInProgress) {
+ this._log.trace(
+ "_shouldNotify - User not notified, notification already in progress."
+ );
+ return false;
+ }
+
+ return true;
+ },
+
+ /**
+ * Show the data choices infobar if needed.
+ */
+ _showInfobar() {
+ if (!this._shouldNotify()) {
+ return;
+ }
+
+ this._log.trace("_showInfobar - User not notified, notifying now.");
+ this._notificationInProgress = true;
+ let request = new NotifyPolicyRequest(this._log);
+ Observers.notify("datareporting:notify-data-policy:request", request);
+ },
+
+ /**
+ * Called when the user is notified with the infobar or otherwise.
+ */
+ _userNotified() {
+ this._log.trace("_userNotified");
+ this._recordNotificationData();
+ TelemetrySend.notifyCanUpload();
+ },
+
+ /**
+ * Record date and the version of the accepted policy.
+ */
+ _recordNotificationData() {
+ this._log.trace("_recordNotificationData");
+ this.dataSubmissionPolicyNotifiedDate = Policy.now();
+ this.dataSubmissionPolicyAcceptedVersion = this.currentPolicyVersion;
+ // The user was notified and the notification data saved: the notification
+ // is no longer in progress.
+ this._notificationInProgress = false;
+ },
+
+ /**
+ * Try to open the privacy policy in a background tab instead of showing the infobar.
+ */
+ _openFirstRunPage() {
+ if (!this._shouldNotify()) {
+ return false;
+ }
+
+ let firstRunPolicyURL = Services.prefs.getStringPref(
+ TelemetryUtils.Preferences.FirstRunURL,
+ ""
+ );
+ if (!firstRunPolicyURL) {
+ return false;
+ }
+ firstRunPolicyURL = Services.urlFormatter.formatURL(firstRunPolicyURL);
+
+ const { BrowserWindowTracker } = ChromeUtils.import(
+ "resource:///modules/BrowserWindowTracker.jsm"
+ );
+ let win = BrowserWindowTracker.getTopWindow();
+
+ if (!win) {
+ this._log.info(
+ "Couldn't find browser window to open first-run page. Falling back to infobar."
+ );
+ return false;
+ }
+
+ // We'll consider the user notified once the privacy policy has been loaded
+ // in a background tab even if that tab hasn't been selected.
+ let tab;
+ let progressListener = {};
+ progressListener.onStateChange = (
+ aBrowser,
+ aWebProgress,
+ aRequest,
+ aStateFlags,
+ aStatus
+ ) => {
+ if (
+ aWebProgress.isTopLevel &&
+ tab &&
+ tab.linkedBrowser == aBrowser &&
+ aStateFlags & Ci.nsIWebProgressListener.STATE_STOP &&
+ aStateFlags & Ci.nsIWebProgressListener.STATE_IS_NETWORK
+ ) {
+ let uri = aBrowser.documentURI;
+ if (
+ uri &&
+ !/^about:(blank|neterror|certerror|blocked)/.test(uri.spec)
+ ) {
+ this._userNotified();
+ } else {
+ this._log.info(
+ "Failed to load first-run page. Falling back to infobar."
+ );
+ this._showInfobar();
+ }
+ removeListeners();
+ }
+ };
+
+ let removeListeners = () => {
+ win.removeEventListener("unload", removeListeners);
+ win.gBrowser.removeTabsProgressListener(progressListener);
+ };
+
+ win.addEventListener("unload", removeListeners);
+ win.gBrowser.addTabsProgressListener(progressListener);
+
+ tab = win.gBrowser.loadOneTab(firstRunPolicyURL, {
+ inBackground: true,
+ triggeringPrincipal: Services.scriptSecurityManager.getSystemPrincipal(),
+ });
+ return true;
+ },
+
+ observe(aSubject, aTopic, aData) {
+ if (aTopic != "sessionstore-windows-restored") {
+ return;
+ }
+
+ if (this.isFirstRun()) {
+ // We're performing the first run, flip firstRun preference for subsequent runs.
+ Services.prefs.setBoolPref(TelemetryUtils.Preferences.FirstRun, false);
+
+ try {
+ if (this._openFirstRunPage()) {
+ return;
+ }
+ } catch (e) {
+ this._log.error("Failed to open privacy policy tab: " + e);
+ }
+ }
+
+ // Show the info bar.
+ const delay = this.isFirstRun()
+ ? NOTIFICATION_DELAY_FIRST_RUN_MSEC
+ : NOTIFICATION_DELAY_NEXT_RUNS_MSEC;
+
+ this._startupNotificationTimerId = Policy.setShowInfobarTimeout(
+ // Calling |canUpload| eventually shows the infobar, if needed.
+ () => this._showInfobar(),
+ delay
+ );
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryScheduler.jsm b/toolkit/components/telemetry/app/TelemetryScheduler.jsm
new file mode 100644
index 0000000000..81f59083fa
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryScheduler.jsm
@@ -0,0 +1,434 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryScheduler"];
+
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { TelemetrySession } = ChromeUtils.import(
+ "resource://gre/modules/TelemetrySession.jsm"
+);
+const { TelemetryUtils } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryUtils.jsm"
+);
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm",
+ this
+);
+const { clearTimeout, setTimeout } = ChromeUtils.import(
+ "resource://gre/modules/Timer.jsm"
+);
+// Other pings
+const { EcosystemTelemetry } = ChromeUtils.import(
+ "resource://gre/modules/EcosystemTelemetry.jsm"
+);
+const { TelemetryPrioPing } = ChromeUtils.import(
+ "resource://gre/modules/PrioPing.jsm"
+);
+
+XPCOMUtils.defineLazyServiceGetters(this, {
+ idleService: ["@mozilla.org/widget/useridleservice;1", "nsIUserIdleService"],
+});
+
+const MIN_SUBSESSION_LENGTH_MS =
+ Services.prefs.getIntPref("toolkit.telemetry.minSubsessionLength", 5 * 60) *
+ 1000;
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+
+// Seconds of idle time before pinging.
+// On idle-daily a gather-telemetry notification is fired, during it probes can
+// start asynchronous tasks to gather data.
+const IDLE_TIMEOUT_SECONDS = Services.prefs.getIntPref(
+ "toolkit.telemetry.idleTimeout",
+ 5 * 60
+);
+
+// Execute a scheduler tick every 5 minutes.
+const SCHEDULER_TICK_INTERVAL_MS =
+ Services.prefs.getIntPref(
+ "toolkit.telemetry.scheduler.tickInterval",
+ 5 * 60
+ ) * 1000;
+// When user is idle, execute a scheduler tick every 60 minutes.
+const SCHEDULER_TICK_IDLE_INTERVAL_MS =
+ Services.prefs.getIntPref(
+ "toolkit.telemetry.scheduler.idleTickInterval",
+ 60 * 60
+ ) * 1000;
+
+// The maximum time (ms) until the tick should moved from the idle
+// queue to the regular queue if it hasn't been executed yet.
+const SCHEDULER_TICK_MAX_IDLE_DELAY_MS = 60 * 1000;
+
+// The frequency at which we persist session data to the disk to prevent data loss
+// in case of aborted sessions (currently 5 minutes).
+const ABORTED_SESSION_UPDATE_INTERVAL_MS = 5 * 60 * 1000;
+
+// The tolerance we have when checking if it's midnight (15 minutes).
+const SCHEDULER_MIDNIGHT_TOLERANCE_MS = 15 * 60 * 1000;
+
+/**
+ * This is a policy object used to override behavior for testing.
+ */
+var Policy = {
+ now: () => new Date(),
+ setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+ clearSchedulerTickTimeout: id => clearTimeout(id),
+ prioEncode: (batchID, prioParams) => PrioEncoder.encode(batchID, prioParams),
+};
+
+/**
+ * TelemetryScheduler contains a single timer driving all regularly-scheduled
+ * Telemetry related jobs. Having a single place with this logic simplifies
+ * reasoning about scheduling actions in a single place, making it easier to
+ * coordinate jobs and coalesce them.
+ */
+var TelemetryScheduler = {
+ // Tracks the main ping
+ _lastDailyPingTime: 0,
+ // Tracks the aborted session ping
+ _lastSessionCheckpointTime: 0,
+ // Tracks all other pings at regular intervals
+ _lastPeriodicPingTime: 0,
+
+ _log: null,
+
+ // The timer which drives the scheduler.
+ _schedulerTimer: null,
+ // The interval used by the scheduler timer.
+ _schedulerInterval: 0,
+ _shuttingDown: true,
+ _isUserIdle: false,
+
+ /**
+ * Initialises the scheduler and schedules the first daily/aborted session pings.
+ */
+ init() {
+ this._log = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ "TelemetryScheduler::"
+ );
+ this._log.trace("init");
+ this._shuttingDown = false;
+ this._isUserIdle = false;
+
+ // Initialize the last daily ping and aborted session last due times to the current time.
+ // Otherwise, we might end up sending daily pings even if the subsession is not long enough.
+ let now = Policy.now();
+ this._lastDailyPingTime = now.getTime();
+ this._lastPeriodicPingTime = now.getTime();
+ this._lastSessionCheckpointTime = now.getTime();
+ this._rescheduleTimeout();
+
+ idleService.addIdleObserver(this, IDLE_TIMEOUT_SECONDS);
+ Services.obs.addObserver(this, "wake_notification");
+ },
+
+ /**
+ * Stops the scheduler.
+ */
+ shutdown() {
+ if (this._shuttingDown) {
+ if (this._log) {
+ this._log.error("shutdown - Already shut down");
+ } else {
+ Cu.reportError("TelemetryScheduler.shutdown - Already shut down");
+ }
+ return;
+ }
+
+ this._log.trace("shutdown");
+ if (this._schedulerTimer) {
+ Policy.clearSchedulerTickTimeout(this._schedulerTimer);
+ this._schedulerTimer = null;
+ }
+
+ idleService.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS);
+ Services.obs.removeObserver(this, "wake_notification");
+
+ this._shuttingDown = true;
+ },
+
+ _clearTimeout() {
+ if (this._schedulerTimer) {
+ Policy.clearSchedulerTickTimeout(this._schedulerTimer);
+ }
+ },
+
+ /**
+ * Reschedules the tick timer.
+ */
+ _rescheduleTimeout() {
+ this._log.trace("_rescheduleTimeout - isUserIdle: " + this._isUserIdle);
+ if (this._shuttingDown) {
+ this._log.warn("_rescheduleTimeout - already shutdown");
+ return;
+ }
+
+ this._clearTimeout();
+
+ const now = Policy.now();
+ let timeout = SCHEDULER_TICK_INTERVAL_MS;
+
+ // When the user is idle we want to fire the timer less often.
+ if (this._isUserIdle) {
+ timeout = SCHEDULER_TICK_IDLE_INTERVAL_MS;
+ // We need to make sure though that we don't miss sending pings around
+ // midnight when we use the longer idle intervals.
+ const nextMidnight = TelemetryUtils.getNextMidnight(now);
+ timeout = Math.min(timeout, nextMidnight.getTime() - now.getTime());
+ }
+
+ this._log.trace(
+ "_rescheduleTimeout - scheduling next tick for " +
+ new Date(now.getTime() + timeout)
+ );
+ this._schedulerTimer = Policy.setSchedulerTickTimeout(
+ () => this._onSchedulerTick(),
+ timeout
+ );
+ },
+
+ _sentPingToday(pingTime, nowDate) {
+ // This is today's date and also the previous midnight (0:00).
+ const todayDate = TelemetryUtils.truncateToDays(nowDate);
+ // We consider a ping sent for today if it occured after or at 00:00 today.
+ return pingTime >= todayDate.getTime();
+ },
+
+ /**
+ * Checks if we can send a daily ping or not.
+ * @param {Object} nowDate A date object.
+ * @return {Boolean} True if we can send the daily ping, false otherwise.
+ */
+ _isDailyPingDue(nowDate) {
+ // The daily ping is not due if we already sent one today.
+ if (this._sentPingToday(this._lastDailyPingTime, nowDate)) {
+ this._log.trace("_isDailyPingDue - already sent one today");
+ return false;
+ }
+
+ // Avoid overly short sessions.
+ const timeSinceLastDaily = nowDate.getTime() - this._lastDailyPingTime;
+ if (timeSinceLastDaily < MIN_SUBSESSION_LENGTH_MS) {
+ this._log.trace(
+ "_isDailyPingDue - delaying daily to keep minimum session length"
+ );
+ return false;
+ }
+
+ this._log.trace("_isDailyPingDue - is due");
+ return true;
+ },
+
+ /**
+ * Checks if we can send a regular ping or not.
+ * @param {Object} nowDate A date object.
+ * @return {Boolean} True if we can send the regular pings, false otherwise.
+ */
+ _isPeriodicPingDue(nowDate) {
+ // The periodic ping is not due if we already sent one today.
+ if (this._sentPingToday(this._lastPeriodicPingTime, nowDate)) {
+ this._log.trace("_isPeriodicPingDue - already sent one today");
+ return false;
+ }
+
+ this._log.trace("_isPeriodicPingDue - is due");
+ return true;
+ },
+
+ /**
+ * An helper function to save an aborted-session ping.
+ * @param {Number} now The current time, in milliseconds.
+ * @param {Object} [competingPayload=null] If we are coalescing the daily and the
+ * aborted-session pings, this is the payload for the former. Note
+ * that the reason field of this payload will be changed.
+ * @return {Promise} A promise resolved when the ping is saved.
+ */
+ _saveAbortedPing(now, competingPayload = null) {
+ this._lastSessionCheckpointTime = now;
+ return TelemetrySession.saveAbortedSessionPing(competingPayload).catch(e =>
+ this._log.error("_saveAbortedPing - Failed", e)
+ );
+ },
+
+ /**
+ * The notifications handler.
+ */
+ observe(aSubject, aTopic, aData) {
+ this._log.trace("observe - aTopic: " + aTopic);
+ switch (aTopic) {
+ case "idle":
+ // If the user is idle, increase the tick interval.
+ this._isUserIdle = true;
+ return this._onSchedulerTick();
+ case "active":
+ // User is back to work, restore the original tick interval.
+ this._isUserIdle = false;
+ return this._onSchedulerTick(true);
+ case "wake_notification":
+ // The machine woke up from sleep, trigger a tick to avoid sessions
+ // spanning more than a day.
+ // This is needed because sleep time does not count towards timeouts
+ // on Mac & Linux - see bug 1262386, bug 1204823 et al.
+ return this._onSchedulerTick(true);
+ }
+ return undefined;
+ },
+
+ /**
+ * Creates an object with a method `dispatch` that will call `dispatchFn` unless
+ * the method `cancel` is called beforehand.
+ *
+ * This is used to wrap main thread idle dispatch since it does not provide a
+ * cancel mechanism.
+ */
+ _makeIdleDispatch(dispatchFn) {
+ this._log.trace("_makeIdleDispatch");
+ let fn = dispatchFn;
+ let l = msg => this._log.trace(msg); // need to bind `this`
+ return {
+ cancel() {
+ fn = undefined;
+ },
+ dispatch(resolve, reject) {
+ l("_makeIdleDispatch.dispatch - !!fn: " + !!fn);
+ if (!fn) {
+ return Promise.resolve().then(resolve, reject);
+ }
+ return fn(resolve, reject);
+ },
+ };
+ },
+
+ /**
+ * Performs a scheduler tick. This function manages Telemetry recurring operations.
+ * @param {Boolean} [dispatchOnIdle=false] If true, the tick is dispatched in the
+ * next idle cycle of the main thread.
+ * @return {Promise} A promise, only used when testing, resolved when the scheduled
+ * operation completes.
+ */
+ _onSchedulerTick(dispatchOnIdle = false) {
+ this._log.trace("_onSchedulerTick - dispatchOnIdle: " + dispatchOnIdle);
+ // This call might not be triggered from a timeout. In that case we don't want to
+ // leave any previously scheduled timeouts pending.
+ this._clearTimeout();
+
+ if (this._idleDispatch) {
+ this._idleDispatch.cancel();
+ }
+
+ if (this._shuttingDown) {
+ this._log.warn("_onSchedulerTick - already shutdown.");
+ return Promise.reject(new Error("Already shutdown."));
+ }
+
+ let promise = Promise.resolve();
+ try {
+ if (dispatchOnIdle) {
+ this._idleDispatch = this._makeIdleDispatch((resolve, reject) => {
+ this._log.trace(
+ "_onSchedulerTick - ildeDispatchToMainThread dispatch"
+ );
+ return this._schedulerTickLogic().then(resolve, reject);
+ });
+ promise = new Promise((resolve, reject) =>
+ Services.tm.idleDispatchToMainThread(() => {
+ return this._idleDispatch
+ ? this._idleDispatch.dispatch(resolve, reject)
+ : Promise.resolve().then(resolve, reject);
+ }, SCHEDULER_TICK_MAX_IDLE_DELAY_MS)
+ );
+ } else {
+ promise = this._schedulerTickLogic();
+ }
+ } catch (e) {
+ this._log.error("_onSchedulerTick - There was an exception", e);
+ } finally {
+ this._rescheduleTimeout();
+ }
+
+ // This promise is returned to make testing easier.
+ return promise;
+ },
+
+ /**
+ * Implements the scheduler logic.
+ * @return {Promise} Resolved when the scheduled task completes. Only used in tests.
+ */
+ _schedulerTickLogic() {
+ this._log.trace("_schedulerTickLogic");
+
+ let nowDate = Policy.now();
+ let now = nowDate.getTime();
+
+ // Check if the daily ping is due.
+ const shouldSendDaily = this._isDailyPingDue(nowDate);
+ // Check if other regular pings are due.
+ const shouldSendPeriodic = this._isPeriodicPingDue(nowDate);
+
+ if (shouldSendPeriodic) {
+ this._log.trace("_schedulerTickLogic - Periodic ping due.");
+ this._lastPeriodicPingTime = now;
+ // Send other pings.
+ EcosystemTelemetry.periodicPing();
+ TelemetryPrioPing.periodicPing();
+ }
+
+ if (shouldSendDaily) {
+ this._log.trace("_schedulerTickLogic - Daily ping due.");
+ this._lastDailyPingTime = now;
+ return TelemetrySession.sendDailyPing();
+ }
+
+ // Check if the aborted-session ping is due. If a daily ping was saved above, it was
+ // already duplicated as an aborted-session ping.
+ const isAbortedPingDue =
+ now - this._lastSessionCheckpointTime >=
+ ABORTED_SESSION_UPDATE_INTERVAL_MS;
+ if (isAbortedPingDue) {
+ this._log.trace("_schedulerTickLogic - Aborted session ping due.");
+ return this._saveAbortedPing(now);
+ }
+
+ // No ping is due.
+ this._log.trace("_schedulerTickLogic - No ping due.");
+ return Promise.resolve();
+ },
+
+ /**
+ * Re-schedule the daily ping if some other equivalent ping was sent.
+ *
+ * This is only called from TelemetrySession when a main ping with reason 'environment-change'
+ * is sent.
+ *
+ * @param {Object} [payload] The payload of the ping that was sent,
+ * to be stored as an aborted-session ping.
+ */
+ rescheduleDailyPing(payload) {
+ if (this._shuttingDown) {
+ this._log.error("rescheduleDailyPing - already shutdown");
+ return;
+ }
+
+ this._log.trace("rescheduleDailyPing");
+ let now = Policy.now();
+
+ // We just generated an environment-changed ping, save it as an aborted session and
+ // update the schedules.
+ this._saveAbortedPing(now.getTime(), payload);
+
+ // If we're close to midnight, skip today's daily ping and reschedule it for tomorrow.
+ let nearestMidnight = TelemetryUtils.getNearestMidnight(
+ now,
+ SCHEDULER_MIDNIGHT_TOLERANCE_MS
+ );
+ if (nearestMidnight) {
+ this._lastDailyPingTime = now.getTime();
+ }
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetrySend.jsm b/toolkit/components/telemetry/app/TelemetrySend.jsm
new file mode 100644
index 0000000000..caf11f4406
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetrySend.jsm
@@ -0,0 +1,1600 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This module is responsible for uploading pings to the server and persisting
+ * pings that can't be send now.
+ * Those pending pings are persisted on disk and sent at the next opportunity,
+ * newest first.
+ */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetrySend"];
+
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this);
+ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this);
+const { ClientID } = ChromeUtils.import("resource://gre/modules/ClientID.jsm");
+ChromeUtils.import("resource://gre/modules/Log.jsm", this);
+const { PromiseUtils } = ChromeUtils.import(
+ "resource://gre/modules/PromiseUtils.jsm"
+);
+ChromeUtils.import("resource://gre/modules/ServiceRequest.jsm", this);
+ChromeUtils.import("resource://gre/modules/Services.jsm", this);
+ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
+ChromeUtils.import("resource://gre/modules/Timer.jsm", this);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetryStorage",
+ "resource://gre/modules/TelemetryStorage.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetryReportingPolicy",
+ "resource://gre/modules/TelemetryReportingPolicy.jsm"
+);
+ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "Telemetry",
+ "@mozilla.org/base/telemetry;1",
+ "nsITelemetry"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetryHealthPing",
+ "resource://gre/modules/HealthPing.jsm"
+);
+
+const Utils = TelemetryUtils;
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "TelemetrySend::";
+
+const TOPIC_IDLE_DAILY = "idle-daily";
+// The following topics are notified when Firefox is closing
+// because the OS is shutting down.
+const TOPIC_QUIT_APPLICATION_GRANTED = "quit-application-granted";
+const TOPIC_QUIT_APPLICATION_FORCED = "quit-application-forced";
+const PREF_CHANGED_TOPIC = "nsPref:changed";
+const TOPIC_PROFILE_CHANGE_NET_TEARDOWN = "profile-change-net-teardown";
+
+// Whether the FHR/Telemetry unification features are enabled.
+// Changing this pref requires a restart.
+const IS_UNIFIED_TELEMETRY = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.Unified,
+ false
+);
+
+const MS_IN_A_MINUTE = 60 * 1000;
+
+const PING_TYPE_DELETION_REQUEST = "deletion-request";
+
+// We try to spread "midnight" pings out over this interval.
+const MIDNIGHT_FUZZING_INTERVAL_MS = 60 * MS_IN_A_MINUTE;
+// We delay sending "midnight" pings on this client by this interval.
+const MIDNIGHT_FUZZING_DELAY_MS = Math.random() * MIDNIGHT_FUZZING_INTERVAL_MS;
+
+// Timeout after which we consider a ping submission failed.
+const PING_SUBMIT_TIMEOUT_MS = 1.5 * MS_IN_A_MINUTE;
+
+// To keep resource usage in check, we limit ping sending to a maximum number
+// of pings per minute.
+const MAX_PING_SENDS_PER_MINUTE = 10;
+
+// If we have more pending pings then we can send right now, we schedule the next
+// send for after SEND_TICK_DELAY.
+const SEND_TICK_DELAY = 1 * MS_IN_A_MINUTE;
+// If we had any ping send failures since the last ping, we use a backoff timeout
+// for the next ping sends. We increase the delay exponentially up to a limit of
+// SEND_MAXIMUM_BACKOFF_DELAY_MS.
+// This exponential backoff will be reset by external ping submissions & idle-daily.
+const SEND_MAXIMUM_BACKOFF_DELAY_MS = 120 * MS_IN_A_MINUTE;
+
+// Strings to map from XHR.errorCode to TELEMETRY_SEND_FAILURE_TYPE.
+// Echoes XMLHttpRequestMainThread's ErrorType enum.
+// Make sure that any additions done to XHR_ERROR_TYPE enum are also mirrored in
+// TELEMETRY_SEND_FAILURE_TYPE and TELEMETRY_SEND_FAILURE_TYPE_PER_PING's labels.
+const XHR_ERROR_TYPE = [
+ "eOK",
+ "eRequest",
+ "eUnreachable",
+ "eChannelOpen",
+ "eRedirect",
+ "eTerminated",
+];
+
+/**
+ * This is a policy object used to override behavior within this module.
+ * Tests override properties on this object to allow for control of behavior
+ * that would otherwise be very hard to cover.
+ */
+var Policy = {
+ now: () => new Date(),
+ midnightPingFuzzingDelay: () => MIDNIGHT_FUZZING_DELAY_MS,
+ pingSubmissionTimeout: () => PING_SUBMIT_TIMEOUT_MS,
+ setSchedulerTickTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+ clearSchedulerTickTimeout: id => clearTimeout(id),
+ gzipCompressString: data => gzipCompressString(data),
+};
+
+/**
+ * Determine if the ping has the new v4 ping format or the legacy v2 one or earlier.
+ */
+function isV4PingFormat(aPing) {
+ return (
+ "id" in aPing &&
+ "application" in aPing &&
+ "version" in aPing &&
+ aPing.version >= 2
+ );
+}
+
+/**
+ * Check if the provided ping is a deletion-request ping.
+ * @param {Object} aPing The ping to check.
+ * @return {Boolean} True if the ping is a deletion-request ping, false otherwise.
+ */
+function isDeletionRequestPing(aPing) {
+ return isV4PingFormat(aPing) && aPing.type == PING_TYPE_DELETION_REQUEST;
+}
+
+/**
+ * Save the provided ping as a pending ping.
+ * @param {Object} aPing The ping to save.
+ * @return {Promise} A promise resolved when the ping is saved.
+ */
+function savePing(aPing) {
+ return TelemetryStorage.savePendingPing(aPing);
+}
+
+/**
+ * @return {String} This returns a string with the gzip compressed data.
+ */
+function gzipCompressString(string) {
+ let observer = {
+ buffer: "",
+ onStreamComplete(loader, context, status, length, result) {
+ // String.fromCharCode can only deal with 500,000 characters at
+ // a time, so chunk the result into parts of that size.
+ const chunkSize = 500000;
+ for (let offset = 0; offset < result.length; offset += chunkSize) {
+ this.buffer += String.fromCharCode.apply(
+ String,
+ result.slice(offset, offset + chunkSize)
+ );
+ }
+ },
+ };
+
+ let scs = Cc["@mozilla.org/streamConverters;1"].getService(
+ Ci.nsIStreamConverterService
+ );
+ let listener = Cc["@mozilla.org/network/stream-loader;1"].createInstance(
+ Ci.nsIStreamLoader
+ );
+ listener.init(observer);
+ let converter = scs.asyncConvertData("uncompressed", "gzip", listener, null);
+ let stringStream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(
+ Ci.nsIStringInputStream
+ );
+ stringStream.data = string;
+ converter.onStartRequest(null, null);
+ converter.onDataAvailable(null, stringStream, 0, string.length);
+ converter.onStopRequest(null, null, null);
+ return observer.buffer;
+}
+
+var TelemetrySend = {
+ get pendingPingCount() {
+ return TelemetrySendImpl.pendingPingCount;
+ },
+
+ /**
+ * Partial setup that runs immediately at startup. This currently triggers
+ * the crash report annotations.
+ */
+ earlyInit() {
+ TelemetrySendImpl.earlyInit();
+ },
+
+ /**
+ * Initializes this module.
+ *
+ * @param {Boolean} testing Whether this is run in a test. This changes some behavior
+ * to enable proper testing.
+ * @return {Promise} Resolved when setup is finished.
+ */
+ setup(testing = false) {
+ return TelemetrySendImpl.setup(testing);
+ },
+
+ /**
+ * Shutdown this module - this will cancel any pending ping tasks and wait for
+ * outstanding async activity like network and disk I/O.
+ *
+ * @return {Promise} Promise that is resolved when shutdown is finished.
+ */
+ shutdown() {
+ return TelemetrySendImpl.shutdown();
+ },
+
+ /**
+ * Flushes all pings to pingsender that were both
+ * 1. submitted after profile-change-net-teardown, and
+ * 2. wanting to be sent using pingsender.
+ */
+ flushPingSenderBatch() {
+ TelemetrySendImpl.flushPingSenderBatch();
+ },
+
+ /**
+ * Submit a ping for sending. This will:
+ * - send the ping right away if possible or
+ * - save the ping to disk and send it at the next opportunity
+ *
+ * @param {Object} ping The ping data to send, must be serializable to JSON.
+ * @param {Object} [aOptions] Options object.
+ * @param {Boolean} [options.usePingSender=false] if true, send the ping using the PingSender.
+ * @return {Promise} Test-only - a promise that is resolved when the ping is sent or saved.
+ */
+ submitPing(ping, options = {}) {
+ options.usePingSender = options.usePingSender || false;
+ return TelemetrySendImpl.submitPing(ping, options);
+ },
+
+ /**
+ * Check if sending is disabled. If Telemetry is not allowed to upload,
+ * pings are not sent to the server.
+ * If trying to send a deletion-request ping, don't block it.
+ *
+ * @param {Object} [ping=null] A ping to be checked.
+ * @return {Boolean} True if pings can be send to the servers, false otherwise.
+ */
+ sendingEnabled(ping = null) {
+ return TelemetrySendImpl.sendingEnabled(ping);
+ },
+
+ /**
+ * Notify that we can start submitting data to the servers.
+ */
+ notifyCanUpload() {
+ return TelemetrySendImpl.notifyCanUpload();
+ },
+
+ /**
+ * Only used in tests. Used to reset the module data to emulate a restart.
+ */
+ reset() {
+ return TelemetrySendImpl.reset();
+ },
+
+ /**
+ * Only used in tests.
+ */
+ setServer(server) {
+ return TelemetrySendImpl.setServer(server);
+ },
+
+ /**
+ * Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests.
+ */
+ clearCurrentPings() {
+ return TelemetrySendImpl.clearCurrentPings();
+ },
+
+ /**
+ * Only used in tests to wait on outgoing pending pings.
+ */
+ testWaitOnOutgoingPings() {
+ return TelemetrySendImpl.promisePendingPingActivity();
+ },
+
+ /**
+ * Only used in tests to set whether it is too late in shutdown to send pings.
+ */
+ testTooLateToSend(tooLate) {
+ TelemetrySendImpl._tooLateToSend = tooLate;
+ },
+
+ /**
+ * Test-only - this allows overriding behavior to enable ping sending in debug builds.
+ */
+ setTestModeEnabled(testing) {
+ TelemetrySendImpl.setTestModeEnabled(testing);
+ },
+
+ /**
+ * This returns state info for this module for AsyncShutdown timeout diagnostics.
+ */
+ getShutdownState() {
+ return TelemetrySendImpl.getShutdownState();
+ },
+
+ /**
+ * Send a ping using the ping sender.
+ * This method will not wait for the ping to be sent, instead it will return
+ * as soon as the pingsender program has been launched.
+ *
+ * This method is currently exposed here only for testing purposes as it's
+ * only used internally.
+ *
+ * @param {Array}<Object> pings An array of objects holding url / path pairs
+ * for each ping to be sent. The URL represent the telemetry server the
+ * ping will be sent to and the path points to the ping data. The ping
+ * data files will be deleted if the pings have been submitted
+ * successfully.
+ * @param {callback} observer A function called with parameters
+ * (subject, topic, data) and a topic of "process-finished" or
+ * "process-failed" after pingsender completion.
+ *
+ * @throws NS_ERROR_FAILURE if we couldn't find or run the pingsender
+ * executable.
+ * @throws NS_ERROR_NOT_IMPLEMENTED on Android as the pingsender is not
+ * available.
+ */
+ testRunPingSender(pings, observer) {
+ return TelemetrySendImpl.runPingSender(pings, observer);
+ },
+};
+
+var CancellableTimeout = {
+ _deferred: null,
+ _timer: null,
+
+ /**
+ * This waits until either the given timeout passed or the timeout was cancelled.
+ *
+ * @param {Number} timeoutMs The timeout in ms.
+ * @return {Promise<bool>} Promise that is resolved with false if the timeout was cancelled,
+ * false otherwise.
+ */
+ promiseWaitOnTimeout(timeoutMs) {
+ if (!this._deferred) {
+ this._deferred = PromiseUtils.defer();
+ this._timer = Policy.setSchedulerTickTimeout(
+ () => this._onTimeout(),
+ timeoutMs
+ );
+ }
+
+ return this._deferred.promise;
+ },
+
+ _onTimeout() {
+ if (this._deferred) {
+ this._deferred.resolve(false);
+ this._timer = null;
+ this._deferred = null;
+ }
+ },
+
+ cancelTimeout() {
+ if (this._deferred) {
+ Policy.clearSchedulerTickTimeout(this._timer);
+ this._deferred.resolve(true);
+ this._timer = null;
+ this._deferred = null;
+ }
+ },
+};
+
+/**
+ * SendScheduler implements the timer & scheduling behavior for ping sends.
+ */
+var SendScheduler = {
+ // Whether any ping sends failed since the last tick. If yes, we start with our exponential
+ // backoff timeout.
+ _sendsFailed: false,
+ // The current retry delay after ping send failures. We use this for the exponential backoff,
+ // increasing this value everytime we had send failures since the last tick.
+ _backoffDelay: SEND_TICK_DELAY,
+ _shutdown: false,
+ _sendTask: null,
+ // A string that tracks the last seen send task state, null if it never ran.
+ _sendTaskState: null,
+
+ _logger: null,
+
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX + "Scheduler::"
+ );
+ }
+
+ return this._logger;
+ },
+
+ shutdown() {
+ this._log.trace("shutdown");
+ this._shutdown = true;
+ CancellableTimeout.cancelTimeout();
+ return Promise.resolve(this._sendTask);
+ },
+
+ start() {
+ this._log.trace("start");
+ this._sendsFailed = false;
+ this._backoffDelay = SEND_TICK_DELAY;
+ this._shutdown = false;
+ },
+
+ /**
+ * Only used for testing, resets the state to emulate a restart.
+ */
+ reset() {
+ this._log.trace("reset");
+ return this.shutdown().then(() => this.start());
+ },
+
+ /**
+ * Notify the scheduler of a failure in sending out pings that warrants retrying.
+ * This will trigger the exponential backoff timer behavior on the next tick.
+ */
+ notifySendsFailed() {
+ this._log.trace("notifySendsFailed");
+ if (this._sendsFailed) {
+ return;
+ }
+
+ this._sendsFailed = true;
+ this._log.trace("notifySendsFailed - had send failures");
+ },
+
+ /**
+ * Returns whether ping submissions are currently throttled.
+ */
+ isThrottled() {
+ const now = Policy.now();
+ const nextPingSendTime = this._getNextPingSendTime(now);
+ return nextPingSendTime > now.getTime();
+ },
+
+ waitOnSendTask() {
+ return Promise.resolve(this._sendTask);
+ },
+
+ triggerSendingPings(immediately) {
+ this._log.trace(
+ "triggerSendingPings - active send task: " +
+ !!this._sendTask +
+ ", immediately: " +
+ immediately
+ );
+
+ if (!this._sendTask) {
+ this._sendTask = this._doSendTask();
+ let clear = () => (this._sendTask = null);
+ this._sendTask.then(clear, clear);
+ } else if (immediately) {
+ CancellableTimeout.cancelTimeout();
+ }
+
+ return this._sendTask;
+ },
+
+ async _doSendTask() {
+ this._sendTaskState = "send task started";
+ this._backoffDelay = SEND_TICK_DELAY;
+ this._sendsFailed = false;
+
+ const resetBackoffTimer = () => {
+ this._backoffDelay = SEND_TICK_DELAY;
+ };
+
+ for (;;) {
+ this._log.trace("_doSendTask iteration");
+ this._sendTaskState = "start iteration";
+
+ if (this._shutdown) {
+ this._log.trace("_doSendTask - shutting down, bailing out");
+ this._sendTaskState = "bail out - shutdown check";
+ return;
+ }
+
+ // Get a list of pending pings, sorted by last modified, descending.
+ // Filter out all the pings we can't send now. This addresses scenarios like "deletion-request" pings
+ // which can be sent even when upload is disabled.
+ let pending = TelemetryStorage.getPendingPingList();
+ let current = TelemetrySendImpl.getUnpersistedPings();
+ this._log.trace(
+ "_doSendTask - pending: " +
+ pending.length +
+ ", current: " +
+ current.length
+ );
+ // Note that the two lists contain different kind of data. |pending| only holds ping
+ // info, while |current| holds actual ping data.
+ if (!TelemetrySendImpl.sendingEnabled()) {
+ // If sending is disabled, only handle deletion-request pings
+ pending = [];
+ current = current.filter(p => isDeletionRequestPing(p));
+ }
+ this._log.trace(
+ "_doSendTask - can send - pending: " +
+ pending.length +
+ ", current: " +
+ current.length
+ );
+
+ // Bail out if there is nothing to send.
+ if (!pending.length && !current.length) {
+ this._log.trace("_doSendTask - no pending pings, bailing out");
+ this._sendTaskState = "bail out - no pings to send";
+ return;
+ }
+
+ // If we are currently throttled (e.g. fuzzing to avoid midnight spikes), wait for the next send window.
+ const now = Policy.now();
+ if (this.isThrottled()) {
+ const nextPingSendTime = this._getNextPingSendTime(now);
+ this._log.trace(
+ "_doSendTask - throttled, delaying ping send to " +
+ new Date(nextPingSendTime)
+ );
+ this._sendTaskState = "wait for throttling to pass";
+
+ const delay = nextPingSendTime - now.getTime();
+ const cancelled = await CancellableTimeout.promiseWaitOnTimeout(delay);
+ if (cancelled) {
+ this._log.trace(
+ "_doSendTask - throttling wait was cancelled, resetting backoff timer"
+ );
+ resetBackoffTimer();
+ }
+
+ continue;
+ }
+
+ let sending = pending.slice(0, MAX_PING_SENDS_PER_MINUTE);
+ pending = pending.slice(MAX_PING_SENDS_PER_MINUTE);
+ this._log.trace(
+ "_doSendTask - triggering sending of " +
+ sending.length +
+ " pings now" +
+ ", " +
+ pending.length +
+ " pings waiting"
+ );
+
+ this._sendsFailed = false;
+ const sendStartTime = Policy.now();
+ this._sendTaskState = "wait on ping sends";
+ await TelemetrySendImpl.sendPings(
+ current,
+ sending.map(p => p.id)
+ );
+ if (this._shutdown || TelemetrySend.pendingPingCount == 0) {
+ this._log.trace(
+ "_doSendTask - bailing out after sending, shutdown: " +
+ this._shutdown +
+ ", pendingPingCount: " +
+ TelemetrySend.pendingPingCount
+ );
+ this._sendTaskState = "bail out - shutdown & pending check after send";
+ return;
+ }
+
+ // Calculate the delay before sending the next batch of pings.
+ // We start with a delay that makes us send max. 1 batch per minute.
+ // If we had send failures in the last batch, we will override this with
+ // a backoff delay.
+ const timeSinceLastSend = Policy.now() - sendStartTime;
+ let nextSendDelay = Math.max(0, SEND_TICK_DELAY - timeSinceLastSend);
+
+ if (!this._sendsFailed) {
+ this._log.trace(
+ "_doSendTask - had no send failures, resetting backoff timer"
+ );
+ resetBackoffTimer();
+ } else {
+ const newDelay = Math.min(
+ SEND_MAXIMUM_BACKOFF_DELAY_MS,
+ this._backoffDelay * 2
+ );
+ this._log.trace(
+ "_doSendTask - had send failures, backing off -" +
+ " old timeout: " +
+ this._backoffDelay +
+ ", new timeout: " +
+ newDelay
+ );
+ this._backoffDelay = newDelay;
+ nextSendDelay = this._backoffDelay;
+ }
+
+ this._log.trace(
+ "_doSendTask - waiting for next send opportunity, timeout is " +
+ nextSendDelay
+ );
+ this._sendTaskState = "wait on next send opportunity";
+ const cancelled = await CancellableTimeout.promiseWaitOnTimeout(
+ nextSendDelay
+ );
+ if (cancelled) {
+ this._log.trace(
+ "_doSendTask - batch send wait was cancelled, resetting backoff timer"
+ );
+ resetBackoffTimer();
+ }
+ }
+ },
+
+ /**
+ * This helper calculates the next time that we can send pings at.
+ * Currently this mostly redistributes ping sends from midnight until one hour after
+ * to avoid submission spikes around local midnight for daily pings.
+ *
+ * @param now Date The current time.
+ * @return Number The next time (ms from UNIX epoch) when we can send pings.
+ */
+ _getNextPingSendTime(now) {
+ // 1. First we check if the pref is set to skip any delay and send immediately.
+ // 2. Next we check if the time is between 0am and 1am. If it's not, we send
+ // immediately.
+ // 3. If we confirmed the time is indeed between 0am and 1am in step 1, we disallow
+ // sending before (midnight + fuzzing delay), which is a random time between 0am-1am
+ // (decided at startup).
+
+ let disableFuzzingDelay = Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.DisableFuzzingDelay,
+ false
+ );
+ if (disableFuzzingDelay) {
+ return now.getTime();
+ }
+
+ const midnight = Utils.truncateToDays(now);
+ // Don't delay pings if we are not within the fuzzing interval.
+ if (now.getTime() - midnight.getTime() > MIDNIGHT_FUZZING_INTERVAL_MS) {
+ return now.getTime();
+ }
+
+ // Delay ping send if we are within the midnight fuzzing range.
+ // We spread those ping sends out between |midnight| and |midnight + midnightPingFuzzingDelay|.
+ return midnight.getTime() + Policy.midnightPingFuzzingDelay();
+ },
+
+ getShutdownState() {
+ return {
+ shutdown: this._shutdown,
+ hasSendTask: !!this._sendTask,
+ sendsFailed: this._sendsFailed,
+ sendTaskState: this._sendTaskState,
+ backoffDelay: this._backoffDelay,
+ };
+ },
+};
+
+var TelemetrySendImpl = {
+ _sendingEnabled: false,
+ // Tracks the shutdown state.
+ _shutdown: false,
+ _logger: null,
+ // This tracks all pending ping requests to the server.
+ _pendingPingRequests: new Map(),
+ // This tracks all the pending async ping activity.
+ _pendingPingActivity: new Set(),
+ // This is true when running in the test infrastructure.
+ _testMode: false,
+ // This holds pings that we currently try and haven't persisted yet.
+ _currentPings: new Map(),
+ // Used to skip spawning the pingsender if OS is shutting down.
+ _isOSShutdown: false,
+ // Has the network shut down, making it too late to send pings?
+ _tooLateToSend: false,
+ // Array of {url, path} awaiting flushPingSenderBatch().
+ _pingSenderBatch: [],
+
+ OBSERVER_TOPICS: [
+ TOPIC_IDLE_DAILY,
+ TOPIC_QUIT_APPLICATION_GRANTED,
+ TOPIC_QUIT_APPLICATION_FORCED,
+ TOPIC_PROFILE_CHANGE_NET_TEARDOWN,
+ ],
+
+ OBSERVED_PREFERENCES: [
+ TelemetryUtils.Preferences.TelemetryEnabled,
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ ],
+
+ // Whether sending pings has been overridden.
+ get _overrideOfficialCheck() {
+ return Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.OverrideOfficialCheck,
+ false
+ );
+ },
+
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+
+ return this._logger;
+ },
+
+ get pendingPingRequests() {
+ return this._pendingPingRequests;
+ },
+
+ get pendingPingCount() {
+ return (
+ TelemetryStorage.getPendingPingList().length + this._currentPings.size
+ );
+ },
+
+ setTestModeEnabled(testing) {
+ this._testMode = testing;
+ },
+
+ earlyInit() {
+ this._annotateCrashReport();
+
+ // Install the observer to detect OS shutdown early enough, so
+ // that we catch this before the delayed setup happens.
+ Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_FORCED);
+ Services.obs.addObserver(this, TOPIC_QUIT_APPLICATION_GRANTED);
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
+
+ async setup(testing) {
+ this._log.trace("setup");
+
+ this._testMode = testing;
+
+ Services.obs.addObserver(this, TOPIC_IDLE_DAILY);
+ Services.obs.addObserver(this, TOPIC_PROFILE_CHANGE_NET_TEARDOWN);
+
+ this._server = Services.prefs.getStringPref(
+ TelemetryUtils.Preferences.Server,
+ undefined
+ );
+ this._sendingEnabled = true;
+
+ // Annotate crash reports so that crash pings are sent correctly and listen
+ // to pref changes to adjust the annotations accordingly.
+ for (let pref of this.OBSERVED_PREFERENCES) {
+ Services.prefs.addObserver(pref, this, true);
+ }
+ this._annotateCrashReport();
+
+ // Check the pending pings on disk now.
+ try {
+ await this._checkPendingPings();
+ } catch (ex) {
+ this._log.error("setup - _checkPendingPings rejected", ex);
+ }
+
+ // Enforce the pending pings storage quota. It could take a while so don't
+ // block on it.
+ TelemetryStorage.runEnforcePendingPingsQuotaTask();
+
+ // Start sending pings, but don't block on this.
+ SendScheduler.triggerSendingPings(true);
+ },
+
+ /**
+ * Triggers the crash report annotations depending on the current
+ * configuration. This communicates to the crash reporter if it can send a
+ * crash ping or not. This method can be called safely before setup() has
+ * been called.
+ */
+ _annotateCrashReport() {
+ try {
+ const cr = Cc["@mozilla.org/toolkit/crash-reporter;1"];
+ if (cr) {
+ const crs = cr.getService(Ci.nsICrashReporter);
+
+ let clientId = ClientID.getCachedClientID();
+ let server =
+ this._server ||
+ Services.prefs.getStringPref(
+ TelemetryUtils.Preferences.Server,
+ undefined
+ );
+
+ if (!this.sendingEnabled() || !TelemetryReportingPolicy.canUpload()) {
+ // If we cannot send pings then clear the crash annotations
+ crs.removeCrashReportAnnotation("TelemetryClientId");
+ crs.removeCrashReportAnnotation("TelemetryServerURL");
+ } else {
+ crs.annotateCrashReport("TelemetryClientId", clientId);
+ crs.annotateCrashReport("TelemetryServerURL", server);
+ }
+ }
+ } catch (e) {
+ // Ignore errors when crash reporting is disabled
+ }
+ },
+
+ /**
+ * Discard old pings from the pending pings and detect overdue ones.
+ * @return {Boolean} True if we have overdue pings, false otherwise.
+ */
+ async _checkPendingPings() {
+ // Scan the pending pings - that gives us a list sorted by last modified, descending.
+ let infos = await TelemetryStorage.loadPendingPingList();
+ this._log.info("_checkPendingPings - pending ping count: " + infos.length);
+ if (!infos.length) {
+ this._log.trace("_checkPendingPings - no pending pings");
+ return;
+ }
+
+ const now = Policy.now();
+
+ // Submit the age of the pending pings.
+ for (let pingInfo of infos) {
+ const ageInDays = Utils.millisecondsToDays(
+ Math.abs(now.getTime() - pingInfo.lastModificationDate)
+ );
+ Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_AGE").add(ageInDays);
+ }
+ },
+
+ async shutdown() {
+ this._shutdown = true;
+
+ for (let pref of this.OBSERVED_PREFERENCES) {
+ // FIXME: When running tests this causes errors to be printed out if
+ // TelemetrySend.shutdown() is called twice in a row without calling
+ // TelemetrySend.setup() in-between.
+ Services.prefs.removeObserver(pref, this);
+ }
+
+ for (let topic of this.OBSERVER_TOPICS) {
+ try {
+ Services.obs.removeObserver(this, topic);
+ } catch (ex) {
+ this._log.error(
+ "shutdown - failed to remove observer for " + topic,
+ ex
+ );
+ }
+ }
+
+ // We can't send anymore now.
+ this._sendingEnabled = false;
+
+ // Cancel any outgoing requests.
+ await this._cancelOutgoingRequests();
+
+ // Stop any active send tasks.
+ await SendScheduler.shutdown();
+
+ // Wait for any outstanding async ping activity.
+ await this.promisePendingPingActivity();
+
+ // Save any outstanding pending pings to disk.
+ await this._persistCurrentPings();
+ },
+
+ flushPingSenderBatch() {
+ if (this._pingSenderBatch.length === 0) {
+ return;
+ }
+ this._log.trace(
+ `flushPingSenderBatch - Sending ${this._pingSenderBatch.length} pings.`
+ );
+ this.runPingSender(this._pingSenderBatch);
+ },
+
+ reset() {
+ this._log.trace("reset");
+
+ this._shutdown = false;
+ this._currentPings = new Map();
+ this._tooLateToSend = false;
+ this._isOSShutdown = false;
+ this._sendingEnabled = true;
+
+ const histograms = [
+ "TELEMETRY_SUCCESS",
+ "TELEMETRY_SEND_SUCCESS",
+ "TELEMETRY_SEND_FAILURE",
+ "TELEMETRY_SEND_FAILURE_TYPE",
+ ];
+
+ histograms.forEach(h => Telemetry.getHistogramById(h).clear());
+
+ const keyedHistograms = ["TELEMETRY_SEND_FAILURE_TYPE_PER_PING"];
+
+ keyedHistograms.forEach(h => Telemetry.getKeyedHistogramById(h).clear());
+
+ return SendScheduler.reset();
+ },
+
+ /**
+ * Notify that we can start submitting data to the servers.
+ */
+ notifyCanUpload() {
+ if (!this._sendingEnabled) {
+ this._log.trace(
+ "notifyCanUpload - notifying before sending is enabled. Ignoring."
+ );
+ return Promise.resolve();
+ }
+ // Let the scheduler trigger sending pings if possible, also inform the
+ // crash reporter that it can send crash pings if appropriate.
+ SendScheduler.triggerSendingPings(true);
+ this._annotateCrashReport();
+
+ return this.promisePendingPingActivity();
+ },
+
+ observe(subject, topic, data) {
+ let setOSShutdown = () => {
+ this._log.trace("setOSShutdown - in OS shutdown");
+ this._isOSShutdown = true;
+ };
+
+ switch (topic) {
+ case TOPIC_IDLE_DAILY:
+ SendScheduler.triggerSendingPings(true);
+ break;
+ case TOPIC_QUIT_APPLICATION_FORCED:
+ setOSShutdown();
+ break;
+ case TOPIC_QUIT_APPLICATION_GRANTED:
+ if (data == "syncShutdown") {
+ setOSShutdown();
+ }
+ break;
+ case PREF_CHANGED_TOPIC:
+ if (this.OBSERVED_PREFERENCES.includes(data)) {
+ this._annotateCrashReport();
+ }
+ break;
+ case TOPIC_PROFILE_CHANGE_NET_TEARDOWN:
+ this._tooLateToSend = true;
+ break;
+ }
+ },
+
+ /**
+ * Spawn the PingSender process that sends a ping. This function does
+ * not return an error or throw, it only logs an error.
+ *
+ * Even if the function doesn't fail, it doesn't mean that the ping was
+ * successfully sent, as we have no control over the spawned process. If it,
+ * succeeds, the ping is eventually removed from the disk to prevent duplicated
+ * submissions.
+ *
+ * @param {String} pingId The id of the ping to send.
+ * @param {String} submissionURL The complete Telemetry-compliant URL for the ping.
+ */
+ _sendWithPingSender(pingId, submissionURL) {
+ this._log.trace(
+ "_sendWithPingSender - sending " + pingId + " to " + submissionURL
+ );
+ try {
+ const pingPath = OS.Path.join(TelemetryStorage.pingDirectoryPath, pingId);
+ if (this._tooLateToSend) {
+ // We're in shutdown. Batch pings destined for pingsender.
+ this._log.trace("_sendWithPingSender - too late to send. Batching.");
+ this._pingSenderBatch.push({ url: submissionURL, path: pingPath });
+ return;
+ }
+ this.runPingSender([{ url: submissionURL, path: pingPath }]);
+ } catch (e) {
+ this._log.error("_sendWithPingSender - failed to submit ping", e);
+ }
+ },
+
+ submitPing(ping, options) {
+ this._log.trace(
+ "submitPing - ping id: " +
+ ping.id +
+ ", options: " +
+ JSON.stringify(options)
+ );
+
+ if (!this.sendingEnabled(ping)) {
+ this._log.trace("submitPing - Telemetry is not allowed to send pings.");
+ return Promise.resolve();
+ }
+
+ // Send the ping using the PingSender, if requested and the user was
+ // notified of our policy. We don't support the pingsender on Android,
+ // so ignore this option on that platform (see bug 1335917).
+ // Moreover, if the OS is shutting down, we don't want to spawn the
+ // pingsender as it could unnecessarily slow down OS shutdown.
+ // Additionally, it could be be killed before it can complete its tasks,
+ // for example after successfully sending the ping but before removing
+ // the copy from the disk, resulting in receiving duplicate pings when
+ // Firefox restarts.
+ if (
+ options.usePingSender &&
+ !this._isOSShutdown &&
+ TelemetryReportingPolicy.canUpload() &&
+ AppConstants.platform != "android"
+ ) {
+ const url = this._buildSubmissionURL(ping);
+ // Serialize the ping to the disk and then spawn the PingSender.
+ return savePing(ping).then(() => this._sendWithPingSender(ping.id, url));
+ }
+
+ if (!this.canSendNow) {
+ // Sending is disabled or throttled, add this to the persisted pending pings.
+ this._log.trace(
+ "submitPing - can't send ping now, persisting to disk - " +
+ "canSendNow: " +
+ this.canSendNow
+ );
+ return savePing(ping);
+ }
+
+ // Let the scheduler trigger sending pings if possible.
+ // As a safety mechanism, this resets any currently active throttling.
+ this._log.trace("submitPing - can send pings, trying to send now");
+ this._currentPings.set(ping.id, ping);
+ SendScheduler.triggerSendingPings(true);
+ return Promise.resolve();
+ },
+
+ /**
+ * Only used in tests.
+ */
+ setServer(server) {
+ this._log.trace("setServer", server);
+ this._server = server;
+ },
+
+ /**
+ * Clear out unpersisted, yet to be sent, pings and cancel outgoing ping requests.
+ */
+ async clearCurrentPings() {
+ if (this._shutdown) {
+ this._log.trace("clearCurrentPings - in shutdown, bailing out");
+ return;
+ }
+
+ // Temporarily disable the scheduler. It must not try to reschedule ping sending
+ // while we're deleting them.
+ await SendScheduler.shutdown();
+
+ // Now that the ping activity has settled, abort outstanding ping requests.
+ this._cancelOutgoingRequests();
+
+ // Also, purge current pings.
+ this._currentPings.clear();
+
+ // We might have been interrupted and shutdown could have been started.
+ // We need to bail out in that case to avoid triggering send activity etc.
+ // at unexpected times.
+ if (this._shutdown) {
+ this._log.trace(
+ "clearCurrentPings - in shutdown, not spinning SendScheduler up again"
+ );
+ return;
+ }
+
+ // Enable the scheduler again and spin the send task.
+ SendScheduler.start();
+ SendScheduler.triggerSendingPings(true);
+ },
+
+ _cancelOutgoingRequests() {
+ // Abort any pending ping XHRs.
+ for (let [id, request] of this._pendingPingRequests) {
+ this._log.trace(
+ "_cancelOutgoingRequests - aborting ping request for id " + id
+ );
+ try {
+ request.abort();
+ } catch (e) {
+ this._log.error(
+ "_cancelOutgoingRequests - failed to abort request for id " + id,
+ e
+ );
+ }
+ }
+ this._pendingPingRequests.clear();
+ },
+
+ sendPings(currentPings, persistedPingIds) {
+ let pingSends = [];
+
+ // Prioritize health pings to enable low-latency monitoring.
+ currentPings = [
+ ...currentPings.filter(ping => ping.type === "health"),
+ ...currentPings.filter(ping => ping.type !== "health"),
+ ];
+
+ for (let current of currentPings) {
+ let ping = current;
+ let p = (async () => {
+ try {
+ await this._doPing(ping, ping.id, false);
+ } catch (ex) {
+ this._log.info(
+ "sendPings - ping " + ping.id + " not sent, saving to disk",
+ ex
+ );
+ await savePing(ping);
+ } finally {
+ this._currentPings.delete(ping.id);
+ }
+ })();
+
+ this._trackPendingPingTask(p);
+ pingSends.push(p);
+ }
+
+ if (persistedPingIds.length) {
+ pingSends.push(
+ this._sendPersistedPings(persistedPingIds).catch(ex => {
+ this._log.info("sendPings - persisted pings not sent", ex);
+ })
+ );
+ }
+
+ return Promise.all(pingSends);
+ },
+
+ /**
+ * Send the persisted pings to the server.
+ *
+ * @param {Array<string>} List of ping ids that should be sent.
+ *
+ * @return Promise A promise that is resolved when all pings finished sending or failed.
+ */
+ async _sendPersistedPings(pingIds) {
+ this._log.trace("sendPersistedPings");
+
+ if (this.pendingPingCount < 1) {
+ this._log.trace("_sendPersistedPings - no pings to send");
+ return;
+ }
+
+ if (pingIds.length < 1) {
+ this._log.trace("sendPersistedPings - no pings to send");
+ return;
+ }
+
+ // We can send now.
+ // If there are any send failures, _doPing() sets up handlers that e.g. trigger backoff timer behavior.
+ this._log.trace(
+ "sendPersistedPings - sending " + pingIds.length + " pings"
+ );
+ let pingSendPromises = [];
+ for (let pingId of pingIds) {
+ const id = pingId;
+ pingSendPromises.push(
+ TelemetryStorage.loadPendingPing(id)
+ .then(data => this._doPing(data, id, true))
+ .catch(e =>
+ this._log.error("sendPersistedPings - failed to send ping " + id, e)
+ )
+ );
+ }
+
+ let promise = Promise.all(pingSendPromises);
+ this._trackPendingPingTask(promise);
+ await promise;
+ },
+
+ _onPingRequestFinished(success, startTime, id, isPersisted) {
+ this._log.trace(
+ "_onPingRequestFinished - success: " +
+ success +
+ ", persisted: " +
+ isPersisted
+ );
+
+ let sendId = success ? "TELEMETRY_SEND_SUCCESS" : "TELEMETRY_SEND_FAILURE";
+ let hsend = Telemetry.getHistogramById(sendId);
+ let hsuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
+
+ hsend.add(Utils.monotonicNow() - startTime);
+ hsuccess.add(success);
+
+ if (!success) {
+ // Let the scheduler know about send failures for triggering backoff timeouts.
+ SendScheduler.notifySendsFailed();
+ }
+
+ if (success && isPersisted) {
+ return TelemetryStorage.removePendingPing(id);
+ }
+ return Promise.resolve();
+ },
+
+ _buildSubmissionURL(ping) {
+ const version = isV4PingFormat(ping)
+ ? AppConstants.TELEMETRY_PING_FORMAT_VERSION
+ : 1;
+ return this._server + this._getSubmissionPath(ping) + "?v=" + version;
+ },
+
+ _getSubmissionPath(ping) {
+ // The new ping format contains an "application" section, the old one doesn't.
+ let pathComponents;
+ if (isV4PingFormat(ping)) {
+ // We insert the Ping id in the URL to simplify server handling of duplicated
+ // pings.
+ let app = ping.application;
+ pathComponents = [
+ ping.id,
+ ping.type,
+ app.name,
+ app.version,
+ app.channel,
+ app.buildId,
+ ];
+ } else {
+ // This is a ping in the old format.
+ if (!("slug" in ping)) {
+ // That's odd, we don't have a slug. Generate one so that TelemetryStorage.jsm works.
+ ping.slug = Utils.generateUUID();
+ }
+
+ // Do we have enough info to build a submission URL?
+ let payload = "payload" in ping ? ping.payload : null;
+ if (payload && "info" in payload) {
+ let info = ping.payload.info;
+ pathComponents = [
+ ping.slug,
+ info.reason,
+ info.appName,
+ info.appVersion,
+ info.appUpdateChannel,
+ info.appBuildID,
+ ];
+ } else {
+ // Only use the UUID as the slug.
+ pathComponents = [ping.slug];
+ }
+ }
+
+ let slug = pathComponents.join("/");
+ return "/submit/telemetry/" + slug;
+ },
+
+ _doPing(ping, id, isPersisted) {
+ if (!this.sendingEnabled(ping)) {
+ // We can't send the pings to the server, so don't try to.
+ this._log.trace("_doPing - Can't send ping " + ping.id);
+ return Promise.resolve();
+ }
+
+ if (this._tooLateToSend) {
+ // Too late to send now. Reject so we pend the ping to send it next time.
+ this._log.trace("_doPing - Too late to send ping " + ping.id);
+ Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE").add("eTooLate");
+ Telemetry.getKeyedHistogramById(
+ "TELEMETRY_SEND_FAILURE_TYPE_PER_PING"
+ ).add(ping.type, "eTooLate");
+ return Promise.reject();
+ }
+
+ this._log.trace(
+ "_doPing - server: " +
+ this._server +
+ ", persisted: " +
+ isPersisted +
+ ", id: " +
+ id
+ );
+
+ const url = this._buildSubmissionURL(ping);
+
+ // Don't send cookies with these requests.
+ let request = new ServiceRequest({ mozAnon: true });
+ request.mozBackgroundRequest = true;
+ request.timeout = Policy.pingSubmissionTimeout();
+
+ request.open("POST", url, true);
+ request.overrideMimeType("text/plain");
+ request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
+ request.setRequestHeader("Date", Policy.now().toUTCString());
+
+ this._pendingPingRequests.set(id, request);
+
+ const monotonicStartTime = Utils.monotonicNow();
+ let deferred = PromiseUtils.defer();
+
+ let onRequestFinished = (success, event) => {
+ let onCompletion = () => {
+ if (success) {
+ deferred.resolve();
+ } else {
+ deferred.reject(event);
+ }
+ };
+
+ this._pendingPingRequests.delete(id);
+ this._onPingRequestFinished(
+ success,
+ monotonicStartTime,
+ id,
+ isPersisted
+ ).then(
+ () => onCompletion(),
+ error => {
+ this._log.error(
+ "_doPing - request success: " + success + ", error: " + error
+ );
+ onCompletion();
+ }
+ );
+ };
+
+ let errorhandler = event => {
+ let failure = event.type;
+ if (failure === "error") {
+ failure = XHR_ERROR_TYPE[request.errorCode];
+ }
+
+ TelemetryHealthPing.recordSendFailure(failure);
+
+ if (this.fallbackHttp) {
+ // only one attempt
+ this.fallbackHttp = false;
+
+ request.channel.securityInfo
+ .QueryInterface(Ci.nsITransportSecurityInfo)
+ .QueryInterface(Ci.nsISerializable);
+ if (request.channel.securityInfo.errorCodeString.startsWith("SEC_")) {
+ // re-open the request with the HTTP version of the URL
+ let fallbackUrl = new URL(url);
+ fallbackUrl.protocol = "http:";
+ // TODO encrypt payload
+ request.open("POST", fallbackUrl, true);
+ request.sendInputStream(this.payloadStream);
+ }
+ }
+
+ Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE").add(failure);
+ Telemetry.getKeyedHistogramById(
+ "TELEMETRY_SEND_FAILURE_TYPE_PER_PING"
+ ).add(ping.type, failure);
+
+ this._log.error(
+ "_doPing - error making request to " + url + ": " + failure
+ );
+ onRequestFinished(false, event);
+ };
+ request.onerror = errorhandler;
+ request.ontimeout = errorhandler;
+ request.onabort = errorhandler;
+
+ request.onload = event => {
+ let status = request.status;
+ let statusClass = status - (status % 100);
+ let success = false;
+
+ if (statusClass === 200) {
+ // We can treat all 2XX as success.
+ this._log.info("_doPing - successfully loaded, status: " + status);
+ success = true;
+ } else if (statusClass === 400) {
+ // 4XX means that something with the request was broken.
+ this._log.error(
+ "_doPing - error submitting to " +
+ url +
+ ", status: " +
+ status +
+ " - ping request broken?"
+ );
+ Telemetry.getHistogramById(
+ "TELEMETRY_PING_EVICTED_FOR_SERVER_ERRORS"
+ ).add();
+ // TODO: we should handle this better, but for now we should avoid resubmitting
+ // broken requests by pretending success.
+ success = true;
+ } else if (statusClass === 500) {
+ // 5XX means there was a server-side error and we should try again later.
+ this._log.error(
+ "_doPing - error submitting to " +
+ url +
+ ", status: " +
+ status +
+ " - server error, should retry later"
+ );
+ } else {
+ // We received an unexpected status code.
+ this._log.error(
+ "_doPing - error submitting to " +
+ url +
+ ", status: " +
+ status +
+ ", type: " +
+ event.type
+ );
+ }
+
+ onRequestFinished(success, event);
+ };
+
+ // If that's a legacy ping format, just send its payload.
+ let networkPayload = isV4PingFormat(ping) ? ping : ping.payload;
+ request.setRequestHeader("Content-Encoding", "gzip");
+ let converter = Cc[
+ "@mozilla.org/intl/scriptableunicodeconverter"
+ ].createInstance(Ci.nsIScriptableUnicodeConverter);
+ converter.charset = "UTF-8";
+ let startTime = Utils.monotonicNow();
+ let utf8Payload = converter.ConvertFromUnicode(
+ JSON.stringify(networkPayload)
+ );
+ utf8Payload += converter.Finish();
+ Telemetry.getHistogramById("TELEMETRY_STRINGIFY").add(
+ Utils.monotonicNow() - startTime
+ );
+
+ let payloadStream = Cc[
+ "@mozilla.org/io/string-input-stream;1"
+ ].createInstance(Ci.nsIStringInputStream);
+ startTime = Utils.monotonicNow();
+ payloadStream.data = Policy.gzipCompressString(utf8Payload);
+
+ // Check the size and drop pings which are too big.
+ const compressedPingSizeBytes = payloadStream.data.length;
+ if (compressedPingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
+ this._log.error(
+ "_doPing - submitted ping exceeds the size limit, size: " +
+ compressedPingSizeBytes
+ );
+ Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").add();
+ Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB").add(
+ Math.floor(compressedPingSizeBytes / 1024 / 1024)
+ );
+ // We don't need to call |request.abort()| as it was not sent yet.
+ this._pendingPingRequests.delete(id);
+
+ TelemetryHealthPing.recordDiscardedPing(ping.type);
+ return TelemetryStorage.removePendingPing(id);
+ }
+
+ Telemetry.getHistogramById("TELEMETRY_COMPRESS").add(
+ Utils.monotonicNow() - startTime
+ );
+ request.sendInputStream(payloadStream);
+
+ this.payloadStream = payloadStream;
+
+ return deferred.promise;
+ },
+
+ /**
+ * Check if sending is temporarily disabled.
+ * @return {Boolean} True if we can send pings to the server right now, false if
+ * sending is temporarily disabled.
+ */
+ get canSendNow() {
+ // If the reporting policy was not accepted yet, don't send pings.
+ if (!TelemetryReportingPolicy.canUpload()) {
+ return false;
+ }
+
+ return this._sendingEnabled;
+ },
+
+ /**
+ * Check if sending is disabled. If Telemetry is not allowed to upload,
+ * pings are not sent to the server.
+ * If trying to send a "deletion-request" ping, don't block it.
+ * If unified telemetry is off, don't send pings if Telemetry is disabled.
+ *
+ * @param {Object} [ping=null] A ping to be checked.
+ * @return {Boolean} True if pings can be send to the servers, false otherwise.
+ */
+ sendingEnabled(ping = null) {
+ // We only send pings from official builds, but allow overriding this for tests.
+ if (
+ !Telemetry.isOfficialTelemetry &&
+ !this._testMode &&
+ !this._overrideOfficialCheck
+ ) {
+ return false;
+ }
+
+ // With unified Telemetry, the FHR upload setting controls whether we can send pings.
+ // The Telemetry pref enables sending extended data sets instead.
+ if (IS_UNIFIED_TELEMETRY) {
+ // "deletion-request" pings are sent once even if the upload is disabled.
+ if (ping && isDeletionRequestPing(ping)) {
+ return true;
+ }
+ return Services.prefs.getBoolPref(
+ TelemetryUtils.Preferences.FhrUploadEnabled,
+ false
+ );
+ }
+
+ // Without unified Telemetry, the Telemetry enabled pref controls ping sending.
+ return Utils.isTelemetryEnabled;
+ },
+
+ /**
+ * Track any pending ping send and save tasks through the promise passed here.
+ * This is needed to block shutdown on any outstanding ping activity.
+ */
+ _trackPendingPingTask(promise) {
+ let clear = () => this._pendingPingActivity.delete(promise);
+ promise.then(clear, clear);
+ this._pendingPingActivity.add(promise);
+ },
+
+ /**
+ * Return a promise that allows to wait on pending pings.
+ * @return {Object<Promise>} A promise resolved when all the pending pings promises
+ * are resolved.
+ */
+ promisePendingPingActivity() {
+ this._log.trace("promisePendingPingActivity - Waiting for ping task");
+ let p = Array.from(this._pendingPingActivity, p =>
+ p.catch(ex => {
+ this._log.error(
+ "promisePendingPingActivity - ping activity had an error",
+ ex
+ );
+ })
+ );
+ p.push(SendScheduler.waitOnSendTask());
+ return Promise.all(p);
+ },
+
+ async _persistCurrentPings() {
+ for (let [id, ping] of this._currentPings) {
+ try {
+ await savePing(ping);
+ this._log.trace("_persistCurrentPings - saved ping " + id);
+ } catch (ex) {
+ this._log.error("_persistCurrentPings - failed to save ping " + id, ex);
+ } finally {
+ this._currentPings.delete(id);
+ }
+ }
+ },
+
+ /**
+ * Returns the current pending, not yet persisted, pings, newest first.
+ */
+ getUnpersistedPings() {
+ let current = [...this._currentPings.values()];
+ current.reverse();
+ return current;
+ },
+
+ getShutdownState() {
+ return {
+ sendingEnabled: this._sendingEnabled,
+ pendingPingRequestCount: this._pendingPingRequests.size,
+ pendingPingActivityCount: this._pendingPingActivity.size,
+ unpersistedPingCount: this._currentPings.size,
+ persistedPingCount: TelemetryStorage.getPendingPingList().length,
+ schedulerState: SendScheduler.getShutdownState(),
+ };
+ },
+
+ runPingSender(pings, observer) {
+ if (AppConstants.platform === "android") {
+ throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);
+ }
+
+ const exeName =
+ AppConstants.platform === "win" ? "pingsender.exe" : "pingsender";
+
+ let exe = Services.dirsvc.get("GreBinD", Ci.nsIFile);
+ exe.append(exeName);
+
+ let params = pings.flatMap(ping => [ping.url, ping.path]);
+ let process = Cc["@mozilla.org/process/util;1"].createInstance(
+ Ci.nsIProcess
+ );
+ process.init(exe);
+ process.startHidden = true;
+ process.noShell = true;
+ process.runAsync(params, params.length, observer);
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryStorage.jsm b/toolkit/components/telemetry/app/TelemetryStorage.jsm
new file mode 100644
index 0000000000..bc65282fed
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryStorage.jsm
@@ -0,0 +1,2221 @@
+/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryStorage"];
+
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this);
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+ChromeUtils.import("resource://gre/modules/Services.jsm", this);
+ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this);
+ChromeUtils.import("resource://gre/modules/osfile.jsm", this);
+ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
+ChromeUtils.import("resource://gre/modules/Preferences.jsm", this);
+
+const LOGGER_NAME = "Toolkit.Telemetry";
+const LOGGER_PREFIX = "TelemetryStorage::";
+
+const Telemetry = Services.telemetry;
+const Utils = TelemetryUtils;
+
+// Compute the path of the pings archive on the first use.
+const DATAREPORTING_DIR = "datareporting";
+const PINGS_ARCHIVE_DIR = "archived";
+const ABORTED_SESSION_FILE_NAME = "aborted-session-ping";
+const SESSION_STATE_FILE_NAME = "session-state.json";
+
+XPCOMUtils.defineLazyGetter(this, "gDataReportingDir", function() {
+ return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);
+});
+XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {
+ return OS.Path.join(gDataReportingDir, PINGS_ARCHIVE_DIR);
+});
+XPCOMUtils.defineLazyGetter(this, "gAbortedSessionFilePath", function() {
+ return OS.Path.join(gDataReportingDir, ABORTED_SESSION_FILE_NAME);
+});
+ChromeUtils.defineModuleGetter(
+ this,
+ "CommonUtils",
+ "resource://services-common/utils.js"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "TelemetryHealthPing",
+ "resource://gre/modules/HealthPing.jsm"
+);
+// Maxmimum time, in milliseconds, archive pings should be retained.
+const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000; // 60 days
+
+// Maximum space the archive can take on disk (in Bytes).
+const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB
+// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes).
+const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB
+// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).
+const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB
+
+// The maximum size a pending/archived ping can take on disk.
+const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB
+
+// This special value is submitted when the archive is outside of the quota.
+const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;
+
+// This special value is submitted when the pending pings is outside of the quota, as
+// we don't know the size of the pings above the quota.
+const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17;
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/**
+ * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping
+ * from the disk fails.
+ */
+function PingReadError(
+ message = "Error reading the ping file",
+ becauseNoSuchFile = false
+) {
+ Error.call(this, message);
+ let error = new Error();
+ this.name = "PingReadError";
+ this.message = message;
+ this.stack = error.stack;
+ this.becauseNoSuchFile = becauseNoSuchFile;
+}
+PingReadError.prototype = Object.create(Error.prototype);
+PingReadError.prototype.constructor = PingReadError;
+
+/**
+ * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON
+ * content fails.
+ */
+function PingParseError(message = "Error parsing ping content") {
+ Error.call(this, message);
+ let error = new Error();
+ this.name = "PingParseError";
+ this.message = message;
+ this.stack = error.stack;
+}
+PingParseError.prototype = Object.create(Error.prototype);
+PingParseError.prototype.constructor = PingParseError;
+
+/**
+ * This is a policy object used to override behavior for testing.
+ */
+var Policy = {
+ now: () => new Date(),
+ getArchiveQuota: () => ARCHIVE_QUOTA_BYTES,
+ getPendingPingsQuota: () =>
+ AppConstants.platform == "android"
+ ? PENDING_PINGS_QUOTA_BYTES_MOBILE
+ : PENDING_PINGS_QUOTA_BYTES_DESKTOP,
+ /**
+ * @param {string} id The ID of the ping that will be written into the file. Can be "*" to
+ * make a pattern to find all pings for this installation.
+ * @return
+ * {
+ * directory: <nsIFile>, // Directory to save pings
+ * file: <string>, // File name for this ping (or pattern for all pings)
+ * }
+ */
+ getUninstallPingPath: id => {
+ // UpdRootD is e.g. C:\ProgramData\Mozilla\updates\<PATH HASH>
+ const updateDirectory = Services.dirsvc.get("UpdRootD", Ci.nsIFile);
+ const installPathHash = updateDirectory.leafName;
+
+ return {
+ // e.g. C:\ProgramData\Mozilla
+ directory: updateDirectory.parent.parent.clone(),
+ file: `uninstall_ping_${installPathHash}_${id}.json`,
+ };
+ },
+};
+
+/**
+ * Wait for all promises in iterable to resolve or reject. This function
+ * always resolves its promise with undefined, and never rejects.
+ */
+function waitForAll(it) {
+ let dummy = () => {};
+ let promises = Array.from(it, p => p.catch(dummy));
+ return Promise.all(promises);
+}
+
+/**
+ * Permanently intern the given string. This is mainly used for the ping.type
+ * strings that can be excessively duplicated in the _archivedPings map. Do not
+ * pass large or temporary strings to this function.
+ */
+function internString(str) {
+ return Symbol.keyFor(Symbol.for(str));
+}
+
+var TelemetryStorage = {
+ get pingDirectoryPath() {
+ return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
+ },
+
+ /**
+ * The maximum size a ping can have, in bytes.
+ */
+ get MAXIMUM_PING_SIZE() {
+ return PING_FILE_MAXIMUM_SIZE_BYTES;
+ },
+ /**
+ * Shutdown & block on any outstanding async activity in this module.
+ *
+ * @return {Promise} Promise that is resolved when shutdown is complete.
+ */
+ shutdown() {
+ return TelemetryStorageImpl.shutdown();
+ },
+
+ /**
+ * Save an archived ping to disk.
+ *
+ * @param {object} ping The ping data to archive.
+ * @return {promise} Promise that is resolved when the ping is successfully archived.
+ */
+ saveArchivedPing(ping) {
+ return TelemetryStorageImpl.saveArchivedPing(ping);
+ },
+
+ /**
+ * Load an archived ping from disk.
+ *
+ * @param {string} id The pings id.
+ * @return {promise<object>} Promise that is resolved with the ping data.
+ */
+ loadArchivedPing(id) {
+ return TelemetryStorageImpl.loadArchivedPing(id);
+ },
+
+ /**
+ * Get a list of info on the archived pings.
+ * This will scan the archive directory and grab basic data about the existing
+ * pings out of their filename.
+ *
+ * @return {promise<sequence<object>>}
+ */
+ loadArchivedPingList() {
+ return TelemetryStorageImpl.loadArchivedPingList();
+ },
+
+ /**
+ * Clean the pings archive by removing old pings.
+ * This will scan the archive directory.
+ *
+ * @return {Promise} Resolved when the cleanup task completes.
+ */
+ runCleanPingArchiveTask() {
+ return TelemetryStorageImpl.runCleanPingArchiveTask();
+ },
+
+ /**
+ * Run the task to enforce the pending pings quota.
+ *
+ * @return {Promise} Resolved when the cleanup task completes.
+ */
+ runEnforcePendingPingsQuotaTask() {
+ return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask();
+ },
+
+ /**
+ * Run the task to remove all the pending pings
+ *
+ * @return {Promise} Resolved when the pings are removed.
+ */
+ runRemovePendingPingsTask() {
+ return TelemetryStorageImpl.runRemovePendingPingsTask();
+ },
+
+ /**
+ * Remove all pings that are stored in the userApplicationDataDir
+ * under the "Pending Pings" sub-directory.
+ */
+ removeAppDataPings() {
+ return TelemetryStorageImpl.removeAppDataPings();
+ },
+
+ /**
+ * Reset the storage state in tests.
+ */
+ reset() {
+ return TelemetryStorageImpl.reset();
+ },
+
+ /**
+ * Test method that allows waiting on the archive clean task to finish.
+ */
+ testCleanupTaskPromise() {
+ return TelemetryStorageImpl._cleanArchiveTask || Promise.resolve();
+ },
+
+ /**
+ * Test method that allows waiting on the pending pings quota task to finish.
+ */
+ testPendingQuotaTaskPromise() {
+ return (
+ TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve()
+ );
+ },
+
+ /**
+ * Save a pending - outgoing - ping to disk and track it.
+ *
+ * @param {Object} ping The ping data.
+ * @return {Promise} Resolved when the ping was saved.
+ */
+ savePendingPing(ping) {
+ return TelemetryStorageImpl.savePendingPing(ping);
+ },
+
+ /**
+ * Saves session data to disk.
+ * @param {Object} sessionData The session data.
+ * @return {Promise} Resolved when the data was saved.
+ */
+ saveSessionData(sessionData) {
+ return TelemetryStorageImpl.saveSessionData(sessionData);
+ },
+
+ /**
+ * Loads session data from a session data file.
+ * @return {Promise<object>} Resolved with the session data in object form.
+ */
+ loadSessionData() {
+ return TelemetryStorageImpl.loadSessionData();
+ },
+
+ /**
+ * Load a pending ping from disk by id.
+ *
+ * @param {String} id The pings id.
+ * @return {Promise} Resolved with the loaded ping data.
+ */
+ loadPendingPing(id) {
+ return TelemetryStorageImpl.loadPendingPing(id);
+ },
+
+ /**
+ * Remove a pending ping from disk by id.
+ *
+ * @param {String} id The pings id.
+ * @return {Promise} Resolved when the ping was removed.
+ */
+ removePendingPing(id) {
+ return TelemetryStorageImpl.removePendingPing(id);
+ },
+
+ /**
+ * Returns a list of the currently pending pings in the format:
+ * {
+ * id: <string>, // The pings UUID.
+ * lastModificationDate: <number>, // Timestamp of the pings last modification.
+ * }
+ * This populates the list by scanning the disk.
+ *
+ * @return {Promise<sequence>} Resolved with the ping list.
+ */
+ loadPendingPingList() {
+ return TelemetryStorageImpl.loadPendingPingList();
+ },
+
+ /**
+ * Returns a list of the currently pending pings in the format:
+ * {
+ * id: <string>, // The pings UUID.
+ * lastModificationDate: <number>, // Timestamp of the pings last modification.
+ * }
+ * This does not scan pending pings on disk.
+ *
+ * @return {sequence} The current pending ping list.
+ */
+ getPendingPingList() {
+ return TelemetryStorageImpl.getPendingPingList();
+ },
+
+ /**
+ * Save an aborted-session ping to disk. This goes to a special location so
+ * it is not picked up as a pending ping.
+ *
+ * @param {object} ping The ping data to save.
+ * @return {promise} Promise that is resolved when the ping is successfully saved.
+ */
+ saveAbortedSessionPing(ping) {
+ return TelemetryStorageImpl.saveAbortedSessionPing(ping);
+ },
+
+ /**
+ * Load the aborted-session ping from disk if present.
+ *
+ * @return {promise<object>} Promise that is resolved with the ping data if found.
+ * Otherwise returns null.
+ */
+ loadAbortedSessionPing() {
+ return TelemetryStorageImpl.loadAbortedSessionPing();
+ },
+
+ /**
+ * Remove the aborted-session ping if present.
+ *
+ * @return {promise} Promise that is resolved once the ping is removed.
+ */
+ removeAbortedSessionPing() {
+ return TelemetryStorageImpl.removeAbortedSessionPing();
+ },
+
+ /**
+ * Save an uninstall ping to disk, removing any old ones from this
+ * installation first.
+ * This is stored independently from other pings, and only read by
+ * the Windows uninstaller.
+ *
+ * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+ *
+ * @return {promise} Promise that is resolved when the ping has been saved.
+ */
+ saveUninstallPing(ping) {
+ return TelemetryStorageImpl.saveUninstallPing(ping);
+ },
+
+ /**
+ * Remove all uninstall pings from this installation.
+ *
+ * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+ *
+ * @return {promise} Promise that is resolved when the pings have been removed.
+ */
+ removeUninstallPings() {
+ return TelemetryStorageImpl.removeUninstallPings();
+ },
+
+ /**
+ * Save a single ping to a file.
+ *
+ * @param {object} ping The content of the ping to save.
+ * @param {string} file The destination file.
+ * @param {bool} overwrite If |true|, the file will be overwritten if it exists,
+ * if |false| the file will not be overwritten and no error will be reported if
+ * the file exists.
+ * @returns {promise}
+ */
+ savePingToFile(ping, file, overwrite) {
+ return TelemetryStorageImpl.savePingToFile(ping, file, overwrite);
+ },
+
+ /**
+ * Save a ping to its file.
+ *
+ * @param {object} ping The content of the ping to save.
+ * @param {bool} overwrite If |true|, the file will be overwritten
+ * if it exists.
+ * @returns {promise}
+ */
+ savePing(ping, overwrite) {
+ return TelemetryStorageImpl.savePing(ping, overwrite);
+ },
+
+ /**
+ * Add a ping to the saved pings directory so that it gets saved
+ * and sent along with other pings.
+ *
+ * @param {Object} pingData The ping object.
+ * @return {Promise} A promise resolved when the ping is saved to the pings directory.
+ */
+ addPendingPing(pingData) {
+ return TelemetryStorageImpl.addPendingPing(pingData);
+ },
+
+ /**
+ * Remove the file for a ping
+ *
+ * @param {object} ping The ping.
+ * @returns {promise}
+ */
+ cleanupPingFile(ping) {
+ return TelemetryStorageImpl.cleanupPingFile(ping);
+ },
+
+ /**
+ * Loads a ping file.
+ * @param {String} aFilePath The path of the ping file.
+ * @return {Promise<Object>} A promise resolved with the ping content or rejected if the
+ * ping contains invalid data.
+ */
+ async loadPingFile(aFilePath) {
+ return TelemetryStorageImpl.loadPingFile(aFilePath);
+ },
+
+ /**
+ * Remove FHR database files. This is temporary and will be dropped in
+ * the future.
+ * @return {Promise} Resolved when the database files are deleted.
+ */
+ removeFHRDatabase() {
+ return TelemetryStorageImpl.removeFHRDatabase();
+ },
+
+ /**
+ * Only used in tests, builds an archived ping path from the ping metadata.
+ * @param {String} aPingId The ping id.
+ * @param {Object} aDate The ping creation date.
+ * @param {String} aType The ping type.
+ * @return {String} The full path to the archived ping.
+ */
+ _testGetArchivedPingPath(aPingId, aDate, aType) {
+ return getArchivedPingPath(aPingId, aDate, aType);
+ },
+
+ /**
+ * Only used in tests, this helper extracts ping metadata from a given filename.
+ *
+ * @param fileName {String} The filename.
+ * @return {Object} Null if the filename didn't match the expected form.
+ * Otherwise an object with the extracted data in the form:
+ * { timestamp: <number>,
+ * id: <string>,
+ * type: <string> }
+ */
+ _testGetArchivedPingDataFromFileName(aFileName) {
+ return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName);
+ },
+
+ /**
+ * Only used in tests, this helper allows cleaning up the pending ping storage.
+ */
+ testClearPendingPings() {
+ return TelemetryStorageImpl.runRemovePendingPingsTask();
+ },
+};
+
+/**
+ * This object allows the serialisation of asynchronous tasks. This is particularly
+ * useful to serialise write access to the disk in order to prevent race conditions
+ * to corrupt the data being written.
+ * We are using this to synchronize saving to the file that TelemetrySession persists
+ * its state in.
+ */
+function SaveSerializer() {
+ this._queuedOperations = [];
+ this._queuedInProgress = false;
+ this._log = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+}
+
+SaveSerializer.prototype = {
+ /**
+ * Enqueues an operation to a list to serialise their execution in order to prevent race
+ * conditions. Useful to serialise access to disk.
+ *
+ * @param {Function} aFunction The task function to enqueue. It must return a promise.
+ * @return {Promise} A promise resolved when the enqueued task completes.
+ */
+ enqueueTask(aFunction) {
+ let promise = new Promise((resolve, reject) =>
+ this._queuedOperations.push([aFunction, resolve, reject])
+ );
+
+ if (this._queuedOperations.length == 1) {
+ this._popAndPerformQueuedOperation();
+ }
+ return promise;
+ },
+
+ /**
+ * Make sure to flush all the pending operations.
+ * @return {Promise} A promise resolved when all the pending operations have completed.
+ */
+ flushTasks() {
+ let dummyTask = () => new Promise(resolve => resolve());
+ return this.enqueueTask(dummyTask);
+ },
+
+ /**
+ * Pop a task from the queue, executes it and continue to the next one.
+ * This function recursively pops all the tasks.
+ */
+ _popAndPerformQueuedOperation() {
+ if (!this._queuedOperations.length || this._queuedInProgress) {
+ return;
+ }
+
+ this._log.trace(
+ "_popAndPerformQueuedOperation - Performing queued operation."
+ );
+ let [func, resolve, reject] = this._queuedOperations.shift();
+ let promise;
+
+ try {
+ this._queuedInProgress = true;
+ promise = func();
+ } catch (ex) {
+ this._log.warn(
+ "_popAndPerformQueuedOperation - Queued operation threw during execution. ",
+ ex
+ );
+ this._queuedInProgress = false;
+ reject(ex);
+ this._popAndPerformQueuedOperation();
+ return;
+ }
+
+ if (!promise || typeof promise.then != "function") {
+ let msg = "Queued operation did not return a promise: " + func;
+ this._log.warn("_popAndPerformQueuedOperation - " + msg);
+
+ this._queuedInProgress = false;
+ reject(new Error(msg));
+ this._popAndPerformQueuedOperation();
+ return;
+ }
+
+ promise.then(
+ result => {
+ this._queuedInProgress = false;
+ resolve(result);
+ this._popAndPerformQueuedOperation();
+ },
+ error => {
+ this._log.warn(
+ "_popAndPerformQueuedOperation - Failure when performing queued operation.",
+ error
+ );
+ this._queuedInProgress = false;
+ reject(error);
+ this._popAndPerformQueuedOperation();
+ }
+ );
+ },
+};
+
+var TelemetryStorageImpl = {
+ _logger: null,
+ // Used to serialize aborted session ping writes to disk.
+ _abortedSessionSerializer: new SaveSerializer(),
+ // Used to serialize session state writes to disk.
+ _stateSaveSerializer: new SaveSerializer(),
+
+ // Tracks the archived pings in a Map of (id -> {timestampCreated, type}).
+ // We use this to cache info on archived pings to avoid scanning the disk more than once.
+ _archivedPings: new Map(),
+ // A set of promises for pings currently being archived
+ _activelyArchiving: new Set(),
+ // Track the archive loading task to prevent multiple tasks from being executed.
+ _scanArchiveTask: null,
+ // Track the archive cleanup task.
+ _cleanArchiveTask: null,
+ // Whether we already scanned the archived pings on disk.
+ _scannedArchiveDirectory: false,
+
+ // Track the pending ping removal task.
+ _removePendingPingsTask: null,
+
+ // This tracks all the pending async ping save activity.
+ _activePendingPingSaves: new Set(),
+
+ // Tracks the pending pings in a Map of (id -> {timestampCreated, type}).
+ // We use this to cache info on pending pings to avoid scanning the disk more than once.
+ _pendingPings: new Map(),
+
+ // Track the pending pings enforce quota task.
+ _enforcePendingPingsQuotaTask: null,
+
+ // Track the shutdown process to bail out of the clean up task quickly.
+ _shutdown: false,
+
+ get _log() {
+ if (!this._logger) {
+ this._logger = Log.repository.getLoggerWithMessagePrefix(
+ LOGGER_NAME,
+ LOGGER_PREFIX
+ );
+ }
+
+ return this._logger;
+ },
+
+ /**
+ * Shutdown & block on any outstanding async activity in this module.
+ *
+ * @return {Promise} Promise that is resolved when shutdown is complete.
+ */
+ async shutdown() {
+ this._shutdown = true;
+
+ // If the following tasks are still running, block on them. They will bail out as soon
+ // as possible.
+ await this._abortedSessionSerializer.flushTasks().catch(ex => {
+ this._log.error("shutdown - failed to flush aborted-session writes", ex);
+ });
+
+ if (this._cleanArchiveTask) {
+ await this._cleanArchiveTask.catch(ex => {
+ this._log.error("shutdown - the archive cleaning task failed", ex);
+ });
+ }
+
+ if (this._enforcePendingPingsQuotaTask) {
+ await this._enforcePendingPingsQuotaTask.catch(ex => {
+ this._log.error("shutdown - the pending pings quota task failed", ex);
+ });
+ }
+
+ if (this._removePendingPingsTask) {
+ await this._removePendingPingsTask.catch(ex => {
+ this._log.error("shutdown - the pending pings removal task failed", ex);
+ });
+ }
+
+ // Wait on pending pings still being saved. While OS.File should have shutdown
+ // blockers in place, we a) have seen weird errors being reported that might
+ // indicate a bad shutdown path and b) might have completion handlers hanging
+ // off the save operations that don't expect to be late in shutdown.
+ await this.promisePendingPingSaves();
+ },
+
+ /**
+ * Save an archived ping to disk.
+ *
+ * @param {object} ping The ping data to archive.
+ * @return {promise} Promise that is resolved when the ping is successfully archived.
+ */
+ saveArchivedPing(ping) {
+ let promise = this._saveArchivedPingTask(ping);
+ this._activelyArchiving.add(promise);
+ promise.then(
+ r => {
+ this._activelyArchiving.delete(promise);
+ },
+ e => {
+ this._activelyArchiving.delete(promise);
+ }
+ );
+ return promise;
+ },
+
+ async _saveArchivedPingTask(ping) {
+ const creationDate = new Date(ping.creationDate);
+ if (this._archivedPings.has(ping.id)) {
+ const data = this._archivedPings.get(ping.id);
+ if (data.timestampCreated > creationDate.getTime()) {
+ this._log.error(
+ "saveArchivedPing - trying to overwrite newer ping with the same id"
+ );
+ return Promise.reject(
+ new Error("trying to overwrite newer ping with the same id")
+ );
+ }
+ this._log.warn(
+ "saveArchivedPing - overwriting older ping with the same id"
+ );
+ }
+
+ // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4').
+ const filePath =
+ getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4";
+ await OS.File.makeDir(OS.Path.dirname(filePath), {
+ ignoreExisting: true,
+ from: OS.Constants.Path.profileDir,
+ });
+ await this.savePingToFile(
+ ping,
+ filePath,
+ /* overwrite*/ true,
+ /* compressed*/ true
+ );
+
+ this._archivedPings.set(ping.id, {
+ timestampCreated: creationDate.getTime(),
+ type: internString(ping.type),
+ });
+
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add();
+ return undefined;
+ },
+
+ /**
+ * Load an archived ping from disk.
+ *
+ * @param {string} id The pings id.
+ * @return {promise<object>} Promise that is resolved with the ping data.
+ */
+ async loadArchivedPing(id) {
+ const data = this._archivedPings.get(id);
+ if (!data) {
+ this._log.trace("loadArchivedPing - no ping with id: " + id);
+ return Promise.reject(
+ new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id)
+ );
+ }
+
+ const path = getArchivedPingPath(
+ id,
+ new Date(data.timestampCreated),
+ data.type
+ );
+ const pathCompressed = path + "lz4";
+
+ // Purge pings which are too big.
+ let checkSize = async function(path) {
+ const fileSize = (await OS.File.stat(path)).size;
+ if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
+ Telemetry.getHistogramById(
+ "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB"
+ ).add(Math.floor(fileSize / 1024 / 1024));
+ Telemetry.getHistogramById(
+ "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED"
+ ).add();
+ await OS.File.remove(path, { ignoreAbsent: true });
+ throw new Error(
+ "loadArchivedPing - exceeded the maximum ping size: " + fileSize
+ );
+ }
+ };
+
+ let ping;
+ try {
+ // Try to load a compressed version of the archived ping first.
+ this._log.trace(
+ "loadArchivedPing - loading ping from: " + pathCompressed
+ );
+ await checkSize(pathCompressed);
+ ping = await this.loadPingFile(pathCompressed, /* compressed*/ true);
+ } catch (ex) {
+ if (!ex.becauseNoSuchFile) {
+ throw ex;
+ }
+ // If that fails, look for the uncompressed version.
+ this._log.trace(
+ "loadArchivedPing - compressed ping not found, loading: " + path
+ );
+ await checkSize(path);
+ ping = await this.loadPingFile(path, /* compressed*/ false);
+ }
+
+ return ping;
+ },
+
+ /**
+ * Saves session data to disk.
+ */
+ saveSessionData(sessionData) {
+ return this._stateSaveSerializer.enqueueTask(() =>
+ this._saveSessionData(sessionData)
+ );
+ },
+
+ async _saveSessionData(sessionData) {
+ let dataDir = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);
+ await OS.File.makeDir(dataDir);
+
+ let filePath = OS.Path.join(gDataReportingDir, SESSION_STATE_FILE_NAME);
+ try {
+ await CommonUtils.writeJSON(sessionData, filePath);
+ } catch (e) {
+ this._log.error(
+ "_saveSessionData - Failed to write session data to " + filePath,
+ e
+ );
+ Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1);
+ }
+ },
+
+ /**
+ * Loads session data from the session data file.
+ * @return {Promise<Object>} A promise resolved with an object on success,
+ * with null otherwise.
+ */
+ loadSessionData() {
+ return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData());
+ },
+
+ async _loadSessionData() {
+ const dataFile = OS.Path.join(
+ OS.Constants.Path.profileDir,
+ DATAREPORTING_DIR,
+ SESSION_STATE_FILE_NAME
+ );
+ let content;
+ try {
+ content = await OS.File.read(dataFile, { encoding: "utf-8" });
+ } catch (ex) {
+ this._log.info("_loadSessionData - can not load session data file", ex);
+ Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1);
+ return null;
+ }
+
+ let data;
+ try {
+ data = JSON.parse(content);
+ } catch (ex) {
+ this._log.error("_loadSessionData - failed to parse session data", ex);
+ Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1);
+ return null;
+ }
+
+ return data;
+ },
+
+ /**
+ * Remove an archived ping from disk.
+ *
+ * @param {string} id The pings id.
+ * @param {number} timestampCreated The pings creation timestamp.
+ * @param {string} type The pings type.
+ * @return {promise<object>} Promise that is resolved when the pings is removed.
+ */
+ async _removeArchivedPing(id, timestampCreated, type) {
+ this._log.trace(
+ "_removeArchivedPing - id: " +
+ id +
+ ", timestampCreated: " +
+ timestampCreated +
+ ", type: " +
+ type
+ );
+ const path = getArchivedPingPath(id, new Date(timestampCreated), type);
+ const pathCompressed = path + "lz4";
+
+ this._log.trace("_removeArchivedPing - removing ping from: " + path);
+ await OS.File.remove(path, { ignoreAbsent: true });
+ await OS.File.remove(pathCompressed, { ignoreAbsent: true });
+ // Remove the ping from the cache.
+ this._archivedPings.delete(id);
+ },
+
+ /**
+ * Clean the pings archive by removing old pings.
+ *
+ * @return {Promise} Resolved when the cleanup task completes.
+ */
+ runCleanPingArchiveTask() {
+ // If there's an archive cleaning task already running, return it.
+ if (this._cleanArchiveTask) {
+ return this._cleanArchiveTask;
+ }
+
+ // Make sure to clear |_cleanArchiveTask| once done.
+ let clear = () => (this._cleanArchiveTask = null);
+ // Since there's no archive cleaning task running, start it.
+ this._cleanArchiveTask = this._cleanArchive().then(clear, clear);
+ return this._cleanArchiveTask;
+ },
+
+ /**
+ * Removes pings which are too old from the pings archive.
+ * @return {Promise} Resolved when the ping age check is complete.
+ */
+ async _purgeOldPings() {
+ this._log.trace("_purgeOldPings");
+
+ const nowDate = Policy.now();
+ const startTimeStamp = nowDate.getTime();
+ let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
+ let subdirs = (await dirIterator.nextBatch()).filter(e => e.isDir);
+ dirIterator.close();
+
+ // Keep track of the newest removed month to update the cache, if needed.
+ let newestRemovedMonthTimestamp = null;
+ let evictedDirsCount = 0;
+ let maxDirAgeInMonths = 0;
+
+ // Walk through the monthly subdirs of the form <YYYY-MM>/
+ for (let dir of subdirs) {
+ if (this._shutdown) {
+ this._log.trace(
+ "_purgeOldPings - Terminating the clean up task due to shutdown"
+ );
+ return;
+ }
+
+ if (!isValidArchiveDir(dir.name)) {
+ this._log.warn(
+ "_purgeOldPings - skipping invalidly named subdirectory " + dir.path
+ );
+ continue;
+ }
+
+ const archiveDate = getDateFromArchiveDir(dir.name);
+ if (!archiveDate) {
+ this._log.warn(
+ "_purgeOldPings - skipping invalid subdirectory date " + dir.path
+ );
+ continue;
+ }
+
+ // If this archive directory is older than allowed, remove it.
+ if (
+ startTimeStamp - archiveDate.getTime() >
+ MAX_ARCHIVED_PINGS_RETENTION_MS
+ ) {
+ try {
+ await OS.File.removeDir(dir.path);
+ evictedDirsCount++;
+
+ // Update the newest removed month.
+ newestRemovedMonthTimestamp = Math.max(
+ archiveDate,
+ newestRemovedMonthTimestamp
+ );
+ } catch (ex) {
+ this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex);
+ }
+ } else {
+ // We're not removing this directory, so record the age for the oldest directory.
+ const dirAgeInMonths = Utils.getElapsedTimeInMonths(
+ archiveDate,
+ nowDate
+ );
+ maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths);
+ }
+ }
+
+ // Trigger scanning of the archived pings.
+ await this.loadArchivedPingList();
+
+ // Refresh the cache: we could still skip this, but it's cheap enough to keep it
+ // to avoid introducing task dependencies.
+ if (newestRemovedMonthTimestamp) {
+ // Scan the archive cache for pings older than the newest directory pruned above.
+ for (let [id, info] of this._archivedPings) {
+ const timestampCreated = new Date(info.timestampCreated);
+ if (timestampCreated.getTime() > newestRemovedMonthTimestamp) {
+ continue;
+ }
+ // Remove outdated pings from the cache.
+ this._archivedPings.delete(id);
+ }
+ }
+
+ const endTimeStamp = Policy.now().getTime();
+
+ // Save the time it takes to evict old directories and the eviction count.
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").add(
+ evictedDirsCount
+ );
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS").add(
+ Math.ceil(endTimeStamp - startTimeStamp)
+ );
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE").add(
+ maxDirAgeInMonths
+ );
+ },
+
+ /**
+ * Enforce a disk quota for the pings archive.
+ * @return {Promise} Resolved when the quota check is complete.
+ */
+ async _enforceArchiveQuota() {
+ this._log.trace("_enforceArchiveQuota");
+ let startTimeStamp = Policy.now().getTime();
+
+ // Build an ordered list, from newer to older, of archived pings.
+ let pingList = Array.from(this._archivedPings, p => ({
+ id: p[0],
+ timestampCreated: p[1].timestampCreated,
+ type: p[1].type,
+ }));
+
+ pingList.sort((a, b) => b.timestampCreated - a.timestampCreated);
+
+ // If our archive is too big, we should reduce it to reach 90% of the quota.
+ const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9;
+ // The index of the last ping to keep. Pings older than this one will be deleted if
+ // the archive exceeds the quota.
+ let lastPingIndexToKeep = null;
+ let archiveSizeInBytes = 0;
+
+ // Find the disk size of the archive.
+ for (let i = 0; i < pingList.length; i++) {
+ if (this._shutdown) {
+ this._log.trace(
+ "_enforceArchiveQuota - Terminating the clean up task due to shutdown"
+ );
+ return;
+ }
+
+ let ping = pingList[i];
+
+ // Get the size for this ping.
+ const fileSize = await getArchivedPingSize(
+ ping.id,
+ new Date(ping.timestampCreated),
+ ping.type
+ );
+ if (!fileSize) {
+ this._log.warn(
+ "_enforceArchiveQuota - Unable to find the size of ping " + ping.id
+ );
+ continue;
+ }
+
+ // Enforce a maximum file size limit on archived pings.
+ if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
+ this._log.error(
+ "_enforceArchiveQuota - removing file exceeding size limit, size: " +
+ fileSize
+ );
+ // We just remove the ping from the disk, we don't bother removing it from pingList
+ // since it won't contribute to the quota.
+ await this._removeArchivedPing(
+ ping.id,
+ ping.timestampCreated,
+ ping.type
+ ).catch(e =>
+ this._log.error(
+ "_enforceArchiveQuota - failed to remove archived ping" + ping.id
+ )
+ );
+ Telemetry.getHistogramById(
+ "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB"
+ ).add(Math.floor(fileSize / 1024 / 1024));
+ Telemetry.getHistogramById(
+ "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED"
+ ).add();
+ continue;
+ }
+
+ archiveSizeInBytes += fileSize;
+
+ if (archiveSizeInBytes < SAFE_QUOTA) {
+ // We save the index of the last ping which is ok to keep in order to speed up ping
+ // pruning.
+ lastPingIndexToKeep = i;
+ } else if (archiveSizeInBytes > Policy.getArchiveQuota()) {
+ // Ouch, our ping archive is too big. Bail out and start pruning!
+ break;
+ }
+ }
+
+ // Save the time it takes to check if the archive is over-quota.
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS").add(
+ Math.round(Policy.now().getTime() - startTimeStamp)
+ );
+
+ let submitProbes = (sizeInMB, evictedPings, elapsedMs) => {
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB);
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(
+ evictedPings
+ );
+ Telemetry.getHistogramById(
+ "TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS"
+ ).add(elapsedMs);
+ };
+
+ // Check if we're using too much space. If not, submit the archive size and bail out.
+ if (archiveSizeInBytes < Policy.getArchiveQuota()) {
+ submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0);
+ return;
+ }
+
+ this._log.info(
+ "_enforceArchiveQuota - archive size: " +
+ archiveSizeInBytes +
+ "bytes" +
+ ", safety quota: " +
+ SAFE_QUOTA +
+ "bytes"
+ );
+
+ startTimeStamp = Policy.now().getTime();
+ let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);
+
+ // Remove all the pings older than the last one which we are safe to keep.
+ for (let ping of pingsToPurge) {
+ if (this._shutdown) {
+ this._log.trace(
+ "_enforceArchiveQuota - Terminating the clean up task due to shutdown"
+ );
+ return;
+ }
+
+ // This list is guaranteed to be in order, so remove the pings at its
+ // beginning (oldest).
+ await this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);
+ }
+
+ const endTimeStamp = Policy.now().getTime();
+ submitProbes(
+ ARCHIVE_SIZE_PROBE_SPECIAL_VALUE,
+ pingsToPurge.length,
+ Math.ceil(endTimeStamp - startTimeStamp)
+ );
+ },
+
+ async _cleanArchive() {
+ this._log.trace("cleanArchiveTask");
+
+ if (!(await OS.File.exists(gPingsArchivePath))) {
+ return;
+ }
+
+ // Remove pings older than allowed.
+ try {
+ await this._purgeOldPings();
+ } catch (ex) {
+ this._log.error(
+ "_cleanArchive - There was an error removing old directories",
+ ex
+ );
+ }
+
+ // Make sure we respect the archive disk quota.
+ await this._enforceArchiveQuota();
+ },
+
+ /**
+ * Run the task to enforce the pending pings quota.
+ *
+ * @return {Promise} Resolved when the cleanup task completes.
+ */
+ async runEnforcePendingPingsQuotaTask() {
+ // If there's a cleaning task already running, return it.
+ if (this._enforcePendingPingsQuotaTask) {
+ return this._enforcePendingPingsQuotaTask;
+ }
+
+ // Since there's no quota enforcing task running, start it.
+ try {
+ this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota();
+ await this._enforcePendingPingsQuotaTask;
+ } finally {
+ this._enforcePendingPingsQuotaTask = null;
+ }
+ return undefined;
+ },
+
+ /**
+ * Enforce a disk quota for the pending pings.
+ * @return {Promise} Resolved when the quota check is complete.
+ */
+ async _enforcePendingPingsQuota() {
+ this._log.trace("_enforcePendingPingsQuota");
+ let startTimeStamp = Policy.now().getTime();
+
+ // Build an ordered list, from newer to older, of pending pings.
+ let pingList = Array.from(this._pendingPings, p => ({
+ id: p[0],
+ lastModificationDate: p[1].lastModificationDate,
+ }));
+
+ pingList.sort((a, b) => b.lastModificationDate - a.lastModificationDate);
+
+ // If our pending pings directory is too big, we should reduce it to reach 90% of the quota.
+ const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9;
+ // The index of the last ping to keep. Pings older than this one will be deleted if
+ // the pending pings directory size exceeds the quota.
+ let lastPingIndexToKeep = null;
+ let pendingPingsSizeInBytes = 0;
+
+ // Find the disk size of the pending pings directory.
+ for (let i = 0; i < pingList.length; i++) {
+ if (this._shutdown) {
+ this._log.trace(
+ "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"
+ );
+ return;
+ }
+
+ let ping = pingList[i];
+
+ // Get the size for this ping.
+ const fileSize = await getPendingPingSize(ping.id);
+ if (!fileSize) {
+ this._log.warn(
+ "_enforcePendingPingsQuota - Unable to find the size of ping " +
+ ping.id
+ );
+ continue;
+ }
+
+ pendingPingsSizeInBytes += fileSize;
+ if (pendingPingsSizeInBytes < SAFE_QUOTA) {
+ // We save the index of the last ping which is ok to keep in order to speed up ping
+ // pruning.
+ lastPingIndexToKeep = i;
+ } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) {
+ // Ouch, our pending pings directory size is too big. Bail out and start pruning!
+ break;
+ }
+ }
+
+ // Save the time it takes to check if the pending pings are over-quota.
+ Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS").add(
+ Math.round(Policy.now().getTime() - startTimeStamp)
+ );
+
+ let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => {
+ Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add(
+ sizeInMB
+ );
+ Telemetry.getHistogramById(
+ "TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA"
+ ).add(evictedPings);
+ Telemetry.getHistogramById(
+ "TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS"
+ ).add(elapsedMs);
+ };
+
+ // Check if we're using too much space. If not, bail out.
+ if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) {
+ recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0);
+ return;
+ }
+
+ this._log.info(
+ "_enforcePendingPingsQuota - size: " +
+ pendingPingsSizeInBytes +
+ "bytes" +
+ ", safety quota: " +
+ SAFE_QUOTA +
+ "bytes"
+ );
+
+ startTimeStamp = Policy.now().getTime();
+ let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);
+
+ // Remove all the pings older than the last one which we are safe to keep.
+ for (let ping of pingsToPurge) {
+ if (this._shutdown) {
+ this._log.trace(
+ "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"
+ );
+ return;
+ }
+
+ // This list is guaranteed to be in order, so remove the pings at its
+ // beginning (oldest).
+ await this.removePendingPing(ping.id);
+ }
+
+ const endTimeStamp = Policy.now().getTime();
+ // We don't know the size of the pending pings directory if we are above the quota,
+ // since we stop scanning once we reach the quota. We use a special value to show
+ // this condition.
+ recordHistograms(
+ PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE,
+ pingsToPurge.length,
+ Math.ceil(endTimeStamp - startTimeStamp)
+ );
+ },
+
+ /**
+ * Reset the storage state in tests.
+ */
+ reset() {
+ this._shutdown = false;
+ this._scannedArchiveDirectory = false;
+ this._archivedPings = new Map();
+ this._scannedPendingDirectory = false;
+ this._pendingPings = new Map();
+ },
+
+ /**
+ * Get a list of info on the archived pings.
+ * This will scan the archive directory and grab basic data about the existing
+ * pings out of their filename.
+ *
+ * @return {promise<sequence<object>>}
+ */
+ async loadArchivedPingList() {
+ // If there's an archive loading task already running, return it.
+ if (this._scanArchiveTask) {
+ return this._scanArchiveTask;
+ }
+
+ await waitForAll(this._activelyArchiving);
+
+ if (this._scannedArchiveDirectory) {
+ this._log.trace(
+ "loadArchivedPingList - Archive already scanned, hitting cache."
+ );
+ return this._archivedPings;
+ }
+
+ // Since there's no archive loading task running, start it.
+ let result;
+ try {
+ this._scanArchiveTask = this._scanArchive();
+ result = await this._scanArchiveTask;
+ } finally {
+ this._scanArchiveTask = null;
+ }
+ return result;
+ },
+
+ async _scanArchive() {
+ this._log.trace("_scanArchive");
+
+ let submitProbes = (pingCount, dirCount) => {
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").add(
+ pingCount
+ );
+ Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT").add(
+ dirCount
+ );
+ };
+
+ if (!(await OS.File.exists(gPingsArchivePath))) {
+ submitProbes(0, 0);
+ return new Map();
+ }
+
+ let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
+ let subdirs = (await dirIterator.nextBatch())
+ .filter(e => e.isDir)
+ .filter(e => isValidArchiveDir(e.name));
+ dirIterator.close();
+
+ // Walk through the monthly subdirs of the form <YYYY-MM>/
+ for (let dir of subdirs) {
+ this._log.trace("_scanArchive - checking in subdir: " + dir.path);
+ let pingIterator = new OS.File.DirectoryIterator(dir.path);
+ let pings = (await pingIterator.nextBatch()).filter(e => !e.isDir);
+ pingIterator.close();
+
+ // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]".
+ for (let p of pings) {
+ // data may be null if the filename doesn't match the above format.
+ let data = this._getArchivedPingDataFromFileName(p.name);
+ if (!data) {
+ continue;
+ }
+
+ // In case of conflicts, overwrite only with newer pings.
+ if (this._archivedPings.has(data.id)) {
+ const overwrite =
+ data.timestamp > this._archivedPings.get(data.id).timestampCreated;
+ this._log.warn(
+ "_scanArchive - have seen this id before: " +
+ data.id +
+ ", overwrite: " +
+ overwrite
+ );
+ if (!overwrite) {
+ continue;
+ }
+
+ await this._removeArchivedPing(
+ data.id,
+ data.timestampCreated,
+ data.type
+ ).catch(e =>
+ this._log.warn("_scanArchive - failed to remove ping", e)
+ );
+ }
+
+ this._archivedPings.set(data.id, {
+ timestampCreated: data.timestamp,
+ type: internString(data.type),
+ });
+ }
+ }
+
+ // Mark the archive as scanned, so we no longer hit the disk.
+ this._scannedArchiveDirectory = true;
+ // Update the ping and directories count histograms.
+ submitProbes(this._archivedPings.size, subdirs.length);
+ return this._archivedPings;
+ },
+
+ /**
+ * Save a single ping to a file.
+ *
+ * @param {object} ping The content of the ping to save.
+ * @param {string} file The destination file.
+ * @param {bool} overwrite If |true|, the file will be overwritten if it exists,
+ * if |false| the file will not be overwritten and no error will be reported if
+ * the file exists.
+ * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no
+ * compression will be used.
+ * @returns {promise}
+ */
+ async savePingToFile(ping, filePath, overwrite, compress = false) {
+ try {
+ this._log.trace("savePingToFile - path: " + filePath);
+ let pingString = JSON.stringify(ping);
+ let options = { tmpPath: filePath + ".tmp", noOverwrite: !overwrite };
+ if (compress) {
+ options.compression = "lz4";
+ }
+ await OS.File.writeAtomic(filePath, pingString, options);
+ } catch (e) {
+ if (!e.becauseExists) {
+ throw e;
+ }
+ }
+ },
+
+ /**
+ * Save a ping to its file.
+ *
+ * @param {object} ping The content of the ping to save.
+ * @param {bool} overwrite If |true|, the file will be overwritten
+ * if it exists.
+ * @returns {promise}
+ */
+ async savePing(ping, overwrite) {
+ await getPingDirectory();
+ let file = pingFilePath(ping);
+ await this.savePingToFile(ping, file, overwrite);
+ return file;
+ },
+
+ /**
+ * Add a ping to the saved pings directory so that it gets saved
+ * and sent along with other pings.
+ * Note: that the original ping file will not be modified.
+ *
+ * @param {Object} ping The ping object.
+ * @return {Promise} A promise resolved when the ping is saved to the pings directory.
+ */
+ addPendingPing(ping) {
+ return this.savePendingPing(ping);
+ },
+
+ /**
+ * Remove the file for a ping
+ *
+ * @param {object} ping The ping.
+ * @returns {promise}
+ */
+ cleanupPingFile(ping) {
+ return OS.File.remove(pingFilePath(ping));
+ },
+
+ savePendingPing(ping) {
+ let p = this.savePing(ping, true).then(path => {
+ this._pendingPings.set(ping.id, {
+ path,
+ lastModificationDate: Policy.now().getTime(),
+ });
+ this._log.trace("savePendingPing - saved ping with id " + ping.id);
+ });
+ this._trackPendingPingSaveTask(p);
+ return p;
+ },
+
+ async loadPendingPing(id) {
+ this._log.trace("loadPendingPing - id: " + id);
+ let info = this._pendingPings.get(id);
+ if (!info) {
+ this._log.trace("loadPendingPing - unknown id " + id);
+ throw new Error(
+ "TelemetryStorage.loadPendingPing - no ping with id " + id
+ );
+ }
+
+ // Try to get the dimension of the ping. If that fails, update the histograms.
+ let fileSize = 0;
+ try {
+ fileSize = (await OS.File.stat(info.path)).size;
+ } catch (e) {
+ if (!(e instanceof OS.File.Error) || !e.becauseNoSuchFile) {
+ throw e;
+ }
+ // Fall through and let |loadPingFile| report the error.
+ }
+
+ // Purge pings which are too big.
+ if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
+ await this.removePendingPing(id);
+ Telemetry.getHistogramById(
+ "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB"
+ ).add(Math.floor(fileSize / 1024 / 1024));
+ Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();
+
+ // Currently we don't have the ping type available without loading the ping from disk.
+ // Bug 1384903 will fix that.
+ TelemetryHealthPing.recordDiscardedPing("<unknown>");
+ throw new Error(
+ "loadPendingPing - exceeded the maximum ping size: " + fileSize
+ );
+ }
+
+ // Try to load the ping file. Update the related histograms on failure.
+ let ping;
+ try {
+ ping = await this.loadPingFile(info.path, false);
+ } catch (e) {
+ // If we failed to load the ping, check what happened and update the histogram.
+ if (e instanceof PingReadError) {
+ Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add();
+ } else if (e instanceof PingParseError) {
+ Telemetry.getHistogramById(
+ "TELEMETRY_PENDING_LOAD_FAILURE_PARSE"
+ ).add();
+ }
+
+ // Remove the ping from the cache, so we don't try to load it again.
+ this._pendingPings.delete(id);
+ // Then propagate the rejection.
+ throw e;
+ }
+
+ return ping;
+ },
+
+ removePendingPing(id) {
+ let info = this._pendingPings.get(id);
+ if (!info) {
+ this._log.trace("removePendingPing - unknown id " + id);
+ return Promise.resolve();
+ }
+
+ this._log.trace(
+ "removePendingPing - deleting ping with id: " +
+ id +
+ ", path: " +
+ info.path
+ );
+ this._pendingPings.delete(id);
+ return OS.File.remove(info.path).catch(ex =>
+ this._log.error("removePendingPing - failed to remove ping", ex)
+ );
+ },
+
+ /**
+ * Track any pending ping save tasks through the promise passed here.
+ * This is needed to block on any outstanding ping save activity.
+ *
+ * @param {Object<Promise>} The save promise to track.
+ */
+ _trackPendingPingSaveTask(promise) {
+ let clear = () => this._activePendingPingSaves.delete(promise);
+ promise.then(clear, clear);
+ this._activePendingPingSaves.add(promise);
+ },
+
+ /**
+ * Return a promise that allows to wait on pending pings being saved.
+ * @return {Object<Promise>} A promise resolved when all the pending pings save promises
+ * are resolved.
+ */
+ promisePendingPingSaves() {
+ // Make sure to wait for all the promises, even if they reject. We don't need to log
+ // the failures here, as they are already logged elsewhere.
+ return waitForAll(this._activePendingPingSaves);
+ },
+
+ /**
+ * Run the task to remove all the pending pings
+ *
+ * @return {Promise} Resolved when the pings are removed.
+ */
+ async runRemovePendingPingsTask() {
+ // If we already have a pending pings removal task active, return that.
+ if (this._removePendingPingsTask) {
+ return this._removePendingPingsTask;
+ }
+
+ // Start the task to remove all pending pings. Also make sure to clear the task once done.
+ try {
+ this._removePendingPingsTask = this.removePendingPings();
+ await this._removePendingPingsTask;
+ } finally {
+ this._removePendingPingsTask = null;
+ }
+ return undefined;
+ },
+
+ async removePendingPings() {
+ this._log.trace("removePendingPings - removing all pending pings");
+
+ // Wait on pending pings still being saved, so so we don't miss removing them.
+ await this.promisePendingPingSaves();
+
+ // Individually remove existing pings, so we don't interfere with operations expecting
+ // the pending pings directory to exist.
+ const directory = TelemetryStorage.pingDirectoryPath;
+ let iter = new OS.File.DirectoryIterator(directory);
+
+ try {
+ if (!(await iter.exists())) {
+ this._log.trace(
+ "removePendingPings - the pending pings directory doesn't exist"
+ );
+ return;
+ }
+
+ let files = (await iter.nextBatch()).filter(e => !e.isDir);
+ for (let file of files) {
+ try {
+ await OS.File.remove(file.path);
+ } catch (ex) {
+ this._log.error(
+ "removePendingPings - failed to remove file " + file.path,
+ ex
+ );
+ continue;
+ }
+ }
+ } finally {
+ await iter.close();
+ }
+ },
+
+ /**
+ * Iterate through all pings in the userApplicationDataDir under the "Pending Pings" sub-directory
+ * and yield each file.
+ */
+ async *_iterateAppDataPings() {
+ this._log.trace("_iterateAppDataPings");
+
+ // The test suites might not create and define the "UAppData" directory.
+ // We account for that here instead of manually going through each test using
+ // telemetry to manually create the directory and define the constant.
+ if (!OS.Constants.Path.userApplicationDataDir) {
+ this._log.trace(
+ "_iterateAppDataPings - userApplicationDataDir is not defined. Is this a test?"
+ );
+ return;
+ }
+
+ const appDataPendingPings = OS.Path.join(
+ OS.Constants.Path.userApplicationDataDir,
+ "Pending Pings"
+ );
+
+ // Iterate through the pending ping files.
+ let iter = new OS.File.DirectoryIterator(appDataPendingPings);
+ try {
+ // Check if appDataPendingPings exists and bail out if it doesn't.
+ if (!(await iter.exists())) {
+ this._log.trace(
+ "_iterateAppDataPings - the AppData pending pings directory doesn't exist."
+ );
+ return;
+ }
+
+ let files = (await iter.nextBatch()).filter(e => !e.isDir);
+ for (let file of files) {
+ yield file;
+ }
+ } finally {
+ await iter.close();
+ }
+ },
+
+ /**
+ * Remove all pings that are stored in the userApplicationDataDir
+ * under the "Pending Pings" sub-directory.
+ */
+ async removeAppDataPings() {
+ this._log.trace("removeAppDataPings");
+
+ for await (const file of this._iterateAppDataPings()) {
+ try {
+ await OS.File.remove(file.path);
+ } catch (ex) {
+ this._log.error(
+ "removeAppDataPings - failed to remove file " + file.path,
+ ex
+ );
+ }
+ }
+ },
+
+ /**
+ * Migrate pings that are stored in the userApplicationDataDir
+ * under the "Pending Pings" sub-directory.
+ */
+ async _migrateAppDataPings() {
+ this._log.trace("_migrateAppDataPings");
+
+ for await (const file of this._iterateAppDataPings()) {
+ try {
+ // Load the ping data from the original file.
+ const pingData = await this.loadPingFile(file.path);
+
+ // Save it among the pending pings in the user profile, overwrite on
+ // ping id collision.
+ await TelemetryStorage.savePing(pingData, true);
+ } catch (ex) {
+ this._log.error(
+ "_migrateAppDataPings - failed to load or migrate file. Removing. " +
+ file.path,
+ ex
+ );
+ }
+
+ try {
+ // Finally remove the file.
+ await OS.File.remove(file.path);
+ } catch (ex) {
+ this._log.error(
+ "_migrateAppDataPings - failed to remove file " + file.path,
+ ex
+ );
+ }
+ }
+ },
+
+ loadPendingPingList() {
+ // If we already have a pending scanning task active, return that.
+ if (this._scanPendingPingsTask) {
+ return this._scanPendingPingsTask;
+ }
+
+ if (this._scannedPendingDirectory) {
+ this._log.trace(
+ "loadPendingPingList - Pending already scanned, hitting cache."
+ );
+ return Promise.resolve(this._buildPingList());
+ }
+
+ // Since there's no pending pings scan task running, start it.
+ // Also make sure to clear the task once done.
+ this._scanPendingPingsTask = this._scanPendingPings().then(
+ pings => {
+ this._scanPendingPingsTask = null;
+ return pings;
+ },
+ ex => {
+ this._scanPendingPingsTask = null;
+ throw ex;
+ }
+ );
+ return this._scanPendingPingsTask;
+ },
+
+ getPendingPingList() {
+ return this._buildPingList();
+ },
+
+ async _scanPendingPings() {
+ this._log.trace("_scanPendingPings");
+
+ // Before pruning the pending pings, migrate over the ones from the user
+ // application data directory (mainly crash pings that failed to be sent).
+ await this._migrateAppDataPings();
+
+ let directory = TelemetryStorage.pingDirectoryPath;
+ let iter = new OS.File.DirectoryIterator(directory);
+ let exists = await iter.exists();
+
+ try {
+ if (!exists) {
+ return [];
+ }
+
+ let files = (await iter.nextBatch()).filter(e => !e.isDir);
+
+ for (let file of files) {
+ if (this._shutdown) {
+ return [];
+ }
+
+ let info;
+ try {
+ info = await OS.File.stat(file.path);
+ } catch (ex) {
+ this._log.error(
+ "_scanPendingPings - failed to stat file " + file.path,
+ ex
+ );
+ continue;
+ }
+
+ // Enforce a maximum file size limit on pending pings.
+ if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {
+ this._log.error(
+ "_scanPendingPings - removing file exceeding size limit " +
+ file.path
+ );
+ try {
+ await OS.File.remove(file.path);
+ } catch (ex) {
+ this._log.error(
+ "_scanPendingPings - failed to remove file " + file.path,
+ ex
+ );
+ } finally {
+ Telemetry.getHistogramById(
+ "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB"
+ ).add(Math.floor(info.size / 1024 / 1024));
+ Telemetry.getHistogramById(
+ "TELEMETRY_PING_SIZE_EXCEEDED_PENDING"
+ ).add();
+
+ // Currently we don't have the ping type available without loading the ping from disk.
+ // Bug 1384903 will fix that.
+ TelemetryHealthPing.recordDiscardedPing("<unknown>");
+ }
+ continue;
+ }
+
+ let id = OS.Path.basename(file.path);
+ if (!UUID_REGEX.test(id)) {
+ this._log.trace("_scanPendingPings - filename is not a UUID: " + id);
+ id = Utils.generateUUID();
+ }
+
+ this._pendingPings.set(id, {
+ path: file.path,
+ lastModificationDate: info.lastModificationDate.getTime(),
+ });
+ }
+ } finally {
+ await iter.close();
+ }
+
+ this._scannedPendingDirectory = true;
+ return this._buildPingList();
+ },
+
+ _buildPingList() {
+ const list = Array.from(this._pendingPings, p => ({
+ id: p[0],
+ lastModificationDate: p[1].lastModificationDate,
+ }));
+
+ list.sort((a, b) => b.lastModificationDate - a.lastModificationDate);
+ return list;
+ },
+
+ /**
+ * Loads a ping file.
+ * @param {String} aFilePath The path of the ping file.
+ * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4.
+ * @return {Promise<Object>} A promise resolved with the ping content or rejected if the
+ * ping contains invalid data.
+ * @throws {PingReadError} There was an error while reading the ping file from the disk.
+ * @throws {PingParseError} There was an error while parsing the JSON content of the ping file.
+ */
+ async loadPingFile(aFilePath, aCompressed = false) {
+ let options = {};
+ if (aCompressed) {
+ options.compression = "lz4";
+ }
+
+ let array;
+ try {
+ array = await OS.File.read(aFilePath, options);
+ } catch (e) {
+ this._log.trace("loadPingfile - unreadable ping " + aFilePath, e);
+ throw new PingReadError(e.message, e.becauseNoSuchFile);
+ }
+
+ let decoder = new TextDecoder();
+ let string = decoder.decode(array);
+ let ping;
+ try {
+ ping = JSON.parse(string);
+ } catch (e) {
+ this._log.trace("loadPingfile - unparseable ping " + aFilePath, e);
+ await OS.File.remove(aFilePath).catch(ex => {
+ this._log.error(
+ "loadPingFile - failed removing unparseable ping file",
+ ex
+ );
+ });
+ throw new PingParseError(e.message);
+ }
+
+ return ping;
+ },
+
+ /**
+ * Archived pings are saved with file names of the form:
+ * "<timestamp>.<uuid>.<type>.[json|jsonlz4]"
+ * This helper extracts that data from a given filename.
+ *
+ * @param fileName {String} The filename.
+ * @return {Object} Null if the filename didn't match the expected form.
+ * Otherwise an object with the extracted data in the form:
+ * { timestamp: <number>,
+ * id: <string>,
+ * type: <string> }
+ */
+ _getArchivedPingDataFromFileName(fileName) {
+ // Extract the parts.
+ let parts = fileName.split(".");
+ if (parts.length != 4) {
+ this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");
+ return null;
+ }
+
+ let [timestamp, uuid, type, extension] = parts;
+ if (extension != "json" && extension != "jsonlz4") {
+ this._log.trace(
+ "_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension"
+ );
+ return null;
+ }
+
+ // Check for a valid timestamp.
+ timestamp = parseInt(timestamp);
+ if (Number.isNaN(timestamp)) {
+ this._log.trace(
+ "_getArchivedPingDataFromFileName - should have a valid timestamp"
+ );
+ return null;
+ }
+
+ // Check for a valid UUID.
+ if (!UUID_REGEX.test(uuid)) {
+ this._log.trace(
+ "_getArchivedPingDataFromFileName - should have a valid id"
+ );
+ return null;
+ }
+
+ // Check for a valid type string.
+ const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
+ if (!typeRegex.test(type)) {
+ this._log.trace(
+ "_getArchivedPingDataFromFileName - should have a valid type"
+ );
+ return null;
+ }
+
+ return {
+ timestamp,
+ id: uuid,
+ type,
+ };
+ },
+
+ async saveAbortedSessionPing(ping) {
+ this._log.trace(
+ "saveAbortedSessionPing - ping path: " + gAbortedSessionFilePath
+ );
+ await OS.File.makeDir(gDataReportingDir, { ignoreExisting: true });
+
+ return this._abortedSessionSerializer.enqueueTask(() =>
+ this.savePingToFile(ping, gAbortedSessionFilePath, true)
+ );
+ },
+
+ async loadAbortedSessionPing() {
+ let ping = null;
+ try {
+ ping = await this.loadPingFile(gAbortedSessionFilePath);
+ } catch (ex) {
+ if (ex.becauseNoSuchFile) {
+ this._log.trace("loadAbortedSessionPing - no such file");
+ } else {
+ this._log.error("loadAbortedSessionPing - error loading ping", ex);
+ }
+ }
+ return ping;
+ },
+
+ removeAbortedSessionPing() {
+ return this._abortedSessionSerializer.enqueueTask(async () => {
+ try {
+ await OS.File.remove(gAbortedSessionFilePath, { ignoreAbsent: false });
+ this._log.trace("removeAbortedSessionPing - success");
+ } catch (ex) {
+ if (ex.becauseNoSuchFile) {
+ this._log.trace("removeAbortedSessionPing - no such file");
+ } else {
+ this._log.error("removeAbortedSessionPing - error removing ping", ex);
+ }
+ }
+ });
+ },
+
+ async saveUninstallPing(ping) {
+ if (AppConstants.platform != "win") {
+ return;
+ }
+
+ // Remove any old pings from this install first.
+ await this.removeUninstallPings();
+
+ let { directory: pingFile, file } = Policy.getUninstallPingPath(ping.id);
+ pingFile.append(file);
+
+ await this.savePingToFile(ping, pingFile.path, /* overwrite */ true);
+ },
+
+ async removeUninstallPings() {
+ if (AppConstants.platform != "win") {
+ return;
+ }
+
+ const { directory, file } = Policy.getUninstallPingPath("*");
+
+ const iteratorOptions = { winPattern: file };
+ const iterator = new OS.File.DirectoryIterator(
+ directory.path,
+ iteratorOptions
+ );
+
+ await iterator.forEach(async entry => {
+ this._log.trace("removeUninstallPings - removing", entry.path);
+ try {
+ await OS.File.remove(entry.path);
+ this._log.trace("removeUninstallPings - success");
+ } catch (ex) {
+ if (ex.becauseNoSuchFile) {
+ this._log.trace("removeUninstallPings - no such file");
+ } else {
+ this._log.error("removeUninstallPings - error removing ping", ex);
+ }
+ }
+ });
+ iterator.close();
+ },
+
+ /**
+ * Remove FHR database files. This is temporary and will be dropped in
+ * the future.
+ * @return {Promise} Resolved when the database files are deleted.
+ */
+ async removeFHRDatabase() {
+ this._log.trace("removeFHRDatabase");
+
+ // Let's try to remove the FHR DB with the default filename first.
+ const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite";
+
+ // Even if it's uncommon, there may be 2 additional files: - a "write ahead log"
+ // (-wal) file and a "shared memory file" (-shm). We need to remove them as well.
+ let FILES_TO_REMOVE = [
+ OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME),
+ OS.Path.join(
+ OS.Constants.Path.profileDir,
+ FHR_DB_DEFAULT_FILENAME + "-wal"
+ ),
+ OS.Path.join(
+ OS.Constants.Path.profileDir,
+ FHR_DB_DEFAULT_FILENAME + "-shm"
+ ),
+ ];
+
+ // FHR could have used either the default DB file name or a custom one
+ // through this preference.
+ const FHR_DB_CUSTOM_FILENAME = Preferences.get(
+ "datareporting.healthreport.dbName",
+ undefined
+ );
+ if (FHR_DB_CUSTOM_FILENAME) {
+ FILES_TO_REMOVE.push(
+ OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME),
+ OS.Path.join(
+ OS.Constants.Path.profileDir,
+ FHR_DB_CUSTOM_FILENAME + "-wal"
+ ),
+ OS.Path.join(
+ OS.Constants.Path.profileDir,
+ FHR_DB_CUSTOM_FILENAME + "-shm"
+ )
+ );
+ }
+
+ for (let f of FILES_TO_REMOVE) {
+ await OS.File.remove(f, { ignoreAbsent: true }).catch(e =>
+ this._log.error("removeFHRDatabase - failed to remove " + f, e)
+ );
+ }
+ },
+};
+
+// Utility functions
+
+function pingFilePath(ping) {
+ // Support legacy ping formats, who don't have an "id" field, but a "slug" field.
+ let pingIdentifier = ping.slug ? ping.slug : ping.id;
+ return OS.Path.join(TelemetryStorage.pingDirectoryPath, pingIdentifier);
+}
+
+function getPingDirectory() {
+ return (async function() {
+ let directory = TelemetryStorage.pingDirectoryPath;
+
+ if (!(await OS.File.exists(directory))) {
+ await OS.File.makeDir(directory, { unixMode: OS.Constants.S_IRWXU });
+ }
+
+ return directory;
+ })();
+}
+
+/**
+ * Build the path to the archived ping.
+ * @param {String} aPingId The ping id.
+ * @param {Object} aDate The ping creation date.
+ * @param {String} aType The ping type.
+ * @return {String} The full path to the archived ping.
+ */
+function getArchivedPingPath(aPingId, aDate, aType) {
+ // Get the ping creation date and generate the archive directory to hold it. Note
+ // that getMonth returns a 0-based month, so we need to add an offset.
+ let month = String(aDate.getMonth() + 1);
+ let archivedPingDir = OS.Path.join(
+ gPingsArchivePath,
+ aDate.getFullYear() + "-" + month.padStart(2, "0")
+ );
+ // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json
+ let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");
+ return OS.Path.join(archivedPingDir, fileName);
+}
+
+/**
+ * Get the size of the ping file on the disk.
+ * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.
+ */
+var getArchivedPingSize = async function(aPingId, aDate, aType) {
+ const path = getArchivedPingPath(aPingId, aDate, aType);
+ let filePaths = [path + "lz4", path];
+
+ for (let path of filePaths) {
+ try {
+ return (await OS.File.stat(path)).size;
+ } catch (e) {}
+ }
+
+ // That's odd, this ping doesn't seem to exist.
+ return 0;
+};
+
+/**
+ * Get the size of the pending ping file on the disk.
+ * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.
+ */
+var getPendingPingSize = async function(aPingId) {
+ const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, aPingId);
+ try {
+ return (await OS.File.stat(path)).size;
+ } catch (e) {}
+
+ // That's odd, this ping doesn't seem to exist.
+ return 0;
+};
+
+/**
+ * Check if a directory name is in the "YYYY-MM" format.
+ * @param {String} aDirName The name of the pings archive directory.
+ * @return {Boolean} True if the directory name is in the right format, false otherwise.
+ */
+function isValidArchiveDir(aDirName) {
+ const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;
+ return dirRegEx.test(aDirName);
+}
+
+/**
+ * Gets a date object from an archive directory name.
+ * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM
+ * format.
+ * @return {Object} A Date object or null if the dir name is not valid.
+ */
+function getDateFromArchiveDir(aDirName) {
+ let [year, month] = aDirName.split("-");
+ year = parseInt(year);
+ month = parseInt(month);
+ // Make sure to have sane numbers.
+ if (
+ !Number.isFinite(month) ||
+ !Number.isFinite(year) ||
+ month < 1 ||
+ month > 12
+ ) {
+ return null;
+ }
+ return new Date(year, month - 1, 1, 0, 0, 0);
+}
diff --git a/toolkit/components/telemetry/app/TelemetryTimestamps.jsm b/toolkit/components/telemetry/app/TelemetryTimestamps.jsm
new file mode 100644
index 0000000000..b0cc72a725
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryTimestamps.jsm
@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["TelemetryTimestamps"];
+
+/**
+ * This module's purpose is to collect timestamps for important
+ * application-specific events.
+ *
+ * The TelemetryController component attaches the timestamps stored by this module to
+ * the telemetry submission, substracting the process lifetime so that the times
+ * are relative to process startup. The overall goal is to produce a basic
+ * timeline of the startup process.
+ */
+var timeStamps = {};
+
+var TelemetryTimestamps = {
+ /**
+ * Adds a timestamp to the list. The addition of TimeStamps that already have
+ * a value stored is ignored.
+ *
+ * @param name must be a unique, generally "camelCase" descriptor of what the
+ * timestamp represents. e.g.: "delayedStartupStarted"
+ * @param value is a timeStamp in milliseconds since the epoch. If omitted,
+ * defaults to Date.now().
+ */
+ add: function TT_add(name, value) {
+ // Default to "now" if not specified
+ if (value == null) {
+ value = Date.now();
+ }
+
+ if (isNaN(value)) {
+ throw new Error("Value must be a timestamp");
+ }
+
+ // If there's an existing value, just ignore the new value.
+ if (timeStamps.hasOwnProperty(name)) {
+ return;
+ }
+
+ timeStamps[name] = value;
+ },
+
+ /**
+ * Returns a JS object containing all of the timeStamps as properties (can be
+ * easily serialized to JSON). Used by TelemetryController to retrieve the data
+ * to attach to the telemetry submission.
+ */
+ get: function TT_get() {
+ // Return a copy of the object.
+ return Cu.cloneInto(timeStamps, {});
+ },
+};
diff --git a/toolkit/components/telemetry/app/TelemetryUtils.jsm b/toolkit/components/telemetry/app/TelemetryUtils.jsm
new file mode 100644
index 0000000000..5788a3d269
--- /dev/null
+++ b/toolkit/components/telemetry/app/TelemetryUtils.jsm
@@ -0,0 +1,301 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["TelemetryUtils"];
+
+ChromeUtils.import("resource://gre/modules/Services.jsm", this);
+const { TelemetryControllerBase } = ChromeUtils.import(
+ "resource://gre/modules/TelemetryControllerBase.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "UpdateUtils",
+ "resource://gre/modules/UpdateUtils.jsm"
+);
+
+const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
+
+const IS_CONTENT_PROCESS = (function() {
+ // We cannot use Services.appinfo here because in telemetry xpcshell tests,
+ // appinfo is initially unavailable, and becomes available only later on.
+ // eslint-disable-next-line mozilla/use-services
+ let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
+ return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT;
+})();
+
+var TelemetryUtils = {
+ /**
+ * When telemetry is disabled, identifying information (such as client ID)
+ * should be removed. A topic event is emitted with a subject that matches
+ * this constant. When this happens, other systems that store identifying
+ * information about the client should delete that data. Please ask the
+ * Firefox Telemetry Team before relying on this topic.
+ *
+ * Here is an example of listening for that event:
+ *
+ * const { TelemetryUtils } = ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm");
+ *
+ * class YourClass {
+ * constructor() {
+ * Services.obs.addObserver(this, TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC);
+ * }
+ *
+ * observe(subject, topic, data) {
+ * if (topic == TelemetryUtils.TELEMETRY_UPLOAD_DISABLED_TOPIC) {
+ * // Telemetry was disabled
+ * // subject and data are both unused
+ * }
+ * }
+ * }
+ */
+ TELEMETRY_UPLOAD_DISABLED_TOPIC: "telemetry.upload.disabled",
+
+ Preferences: Object.freeze({
+ ...TelemetryControllerBase.Preferences,
+
+ // General Preferences
+ ArchiveEnabled: "toolkit.telemetry.archive.enabled",
+ CachedClientId: "toolkit.telemetry.cachedClientID",
+ DisableFuzzingDelay: "toolkit.telemetry.testing.disableFuzzingDelay",
+ FirstRun: "toolkit.telemetry.reportingpolicy.firstRun",
+ FirstShutdownPingEnabled: "toolkit.telemetry.firstShutdownPing.enabled",
+ HealthPingEnabled: "toolkit.telemetry.healthping.enabled",
+ IPCBatchTimeout: "toolkit.telemetry.ipcBatchTimeout",
+ OverrideOfficialCheck: "toolkit.telemetry.send.overrideOfficialCheck",
+ OverrideUpdateChannel: "toolkit.telemetry.overrideUpdateChannel",
+ Server: "toolkit.telemetry.server",
+ ShutdownPingSender: "toolkit.telemetry.shutdownPingSender.enabled",
+ ShutdownPingSenderFirstSession:
+ "toolkit.telemetry.shutdownPingSender.enabledFirstSession",
+ TelemetryEnabled: "toolkit.telemetry.enabled",
+ UntrustedModulesPingFrequency:
+ "toolkit.telemetry.untrustedModulesPing.frequency",
+ UpdatePing: "toolkit.telemetry.updatePing.enabled",
+ NewProfilePingEnabled: "toolkit.telemetry.newProfilePing.enabled",
+ NewProfilePingDelay: "toolkit.telemetry.newProfilePing.delay",
+ PreviousBuildID: "toolkit.telemetry.previousBuildID",
+
+ // Event Ping Preferences
+ EventPingMinimumFrequency: "toolkit.telemetry.eventping.minimumFrequency",
+ EventPingMaximumFrequency: "toolkit.telemetry.eventping.maximumFrequency",
+
+ // Ecosystem Telemetry Preferences
+ EcosystemTelemetryEnabled: "toolkit.telemetry.ecosystemtelemetry.enabled",
+ EcosystemTelemetryAllowForNonProductionFxA:
+ "toolkit.telemetry.ecosystemtelemetry.allowForNonProductionFxA",
+
+ // Prio Ping Preferences
+ PrioPingEnabled: "toolkit.telemetry.prioping.enabled",
+ PrioPingDataLimit: "toolkit.telemetry.prioping.dataLimit",
+
+ // Data reporting Preferences
+ AcceptedPolicyDate: "datareporting.policy.dataSubmissionPolicyNotifiedTime",
+ AcceptedPolicyVersion:
+ "datareporting.policy.dataSubmissionPolicyAcceptedVersion",
+ BypassNotification:
+ "datareporting.policy.dataSubmissionPolicyBypassNotification",
+ CurrentPolicyVersion: "datareporting.policy.currentPolicyVersion",
+ DataSubmissionEnabled: "datareporting.policy.dataSubmissionEnabled",
+ FhrUploadEnabled: "datareporting.healthreport.uploadEnabled",
+ MinimumPolicyVersion: "datareporting.policy.minimumPolicyVersion",
+ FirstRunURL: "datareporting.policy.firstRunURL",
+ }),
+
+ /**
+ * A fixed valid client ID used when Telemetry upload is disabled.
+ */
+ get knownClientID() {
+ return "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0";
+ },
+
+ /**
+ * True if this is a content process.
+ */
+ get isContentProcess() {
+ return IS_CONTENT_PROCESS;
+ },
+
+ /**
+ * Returns the state of the Telemetry enabled preference, making sure
+ * it correctly evaluates to a boolean type.
+ */
+ get isTelemetryEnabled() {
+ return TelemetryControllerBase.isTelemetryEnabled;
+ },
+
+ /**
+ * Turn a millisecond timestamp into a day timestamp.
+ *
+ * @param aMsec A number of milliseconds since Unix epoch.
+ * @return The number of whole days since Unix epoch.
+ */
+ millisecondsToDays(aMsec) {
+ return Math.floor(aMsec / MILLISECONDS_PER_DAY);
+ },
+
+ /**
+ * Takes a date and returns it truncated to a date with daily precision.
+ */
+ truncateToDays(date) {
+ return new Date(
+ date.getFullYear(),
+ date.getMonth(),
+ date.getDate(),
+ 0,
+ 0,
+ 0,
+ 0
+ );
+ },
+
+ /**
+ * Takes a date and returns it truncated to a date with hourly precision.
+ */
+ truncateToHours(date) {
+ return new Date(
+ date.getFullYear(),
+ date.getMonth(),
+ date.getDate(),
+ date.getHours(),
+ 0,
+ 0,
+ 0
+ );
+ },
+
+ /**
+ * Check if the difference between the times is within the provided tolerance.
+ * @param {Number} t1 A time in milliseconds.
+ * @param {Number} t2 A time in milliseconds.
+ * @param {Number} tolerance The tolerance, in milliseconds.
+ * @return {Boolean} True if the absolute time difference is within the tolerance, false
+ * otherwise.
+ */
+ areTimesClose(t1, t2, tolerance) {
+ return Math.abs(t1 - t2) <= tolerance;
+ },
+
+ /**
+ * Get the next midnight for a date.
+ * @param {Object} date The date object to check.
+ * @return {Object} The Date object representing the next midnight.
+ */
+ getNextMidnight(date) {
+ let nextMidnight = new Date(this.truncateToDays(date));
+ nextMidnight.setDate(nextMidnight.getDate() + 1);
+ return nextMidnight;
+ },
+
+ /**
+ * Get the midnight which is closer to the provided date.
+ * @param {Object} date The date object to check.
+ * @param {Number} tolerance The tolerance within we find the closest midnight.
+ * @return {Object} The Date object representing the closes midnight, or null if midnight
+ * is not within the midnight tolerance.
+ */
+ getNearestMidnight(date, tolerance) {
+ let lastMidnight = this.truncateToDays(date);
+ if (this.areTimesClose(date.getTime(), lastMidnight.getTime(), tolerance)) {
+ return lastMidnight;
+ }
+
+ const nextMidnightDate = this.getNextMidnight(date);
+ if (
+ this.areTimesClose(date.getTime(), nextMidnightDate.getTime(), tolerance)
+ ) {
+ return nextMidnightDate;
+ }
+ return null;
+ },
+
+ generateUUID() {
+ let str = Cc["@mozilla.org/uuid-generator;1"]
+ .getService(Ci.nsIUUIDGenerator)
+ .generateUUID()
+ .toString();
+ // strip {}
+ return str.substring(1, str.length - 1);
+ },
+
+ /**
+ * Find how many months passed between two dates.
+ * @param {Object} aStartDate The starting date.
+ * @param {Object} aEndDate The ending date.
+ * @return {Integer} The number of months between the two dates.
+ */
+ getElapsedTimeInMonths(aStartDate, aEndDate) {
+ return (
+ aEndDate.getMonth() -
+ aStartDate.getMonth() +
+ 12 * (aEndDate.getFullYear() - aStartDate.getFullYear())
+ );
+ },
+
+ /**
+ * Date.toISOString() gives us UTC times, this gives us local times in
+ * the ISO date format. See http://www.w3.org/TR/NOTE-datetime
+ * @param {Object} date The input date.
+ * @return {String} The local time ISO string.
+ */
+ toLocalTimeISOString(date) {
+ function padNumber(number, length) {
+ return number.toString().padStart(length, "0");
+ }
+
+ let sign = n => (n >= 0 ? "+" : "-");
+ // getTimezoneOffset counter-intuitively returns -60 for UTC+1.
+ let tzOffset = -date.getTimezoneOffset();
+
+ // YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00)
+ return (
+ padNumber(date.getFullYear(), 4) +
+ "-" +
+ padNumber(date.getMonth() + 1, 2) +
+ "-" +
+ padNumber(date.getDate(), 2) +
+ "T" +
+ padNumber(date.getHours(), 2) +
+ ":" +
+ padNumber(date.getMinutes(), 2) +
+ ":" +
+ padNumber(date.getSeconds(), 2) +
+ "." +
+ date.getMilliseconds() +
+ sign(tzOffset) +
+ padNumber(Math.floor(Math.abs(tzOffset / 60)), 2) +
+ ":" +
+ padNumber(Math.abs(tzOffset % 60), 2)
+ );
+ },
+
+ /**
+ * @returns {number} The monotonic time since the process start
+ * or (non-monotonic) Date value if this fails back.
+ */
+ monotonicNow() {
+ return Services.telemetry.msSinceProcessStart();
+ },
+
+ /**
+ * @returns {string} The name of the update channel to report
+ * in telemetry.
+ * By default, this is the same as the name of the channel that
+ * the browser uses to download its updates. However in certain
+ * situations, a single update channel provides multiple (distinct)
+ * build types, that need to be distinguishable on Telemetry.
+ */
+ getUpdateChannel() {
+ let overrideChannel = Services.prefs.getCharPref(
+ this.Preferences.OverrideUpdateChannel,
+ undefined
+ );
+ if (overrideChannel) {
+ return overrideChannel;
+ }
+
+ return UpdateUtils.getUpdateChannel(false);
+ },
+};