path: root/toolkit/components/telemetry/app/TelemetryStorage.sys.mjs

/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
import { Log } from "resource://gre/modules/Log.sys.mjs";
import { TelemetryUtils } from "resource://gre/modules/TelemetryUtils.sys.mjs";

const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryStorage::";

const Telemetry = Services.telemetry;
const Utils = TelemetryUtils;

// Compute the path of the pings archive on the first use.
const DATAREPORTING_DIR = "datareporting";
const PINGS_ARCHIVE_DIR = "archived";
const ABORTED_SESSION_FILE_NAME = "aborted-session-ping";
const SESSION_STATE_FILE_NAME = "session-state.json";

const lazy = {};

ChromeUtils.defineLazyGetter(lazy, "gDataReportingDir", function () {
  return PathUtils.join(PathUtils.profileDir, DATAREPORTING_DIR);
});
ChromeUtils.defineLazyGetter(lazy, "gPingsArchivePath", function () {
  return PathUtils.join(lazy.gDataReportingDir, PINGS_ARCHIVE_DIR);
});
ChromeUtils.defineLazyGetter(lazy, "gAbortedSessionFilePath", function () {
  return PathUtils.join(lazy.gDataReportingDir, ABORTED_SESSION_FILE_NAME);
});
ChromeUtils.defineESModuleGetters(lazy, {
  TelemetryHealthPing: "resource://gre/modules/HealthPing.sys.mjs",
});
// Maxmimum time, in milliseconds, archive pings should be retained.
const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000; // 60 days

// Maximum space the archive can take on disk (in Bytes).
const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB
// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes).
const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB
// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes).
const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB

// The maximum size a pending/archived ping can take on disk.
const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB

// This special value is submitted when the archive is outside of the quota.
const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300;

// This special value is submitted when the pending pings is outside of the quota, as
// we don't know the size of the pings above the quota.
const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17;

const UUID_REGEX =
  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

/**
 * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping
 * from the disk fails.
 */
function PingReadError(
  message = "Error reading the ping file",
  becauseNoSuchFile = false
) {
  Error.call(this, message);
  let error = new Error();
  this.name = "PingReadError";
  this.message = message;
  this.stack = error.stack;
  this.becauseNoSuchFile = becauseNoSuchFile;
}
PingReadError.prototype = Object.create(Error.prototype);
PingReadError.prototype.constructor = PingReadError;

/**
 * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON
 * content fails.
 */
function PingParseError(message = "Error parsing ping content") {
  Error.call(this, message);
  let error = new Error();
  this.name = "PingParseError";
  this.message = message;
  this.stack = error.stack;
}
PingParseError.prototype = Object.create(Error.prototype);
PingParseError.prototype.constructor = PingParseError;

/**
 * This is a policy object used to override behavior for testing.
 */
export var Policy = {
  now: () => new Date(),
  getArchiveQuota: () => ARCHIVE_QUOTA_BYTES,
  getPendingPingsQuota: () =>
    AppConstants.platform == "android"
      ? PENDING_PINGS_QUOTA_BYTES_MOBILE
      : PENDING_PINGS_QUOTA_BYTES_DESKTOP,
  /**
   * @param {string} id The ID of the ping that will be written into the file. Can be "*" to
   *                    make a pattern to find all pings for this installation.
   * @return
   *         {
   *           directory: <nsIFile>, // Directory to save pings
   *           file: <string>, // File name for this ping (or pattern for all pings)
   *         }
   */
  getUninstallPingPath: id => {
    // UpdRootD is e.g. C:\ProgramData\Mozilla\updates\<PATH HASH>
    const updateDirectory = Services.dirsvc.get("UpdRootD", Ci.nsIFile);
    const installPathHash = updateDirectory.leafName;

    return {
      // e.g. C:\ProgramData\Mozilla
      directory: updateDirectory.parent.parent.clone(),
      file: `uninstall_ping_${installPathHash}_${id}.json`,
    };
  },
};

/**
 * Wait for all promises in iterable to resolve or reject. This function
 * always resolves its promise with undefined, and never rejects.
 */
function waitForAll(it) {
  let dummy = () => {};
  let promises = Array.from(it, p => p.catch(dummy));
  return Promise.all(promises);
}

/**
 * Permanently intern the given string. This is mainly used for the ping.type
 * strings that can be excessively duplicated in the _archivedPings map. Do not
 * pass large or temporary strings to this function.
 */
function internString(str) {
  return Symbol.keyFor(Symbol.for(str));
}

export var TelemetryStorage = {
  get pingDirectoryPath() {
    return PathUtils.join(PathUtils.profileDir, "saved-telemetry-pings");
  },

  /**
   * The maximum size a ping can have, in bytes.
   */
  get MAXIMUM_PING_SIZE() {
    return PING_FILE_MAXIMUM_SIZE_BYTES;
  },
  /**
   * Shutdown & block on any outstanding async activity in this module.
   *
   * @return {Promise} Promise that is resolved when shutdown is complete.
   */
  shutdown() {
    return TelemetryStorageImpl.shutdown();
  },

  /**
   * Save an archived ping to disk.
   *
   * @param {object} ping The ping data to archive.
   * @return {promise} Promise that is resolved when the ping is successfully archived.
   */
  saveArchivedPing(ping) {
    return TelemetryStorageImpl.saveArchivedPing(ping);
  },

  /**
   * Load an archived ping from disk.
   *
   * @param {string} id The pings id.
   * @return {promise<object>} Promise that is resolved with the ping data.
   */
  loadArchivedPing(id) {
    return TelemetryStorageImpl.loadArchivedPing(id);
  },

  /**
   * Get a list of info on the archived pings.
   * This will scan the archive directory and grab basic data about the existing
   * pings out of their filename.
   *
   * @return {promise<sequence<object>>}
   */
  loadArchivedPingList() {
    return TelemetryStorageImpl.loadArchivedPingList();
  },

  /**
   * Clean the pings archive by removing old pings.
   * This will scan the archive directory.
   *
   * @return {Promise} Resolved when the cleanup task completes.
   */
  runCleanPingArchiveTask() {
    return TelemetryStorageImpl.runCleanPingArchiveTask();
  },

  /**
   * Run the task to enforce the pending pings quota.
   *
   * @return {Promise} Resolved when the cleanup task completes.
   */
  runEnforcePendingPingsQuotaTask() {
    return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask();
  },

  /**
   * Run the task to remove all the pending pings
   *
   * @return {Promise} Resolved when the pings are removed.
   */
  runRemovePendingPingsTask() {
    return TelemetryStorageImpl.runRemovePendingPingsTask();
  },

  /**
   * Remove all pings that are stored in the userApplicationDataDir
   * under the "Pending Pings" sub-directory.
   */
  removeAppDataPings() {
    return TelemetryStorageImpl.removeAppDataPings();
  },

  /**
   * Reset the storage state in tests.
   */
  reset() {
    return TelemetryStorageImpl.reset();
  },

  /**
   * Test method that allows waiting on the archive clean task to finish.
   */
  testCleanupTaskPromise() {
    return TelemetryStorageImpl._cleanArchiveTask || Promise.resolve();
  },

  /**
   * Test method that allows waiting on the pending pings quota task to finish.
   */
  testPendingQuotaTaskPromise() {
    return (
      TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve()
    );
  },

  /**
   * Save a pending - outgoing - ping to disk and track it.
   *
   * @param {Object} ping The ping data.
   * @return {Promise} Resolved when the ping was saved.
   */
  savePendingPing(ping) {
    return TelemetryStorageImpl.savePendingPing(ping);
  },

  /**
   * Saves session data to disk.
   * @param {Object}  sessionData The session data.
   * @return {Promise} Resolved when the data was saved.
   */
  saveSessionData(sessionData) {
    return TelemetryStorageImpl.saveSessionData(sessionData);
  },

  /**
   * Loads session data from a session data file.
   * @return {Promise<object>} Resolved with the session data in object form.
   */
  loadSessionData() {
    return TelemetryStorageImpl.loadSessionData();
  },

  /**
   * Load a pending ping from disk by id.
   *
   * @param {String} id The pings id.
   * @return {Promise} Resolved with the loaded ping data.
   */
  loadPendingPing(id) {
    return TelemetryStorageImpl.loadPendingPing(id);
  },

  /**
   * Remove a pending ping from disk by id.
   *
   * @param {String} id The pings id.
   * @return {Promise} Resolved when the ping was removed.
   */
  removePendingPing(id) {
    return TelemetryStorageImpl.removePendingPing(id);
  },

  /**
   * Returns a list of the currently pending pings in the format:
   * {
   *   id: <string>, // The pings UUID.
   *   lastModified: <number>, // Timestamp of the pings last modification.
   * }
   * This populates the list by scanning the disk.
   *
   * @return {Promise<sequence>} Resolved with the ping list.
   */
  loadPendingPingList() {
    return TelemetryStorageImpl.loadPendingPingList();
  },

  /**
   * Returns a list of the currently pending pings in the format:
   * {
   *   id: <string>, // The pings UUID.
   *   lastModified: <number>, // Timestamp of the pings last modification.
   * }
   * This does not scan pending pings on disk.
   *
   * @return {sequence} The current pending ping list.
   */
  getPendingPingList() {
    return TelemetryStorageImpl.getPendingPingList();
  },

  /**
   * Save an aborted-session ping to disk. This goes to a special location so
   * it is not picked up as a pending ping.
   *
   * @param {object} ping The ping data to save.
   * @return {promise} Promise that is resolved when the ping is successfully saved.
   */
  saveAbortedSessionPing(ping) {
    return TelemetryStorageImpl.saveAbortedSessionPing(ping);
  },

  /**
   * Load the aborted-session ping from disk if present.
   *
   * @return {promise<object>} Promise that is resolved with the ping data if found.
   *                           Otherwise returns null.
   */
  loadAbortedSessionPing() {
    return TelemetryStorageImpl.loadAbortedSessionPing();
  },

  /**
   * Remove the aborted-session ping if present.
   *
   * @return {promise} Promise that is resolved once the ping is removed.
   */
  removeAbortedSessionPing() {
    return TelemetryStorageImpl.removeAbortedSessionPing();
  },

  /**
   * Save an uninstall ping to disk, removing any old ones from this
   * installation first.
   * This is stored independently from other pings, and only read by
   * the Windows uninstaller.
   *
   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
   *
   * @return {promise} Promise that is resolved when the ping has been saved.
   */
  saveUninstallPing(ping) {
    return TelemetryStorageImpl.saveUninstallPing(ping);
  },

  /**
   * Remove all uninstall pings from this installation.
   *
   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
   *
   * @return {promise} Promise that is resolved when the pings have been removed.
   */
  removeUninstallPings() {
    return TelemetryStorageImpl.removeUninstallPings();
  },

  /**
   * Save a single ping to a file.
   *
   * @param {object} ping The content of the ping to save.
   * @param {string} file The destination file.
   * @param {bool} overwrite If |true|, the file will be overwritten if it exists,
   * if |false| the file will not be overwritten and no error will be reported if
   * the file exists.
   * @returns {promise}
   */
  savePingToFile(ping, file, overwrite) {
    return TelemetryStorageImpl.savePingToFile(ping, file, overwrite);
  },

  /**
   * Save a ping to its file.
   *
   * @param {object} ping The content of the ping to save.
   * @param {bool} overwrite If |true|, the file will be overwritten
   * if it exists.
   * @returns {promise}
   */
  savePing(ping, overwrite) {
    return TelemetryStorageImpl.savePing(ping, overwrite);
  },

  /**
   * Remove the file for a ping
   *
   * @param {object} ping The ping.
   * @returns {promise}
   */
  cleanupPingFile(ping) {
    return TelemetryStorageImpl.cleanupPingFile(ping);
  },

  /**
   * Loads a ping file.
   * @param {String} aFilePath The path of the ping file.
   * @return {Promise<Object>} A promise resolved with the ping content or rejected if the
   *                           ping contains invalid data.
   */
  async loadPingFile(aFilePath) {
    return TelemetryStorageImpl.loadPingFile(aFilePath);
  },

  /**
   * Remove FHR database files. This is temporary and will be dropped in
   * the future.
   * @return {Promise} Resolved when the database files are deleted.
   */
  removeFHRDatabase() {
    return TelemetryStorageImpl.removeFHRDatabase();
  },

  /**
   * Only used in tests, builds an archived ping path from the ping metadata.
   * @param {String} aPingId The ping id.
   * @param {Object} aDate The ping creation date.
   * @param {String} aType The ping type.
   * @return {String} The full path to the archived ping.
   */
  _testGetArchivedPingPath(aPingId, aDate, aType) {
    return getArchivedPingPath(aPingId, aDate, aType);
  },

  /**
   * Only used in tests, this helper extracts ping metadata from a given filename.
   *
   * @param fileName {String} The filename.
   * @return {Object} Null if the filename didn't match the expected form.
   *                  Otherwise an object with the extracted data in the form:
   *                  { timestamp: <number>,
   *                    id: <string>,
   *                    type: <string> }
   */
  _testGetArchivedPingDataFromFileName(aFileName) {
    return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName);
  },

  /**
   * Only used in tests, this helper allows cleaning up the pending ping storage.
   */
  testClearPendingPings() {
    return TelemetryStorageImpl.runRemovePendingPingsTask();
  },
};

/**
 * This object allows the serialisation of asynchronous tasks. This is particularly
 * useful to serialise write access to the disk in order to prevent race conditions
 * to corrupt the data being written.
 * We are using this to synchronize saving to the file that TelemetrySession persists
 * its state in.
 */
function SaveSerializer() {
  this._queuedOperations = [];
  this._queuedInProgress = false;
  this._log = Log.repository.getLoggerWithMessagePrefix(
    LOGGER_NAME,
    LOGGER_PREFIX
  );
}

SaveSerializer.prototype = {
  /**
   * Enqueues an operation to a list to serialise their execution in order to prevent race
   * conditions. Useful to serialise access to disk.
   *
   * @param {Function} aFunction The task function to enqueue. It must return a promise.
   * @return {Promise} A promise resolved when the enqueued task completes.
   */
  enqueueTask(aFunction) {
    let promise = new Promise((resolve, reject) =>
      this._queuedOperations.push([aFunction, resolve, reject])
    );

    if (this._queuedOperations.length == 1) {
      this._popAndPerformQueuedOperation();
    }
    return promise;
  },

  /**
   * Make sure to flush all the pending operations.
   * @return {Promise} A promise resolved when all the pending operations have completed.
   */
  flushTasks() {
    let dummyTask = () => new Promise(resolve => resolve());
    return this.enqueueTask(dummyTask);
  },

  /**
   * Pop a task from the queue, executes it and continue to the next one.
   * This function recursively pops all the tasks.
   */
  _popAndPerformQueuedOperation() {
    if (!this._queuedOperations.length || this._queuedInProgress) {
      return;
    }

    this._log.trace(
      "_popAndPerformQueuedOperation - Performing queued operation."
    );
    let [func, resolve, reject] = this._queuedOperations.shift();
    let promise;

    try {
      this._queuedInProgress = true;
      promise = func();
    } catch (ex) {
      this._log.warn(
        "_popAndPerformQueuedOperation - Queued operation threw during execution. ",
        ex
      );
      this._queuedInProgress = false;
      reject(ex);
      this._popAndPerformQueuedOperation();
      return;
    }

    if (!promise || typeof promise.then != "function") {
      let msg = "Queued operation did not return a promise: " + func;
      this._log.warn("_popAndPerformQueuedOperation - " + msg);

      this._queuedInProgress = false;
      reject(new Error(msg));
      this._popAndPerformQueuedOperation();
      return;
    }

    promise.then(
      result => {
        this._queuedInProgress = false;
        resolve(result);
        this._popAndPerformQueuedOperation();
      },
      error => {
        this._log.warn(
          "_popAndPerformQueuedOperation - Failure when performing queued operation.",
          error
        );
        this._queuedInProgress = false;
        reject(error);
        this._popAndPerformQueuedOperation();
      }
    );
  },
};

var TelemetryStorageImpl = {
  _logger: null,
  // Used to serialize aborted session ping writes to disk.
  _abortedSessionSerializer: new SaveSerializer(),
  // Used to serialize session state writes to disk.
  _stateSaveSerializer: new SaveSerializer(),

  // Tracks the archived pings in a Map of (id -> {timestampCreated, type}).
  // We use this to cache info on archived pings to avoid scanning the disk more than once.
  _archivedPings: new Map(),
  // A set of promises for pings currently being archived
  _activelyArchiving: new Set(),
  // Track the archive loading task to prevent multiple tasks from being executed.
  _scanArchiveTask: null,
  // Track the archive cleanup task.
  _cleanArchiveTask: null,
  // Whether we already scanned the archived pings on disk.
  _scannedArchiveDirectory: false,

  // Track the pending ping removal task.
  _removePendingPingsTask: null,

  // This tracks all the pending async ping save activity.
  _activePendingPingSaves: new Set(),

  // Tracks the pending pings in a Map of (id -> {timestampCreated, type}).
  // We use this to cache info on pending pings to avoid scanning the disk more than once.
  _pendingPings: new Map(),

  // Track the pending pings enforce quota task.
  _enforcePendingPingsQuotaTask: null,

  // Track the shutdown process to bail out of the clean up task quickly.
  _shutdown: false,

  get _log() {
    if (!this._logger) {
      this._logger = Log.repository.getLoggerWithMessagePrefix(
        LOGGER_NAME,
        LOGGER_PREFIX
      );
    }

    return this._logger;
  },

  /**
   * Shutdown & block on any outstanding async activity in this module.
   *
   * @return {Promise} Promise that is resolved when shutdown is complete.
   */
  async shutdown() {
    this._shutdown = true;

    // If the following tasks are still running, block on them. They will bail out as soon
    // as possible.
    await this._abortedSessionSerializer.flushTasks().catch(ex => {
      this._log.error("shutdown - failed to flush aborted-session writes", ex);
    });

    if (this._cleanArchiveTask) {
      await this._cleanArchiveTask.catch(ex => {
        this._log.error("shutdown - the archive cleaning task failed", ex);
      });
    }

    if (this._enforcePendingPingsQuotaTask) {
      await this._enforcePendingPingsQuotaTask.catch(ex => {
        this._log.error("shutdown - the pending pings quota task failed", ex);
      });
    }

    if (this._removePendingPingsTask) {
      await this._removePendingPingsTask.catch(ex => {
        this._log.error("shutdown - the pending pings removal task failed", ex);
      });
    }

    // Wait on pending pings still being saved. While IOUtils should have shutdown
    // blockers in place, we a) have seen weird errors being reported that might
    // indicate a bad shutdown path and b) might have completion handlers hanging
    // off the save operations that don't expect to be late in shutdown.
    await this.promisePendingPingSaves();
  },

  /**
   * Save an archived ping to disk.
   *
   * @param {object} ping The ping data to archive.
   * @return {promise} Promise that is resolved when the ping is successfully archived.
   */
  saveArchivedPing(ping) {
    let promise = this._saveArchivedPingTask(ping);
    this._activelyArchiving.add(promise);
    promise.then(
      r => {
        this._activelyArchiving.delete(promise);
      },
      e => {
        this._activelyArchiving.delete(promise);
      }
    );
    return promise;
  },

  async _saveArchivedPingTask(ping) {
    const creationDate = new Date(ping.creationDate);
    if (this._archivedPings.has(ping.id)) {
      const data = this._archivedPings.get(ping.id);
      if (data.timestampCreated > creationDate.getTime()) {
        this._log.error(
          "saveArchivedPing - trying to overwrite newer ping with the same id"
        );
        return Promise.reject(
          new Error("trying to overwrite newer ping with the same id")
        );
      }
      this._log.warn(
        "saveArchivedPing - overwriting older ping with the same id"
      );
    }

    // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4').
    const filePath =
      getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4";
    await IOUtils.makeDirectory(PathUtils.parent(filePath));
    await this.savePingToFile(
      ping,
      filePath,
      /* overwrite*/ true,
      /* compressed*/ true
    );

    this._archivedPings.set(ping.id, {
      timestampCreated: creationDate.getTime(),
      type: internString(ping.type),
    });

    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add();
    return undefined;
  },

  /**
   * Load an archived ping from disk.
   *
   * @param {string} id The pings id.
   * @return {promise<object>} Promise that is resolved with the ping data.
   */
  async loadArchivedPing(id) {
    const data = this._archivedPings.get(id);
    if (!data) {
      this._log.trace("loadArchivedPing - no ping with id: " + id);
      return Promise.reject(
        new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id)
      );
    }

    const path = getArchivedPingPath(
      id,
      new Date(data.timestampCreated),
      data.type
    );
    const pathCompressed = path + "lz4";

    // Purge pings which are too big.
    let checkSize = async function (path) {
      const fileSize = await IOUtils.stat(path).then(info => info.size);
      if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
        Telemetry.getHistogramById(
          "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB"
        ).add(Math.floor(fileSize / 1024 / 1024));
        Telemetry.getHistogramById(
          "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED"
        ).add();
        await IOUtils.remove(path, { ignoreAbsent: true });
        throw new Error(
          `loadArchivedPing - exceeded the maximum ping size: ${fileSize}`
        );
      }
    };

    let ping;
    try {
      // Try to load a compressed version of the archived ping first.
      this._log.trace(
        "loadArchivedPing - loading ping from: " + pathCompressed
      );
      await checkSize(pathCompressed);
      ping = await this.loadPingFile(pathCompressed, /* compressed*/ true);
    } catch (ex) {
      if (!ex.becauseNoSuchFile) {
        throw ex;
      }
      // If that fails, look for the uncompressed version.
      this._log.trace(
        "loadArchivedPing - compressed ping not found, loading: " + path
      );
      await checkSize(path);
      ping = await this.loadPingFile(path, /* compressed*/ false);
    }

    return ping;
  },

  /**
   * Saves session data to disk.
   */
  saveSessionData(sessionData) {
    return this._stateSaveSerializer.enqueueTask(() =>
      this._saveSessionData(sessionData)
    );
  },

  async _saveSessionData(sessionData) {
    await IOUtils.makeDirectory(lazy.gDataReportingDir, {
      createAncestors: false,
    });

    let filePath = PathUtils.join(
      lazy.gDataReportingDir,
      SESSION_STATE_FILE_NAME
    );
    try {
      await IOUtils.writeJSON(filePath, sessionData);
    } catch (e) {
      this._log.error(
        `_saveSessionData - Failed to write session data to ${filePath}`,
        e
      );
      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1);
    }
  },

  /**
   * Loads session data from the session data file.
   * @return {Promise<Object>} A promise resolved with an object on success,
   *                           with null otherwise.
   */
  loadSessionData() {
    return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData());
  },

  async _loadSessionData() {
    const dataFile = PathUtils.join(
      PathUtils.profileDir,
      DATAREPORTING_DIR,
      SESSION_STATE_FILE_NAME
    );
    let content;
    try {
      content = await IOUtils.readUTF8(dataFile);
    } catch (ex) {
      this._log.info("_loadSessionData - can not load session data file", ex);
      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1);
      return null;
    }

    let data;
    try {
      data = JSON.parse(content);
    } catch (ex) {
      this._log.error("_loadSessionData - failed to parse session data", ex);
      Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1);
      return null;
    }

    return data;
  },

  /**
   * Remove an archived ping from disk.
   *
   * @param {string} id The pings id.
   * @param {number} timestampCreated The pings creation timestamp.
   * @param {string} type The pings type.
   * @return {promise<object>} Promise that is resolved when the pings is removed.
   */
  async _removeArchivedPing(id, timestampCreated, type) {
    this._log.trace(
      "_removeArchivedPing - id: " +
        id +
        ", timestampCreated: " +
        timestampCreated +
        ", type: " +
        type
    );
    const path = getArchivedPingPath(id, new Date(timestampCreated), type);
    const pathCompressed = path + "lz4";

    this._log.trace("_removeArchivedPing - removing ping from: " + path);
    await IOUtils.remove(path);
    await IOUtils.remove(pathCompressed);
    // Remove the ping from the cache.
    this._archivedPings.delete(id);
  },

  /**
   * Clean the pings archive by removing old pings.
   *
   * @return {Promise} Resolved when the cleanup task completes.
   */
  runCleanPingArchiveTask() {
    // If there's an archive cleaning task already running, return it.
    if (this._cleanArchiveTask) {
      return this._cleanArchiveTask;
    }

    // Make sure to clear |_cleanArchiveTask| once done.
    let clear = () => (this._cleanArchiveTask = null);
    // Since there's no archive cleaning task running, start it.
    this._cleanArchiveTask = this._cleanArchive().then(clear, clear);
    return this._cleanArchiveTask;
  },

  /**
   * Removes pings which are too old from the pings archive.
   * @return {Promise} Resolved when the ping age check is complete.
   */
  async _purgeOldPings() {
    this._log.trace("_purgeOldPings");

    const nowDate = Policy.now();
    const startTimeStamp = nowDate.getTime();

    // Keep track of the newest removed month to update the cache, if needed.
    let newestRemovedMonthTimestamp = null;
    let evictedDirsCount = 0;
    let maxDirAgeInMonths = 0;

    // Walk through the monthly subdirs of the form <YYYY-MM>/
    for (const path of await IOUtils.getChildren(lazy.gPingsArchivePath)) {
      const info = await IOUtils.stat(path);
      if (info.type !== "directory") {
        continue;
      }

      const name = PathUtils.filename(path);

      if (this._shutdown) {
        this._log.trace(
          "_purgeOldPings - Terminating the clean up task due to shutdown"
        );
        return;
      }

      if (!isValidArchiveDir(name)) {
        this._log.warn(
          `_purgeOldPings - skipping invalidly named subdirectory ${path}`
        );
        continue;
      }

      const archiveDate = getDateFromArchiveDir(name);
      if (!archiveDate) {
        this._log.warn(
          `_purgeOldPings - skipping invalid subdirectory date ${path}`
        );
        continue;
      }

      // If this archive directory is older than allowed, remove it.
      if (
        startTimeStamp - archiveDate.getTime() >
        MAX_ARCHIVED_PINGS_RETENTION_MS
      ) {
        try {
          await IOUtils.remove(path, { recursive: true });
          evictedDirsCount++;

          // Update the newest removed month.
          newestRemovedMonthTimestamp = Math.max(
            archiveDate,
            newestRemovedMonthTimestamp
          );
        } catch (ex) {
          this._log.error(`_purgeOldPings - Unable to remove ${path}`, ex);
        }
      } else {
        // We're not removing this directory, so record the age for the oldest directory.
        const dirAgeInMonths = Utils.getElapsedTimeInMonths(
          archiveDate,
          nowDate
        );
        maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths);
      }
    }

    // Trigger scanning of the archived pings.
    await this.loadArchivedPingList();

    // Refresh the cache: we could still skip this, but it's cheap enough to keep it
    // to avoid introducing task dependencies.
    if (newestRemovedMonthTimestamp) {
      // Scan the archive cache for pings older than the newest directory pruned above.
      for (let [id, info] of this._archivedPings) {
        const timestampCreated = new Date(info.timestampCreated);
        if (timestampCreated.getTime() > newestRemovedMonthTimestamp) {
          continue;
        }
        // Remove outdated pings from the cache.
        this._archivedPings.delete(id);
      }
    }

    const endTimeStamp = Policy.now().getTime();

    // Save the time it takes to evict old directories and the eviction count.
    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").add(
      evictedDirsCount
    );
    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS").add(
      Math.ceil(endTimeStamp - startTimeStamp)
    );
    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE").add(
      maxDirAgeInMonths
    );
  },

  /**
   * Enforce a disk quota for the pings archive.
   * @return {Promise} Resolved when the quota check is complete.
   */
  async _enforceArchiveQuota() {
    this._log.trace("_enforceArchiveQuota");
    let startTimeStamp = Policy.now().getTime();

    // Build an ordered list, from newer to older, of archived pings.
    let pingList = Array.from(this._archivedPings, p => ({
      id: p[0],
      timestampCreated: p[1].timestampCreated,
      type: p[1].type,
    }));

    pingList.sort((a, b) => b.timestampCreated - a.timestampCreated);

    // If our archive is too big, we should reduce it to reach 90% of the quota.
    const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9;
    // The index of the last ping to keep. Pings older than this one will be deleted if
    // the archive exceeds the quota.
    let lastPingIndexToKeep = null;
    let archiveSizeInBytes = 0;

    // Find the disk size of the archive.
    for (let i = 0; i < pingList.length; i++) {
      if (this._shutdown) {
        this._log.trace(
          "_enforceArchiveQuota - Terminating the clean up task due to shutdown"
        );
        return;
      }

      let ping = pingList[i];

      // Get the size for this ping.
      const fileSize = await getArchivedPingSize(
        ping.id,
        new Date(ping.timestampCreated),
        ping.type
      );
      if (!fileSize) {
        this._log.warn(
          "_enforceArchiveQuota - Unable to find the size of ping " + ping.id
        );
        continue;
      }

      // Enforce a maximum file size limit on archived pings.
      if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
        this._log.error(
          "_enforceArchiveQuota - removing file exceeding size limit, size: " +
            fileSize
        );
        // We just remove the ping from the disk, we don't bother removing it from pingList
        // since it won't contribute to the quota.
        await this._removeArchivedPing(
          ping.id,
          ping.timestampCreated,
          ping.type
        ).catch(e =>
          this._log.error(
            "_enforceArchiveQuota - failed to remove archived ping" + ping.id
          )
        );
        Telemetry.getHistogramById(
          "TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB"
        ).add(Math.floor(fileSize / 1024 / 1024));
        Telemetry.getHistogramById(
          "TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED"
        ).add();
        continue;
      }

      archiveSizeInBytes += fileSize;

      if (archiveSizeInBytes < SAFE_QUOTA) {
        // We save the index of the last ping which is ok to keep in order to speed up ping
        // pruning.
        lastPingIndexToKeep = i;
      } else if (archiveSizeInBytes > Policy.getArchiveQuota()) {
        // Ouch, our ping archive is too big. Bail out and start pruning!
        break;
      }
    }

    // Save the time it takes to check if the archive is over-quota.
    Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS").add(
      Math.round(Policy.now().getTime() - startTimeStamp)
    );

    let submitProbes = (sizeInMB, evictedPings, elapsedMs) => {
      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB);
      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(
        evictedPings
      );
      Telemetry.getHistogramById(
        "TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS"
      ).add(elapsedMs);
    };

    // Check if we're using too much space. If not, submit the archive size and bail out.
    if (archiveSizeInBytes < Policy.getArchiveQuota()) {
      submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0);
      return;
    }

    this._log.info(
      "_enforceArchiveQuota - archive size: " +
        archiveSizeInBytes +
        "bytes" +
        ", safety quota: " +
        SAFE_QUOTA +
        "bytes"
    );

    startTimeStamp = Policy.now().getTime();
    let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);

    // Remove all the pings older than the last one which we are safe to keep.
    for (let ping of pingsToPurge) {
      if (this._shutdown) {
        this._log.trace(
          "_enforceArchiveQuota - Terminating the clean up task due to shutdown"
        );
        return;
      }

      // This list is guaranteed to be in order, so remove the pings at its
      // beginning (oldest).
      await this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type);
    }

    const endTimeStamp = Policy.now().getTime();
    submitProbes(
      ARCHIVE_SIZE_PROBE_SPECIAL_VALUE,
      pingsToPurge.length,
      Math.ceil(endTimeStamp - startTimeStamp)
    );
  },

  async _cleanArchive() {
    this._log.trace("cleanArchiveTask");

    if (!(await IOUtils.exists(lazy.gPingsArchivePath))) {
      return;
    }

    // Remove pings older than allowed.
    try {
      await this._purgeOldPings();
    } catch (ex) {
      this._log.error(
        "_cleanArchive - There was an error removing old directories",
        ex
      );
    }

    // Make sure we respect the archive disk quota.
    await this._enforceArchiveQuota();
  },

  /**
   * Run the task to enforce the pending pings quota.
   *
   * @return {Promise} Resolved when the cleanup task completes.
   */
  async runEnforcePendingPingsQuotaTask() {
    // If there's a cleaning task already running, return it.
    if (this._enforcePendingPingsQuotaTask) {
      return this._enforcePendingPingsQuotaTask;
    }

    // Since there's no quota enforcing task running, start it.
    try {
      this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota();
      await this._enforcePendingPingsQuotaTask;
    } finally {
      this._enforcePendingPingsQuotaTask = null;
    }
    return undefined;
  },

  /**
   * Enforce a disk quota for the pending pings.
   * @return {Promise} Resolved when the quota check is complete.
   */
  async _enforcePendingPingsQuota() {
    this._log.trace("_enforcePendingPingsQuota");
    let startTimeStamp = Policy.now().getTime();

    // Build an ordered list, from newer to older, of pending pings.
    let pingList = Array.from(this._pendingPings, p => ({
      id: p[0],
      lastModified: p[1].lastModified,
    }));

    pingList.sort((a, b) => b.lastModified - a.lastModified);

    // If our pending pings directory is too big, we should reduce it to reach 90% of the quota.
    const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9;
    // The index of the last ping to keep. Pings older than this one will be deleted if
    // the pending pings directory size exceeds the quota.
    let lastPingIndexToKeep = null;
    let pendingPingsSizeInBytes = 0;

    // Find the disk size of the pending pings directory.
    for (let i = 0; i < pingList.length; i++) {
      if (this._shutdown) {
        this._log.trace(
          "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"
        );
        return;
      }

      let ping = pingList[i];

      // Get the size for this ping.
      const fileSize = await getPendingPingSize(ping.id);
      if (!fileSize) {
        this._log.warn(
          "_enforcePendingPingsQuota - Unable to find the size of ping " +
            ping.id
        );
        continue;
      }

      pendingPingsSizeInBytes += fileSize;
      if (pendingPingsSizeInBytes < SAFE_QUOTA) {
        // We save the index of the last ping which is ok to keep in order to speed up ping
        // pruning.
        lastPingIndexToKeep = i;
      } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) {
        // Ouch, our pending pings directory size is too big. Bail out and start pruning!
        break;
      }
    }

    // Save the time it takes to check if the pending pings are over-quota.
    Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS").add(
      Math.round(Policy.now().getTime() - startTimeStamp)
    );

    let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => {
      Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add(
        sizeInMB
      );
      Telemetry.getHistogramById(
        "TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA"
      ).add(evictedPings);
      Telemetry.getHistogramById(
        "TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS"
      ).add(elapsedMs);
    };

    // Check if we're using too much space. If not, bail out.
    if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) {
      recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0);
      return;
    }

    this._log.info(
      "_enforcePendingPingsQuota - size: " +
        pendingPingsSizeInBytes +
        "bytes" +
        ", safety quota: " +
        SAFE_QUOTA +
        "bytes"
    );

    startTimeStamp = Policy.now().getTime();
    let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1);

    // Remove all the pings older than the last one which we are safe to keep.
    for (let ping of pingsToPurge) {
      if (this._shutdown) {
        this._log.trace(
          "_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"
        );
        return;
      }

      // This list is guaranteed to be in order, so remove the pings at its
      // beginning (oldest).
      await this.removePendingPing(ping.id);
    }

    const endTimeStamp = Policy.now().getTime();
    // We don't know the size of the pending pings directory if we are above the quota,
    // since we stop scanning once we reach the quota. We use a special value to show
    // this condition.
    recordHistograms(
      PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE,
      pingsToPurge.length,
      Math.ceil(endTimeStamp - startTimeStamp)
    );
  },

  /**
   * Reset the storage state in tests.
   */
  reset() {
    this._shutdown = false;
    this._scannedArchiveDirectory = false;
    this._archivedPings = new Map();
    this._scannedPendingDirectory = false;
    this._pendingPings = new Map();
  },

  /**
   * Get a list of info on the archived pings.
   * This will scan the archive directory and grab basic data about the existing
   * pings out of their filename.
   *
   * @return {promise<sequence<object>>}
   */
  async loadArchivedPingList() {
    // If there's an archive loading task already running, return it.
    if (this._scanArchiveTask) {
      return this._scanArchiveTask;
    }

    await waitForAll(this._activelyArchiving);

    if (this._scannedArchiveDirectory) {
      this._log.trace(
        "loadArchivedPingList - Archive already scanned, hitting cache."
      );
      return this._archivedPings;
    }

    // Since there's no archive loading task running, start it.
    let result;
    try {
      this._scanArchiveTask = this._scanArchive();
      result = await this._scanArchiveTask;
    } finally {
      this._scanArchiveTask = null;
    }
    return result;
  },

  async _scanArchive() {
    this._log.trace("_scanArchive");

    let submitProbes = (pingCount, dirCount) => {
      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").add(
        pingCount
      );
      Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT").add(
        dirCount
      );
    };

    if (!(await IOUtils.exists(lazy.gPingsArchivePath))) {
      submitProbes(0, 0);
      return new Map();
    }

    let subDirCount = 0;
    // Walk through the monthly subdirs of the form <YYYY-MM>/
    for (const path of await IOUtils.getChildren(lazy.gPingsArchivePath)) {
      const info = await IOUtils.stat(path);

      if (info.type !== "directory") {
        continue;
      }

      const name = PathUtils.filename(path);
      if (!isValidArchiveDir(name)) {
        continue;
      }

      subDirCount++;

      this._log.trace(`_scanArchive - checking in subdir: ${path}`);
      const pingPaths = [];
      for (const ping of await IOUtils.getChildren(path)) {
        const info = await IOUtils.stat(ping);
        if (info.type !== "directory") {
          pingPaths.push(ping);
        }
      }

      // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]".
      for (const path of pingPaths) {
        const filename = PathUtils.filename(path);
        // data may be null if the filename doesn't match the above format.
        let data = this._getArchivedPingDataFromFileName(filename);
        if (!data) {
          continue;
        }

        // In case of conflicts, overwrite only with newer pings.
        if (this._archivedPings.has(data.id)) {
          const overwrite =
            data.timestamp > this._archivedPings.get(data.id).timestampCreated;
          this._log.warn(
            `_scanArchive - have seen this id before: ${data.id}, overwrite: ${overwrite}`
          );
          if (!overwrite) {
            continue;
          }

          await this._removeArchivedPing(
            data.id,
            data.timestampCreated,
            data.type
          ).catch(e =>
            this._log.warn("_scanArchive - failed to remove ping", e)
          );
        }

        this._archivedPings.set(data.id, {
          timestampCreated: data.timestamp,
          type: internString(data.type),
        });
      }
    }

    // Mark the archive as scanned, so we no longer hit the disk.
    this._scannedArchiveDirectory = true;
    // Update the ping and directories count histograms.
    submitProbes(this._archivedPings.size, subDirCount);
    return this._archivedPings;
  },

  /**
   * Save a single ping to a file.
   *
   * @param {object} ping The content of the ping to save.
   * @param {string} file The destination file.
   * @param {bool} overwrite If |true|, the file will be overwritten if it exists,
   * if |false| the file will not be overwritten and no error will be reported if
   * the file exists.
   * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no
   * compression will be used.
   * @returns {promise}
   */
  async savePingToFile(ping, filePath, overwrite, compress = false) {
    try {
      this._log.trace("savePingToFile - path: " + filePath);
      await IOUtils.writeJSON(filePath, ping, {
        compress,
        mode: overwrite ? "overwrite" : "create",
        tmpPath: `${filePath}.tmp`,
      });
    } catch (e) {
      if (
        !DOMException.isInstance(e) ||
        e.name !== "NoModificationAllowedError"
      ) {
        throw e;
      }
    }
  },

  /**
   * Save a ping to its file.
   *
   * @param {object} ping The content of the ping to save.
   * @param {bool} overwrite If |true|, the file will be overwritten
   * if it exists.
   * @returns {promise}
   */
  async savePing(ping, overwrite) {
    await getPingDirectory();
    let file = pingFilePath(ping);
    await this.savePingToFile(ping, file, overwrite);
    return file;
  },

  /**
   * Remove the file for a ping
   *
   * @param {object} ping The ping.
   * @returns {promise}
   */
  cleanupPingFile(ping) {
    return IOUtils.remove(pingFilePath(ping));
  },

  savePendingPing(ping) {
    let p = this.savePing(ping, true).then(path => {
      this._pendingPings.set(ping.id, {
        path,
        lastModified: Policy.now().getTime(),
      });
      this._log.trace("savePendingPing - saved ping with id " + ping.id);
    });
    this._trackPendingPingSaveTask(p);
    return p;
  },

  async loadPendingPing(id) {
    this._log.trace("loadPendingPing - id: " + id);
    let info = this._pendingPings.get(id);
    if (!info) {
      this._log.trace("loadPendingPing - unknown id " + id);
      throw new Error(
        "TelemetryStorage.loadPendingPing - no ping with id " + id
      );
    }

    // Try to get the dimension of the ping. If that fails, update the histograms.
    let fileSize = 0;
    try {
      fileSize = await IOUtils.stat(info.path).then(stat => stat.size);
    } catch (e) {
      if (!DOMException.isInstance(e) || e.name !== "NotFoundError") {
        throw e;
      }
      // Fall through and let |loadPingFile| report the error.
    }

    // Purge pings which are too big.
    if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) {
      await this.removePendingPing(id);
      Telemetry.getHistogramById(
        "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB"
      ).add(Math.floor(fileSize / 1024 / 1024));
      Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add();

      // Currently we don't have the ping type available without loading the ping from disk.
      // Bug 1384903 will fix that.
      lazy.TelemetryHealthPing.recordDiscardedPing("<unknown>");
      throw new Error(
        "loadPendingPing - exceeded the maximum ping size: " + fileSize
      );
    }

    // Try to load the ping file. Update the related histograms on failure.
    let ping;
    try {
      ping = await this.loadPingFile(info.path, false);
    } catch (e) {
      // If we failed to load the ping, check what happened and update the histogram.
      if (e instanceof PingReadError) {
        Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add();
      } else if (e instanceof PingParseError) {
        Telemetry.getHistogramById(
          "TELEMETRY_PENDING_LOAD_FAILURE_PARSE"
        ).add();
      }

      // Remove the ping from the cache, so we don't try to load it again.
      this._pendingPings.delete(id);
      // Then propagate the rejection.
      throw e;
    }

    return ping;
  },

  removePendingPing(id) {
    let info = this._pendingPings.get(id);
    if (!info) {
      this._log.trace("removePendingPing - unknown id " + id);
      return Promise.resolve();
    }

    this._log.trace(
      "removePendingPing - deleting ping with id: " +
        id +
        ", path: " +
        info.path
    );
    this._pendingPings.delete(id);
    return IOUtils.remove(info.path).catch(ex =>
      this._log.error("removePendingPing - failed to remove ping", ex)
    );
  },

  /**
   * Track any pending ping save tasks through the promise passed here.
   * This is needed to block on any outstanding ping save activity.
   *
   * @param {Object<Promise>} The save promise to track.
   */
  _trackPendingPingSaveTask(promise) {
    let clear = () => this._activePendingPingSaves.delete(promise);
    promise.then(clear, clear);
    this._activePendingPingSaves.add(promise);
  },

  /**
   * Return a promise that allows to wait on pending pings being saved.
   * @return {Object<Promise>} A promise resolved when all the pending pings save promises
   *         are resolved.
   */
  promisePendingPingSaves() {
    // Make sure to wait for all the promises, even if they reject. We don't need to log
    // the failures here, as they are already logged elsewhere.
    return waitForAll(this._activePendingPingSaves);
  },

  /**
   * Run the task to remove all the pending pings
   *
   * @return {Promise} Resolved when the pings are removed.
   */
  async runRemovePendingPingsTask() {
    // If we already have a pending pings removal task active, return that.
    if (this._removePendingPingsTask) {
      return this._removePendingPingsTask;
    }

    // Start the task to remove all pending pings. Also make sure to clear the task once done.
    try {
      this._removePendingPingsTask = this.removePendingPings();
      await this._removePendingPingsTask;
    } finally {
      this._removePendingPingsTask = null;
    }
    return undefined;
  },

  async removePendingPings() {
    this._log.trace("removePendingPings - removing all pending pings");

    // Wait on pending pings still being saved, so so we don't miss removing them.
    await this.promisePendingPingSaves();

    // Individually remove existing pings, so we don't interfere with operations expecting
    // the pending pings directory to exist.
    const directory = TelemetryStorage.pingDirectoryPath;

    if (!(await IOUtils.exists(directory))) {
      this._log.trace(
        "removePendingPings - the pending pings directory doesn't exist"
      );
      return;
    }

    for (const path of await IOUtils.getChildren(directory)) {
      let info;
      try {
        info = await IOUtils.stat(path);
      } catch (ex) {
        // It is possible there is another task removing a ping in between
        // reading the directory and calling stat.
        //
        // On Windows, attempting to call GetFileAttributesEx() on a file
        // pending deletion will result in ERROR_ACCESS_DENIED, which will
        // propagate to here as a NotAllowedError.
        if (
          DOMException.isInstance(ex) &&
          (ex.name === "NotFoundError" || ex.name === "NotAllowedError")
        ) {
          continue;
        }

        throw ex;
      }

      if (info.type === "directory") {
        continue;
      }

      try {
        await IOUtils.remove(path);
      } catch (ex) {
        this._log.error(
          `removePendingPings - failed to remove file ${path}`,
          ex
        );
        continue;
      }
    }
  },

  /**
   * Iterate through all pings in the userApplicationDataDir under the "Pending Pings" sub-directory
   * and yield each file.
   */
  async *_iterateAppDataPings() {
    this._log.trace("_iterateAppDataPings");

    let uAppDataDir;
    try {
      uAppDataDir = Services.dirsvc.get("UAppData", Ci.nsIFile);
    } catch (ex) {
      // The test suites might not create and define the "UAppData" directory.
      // We account for that here instead of manually going through each test using
      // telemetry to manually create the directory and define the constant.
      this._log.trace(
        "_iterateAppDataPings - userApplicationDataDir is not defined. Is this a test?"
      );
      return;
    }

    const appDataPendingPings = PathUtils.join(
      uAppDataDir.path,
      "Pending Pings"
    );

    // Check if appDataPendingPings exists and bail out if it doesn't.
    if (!(await IOUtils.exists(appDataPendingPings))) {
      this._log.trace(
        "_iterateAppDataPings - the AppData pending pings directory doesn't exist."
      );
      return;
    }

    // Iterate through the pending ping files.
    for (const path of await IOUtils.getChildren(appDataPendingPings)) {
      const info = await IOUtils.stat(path);
      if (info.type !== "directory") {
        yield path;
      }
    }
  },

  /**
   * Remove all pings that are stored in the userApplicationDataDir
   * under the "Pending Pings" sub-directory.
   */
  async removeAppDataPings() {
    this._log.trace("removeAppDataPings");

    for await (const path of this._iterateAppDataPings()) {
      try {
        await IOUtils.remove(path);
      } catch (ex) {
        this._log.error(
          `removeAppDataPings - failed to remove file ${path}`,
          ex
        );
      }
    }
  },

  /**
   * Migrate pings that are stored in the userApplicationDataDir
   * under the "Pending Pings" sub-directory.
   */
  async _migrateAppDataPings() {
    this._log.trace("_migrateAppDataPings");

    for await (const path of this._iterateAppDataPings()) {
      try {
        // Load the ping data from the original file.
        const pingData = await this.loadPingFile(path);

        // Save it among the pending pings in the user profile, overwrite on
        // ping id collision.
        await TelemetryStorage.savePing(pingData, true);
      } catch (ex) {
        this._log.error(
          `_migrateAppDataPings - failed to load or migrate file. Removing ${path}`,
          ex
        );
      }

      try {
        // Finally remove the file.
        await IOUtils.remove(path);
      } catch (ex) {
        this._log.error(
          `_migrateAppDataPings - failed to remove file ${path}`,
          ex
        );
      }
    }
  },

  loadPendingPingList() {
    // If we already have a pending scanning task active, return that.
    if (this._scanPendingPingsTask) {
      return this._scanPendingPingsTask;
    }

    if (this._scannedPendingDirectory) {
      this._log.trace(
        "loadPendingPingList - Pending already scanned, hitting cache."
      );
      return Promise.resolve(this._buildPingList());
    }

    // Since there's no pending pings scan task running, start it.
    // Also make sure to clear the task once done.
    this._scanPendingPingsTask = this._scanPendingPings().then(
      pings => {
        this._scanPendingPingsTask = null;
        return pings;
      },
      ex => {
        this._scanPendingPingsTask = null;
        throw ex;
      }
    );
    return this._scanPendingPingsTask;
  },

  getPendingPingList() {
    return this._buildPingList();
  },

  async _scanPendingPings() {
    this._log.trace("_scanPendingPings");

    // Before pruning the pending pings, migrate over the ones from the user
    // application data directory (mainly crash pings that failed to be sent).
    await this._migrateAppDataPings();

    const directory = TelemetryStorage.pingDirectoryPath;
    if (!(await IOUtils.exists(directory))) {
      return [];
    }

    const files = [];
    for (const path of await IOUtils.getChildren(directory)) {
      if (this._shutdown) {
        return [];
      }

      try {
        const info = await IOUtils.stat(path);
        if (info.type !== "directory") {
          files.push({ path, info });
        }
      } catch (ex) {
        this._log.error(`_scanPendingPings - failed to stat file ${path}`, ex);
        continue;
      }
    }

    for (const { path, info } of files) {
      if (this._shutdown) {
        return [];
      }

      // Enforce a maximum file size limit on pending pings.
      if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) {
        this._log.error(
          `_scanPendingPings - removing file exceeding size limit ${path}`
        );
        try {
          await IOUtils.remove(path);
        } catch (ex) {
          this._log.error(
            `_scanPendingPings - failed to remove file ${path}`,
            ex
          );
        } finally {
          Telemetry.getHistogramById(
            "TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB"
          ).add(Math.floor(info.size / 1024 / 1024));
          Telemetry.getHistogramById(
            "TELEMETRY_PING_SIZE_EXCEEDED_PENDING"
          ).add();

          // Currently we don't have the ping type available without loading the ping from disk.
          // Bug 1384903 will fix that.
          lazy.TelemetryHealthPing.recordDiscardedPing("<unknown>");
        }
        continue;
      }

      let id = PathUtils.filename(path);
      if (!UUID_REGEX.test(id)) {
        this._log.trace(`_scanPendingPings - filename is not a UUID: ${id}`);
        id = Utils.generateUUID();
      }

      this._pendingPings.set(id, {
        path,
        lastModified: info.lastModified,
      });
    }

    this._scannedPendingDirectory = true;
    return this._buildPingList();
  },

  _buildPingList() {
    const list = Array.from(this._pendingPings, p => ({
      id: p[0],
      lastModified: p[1].lastModified,
    }));

    list.sort((a, b) => b.lastModified - a.lastModified);
    return list;
  },

  /**
   * Loads a ping file.
   * @param {String} aFilePath The path of the ping file.
   * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4.
   * @return {Promise<Object>} A promise resolved with the ping content or rejected if the
   *                           ping contains invalid data.
   * @throws {PingReadError} There was an error while reading the ping file from the disk.
   * @throws {PingParseError} There was an error while parsing the JSON content of the ping file.
   */
  async loadPingFile(aFilePath, aCompressed = false) {
    let rawPing;
    try {
      rawPing = await IOUtils.readUTF8(aFilePath, { decompress: aCompressed });
    } catch (e) {
      this._log.trace(`loadPingfile - unreadable ping ${aFilePath}`, e);
      throw new PingReadError(
        e.message,
        DOMException.isInstance(e) && e.name === "NotFoundError"
      );
    }

    let ping;
    try {
      ping = JSON.parse(rawPing);
    } catch (e) {
      this._log.trace(`loadPingfile - unparseable ping ${aFilePath}`, e);
      await IOUtils.remove(aFilePath).catch(ex => {
        this._log.error(
          `loadPingFile - failed removing unparseable ping file ${aFilePath}`,
          ex
        );
      });
      throw new PingParseError(e.message);
    }

    return ping;
  },

  /**
   * Archived pings are saved with file names of the form:
   * "<timestamp>.<uuid>.<type>.[json|jsonlz4]"
   * This helper extracts that data from a given filename.
   *
   * @param fileName {String} The filename.
   * @return {Object} Null if the filename didn't match the expected form.
   *                  Otherwise an object with the extracted data in the form:
   *                  { timestamp: <number>,
   *                    id: <string>,
   *                    type: <string> }
   */
  _getArchivedPingDataFromFileName(fileName) {
    // Extract the parts.
    let parts = fileName.split(".");
    if (parts.length != 4) {
      this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");
      return null;
    }

    let [timestamp, uuid, type, extension] = parts;
    if (extension != "json" && extension != "jsonlz4") {
      this._log.trace(
        "_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension"
      );
      return null;
    }

    // Check for a valid timestamp.
    timestamp = parseInt(timestamp);
    if (Number.isNaN(timestamp)) {
      this._log.trace(
        "_getArchivedPingDataFromFileName - should have a valid timestamp"
      );
      return null;
    }

    // Check for a valid UUID.
    if (!UUID_REGEX.test(uuid)) {
      this._log.trace(
        "_getArchivedPingDataFromFileName - should have a valid id"
      );
      return null;
    }

    // Check for a valid type string.
    const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
    if (!typeRegex.test(type)) {
      this._log.trace(
        "_getArchivedPingDataFromFileName - should have a valid type"
      );
      return null;
    }

    return {
      timestamp,
      id: uuid,
      type,
    };
  },

  async saveAbortedSessionPing(ping) {
    this._log.trace(
      "saveAbortedSessionPing - ping path: " + lazy.gAbortedSessionFilePath
    );
    await IOUtils.makeDirectory(lazy.gDataReportingDir);

    return this._abortedSessionSerializer.enqueueTask(() =>
      this.savePingToFile(ping, lazy.gAbortedSessionFilePath, true)
    );
  },

  async loadAbortedSessionPing() {
    let ping = null;
    try {
      ping = await this.loadPingFile(lazy.gAbortedSessionFilePath);
    } catch (ex) {
      if (ex.becauseNoSuchFile) {
        this._log.trace("loadAbortedSessionPing - no such file");
      } else {
        this._log.error("loadAbortedSessionPing - error loading ping", ex);
      }
    }
    return ping;
  },

  removeAbortedSessionPing() {
    return this._abortedSessionSerializer.enqueueTask(async () => {
      try {
        await IOUtils.remove(lazy.gAbortedSessionFilePath, {
          ignoreAbsent: false,
        });
        this._log.trace("removeAbortedSessionPing - success");
      } catch (ex) {
        if (DOMException.isInstance(ex) && ex.name === "NotFoundError") {
          this._log.trace("removeAbortedSessionPing - no such file");
        } else {
          this._log.error("removeAbortedSessionPing - error removing ping", ex);
        }
      }
    });
  },

  async saveUninstallPing(ping) {
    if (AppConstants.platform != "win") {
      return;
    }

    // Remove any old pings from this install first.
    await this.removeUninstallPings();

    let { directory: pingFile, file } = Policy.getUninstallPingPath(ping.id);
    pingFile.append(file);

    await this.savePingToFile(ping, pingFile.path, /* overwrite */ true);
  },

  async removeUninstallPings() {
    if (AppConstants.platform != "win") {
      return;
    }

    const { directory, file } = Policy.getUninstallPingPath("*");
    const [prefix, suffix] = file.split("*");

    for (const path of await IOUtils.getChildren(directory.path)) {
      const filename = PathUtils.filename(path);
      if (!filename.startsWith(prefix) || !filename.endsWith(suffix)) {
        continue;
      }

      this._log.trace("removeUninstallPings - removing", path);
      try {
        await IOUtils.remove(path);
        this._log.trace("removeUninstallPings - success");
      } catch (ex) {
        if (DOMException.isInstance(ex) && ex.name === "NotFoundError") {
          this._log.trace("removeUninstallPings - no such file");
        } else {
          this._log.error("removeUninstallPings - error removing ping", ex);
        }
      }
    }
  },

  /**
   * Remove FHR database files. This is temporary and will be dropped in
   * the future.
   * @return {Promise} Resolved when the database files are deleted.
   */
  async removeFHRDatabase() {
    this._log.trace("removeFHRDatabase");

    // Let's try to remove the FHR DB with the default filename first.
    const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite";

    // Even if it's uncommon, there may be 2 additional files: - a "write ahead log"
    // (-wal) file and a "shared memory file" (-shm). We need to remove them as well.
    let FILES_TO_REMOVE = [
      PathUtils.join(PathUtils.profileDir, FHR_DB_DEFAULT_FILENAME),
      PathUtils.join(PathUtils.profileDir, FHR_DB_DEFAULT_FILENAME + "-wal"),
      PathUtils.join(PathUtils.profileDir, FHR_DB_DEFAULT_FILENAME + "-shm"),
    ];

    // FHR could have used either the default DB file name or a custom one
    // through this preference.
    const FHR_DB_CUSTOM_FILENAME = Services.prefs.getStringPref(
      "datareporting.healthreport.dbName",
      undefined
    );
    if (FHR_DB_CUSTOM_FILENAME) {
      FILES_TO_REMOVE.push(
        PathUtils.join(PathUtils.profileDir, FHR_DB_CUSTOM_FILENAME),
        PathUtils.join(PathUtils.profileDir, FHR_DB_CUSTOM_FILENAME + "-wal"),
        PathUtils.join(PathUtils.profileDir, FHR_DB_CUSTOM_FILENAME + "-shm")
      );
    }

    for (let f of FILES_TO_REMOVE) {
      await IOUtils.remove(f).catch(e =>
        this._log.error(`removeFHRDatabase - failed to remove ${f}`, e)
      );
    }
  },
};

// Utility functions

function pingFilePath(ping) {
  // Support legacy ping formats, who don't have an "id" field, but a "slug" field.
  let pingIdentifier = ping.slug ? ping.slug : ping.id;

  if (typeof pingIdentifier === "undefined" || pingIdentifier === null) {
    throw new Error(
      "Incompatible ping format -- ping has no slug or id attribute"
    );
  }

  return PathUtils.join(TelemetryStorage.pingDirectoryPath, pingIdentifier);
}

function getPingDirectory() {
  return (async function () {
    let directory = TelemetryStorage.pingDirectoryPath;

    if (!(await IOUtils.exists(directory))) {
      await IOUtils.makeDirectory(directory, { permissions: 0o700 });
    }

    return directory;
  })();
}

/**
 * Build the path to the archived ping.
 * @param {String} aPingId The ping id.
 * @param {Object} aDate The ping creation date.
 * @param {String} aType The ping type.
 * @return {String} The full path to the archived ping.
 */
function getArchivedPingPath(aPingId, aDate, aType) {
  // Get the ping creation date and generate the archive directory to hold it. Note
  // that getMonth returns a 0-based month, so we need to add an offset.
  let month = String(aDate.getMonth() + 1);
  let archivedPingDir = PathUtils.join(
    lazy.gPingsArchivePath,
    aDate.getFullYear() + "-" + month.padStart(2, "0")
  );
  // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json
  let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");
  return PathUtils.join(archivedPingDir, fileName);
}

/**
 * Get the size of the ping file on the disk.
 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.
 */
var getArchivedPingSize = async function (aPingId, aDate, aType) {
  const path = getArchivedPingPath(aPingId, aDate, aType);
  let filePaths = [path + "lz4", path];

  for (let path of filePaths) {
    try {
      return (await IOUtils.stat(path)).size;
    } catch (e) {}
  }

  // That's odd, this ping doesn't seem to exist.
  return 0;
};

/**
 * Get the size of the pending ping file on the disk.
 * @return {Integer} The file size, in bytes, of the ping file or 0 on errors.
 */
var getPendingPingSize = async function (aPingId) {
  const path = PathUtils.join(TelemetryStorage.pingDirectoryPath, aPingId);
  try {
    return (await IOUtils.stat(path)).size;
  } catch (e) {}

  // That's odd, this ping doesn't seem to exist.
  return 0;
};

/**
 * Check if a directory name is in the "YYYY-MM" format.
 * @param {String} aDirName The name of the pings archive directory.
 * @return {Boolean} True if the directory name is in the right format, false otherwise.
 */
function isValidArchiveDir(aDirName) {
  const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;
  return dirRegEx.test(aDirName);
}

/**
 * Gets a date object from an archive directory name.
 * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM
 *        format.
 * @return {Object} A Date object or null if the dir name is not valid.
 */
function getDateFromArchiveDir(aDirName) {
  let [year, month] = aDirName.split("-");
  year = parseInt(year);
  month = parseInt(month);
  // Make sure to have sane numbers.
  if (
    !Number.isFinite(month) ||
    !Number.isFinite(year) ||
    month < 1 ||
    month > 12
  ) {
    return null;
  }
  return new Date(year, month - 1, 1, 0, 0, 0);
}