Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 501 insertions, 0 deletions

diff --git a/toolkit/modules/JSONFile.sys.mjs b/toolkit/modules/JSONFile.sys.mjs
new file mode 100644
index 0000000000..c9bd290de5
--- /dev/null
+++ b/toolkit/modules/JSONFile.sys.mjs
@@ -0,0 +1,501 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Handles serialization of the data and persistence into a file.
+ *
+ * This modules handles the raw data stored in JavaScript serializable objects,
+ * and contains no special validation or query logic, that is handled entirely
+ * by "storage.js" instead.
+ *
+ * The data can be manipulated only after it has been loaded from disk.  The
+ * load process can happen asynchronously, through the "load" method, or
+ * synchronously, through "ensureDataReady".  After any modification, the
+ * "saveSoon" method must be called to flush the data to disk asynchronously.
+ *
+ * The raw data should be manipulated synchronously, without waiting for the
+ * event loop or for promise resolution, so that the saved file is always
+ * consistent.  This synchronous approach also simplifies the query and update
+ * logic.  For example, it is possible to find an object and modify it
+ * immediately without caring whether other code modifies it in the meantime.
+ *
+ * An asynchronous shutdown observer makes sure that data is always saved before
+ * the browser is closed. The data cannot be modified during shutdown.
+ *
+ * The file is stored in JSON format, without indentation, using UTF-8 encoding.
+ */
+
+// Globals
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs",
+  FileUtils: "resource://gre/modules/FileUtils.sys.mjs",
+});
+ChromeUtils.defineModuleGetter(
+  lazy,
+  "NetUtil",
+  "resource://gre/modules/NetUtil.jsm"
+);
+
+XPCOMUtils.defineLazyGetter(lazy, "gTextDecoder", function() {
+  return new TextDecoder();
+});
+
+const FileInputStream = Components.Constructor(
+  "@mozilla.org/network/file-input-stream;1",
+  "nsIFileInputStream",
+  "init"
+);
+
+/**
+ * Delay between a change to the data and the related save operation.
+ */
+const kSaveDelayMs = 1500;
+
+/**
+ * Cleansed basenames of the filenames that telemetry can be recorded for.
+ * Keep synchronized with 'objects' from Events.yaml.
+ */
+const TELEMETRY_BASENAMES = new Set(["logins", "autofillprofiles"]);
+
+// JSONFile
+
+/**
+ * Handles serialization of the data and persistence into a file.
+ *
+ * @param config An object containing following members:
+ *        - path: String containing the file path where data should be saved.
+ *        - sanitizedBasename: Sanitized string identifier used for logging,
+ *                             shutdown debugging, and telemetry.  Defaults to
+ *                             basename of given `path`, sanitized.
+ *        - dataPostProcessor: Function triggered when data is just loaded. The
+ *                             data object will be passed as the first argument
+ *                             and should be returned no matter it's modified or
+ *                             not. Its failure leads to the failure of load()
+ *                             and ensureDataReady().
+ *        - saveDelayMs: Number indicating the delay (in milliseconds) between a
+ *                       change to the data and the related save operation. The
+ *                       default value will be applied if omitted.
+ *        - beforeSave: Promise-returning function triggered just before the
+ *                      data is written to disk. This can be used to create any
+ *                      intermediate directories before saving. The file will
+ *                      not be saved if the promise rejects or the function
+ *                      throws an exception.
+ *        - finalizeAt: An `IOUtils` phase or barrier client that should
+ *                      automatically finalize the file when triggered. Defaults
+ *                      to `profileBeforeChange`; exposed as an option for
+ *                      testing.
+ *        - compression: A compression algorithm to use when reading and
+ *                       writing the data.
+ *        - backupTo: A string value indicating where writeAtomic should create
+ *                    a backup before writing to json files. Note that using this
+ *                    option currently ensures that we automatically restore backed
+ *                    up json files in load() and ensureDataReady() when original
+ *                    files are missing or corrupt.
+ */
+export function JSONFile(config) {
+  this.path = config.path;
+  this.sanitizedBasename =
+    config.sanitizedBasename ??
+    PathUtils.filename(this.path)
+      .replace(/\.json(.lz4)?$/, "")
+      .replaceAll(/[^a-zA-Z0-9_.]/g, "");
+
+  if (typeof config.dataPostProcessor === "function") {
+    this._dataPostProcessor = config.dataPostProcessor;
+  }
+  if (typeof config.beforeSave === "function") {
+    this._beforeSave = config.beforeSave;
+  }
+
+  if (config.saveDelayMs === undefined) {
+    config.saveDelayMs = kSaveDelayMs;
+  }
+  this._saver = new lazy.DeferredTask(() => this._save(), config.saveDelayMs);
+
+  this._options = {};
+  if (config.compression) {
+    this._options.decompress = this._options.compress = true;
+  }
+
+  if (config.backupTo) {
+    this._options.backupFile = this._options.backupTo = config.backupTo;
+  }
+
+  this._finalizeAt = config.finalizeAt || IOUtils.profileBeforeChange;
+  this._finalizeInternalBound = this._finalizeInternal.bind(this);
+  this._finalizeAt.addBlocker(
+    `JSON store: writing data for '${this.sanitizedBasename}'`,
+    this._finalizeInternalBound,
+    () => ({ sanitizedBasename: this.sanitizedBasename })
+  );
+
+  Services.telemetry.setEventRecordingEnabled("jsonfile", true);
+}
+
+JSONFile.prototype = {
+  /**
+   * String containing the file path where data should be saved.
+   */
+  path: "",
+
+  /**
+   * Sanitized identifier used for logging, shutdown debugging, and telemetry.
+   */
+  sanitizedBasename: "",
+
+  /**
+   * True when data has been loaded.
+   */
+  dataReady: false,
+
+  /**
+   * DeferredTask that handles the save operation.
+   */
+  _saver: null,
+
+  /**
+   * Internal data object.
+   */
+  _data: null,
+
+  /**
+   * Internal fields used during finalization.
+   */
+  _finalizeAt: null,
+  _finalizePromise: null,
+  _finalizeInternalBound: null,
+
+  /**
+   * Serializable object containing the data. This is populated directly with
+   * the data loaded from the file, and is saved without modifications.
+   *
+   * The raw data should be manipulated synchronously, without waiting for the
+   * event loop or for promise resolution, so that the saved file is always
+   * consistent.
+   */
+  get data() {
+    if (!this.dataReady) {
+      throw new Error("Data is not ready.");
+    }
+    return this._data;
+  },
+
+  /**
+   * Sets the loaded data to a new object. This will overwrite any persisted
+   * data on the next save.
+   */
+  set data(data) {
+    this._data = data;
+    this.dataReady = true;
+  },
+
+  /**
+   * Loads persistent data from the file to memory.
+   *
+   * @return {Promise}
+   * @resolves When the operation finished successfully.
+   * @rejects JavaScript exception when dataPostProcessor fails. It never fails
+   *          if there is no dataPostProcessor.
+   */
+  async load() {
+    if (this.dataReady) {
+      return;
+    }
+
+    let data = {};
+
+    try {
+      data = await IOUtils.readJSON(this.path, this._options);
+
+      // If synchronous loading happened in the meantime, exit now.
+      if (this.dataReady) {
+        return;
+      }
+    } catch (ex) {
+      // If an exception occurs because the file does not exist or it cannot be read,
+      // we do two things.
+      // 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
+      //    we try to look for and use backed up json files first. If the backup
+      //    is also not found or if the backup is unreadable, we then start with an empty file.
+      // 2. If a consumer does not configure a `backupTo` path option, we just start
+      //    with an empty file.
+
+      // In the event that the file exists, but an exception is thrown because it cannot be read,
+      // we store it as a .corrupt file for debugging purposes.
+
+      let errorNo = ex.winLastError || ex.unixErrno;
+      this._recordTelemetry("load", errorNo ? errorNo.toString() : "");
+      if (!(DOMException.isInstance(ex) && ex.name == "NotFoundError")) {
+        Cu.reportError(ex);
+
+        // Move the original file to a backup location, ignoring errors.
+        try {
+          let uniquePath = await IOUtils.createUniqueFile(
+            PathUtils.parent(this.path),
+            PathUtils.filename(this.path) + ".corrupt",
+            0o600
+          );
+          await IOUtils.move(this.path, uniquePath);
+          this._recordTelemetry("load", "invalid_json");
+        } catch (e2) {
+          Cu.reportError(e2);
+        }
+      }
+
+      if (this._options.backupFile) {
+        // Restore the original file from the backup here so fresh writes to empty
+        // json files don't happen at any time in the future compromising the backup
+        // in the process.
+        try {
+          await IOUtils.copy(this._options.backupFile, this.path);
+        } catch (e) {
+          if (!(DOMException.isInstance(e) && e.name == "NotFoundError")) {
+            Cu.reportError(e);
+          }
+        }
+
+        try {
+          // We still read from the backup file here instead of the original file in case
+          // access to the original file is blocked, e.g. by anti-virus software on the
+          // user's computer.
+          data = await IOUtils.readJSON(
+            this._options.backupFile,
+            this._options
+          );
+          // If synchronous loading happened in the meantime, exit now.
+          if (this.dataReady) {
+            return;
+          }
+          this._recordTelemetry("load", "used_backup");
+        } catch (e3) {
+          if (!(DOMException.isInstance(e3) && e3.name == "NotFoundError")) {
+            Cu.reportError(e3);
+          }
+        }
+      }
+
+      // In some rare cases it's possible for data to have been added to
+      // our database between the call to IOUtils.read and when we've been
+      // notified that there was a problem with it. In that case, leave the
+      // synchronously-added data alone.
+      if (this.dataReady) {
+        return;
+      }
+    }
+
+    this._processLoadedData(data);
+  },
+
+  /**
+   * Loads persistent data from the file to memory, synchronously. An exception
+   * can be thrown only if dataPostProcessor exists and fails.
+   */
+  ensureDataReady() {
+    if (this.dataReady) {
+      return;
+    }
+
+    let data = {};
+
+    try {
+      // This reads the file and automatically detects the UTF-8 encoding.
+      let inputStream = new FileInputStream(
+        new lazy.FileUtils.File(this.path),
+        lazy.FileUtils.MODE_RDONLY,
+        lazy.FileUtils.PERMS_FILE,
+        0
+      );
+      try {
+        let bytes = lazy.NetUtil.readInputStream(
+          inputStream,
+          inputStream.available()
+        );
+        data = JSON.parse(lazy.gTextDecoder.decode(bytes));
+      } finally {
+        inputStream.close();
+      }
+    } catch (ex) {
+      // If an exception occurs because the file does not exist or it cannot be read,
+      // we do two things.
+      // 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
+      //    we try to look for and use backed up json files first. If the backup
+      //    is also not found or if the backup is unreadable, we then start with an empty file.
+      // 2. If a consumer does not configure a `backupTo` path option, we just start
+      //    with an empty file.
+
+      // In the event that the file exists, but an exception is thrown because it cannot be read,
+      // we store it as a .corrupt file for debugging purposes.
+      if (
+        !(
+          ex instanceof Components.Exception &&
+          ex.result == Cr.NS_ERROR_FILE_NOT_FOUND
+        )
+      ) {
+        Cu.reportError(ex);
+        // Move the original file to a backup location, ignoring errors.
+        try {
+          let originalFile = new lazy.FileUtils.File(this.path);
+          let backupFile = originalFile.clone();
+          backupFile.leafName += ".corrupt";
+          backupFile.createUnique(
+            Ci.nsIFile.NORMAL_FILE_TYPE,
+            lazy.FileUtils.PERMS_FILE
+          );
+          backupFile.remove(false);
+          originalFile.moveTo(backupFile.parent, backupFile.leafName);
+        } catch (e2) {
+          Cu.reportError(e2);
+        }
+      }
+
+      if (this._options.backupFile) {
+        // Restore the original file from the backup here so fresh writes to empty
+        // json files don't happen at any time in the future compromising the backup
+        // in the process.
+        try {
+          let basename = PathUtils.filename(this.path);
+          let backupFile = new lazy.FileUtils.File(this._options.backupFile);
+          backupFile.copyTo(null, basename);
+        } catch (e) {
+          if (e.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
+            Cu.reportError(e);
+          }
+        }
+
+        try {
+          // We still read from the backup file here instead of the original file in case
+          // access to the original file is blocked, e.g. by anti-virus software on the
+          // user's computer.
+          // This reads the file and automatically detects the UTF-8 encoding.
+          let inputStream = new FileInputStream(
+            new lazy.FileUtils.File(this._options.backupFile),
+            lazy.FileUtils.MODE_RDONLY,
+            lazy.FileUtils.PERMS_FILE,
+            0
+          );
+          try {
+            let bytes = lazy.NetUtil.readInputStream(
+              inputStream,
+              inputStream.available()
+            );
+            data = JSON.parse(lazy.gTextDecoder.decode(bytes));
+          } finally {
+            inputStream.close();
+          }
+        } catch (e3) {
+          if (e3.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
+            Cu.reportError(e3);
+          }
+        }
+      }
+    }
+
+    this._processLoadedData(data);
+  },
+
+  /**
+   * Called when the data changed, this triggers asynchronous serialization.
+   */
+  saveSoon() {
+    return this._saver.arm();
+  },
+
+  /**
+   * Saves persistent data from memory to the file.
+   *
+   * If an error occurs, the previous file is not deleted.
+   *
+   * @return {Promise}
+   * @resolves When the operation finished successfully.
+   * @rejects JavaScript exception.
+   */
+  async _save() {
+    // Create or overwrite the file.
+    if (this._beforeSave) {
+      await Promise.resolve(this._beforeSave());
+    }
+
+    try {
+      await IOUtils.writeJSON(
+        this.path,
+        this._data,
+        Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
+      );
+    } catch (ex) {
+      if (typeof this._data.toJSONSafe == "function") {
+        // If serialization fails, try fallback safe JSON converter.
+        await IOUtils.writeUTF8(
+          this.path,
+          this._data.toJSONSafe(),
+          Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
+        );
+      }
+    }
+  },
+
+  /**
+   * Synchronously work on the data just loaded into memory.
+   */
+  _processLoadedData(data) {
+    if (this._finalizePromise) {
+      // It's possible for `load` to race with `finalize`. In that case, don't
+      // process or set the loaded data.
+      return;
+    }
+    this.data = this._dataPostProcessor ? this._dataPostProcessor(data) : data;
+  },
+
+  _recordTelemetry(method, value) {
+    if (!TELEMETRY_BASENAMES.has(this.sanitizedBasename)) {
+      // Avoid recording so we don't log an error in the console.
+      return;
+    }
+
+    Services.telemetry.recordEvent(
+      "jsonfile",
+      method,
+      this.sanitizedBasename,
+      value
+    );
+  },
+
+  /**
+   * Finishes persisting data to disk and resets all state for this file.
+   *
+   * @return {Promise}
+   * @resolves When the object is finalized.
+   */
+  _finalizeInternal() {
+    if (this._finalizePromise) {
+      // Finalization already in progress; return the pending promise. This is
+      // possible if `finalize` is called concurrently with shutdown.
+      return this._finalizePromise;
+    }
+    this._finalizePromise = (async () => {
+      await this._saver.finalize();
+      this._data = null;
+      this.dataReady = false;
+    })();
+    return this._finalizePromise;
+  },
+
+  /**
+   * Ensures that all data is persisted to disk, and prevents future calls to
+   * `saveSoon`. This is called automatically on shutdown, but can also be
+   * called explicitly when the file is no longer needed.
+   */
+  async finalize() {
+    if (this._finalizePromise) {
+      throw new Error(`The file ${this.path} has already been finalized`);
+    }
+    // Wait for finalization before removing the shutdown blocker.
+    await this._finalizeInternal();
+    this._finalizeAt.removeBlocker(this._finalizeInternalBound);
+  },
+};