Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 602 insertions, 0 deletions

diff --git a/services/settings/Database.sys.mjs b/services/settings/Database.sys.mjs
new file mode 100644
index 0000000000..e8a89028bc
--- /dev/null
+++ b/services/settings/Database.sys.mjs
@@ -0,0 +1,602 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs",
+  CommonUtils: "resource://services-common/utils.sys.mjs",
+  IDBHelpers: "resource://services-settings/IDBHelpers.sys.mjs",
+  ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs",
+  Utils: "resource://services-settings/Utils.sys.mjs",
+});
+
+ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
+
+/**
+ * Database is a tiny wrapper with the objective
+ * of providing major kinto-offline-client collection API.
+ * (with the objective of getting rid of kinto-offline-client)
+ */
+export class Database {
+  static destroy() {
+    return destroyIDB();
+  }
+
+  constructor(identifier) {
+    ensureShutdownBlocker();
+    this.identifier = identifier;
+  }
+
+  async list(options = {}) {
+    const { filters = {}, order = "" } = options;
+    let results = [];
+    try {
+      await executeIDB(
+        "records",
+        (store, rejectTransaction) => {
+          // Fast-path the (very common) no-filters case
+          if (lazy.ObjectUtils.isEmpty(filters)) {
+            const range = IDBKeyRange.only(this.identifier);
+            const request = store.index("cid").getAll(range);
+            request.onsuccess = e => {
+              results = e.target.result;
+            };
+            return;
+          }
+          const request = store
+            .index("cid")
+            .openCursor(IDBKeyRange.only(this.identifier));
+          const objFilters = transformSubObjectFilters(filters);
+          request.onsuccess = event => {
+            try {
+              const cursor = event.target.result;
+              if (cursor) {
+                const { value } = cursor;
+                if (lazy.Utils.filterObject(objFilters, value)) {
+                  results.push(value);
+                }
+                cursor.continue();
+              }
+            } catch (ex) {
+              rejectTransaction(ex);
+            }
+          };
+        },
+        { mode: "readonly" }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "list()", this.identifier);
+    }
+    // Remove IDB key field from results.
+    for (const result of results) {
+      delete result._cid;
+    }
+    return order ? lazy.Utils.sortObjects(order, results) : results;
+  }
+
+  async importChanges(metadata, timestamp, records = [], options = {}) {
+    const { clear = false } = options;
+    const _cid = this.identifier;
+    try {
+      await executeIDB(
+        ["collections", "timestamps", "records"],
+        (stores, rejectTransaction) => {
+          const [storeMetadata, storeTimestamps, storeRecords] = stores;
+
+          if (clear) {
+            // Our index is over the _cid and id fields. We want to remove
+            // all of the items in the collection for which the object was
+            // created, ie with _cid == this.identifier.
+            // We would like to just tell IndexedDB:
+            // store.index(IDBKeyRange.only(this.identifier)).delete();
+            // to delete all records matching the first part of the 2-part key.
+            // Unfortunately such an API does not exist.
+            // While we could iterate over the index with a cursor, we'd do
+            // a roundtrip to PBackground for each item. Once you have 1000
+            // items, the result is very slow because of all the overhead of
+            // jumping between threads and serializing/deserializing.
+            // So instead, we tell the store to delete everything between
+            // "our" _cid identifier, and what would be the next identifier
+            // (via lexicographical sorting). Unfortunately there does not
+            // seem to be a way to specify bounds for all items that share
+            // the same first part of the key using just that first part, hence
+            // the use of the hypothetical [] for the second part of the end of
+            // the bounds.
+            storeRecords.delete(
+              IDBKeyRange.bound([_cid], [_cid, []], false, true)
+            );
+          }
+
+          // Store or erase metadata.
+          if (metadata === null) {
+            storeMetadata.delete(_cid);
+          } else if (metadata) {
+            storeMetadata.put({ cid: _cid, metadata });
+          }
+          // Store or erase timestamp.
+          if (timestamp === null) {
+            storeTimestamps.delete(_cid);
+          } else if (timestamp) {
+            storeTimestamps.put({ cid: _cid, value: timestamp });
+          }
+
+          if (!records.length) {
+            return;
+          }
+
+          // Separate tombstones from creations/updates.
+          const toDelete = records.filter(r => r.deleted);
+          const toInsert = records.filter(r => !r.deleted);
+          lazy.console.debug(
+            `${_cid} ${toDelete.length} to delete, ${toInsert.length} to insert`
+          );
+          // Delete local records for each tombstone.
+          lazy.IDBHelpers.bulkOperationHelper(
+            storeRecords,
+            {
+              reject: rejectTransaction,
+              completion() {
+                // Overwrite all other data.
+                lazy.IDBHelpers.bulkOperationHelper(
+                  storeRecords,
+                  {
+                    reject: rejectTransaction,
+                  },
+                  "put",
+                  toInsert.map(item => ({ ...item, _cid }))
+                );
+              },
+            },
+            "delete",
+            toDelete.map(item => [_cid, item.id])
+          );
+        },
+        { desc: "importChanges() in " + _cid }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "importChanges()", _cid);
+    }
+  }
+
+  async getLastModified() {
+    let entry = null;
+    try {
+      await executeIDB(
+        "timestamps",
+        store => {
+          store.get(this.identifier).onsuccess = e => (entry = e.target.result);
+        },
+        { mode: "readonly" }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(
+        e,
+        "getLastModified()",
+        this.identifier
+      );
+    }
+    if (!entry) {
+      return null;
+    }
+    // Some distributions where released with a modified dump that did not
+    // contain timestamps for last_modified. Work around this here, and return
+    // the timestamp as zero, so that the entries should get updated.
+    if (isNaN(entry.value)) {
+      lazy.console.warn(`Local timestamp is NaN for ${this.identifier}`);
+      return 0;
+    }
+    return entry.value;
+  }
+
+  async getMetadata() {
+    let entry = null;
+    try {
+      await executeIDB(
+        "collections",
+        store => {
+          store.get(this.identifier).onsuccess = e => (entry = e.target.result);
+        },
+        { mode: "readonly" }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(
+        e,
+        "getMetadata()",
+        this.identifier
+      );
+    }
+    return entry ? entry.metadata : null;
+  }
+
+  async getAttachment(attachmentId) {
+    let entry = null;
+    try {
+      await executeIDB(
+        "attachments",
+        store => {
+          store.get([this.identifier, attachmentId]).onsuccess = e => {
+            entry = e.target.result;
+          };
+        },
+        { mode: "readonly" }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(
+        e,
+        "getAttachment()",
+        this.identifier
+      );
+    }
+    return entry ? entry.attachment : null;
+  }
+
+  async saveAttachment(attachmentId, attachment) {
+    try {
+      await executeIDB(
+        "attachments",
+        store => {
+          if (attachment) {
+            store.put({ cid: this.identifier, attachmentId, attachment });
+          } else {
+            store.delete([this.identifier, attachmentId]);
+          }
+        },
+        { desc: "saveAttachment(" + attachmentId + ") in " + this.identifier }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(
+        e,
+        "saveAttachment()",
+        this.identifier
+      );
+    }
+  }
+
+  /**
+   * Delete all attachments which don't match any record.
+   *
+   * Attachments are linked to records, except when a fixed `attachmentId` is used.
+   * A record can be updated or deleted, potentially by deleting a record and restoring an updated version
+   * of the record with the same ID. Potentially leaving orphaned attachments in the database.
+   * Since we run the pruning logic after syncing, any attachment without a
+   * matching record can be discarded as they will be unreachable forever.
+   *
+   * @param {Array<String>} excludeIds List of attachments IDs to exclude from pruning.
+   */
+  async pruneAttachments(excludeIds) {
+    const _cid = this.identifier;
+    let deletedCount = 0;
+    try {
+      await executeIDB(
+        ["attachments", "records"],
+        async (stores, rejectTransaction) => {
+          const [attachmentsStore, recordsStore] = stores;
+
+          // List all stored attachments.
+          // All keys ≥ [_cid, ..] && < [_cid, []]. See comment in `importChanges()`
+          const rangeAllKeys = IDBKeyRange.bound(
+            [_cid],
+            [_cid, []],
+            false,
+            true
+          );
+          const allAttachments = await new Promise((resolve, reject) => {
+            const request = attachmentsStore.getAll(rangeAllKeys);
+            request.onsuccess = e => resolve(e.target.result);
+            request.onerror = e => reject(e);
+          });
+          if (!allAttachments.length) {
+            lazy.console.debug(
+              `${this.identifier} No attachments in IDB cache. Nothing to do.`
+            );
+            return;
+          }
+
+          // List all stored records.
+          const allRecords = await new Promise((resolve, reject) => {
+            const rangeAllIndexed = IDBKeyRange.only(_cid);
+            const request = recordsStore.index("cid").getAll(rangeAllIndexed);
+            request.onsuccess = e => resolve(e.target.result);
+            request.onerror = e => reject(e);
+          });
+
+          // Compare known records IDs to those stored along the attachments.
+          const currentRecordsIDs = new Set(allRecords.map(r => r.id));
+          const attachmentsToDelete = allAttachments.reduce((acc, entry) => {
+            // Skip excluded attachments.
+            if (excludeIds.includes(entry.attachmentId)) {
+              return acc;
+            }
+            // Delete attachment if associated record does not exist.
+            if (!currentRecordsIDs.has(entry.attachment.record.id)) {
+              acc.push([_cid, entry.attachmentId]);
+            }
+            return acc;
+          }, []);
+
+          // Perform a bulk delete of all obsolete attachments.
+          lazy.console.debug(
+            `${this.identifier} Bulk delete ${attachmentsToDelete.length} obsolete attachments`
+          );
+          lazy.IDBHelpers.bulkOperationHelper(
+            attachmentsStore,
+            {
+              reject: rejectTransaction,
+            },
+            "delete",
+            attachmentsToDelete
+          );
+          deletedCount = attachmentsToDelete.length;
+        },
+        { desc: "pruneAttachments() in " + this.identifier }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(
+        e,
+        "pruneAttachments()",
+        this.identifier
+      );
+    }
+    return deletedCount;
+  }
+
+  async clear() {
+    try {
+      await this.importChanges(null, null, [], { clear: true });
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "clear()", this.identifier);
+    }
+  }
+
+  /*
+   * Methods used by unit tests.
+   */
+
+  async create(record) {
+    if (!("id" in record)) {
+      record = { ...record, id: lazy.CommonUtils.generateUUID() };
+    }
+    try {
+      await executeIDB(
+        "records",
+        store => {
+          store.add({ ...record, _cid: this.identifier });
+        },
+        { desc: "create() in " + this.identifier }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "create()", this.identifier);
+    }
+    return record;
+  }
+
+  async update(record) {
+    try {
+      await executeIDB(
+        "records",
+        store => {
+          store.put({ ...record, _cid: this.identifier });
+        },
+        { desc: "update() in " + this.identifier }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "update()", this.identifier);
+    }
+  }
+
+  async delete(recordId) {
+    try {
+      await executeIDB(
+        "records",
+        store => {
+          store.delete([this.identifier, recordId]); // [_cid, id]
+        },
+        { desc: "delete() in " + this.identifier }
+      );
+    } catch (e) {
+      throw new lazy.IDBHelpers.IndexedDBError(e, "delete()", this.identifier);
+    }
+  }
+}
+
+let gDB = null;
+let gDBPromise = null;
+
+/**
+ * This function attempts to ensure `gDB` points to a valid database value.
+ * If gDB is already a database, it will do no-op (but this may take a
+ * microtask or two).
+ * If opening the database fails, it will throw an IndexedDBError.
+ */
+async function openIDB() {
+  // We can be called multiple times in a race; always ensure that when
+  // we complete, `gDB` is no longer null, but avoid doing the actual
+  // IndexedDB work more than once.
+  if (!gDBPromise) {
+    // Open and initialize/upgrade if needed.
+    gDBPromise = lazy.IDBHelpers.openIDB();
+  }
+  let db = await gDBPromise;
+  if (!gDB) {
+    gDB = db;
+  }
+}
+
+const gPendingReadOnlyTransactions = new Set();
+const gPendingWriteOperations = new Set();
+/**
+ * Helper to wrap some IDBObjectStore operations into a promise.
+ *
+ * @param {IDBDatabase} db
+ * @param {String|String[]} storeNames - either a string or an array of strings.
+ * @param {function} callback
+ * @param {Object} options
+ * @param {String} options.mode
+ * @param {String} options.desc   for shutdown tracking.
+ */
+async function executeIDB(storeNames, callback, options = {}) {
+  if (!gDB) {
+    // Check if we're shutting down. Services.startup.shuttingDown will
+    // be true sooner, but is never true in xpcshell tests, so we check
+    // both that and a bool we set ourselves when `profile-before-change`
+    // starts.
+    if (gShutdownStarted || Services.startup.shuttingDown) {
+      throw new lazy.IDBHelpers.ShutdownError(
+        "The application is shutting down",
+        "execute()"
+      );
+    }
+    await openIDB();
+  } else {
+    // Even if we have a db, wait a tick to avoid making IndexedDB sad.
+    // We should be able to remove this once bug 1626935 is fixed.
+    await Promise.resolve();
+  }
+
+  // Check for shutdown again as we've await'd something...
+  if (!gDB && (gShutdownStarted || Services.startup.shuttingDown)) {
+    throw new lazy.IDBHelpers.ShutdownError(
+      "The application is shutting down",
+      "execute()"
+    );
+  }
+
+  // Start the actual transaction:
+  const { mode = "readwrite", desc = "" } = options;
+  let { promise, transaction } = lazy.IDBHelpers.executeIDB(
+    gDB,
+    storeNames,
+    mode,
+    callback,
+    desc
+  );
+
+  // We track all readonly transactions and abort them at shutdown.
+  // We track all readwrite ones and await their completion at shutdown
+  // (to avoid dataloss when writes fail).
+  // We use a `.finally()` clause for this; it'll run the function irrespective
+  // of whether the promise resolves or rejects, and the promise it returns
+  // will resolve/reject with the same value.
+  let finishedFn;
+  if (mode == "readonly") {
+    gPendingReadOnlyTransactions.add(transaction);
+    finishedFn = () => gPendingReadOnlyTransactions.delete(transaction);
+  } else {
+    let obj = { promise, desc };
+    gPendingWriteOperations.add(obj);
+    finishedFn = () => gPendingWriteOperations.delete(obj);
+  }
+  return promise.finally(finishedFn);
+}
+
+async function destroyIDB() {
+  if (gDB) {
+    if (gShutdownStarted || Services.startup.shuttingDown) {
+      throw new lazy.IDBHelpers.ShutdownError(
+        "The application is shutting down",
+        "destroyIDB()"
+      );
+    }
+
+    // This will return immediately; the actual close will happen once
+    // there are no more running transactions.
+    gDB.close();
+    const allTransactions = new Set([
+      ...gPendingWriteOperations,
+      ...gPendingReadOnlyTransactions,
+    ]);
+    for (let transaction of Array.from(allTransactions)) {
+      try {
+        transaction.abort();
+      } catch (ex) {
+        // Ignore errors to abort transactions, we'll destroy everything.
+      }
+    }
+  }
+  gDB = null;
+  gDBPromise = null;
+  return lazy.IDBHelpers.destroyIDB();
+}
+
+function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
+  const last = arr.length - 1;
+  return arr.reduce((acc, cv, i) => {
+    if (i === last) {
+      return (acc[cv] = val);
+    } else if (Object.prototype.hasOwnProperty.call(acc, cv)) {
+      return acc[cv];
+    }
+    return (acc[cv] = {});
+  }, nestedFiltersObj);
+}
+
+function transformSubObjectFilters(filtersObj) {
+  const transformedFilters = {};
+  for (const [key, val] of Object.entries(filtersObj)) {
+    const keysArr = key.split(".");
+    makeNestedObjectFromArr(keysArr, val, transformedFilters);
+  }
+  return transformedFilters;
+}
+
+// We need to expose this wrapper function so we can test
+// shutdown handling.
+Database._executeIDB = executeIDB;
+
+let gShutdownStarted = false;
+// Test-only helper to be able to test shutdown multiple times:
+Database._cancelShutdown = () => {
+  gShutdownStarted = false;
+};
+
+let gShutdownBlocker = false;
+Database._shutdownHandler = () => {
+  gShutdownStarted = true;
+  const NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR = 0x80660006;
+  // Duplicate the list (to avoid it being modified) and then
+  // abort all read-only transactions.
+  for (let transaction of Array.from(gPendingReadOnlyTransactions)) {
+    try {
+      transaction.abort();
+    } catch (ex) {
+      // Ensure we don't throw/break, because either way we're in shutdown.
+
+      // In particular, `transaction.abort` can throw if the transaction
+      // is complete, ie if we manage to get called in between the
+      // transaction completing, and our completion handler being called
+      // to remove the item from the set. We don't care about that.
+      if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) {
+        // Report any other errors:
+        console.error(ex);
+      }
+    }
+  }
+  if (gDB) {
+    // This will return immediately; the actual close will happen once
+    // there are no more running transactions.
+    gDB.close();
+    gDB = null;
+  }
+  gDBPromise = null;
+  return Promise.allSettled(
+    Array.from(gPendingWriteOperations).map(op => op.promise)
+  );
+};
+
+function ensureShutdownBlocker() {
+  if (gShutdownBlocker) {
+    return;
+  }
+  gShutdownBlocker = true;
+  lazy.AsyncShutdown.profileBeforeChange.addBlocker(
+    "RemoteSettingsClient - finish IDB access.",
+    Database._shutdownHandler,
+    {
+      fetchState() {
+        return Array.from(gPendingWriteOperations).map(op => op.desc);
+      },
+    }
+  );
+}