2 files changed, 447 insertions, 0 deletions
diff --git a/services/settings/RemoteSettingsWorker.js b/services/settings/RemoteSettingsWorker.js
new file mode 100644
index 0000000000..c31a478a0d
--- /dev/null
+++ b/services/settings/RemoteSettingsWorker.js
@@ -0,0 +1,202 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env mozilla/chrome-worker */
+
+"use strict";
+
+/**
+ * A worker dedicated to Remote Settings.
+ */
+
+importScripts(
+  "resource://gre/modules/workers/require.js",
+  "resource://gre/modules/CanonicalJSON.jsm",
+  "resource://services-settings/IDBHelpers.jsm",
+  "resource://services-settings/SharedUtils.jsm",
+  "resource://gre/modules/third_party/jsesc/jsesc.js"
+);
+
+const IDB_RECORDS_STORE = "records";
+const IDB_TIMESTAMPS_STORE = "timestamps";
+
+let gShutdown = false;
+
+const Agent = {
+  /**
+   * Return the canonical JSON serialization of the specified records.
+   * It has to match what is done on the server (See Kinto/kinto-signer).
+   *
+   * @param {Array<Object>} records
+   * @param {String} timestamp
+   * @returns {String}
+   */
+  async canonicalStringify(records, timestamp) {
+    // Sort list by record id.
+    let allRecords = records.sort((a, b) => {
+      if (a.id < b.id) {
+        return -1;
+      }
+      return a.id > b.id ? 1 : 0;
+    });
+    // All existing records are replaced by the version from the server
+    // and deleted records are removed.
+    for (let i = 0; i < allRecords.length /* no increment! */; ) {
+      const rec = allRecords[i];
+      const next = allRecords[i + 1];
+      if ((next && rec.id == next.id) || rec.deleted) {
+        allRecords.splice(i, 1); // remove local record
+      } else {
+        i++;
+      }
+    }
+    const toSerialize = {
+      last_modified: "" + timestamp,
+      data: allRecords,
+    };
+    return CanonicalJSON.stringify(toSerialize, jsesc);
+  },
+
+  /**
+   * If present, import the JSON file into the Remote Settings IndexedDB
+   * for the specified bucket and collection.
+   * (eg. blocklists/certificates, main/onboarding)
+   * @param {String} bucket
+   * @param {String} collection
+   * @returns {int} Number of records loaded from dump or -1 if no dump found.
+   */
+  async importJSONDump(bucket, collection) {
+    const { data: records } = await SharedUtils.loadJSONDump(
+      bucket,
+      collection
+    );
+    if (records === null) {
+      // Return -1 if file is missing.
+      return -1;
+    }
+    if (gShutdown) {
+      throw new Error("Can't import when we've started shutting down.");
+    }
+    await importDumpIDB(bucket, collection, records);
+    return records.length;
+  },
+
+  /**
+   * Check that the specified file matches the expected size and SHA-256 hash.
+   * @param {String} fileUrl file URL to read from
+   * @param {Number} size expected file size
+   * @param {String} size expected file SHA-256 as hex string
+   * @returns {boolean}
+   */
+  async checkFileHash(fileUrl, size, hash) {
+    let resp;
+    try {
+      resp = await fetch(fileUrl);
+    } catch (e) {
+      // File does not exist.
+      return false;
+    }
+    const buffer = await resp.arrayBuffer();
+    return SharedUtils.checkContentHash(buffer, size, hash);
+  },
+
+  async prepareShutdown() {
+    gShutdown = true;
+    // Ensure we can iterate and abort (which may delete items) by cloning
+    // the list.
+    let transactions = Array.from(gPendingTransactions);
+    for (let transaction of transactions) {
+      try {
+        transaction.abort();
+      } catch (ex) {
+        // We can hit this case if the transaction has finished but
+        // we haven't heard about it yet.
+      }
+    }
+  },
+
+  _test_only_import(bucket, collection, records) {
+    return importDumpIDB(bucket, collection, records);
+  },
+};
+
+/**
+ * Wrap worker invocations in order to return the `callbackId` along
+ * the result. This will allow to transform the worker invocations
+ * into promises in `RemoteSettingsWorker.jsm`.
+ */
+self.onmessage = event => {
+  const { callbackId, method, args = [] } = event.data;
+  Agent[method](...args)
+    .then(result => {
+      self.postMessage({ callbackId, result });
+    })
+    .catch(error => {
+      console.log(`RemoteSettingsWorker error: ${error}`);
+      self.postMessage({ callbackId, error: "" + error });
+    });
+};
+
+let gPendingTransactions = new Set();
+
+/**
+ * Import the records into the Remote Settings Chrome IndexedDB.
+ *
+ * Note: This duplicates some logics from `kinto-offline-client.js`.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @param {Array<Object>} records
+ */
+async function importDumpIDB(bucket, collection, records) {
+  // Open the DB. It will exist since if we are running this, it means
+  // we already tried to read the timestamp in `remote-settings.js`
+  const db = await IDBHelpers.openIDB(false /* do not allow upgrades */);
+
+  // try...finally to ensure we always close the db.
+  try {
+    if (gShutdown) {
+      throw new Error("Can't import when we've started shutting down.");
+    }
+
+    // Each entry of the dump will be stored in the records store.
+    // They are indexed by `_cid`.
+    const cid = bucket + "/" + collection;
+    // We can just modify the items in-place, as we got them from SharedUtils.loadJSONDump().
+    records.forEach(item => {
+      item._cid = cid;
+    });
+    // Store the highest timestamp as the collection timestamp (or zero if dump is empty).
+    const timestamp =
+      records.length === 0
+        ? 0
+        : Math.max(...records.map(record => record.last_modified));
+    let { transaction, promise } = IDBHelpers.executeIDB(
+      db,
+      [IDB_RECORDS_STORE, IDB_TIMESTAMPS_STORE],
+      "readwrite",
+      ([recordsStore, timestampStore], rejectTransaction) => {
+        // Wipe before loading
+        recordsStore.delete(IDBKeyRange.bound([cid], [cid, []], false, true));
+        IDBHelpers.bulkOperationHelper(
+          recordsStore,
+          {
+            reject: rejectTransaction,
+            completion() {
+              timestampStore.put({ cid, value: timestamp });
+            },
+          },
+          "put",
+          records
+        );
+      }
+    );
+    gPendingTransactions.add(transaction);
+    promise = promise.finally(() => gPendingTransactions.delete(transaction));
+    await promise;
+  } finally {
+    // Close now that we're done.
+    db.close();
+  }
+}
diff --git a/services/settings/RemoteSettingsWorker.jsm b/services/settings/RemoteSettingsWorker.jsm
new file mode 100644
index 0000000000..147ebb6b13
--- /dev/null
+++ b/services/settings/RemoteSettingsWorker.jsm
@@ -0,0 +1,245 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * Interface to a dedicated thread handling for Remote Settings heavy operations.
+ */
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+const { setTimeout, clearTimeout } = ChromeUtils.import(
+  "resource://gre/modules/Timer.jsm"
+);
+
+var EXPORTED_SYMBOLS = ["RemoteSettingsWorker"];
+
+XPCOMUtils.defineLazyPreferenceGetter(
+  this,
+  "gMaxIdleMilliseconds",
+  "services.settings.worker_idle_max_milliseconds",
+  30 * 1000 // Default of 30 seconds.
+);
+
+ChromeUtils.defineModuleGetter(
+  this,
+  "AsyncShutdown",
+  "resource://gre/modules/AsyncShutdown.jsm"
+);
+
+ChromeUtils.defineModuleGetter(
+  this,
+  "SharedUtils",
+  "resource://services-settings/SharedUtils.jsm"
+);
+
+// Note: we currently only ever construct one instance of Worker.
+// If it stops being a singleton, the AsyncShutdown code at the bottom
+// of this file, as well as these globals, will need adjusting.
+let gShutdown = false;
+let gShutdownResolver = null;
+
+class RemoteSettingsWorkerError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RemoteSettingsWorkerError";
+  }
+}
+
+class Worker {
+  constructor(source) {
+    if (gShutdown) {
+      Cu.reportError("Can't create worker once shutdown has started");
+    }
+    this.source = source;
+    this.worker = null;
+
+    this.callbacks = new Map();
+    this.lastCallbackId = 0;
+    this.idleTimeoutId = null;
+  }
+
+  async _execute(method, args = [], options = {}) {
+    // Check if we're shutting down.
+    if (gShutdown && method != "prepareShutdown") {
+      throw new RemoteSettingsWorkerError("Remote Settings has shut down.");
+    }
+    // Don't instantiate the worker to shut it down.
+    if (method == "prepareShutdown" && !this.worker) {
+      return null;
+    }
+
+    const { mustComplete = false } = options;
+    // (Re)instantiate the worker if it was terminated.
+    if (!this.worker) {
+      this.worker = new ChromeWorker(this.source);
+      this.worker.onmessage = this._onWorkerMessage.bind(this);
+      this.worker.onerror = error => {
+        // Worker crashed. Reject each pending callback.
+        for (const { reject } of this.callbacks.values()) {
+          reject(error);
+        }
+        this.callbacks.clear();
+        // And terminate it.
+        this.stop();
+      };
+    }
+    // New activity: reset the idle timer.
+    if (this.idleTimeoutId) {
+      clearTimeout(this.idleTimeoutId);
+    }
+    let identifier = method + "-";
+    // Include the collection details in the importJSONDump case.
+    if (identifier == "importJSONDump-") {
+      identifier += `${args[0]}-${args[1]}-`;
+    }
+    return new Promise((resolve, reject) => {
+      const callbackId = `${identifier}${++this.lastCallbackId}`;
+      this.callbacks.set(callbackId, { resolve, reject, mustComplete });
+      this.worker.postMessage({ callbackId, method, args });
+    });
+  }
+
+  _onWorkerMessage(event) {
+    const { callbackId, result, error } = event.data;
+    // If we're shutting down, we may have already rejected this operation
+    // and removed its callback from our map:
+    if (!this.callbacks.has(callbackId)) {
+      return;
+    }
+    const { resolve, reject } = this.callbacks.get(callbackId);
+    if (error) {
+      reject(new RemoteSettingsWorkerError(error));
+    } else {
+      resolve(result);
+    }
+    this.callbacks.delete(callbackId);
+
+    // Terminate the worker when it's unused for some time.
+    // But don't terminate it if an operation is pending.
+    if (!this.callbacks.size) {
+      if (gShutdown) {
+        this.stop();
+        if (gShutdownResolver) {
+          gShutdownResolver();
+        }
+      } else {
+        this.idleTimeoutId = setTimeout(() => {
+          this.stop();
+        }, gMaxIdleMilliseconds);
+      }
+    }
+  }
+
+  /**
+   * Called at shutdown to abort anything the worker is doing that isn't
+   * critical.
+   */
+  _abortCancelableRequests() {
+    // End all tasks that we can.
+    const callbackCopy = Array.from(this.callbacks.entries());
+    const error = new Error("Shutdown, aborting read-only worker requests.");
+    for (const [id, { reject, mustComplete }] of callbackCopy) {
+      if (!mustComplete) {
+        this.callbacks.delete(id);
+        reject(error);
+      }
+    }
+    // There might be nothing left now:
+    if (!this.callbacks.size) {
+      this.stop();
+      if (gShutdownResolver) {
+        gShutdownResolver();
+      }
+    }
+    // If there was something left, we'll stop as soon as we get messages from
+    // those tasks, too.
+    // Let's hurry them along a bit:
+    this._execute("prepareShutdown");
+  }
+
+  stop() {
+    this.worker.terminate();
+    this.worker = null;
+    this.idleTimeoutId = null;
+  }
+
+  async canonicalStringify(localRecords, remoteRecords, timestamp) {
+    return this._execute("canonicalStringify", [
+      localRecords,
+      remoteRecords,
+      timestamp,
+    ]);
+  }
+
+  async importJSONDump(bucket, collection) {
+    return this._execute("importJSONDump", [bucket, collection], {
+      mustComplete: true,
+    });
+  }
+
+  async checkFileHash(filepath, size, hash) {
+    return this._execute("checkFileHash", [filepath, size, hash]);
+  }
+
+  async checkContentHash(buffer, size, hash) {
+    // The implementation does little work on the current thread, so run the
+    // task on the current thread instead of the worker thread.
+    return SharedUtils.checkContentHash(buffer, size, hash);
+  }
+}
+
+// Now, first add a shutdown blocker. If that fails, we must have
+// shut down already.
+// We're doing this here rather than in the Worker constructor because in
+// principle having just 1 shutdown blocker for the entire file should be
+// fine. If we ever start creating more than one Worker instance, this
+// code will need adjusting to deal with that.
+try {
+  AsyncShutdown.profileBeforeChange.addBlocker(
+    "Remote Settings profile-before-change",
+    async () => {
+      // First, indicate we've shut down.
+      gShutdown = true;
+      // Then, if we have no worker or no callbacks, we're done.
+      if (
+        !RemoteSettingsWorker.worker ||
+        !RemoteSettingsWorker.callbacks.size
+      ) {
+        return null;
+      }
+      // Otherwise, there's something left to do. Set up a promise:
+      let finishedPromise = new Promise(resolve => {
+        gShutdownResolver = resolve;
+      });
+
+      // Try to cancel most of the work:
+      RemoteSettingsWorker._abortCancelableRequests();
+
+      // Return a promise that the worker will resolve.
+      return finishedPromise;
+    },
+    {
+      fetchState() {
+        const remainingCallbacks = RemoteSettingsWorker.callbacks;
+        const details = Array.from(remainingCallbacks.keys()).join(", ");
+        return `Remaining: ${remainingCallbacks.size} callbacks (${details}).`;
+      },
+    }
+  );
+} catch (ex) {
+  Cu.reportError(
+    "Couldn't add shutdown blocker, assuming shutdown has started."
+  );
+  Cu.reportError(ex);
+  // If AsyncShutdown throws, `profileBeforeChange` has already fired. Ignore it
+  // and mark shutdown. Constructing the worker will report an error and do
+  // nothing.
+  gShutdown = true;
+}
+
+var RemoteSettingsWorker = new Worker(
+  "resource://services-settings/RemoteSettingsWorker.js"
+);