Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1289 insertions, 0 deletions

diff --git a/services/settings/RemoteSettingsClient.jsm b/services/settings/RemoteSettingsClient.jsm
new file mode 100644
index 0000000000..82b16c8a24
--- /dev/null
+++ b/services/settings/RemoteSettingsClient.jsm
@@ -0,0 +1,1289 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["RemoteSettingsClient"];
+
+const { XPCOMUtils } = ChromeUtils.importESModule(
+  "resource://gre/modules/XPCOMUtils.sys.mjs"
+);
+const { AppConstants } = ChromeUtils.importESModule(
+  "resource://gre/modules/AppConstants.sys.mjs"
+);
+const { Downloader } = ChromeUtils.import(
+  "resource://services-settings/Attachments.jsm"
+);
+
+const lazy = {};
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+  ClientEnvironmentBase:
+    "resource://gre/modules/components-utils/ClientEnvironment.jsm",
+  Database: "resource://services-settings/Database.jsm",
+  IDBHelpers: "resource://services-settings/IDBHelpers.jsm",
+  KintoHttpClient: "resource://services-common/kinto-http-client.js",
+  ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
+  RemoteSettingsWorker: "resource://services-settings/RemoteSettingsWorker.jsm",
+  SharedUtils: "resource://services-settings/SharedUtils.jsm",
+  UptakeTelemetry: "resource://services-common/uptake-telemetry.js",
+  Utils: "resource://services-settings/Utils.jsm",
+});
+
+const TELEMETRY_COMPONENT = "remotesettings";
+
+XPCOMUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
+
+/**
+ * cacheProxy returns an object Proxy that will memoize properties of the target.
+ * @param {Object} target the object to wrap.
+ * @returns {Proxy}
+ */
+function cacheProxy(target) {
+  const cache = new Map();
+  return new Proxy(target, {
+    get(target, prop, receiver) {
+      if (!cache.has(prop)) {
+        cache.set(prop, target[prop]);
+      }
+      return cache.get(prop);
+    },
+  });
+}
+
+/**
+ * Minimalist event emitter.
+ *
+ * Note: we don't use `toolkit/modules/EventEmitter` because **we want** to throw
+ * an error when a listener fails to execute.
+ */
+class EventEmitter {
+  constructor(events) {
+    this._listeners = new Map();
+    for (const event of events) {
+      this._listeners.set(event, []);
+    }
+  }
+
+  /**
+   * Event emitter: will execute the registered listeners in the order and
+   * sequentially.
+   *
+   * @param {string} event    the event name
+   * @param {Object} payload  the event payload to call the listeners with
+   */
+  async emit(event, payload) {
+    const callbacks = this._listeners.get(event);
+    let lastError;
+    for (const cb of callbacks) {
+      try {
+        await cb(payload);
+      } catch (e) {
+        lastError = e;
+      }
+    }
+    if (lastError) {
+      throw lastError;
+    }
+  }
+
+  hasListeners(event) {
+    return this._listeners.has(event) && !!this._listeners.get(event).length;
+  }
+
+  on(event, callback) {
+    if (!this._listeners.has(event)) {
+      throw new Error(`Unknown event type ${event}`);
+    }
+    this._listeners.get(event).push(callback);
+  }
+
+  off(event, callback) {
+    if (!this._listeners.has(event)) {
+      throw new Error(`Unknown event type ${event}`);
+    }
+    const callbacks = this._listeners.get(event);
+    const i = callbacks.indexOf(callback);
+    if (i < 0) {
+      throw new Error(`Unknown callback`);
+    } else {
+      callbacks.splice(i, 1);
+    }
+  }
+}
+
+class APIError extends Error {}
+
+class NetworkError extends APIError {
+  constructor(e) {
+    super(`Network error: ${e}`, { cause: e });
+    this.name = "NetworkError";
+  }
+}
+
+class NetworkOfflineError extends APIError {
+  constructor() {
+    super("Network is offline");
+    this.name = "NetworkOfflineError";
+  }
+}
+
+class ServerContentParseError extends APIError {
+  constructor(e) {
+    super(`Cannot parse server content: ${e}`, { cause: e });
+    this.name = "ServerContentParseError";
+  }
+}
+
+class BackendError extends APIError {
+  constructor(e) {
+    super(`Backend error: ${e}`, { cause: e });
+    this.name = "BackendError";
+  }
+}
+
+class BackoffError extends APIError {
+  constructor(e) {
+    super(`Server backoff: ${e}`, { cause: e });
+    this.name = "BackoffError";
+  }
+}
+
+class TimeoutError extends APIError {
+  constructor(e) {
+    super(`API timeout: ${e}`, { cause: e });
+    this.name = "TimeoutError";
+  }
+}
+
+class StorageError extends Error {
+  constructor(e) {
+    super(`Storage error: ${e}`, { cause: e });
+    this.name = "StorageError";
+  }
+}
+
+class InvalidSignatureError extends Error {
+  constructor(cid, x5u) {
+    let message = `Invalid content signature (${cid})`;
+    if (x5u) {
+      const chain = x5u.split("/").pop();
+      message += ` using '${chain}'`;
+    }
+    super(message);
+    this.name = "InvalidSignatureError";
+  }
+}
+
+class MissingSignatureError extends InvalidSignatureError {
+  constructor(cid) {
+    super(cid);
+    this.message = `Missing signature (${cid})`;
+    this.name = "MissingSignatureError";
+  }
+}
+
+class CorruptedDataError extends InvalidSignatureError {
+  constructor(cid) {
+    super(cid);
+    this.message = `Corrupted local data (${cid})`;
+    this.name = "CorruptedDataError";
+  }
+}
+
+class UnknownCollectionError extends Error {
+  constructor(cid) {
+    super(`Unknown Collection "${cid}"`);
+    this.name = "UnknownCollectionError";
+  }
+}
+
+class AttachmentDownloader extends Downloader {
+  constructor(client) {
+    super(client.bucketName, client.collectionName);
+    this._client = client;
+  }
+
+  get cacheImpl() {
+    const cacheImpl = {
+      get: async attachmentId => {
+        return this._client.db.getAttachment(attachmentId);
+      },
+      set: async (attachmentId, attachment) => {
+        return this._client.db.saveAttachment(attachmentId, attachment);
+      },
+      delete: async attachmentId => {
+        return this._client.db.saveAttachment(attachmentId, null);
+      },
+      prune: async excludeIds => {
+        return this._client.db.pruneAttachments(excludeIds);
+      },
+    };
+    Object.defineProperty(this, "cacheImpl", { value: cacheImpl });
+    return cacheImpl;
+  }
+
+  /**
+   * Download attachment and report Telemetry on failure.
+   *
+   * @see Downloader.download
+   */
+  async download(record, options) {
+    try {
+      // Explicitly await here to ensure we catch a network error.
+      return await super.download(record, options);
+    } catch (err) {
+      // Report download error.
+      let status = lazy.UptakeTelemetry.STATUS.DOWNLOAD_ERROR;
+      if (lazy.Utils.isOffline) {
+        status = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
+      } else if (/NetworkError/.test(err.message)) {
+        status = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR;
+      }
+      // If the file failed to be downloaded, report it as such in Telemetry.
+      await lazy.UptakeTelemetry.report(TELEMETRY_COMPONENT, status, {
+        source: this._client.identifier,
+      });
+      throw err;
+    }
+  }
+
+  /**
+   * Delete all downloaded records attachments.
+   *
+   * Note: the list of attachments to be deleted is based on the
+   * current list of records.
+   */
+  async deleteAll() {
+    let allRecords = await this._client.db.list();
+    return Promise.all(
+      allRecords
+        .filter(r => !!r.attachment)
+        .map(r =>
+          Promise.all([this.deleteDownloaded(r), this.deleteFromDisk(r)])
+        )
+    );
+  }
+}
+
+class RemoteSettingsClient extends EventEmitter {
+  static get APIError() {
+    return APIError;
+  }
+  static get NetworkError() {
+    return NetworkError;
+  }
+  static get NetworkOfflineError() {
+    return NetworkOfflineError;
+  }
+  static get ServerContentParseError() {
+    return ServerContentParseError;
+  }
+  static get BackendError() {
+    return BackendError;
+  }
+  static get BackoffError() {
+    return BackoffError;
+  }
+  static get TimeoutError() {
+    return TimeoutError;
+  }
+  static get StorageError() {
+    return StorageError;
+  }
+  static get InvalidSignatureError() {
+    return InvalidSignatureError;
+  }
+  static get MissingSignatureError() {
+    return MissingSignatureError;
+  }
+  static get CorruptedDataError() {
+    return CorruptedDataError;
+  }
+  static get UnknownCollectionError() {
+    return UnknownCollectionError;
+  }
+
+  constructor(
+    collectionName,
+    {
+      bucketName = AppConstants.REMOTE_SETTINGS_DEFAULT_BUCKET,
+      signerName,
+      filterFunc,
+      localFields = [],
+      keepAttachmentsIds = [],
+      lastCheckTimePref,
+    } = {}
+  ) {
+    // Remote Settings cannot be used in child processes (no access to disk,
+    // easily killed, isolated observer notifications etc.).
+    // Since our goal here is to prevent consumers to instantiate while developing their
+    // feature, throwing in Nightly only is enough, and prevents unexpected crashes
+    // in release or beta.
+    if (
+      !AppConstants.RELEASE_OR_BETA &&
+      Services.appinfo.processType !== Services.appinfo.PROCESS_TYPE_DEFAULT
+    ) {
+      throw new Error(
+        "Cannot instantiate Remote Settings client in child processes."
+      );
+    }
+
+    super(["sync"]); // emitted events
+
+    this.collectionName = collectionName;
+    // Client is constructed with the raw bucket name (eg. "main", "security-state", "blocklist")
+    // The `bucketName` will contain the `-preview` suffix if the preview mode is enabled.
+    this.bucketName = lazy.Utils.actualBucketName(bucketName);
+    this.signerName = signerName;
+    this.filterFunc = filterFunc;
+    this.localFields = localFields;
+    this.keepAttachmentsIds = keepAttachmentsIds;
+    this._lastCheckTimePref = lastCheckTimePref;
+    this._verifier = null;
+    this._syncRunning = false;
+
+    // This attribute allows signature verification to be disabled, when running tests
+    // or when pulling data from a dev server.
+    this.verifySignature = AppConstants.REMOTE_SETTINGS_VERIFY_SIGNATURE;
+
+    XPCOMUtils.defineLazyGetter(
+      this,
+      "db",
+      () => new lazy.Database(this.identifier)
+    );
+
+    XPCOMUtils.defineLazyGetter(
+      this,
+      "attachments",
+      () => new AttachmentDownloader(this)
+    );
+  }
+
+  /**
+   * Internal method to refresh the client bucket name after the preview mode
+   * was toggled.
+   *
+   * See `RemoteSettings.enabledPreviewMode()`.
+   */
+  refreshBucketName() {
+    this.bucketName = lazy.Utils.actualBucketName(this.bucketName);
+    this.db.identifier = this.identifier;
+  }
+
+  get identifier() {
+    return `${this.bucketName}/${this.collectionName}`;
+  }
+
+  get lastCheckTimePref() {
+    return (
+      this._lastCheckTimePref ||
+      `services.settings.${this.bucketName}.${this.collectionName}.last_check`
+    );
+  }
+
+  httpClient() {
+    const api = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL, {
+      fetchFunc: lazy.Utils.fetch, // Use fetch() wrapper.
+    });
+    return api.bucket(this.bucketName).collection(this.collectionName);
+  }
+
+  /**
+   * Retrieve the collection timestamp for the last synchronization.
+   * This is an opaque and comparable value assigned automatically by
+   * the server.
+   *
+   * @returns {number}
+   *          The timestamp in milliseconds, returns -1 if retrieving
+   *          the timestamp from the kinto collection fails.
+   */
+  async getLastModified() {
+    let timestamp = -1;
+    try {
+      timestamp = await this.db.getLastModified();
+    } catch (err) {
+      lazy.console.warn(
+        `Error retrieving the getLastModified timestamp from ${this.identifier} RemoteSettingsClient`,
+        err
+      );
+    }
+
+    return timestamp;
+  }
+
+  /**
+   * Lists settings.
+   *
+   * @param  {Object} options                    The options object.
+   * @param  {Object} options.filters            Filter the results (default: `{}`).
+   * @param  {String} options.order              The order to apply (eg. `"-last_modified"`).
+   * @param  {boolean} options.dumpFallback      Fallback to dump data if read of local DB fails (default: `true`).
+   * @param  {boolean} options.emptyListFallback Fallback to empty list if no dump data and read of local DB fails (default: `true`).
+   * @param  {boolean} options.loadDumpIfNewer   Use dump data if it is newer than local data (default: `true`).
+   * @param  {boolean} options.forceSync         Always synchronize from server before returning results (default: `false`).
+   * @param  {boolean} options.syncIfEmpty       Synchronize from server if local data is empty (default: `true`).
+   * @param  {boolean} options.verifySignature   Verify the signature of the local data (default: `false`).
+   * @return {Promise}
+   */
+  async get(options = {}) {
+    const {
+      filters = {},
+      order = "", // not sorted by default.
+      dumpFallback = true,
+      emptyListFallback = true,
+      forceSync = false,
+      loadDumpIfNewer = true,
+      syncIfEmpty = true,
+    } = options;
+    let { verifySignature = false } = options;
+
+    const hasParallelCall = !!this._importingPromise;
+    let data;
+    try {
+      let lastModified = forceSync ? null : await this.db.getLastModified();
+      let hasLocalData = lastModified !== null;
+
+      if (forceSync) {
+        if (!this._importingPromise) {
+          this._importingPromise = (async () => {
+            await this.sync({ sendEvents: false, trigger: "forced" });
+            return true; // No need to re-verify signature after sync.
+          })();
+        }
+      } else if (syncIfEmpty && !hasLocalData) {
+        // .get() was called before we had the chance to synchronize the local database.
+        // We'll try to avoid returning an empty list.
+        if (!this._importingPromise) {
+          // Prevent parallel loading when .get() is called multiple times.
+          this._importingPromise = (async () => {
+            const importedFromDump = lazy.Utils.LOAD_DUMPS
+              ? await this._importJSONDump()
+              : -1;
+            if (importedFromDump < 0) {
+              // There is no JSON dump to load, force a synchronization from the server.
+              // We don't want the "sync" event to be sent, since some consumers use `.get()`
+              // in "sync" callbacks. See Bug 1761953
+              lazy.console.debug(
+                `${this.identifier} Local DB is empty, pull data from server`
+              );
+              await this.sync({ loadDump: false, sendEvents: false });
+            }
+            // Return `true` to indicate we don't need to `verifySignature`,
+            // since a trusted dump was loaded or a signature verification
+            // happened during synchronization.
+            return true;
+          })();
+        } else {
+          lazy.console.debug(`${this.identifier} Awaiting existing import.`);
+        }
+      } else if (hasLocalData && loadDumpIfNewer) {
+        // Check whether the local data is older than the packaged dump.
+        // If it is, load the packaged dump (which overwrites the local data).
+        let lastModifiedDump = await lazy.Utils.getLocalDumpLastModified(
+          this.bucketName,
+          this.collectionName
+        );
+        if (lastModified < lastModifiedDump) {
+          lazy.console.debug(
+            `${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})`
+          );
+          if (!this._importingPromise) {
+            // As part of importing, any existing data is wiped.
+            this._importingPromise = (async () => {
+              const importedFromDump = await this._importJSONDump();
+              // Return `true` to skip signature verification if a dump was found.
+              // The dump can be missing if a collection is listed in the timestamps summary file,
+              // because its dump is present in the source tree, but the dump was not
+              // included in the `package-manifest.in` file. (eg. android, thunderbird)
+              return importedFromDump >= 0;
+            })();
+          } else {
+            lazy.console.debug(`${this.identifier} Awaiting existing import.`);
+          }
+        }
+      }
+
+      if (this._importingPromise) {
+        try {
+          if (await this._importingPromise) {
+            // No need to verify signature, because either we've just loaded a trusted
+            // dump (here or in a parallel call), or it was verified during sync.
+            verifySignature = false;
+          }
+        } catch (e) {
+          if (!hasParallelCall) {
+            // Sync or load dump failed. Throw.
+            throw e;
+          }
+          // Report error, but continue because there could have been data
+          // loaded from a parallel call.
+          console.error(e);
+        } finally {
+          // then delete this promise again, as now we should have local data:
+          delete this._importingPromise;
+        }
+      }
+
+      // Read from the local DB.
+      data = await this.db.list({ filters, order });
+    } catch (e) {
+      // If the local DB cannot be read (for unknown reasons, Bug 1649393)
+      // or sync failed, we fallback to the packaged data, and filter/sort in memory.
+      if (!dumpFallback) {
+        throw e;
+      }
+      console.error(e);
+      let { data } = await lazy.SharedUtils.loadJSONDump(
+        this.bucketName,
+        this.collectionName
+      );
+      if (data !== null) {
+        lazy.console.info(`${this.identifier} falling back to JSON dump`);
+      } else if (emptyListFallback) {
+        lazy.console.info(
+          `${this.identifier} no dump fallback, return empty list`
+        );
+        data = [];
+      } else {
+        // Obtaining the records failed, there is no dump, and we don't fallback
+        // to an empty list. Throw the original error.
+        throw e;
+      }
+      if (!lazy.ObjectUtils.isEmpty(filters)) {
+        data = data.filter(r => lazy.Utils.filterObject(filters, r));
+      }
+      if (order) {
+        data = lazy.Utils.sortObjects(order, data);
+      }
+      // No need to verify signature on JSON dumps.
+      // If local DB cannot be read, then we don't even try to do anything,
+      // we return results early.
+      return this._filterEntries(data);
+    }
+
+    lazy.console.debug(
+      `${this.identifier} ${data.length} records before filtering.`
+    );
+
+    if (verifySignature) {
+      lazy.console.debug(
+        `${this.identifier} verify signature of local data on read`
+      );
+      const allData = lazy.ObjectUtils.isEmpty(filters)
+        ? data
+        : await this.db.list();
+      const localRecords = allData.map(r => this._cleanLocalFields(r));
+      const timestamp = await this.db.getLastModified();
+      let metadata = await this.db.getMetadata();
+      if (syncIfEmpty && lazy.ObjectUtils.isEmpty(metadata)) {
+        // No sync occured yet, may have records from dump but no metadata.
+        // We don't want the "sync" event to be sent, since some consumers use `.get()`
+        // in "sync" callbacks. See Bug 1761953
+        await this.sync({ loadDump: false, sendEvents: false });
+        metadata = await this.db.getMetadata();
+      }
+      // Will throw MissingSignatureError if no metadata and `syncIfEmpty` is false.
+      await this._validateCollectionSignature(
+        localRecords,
+        timestamp,
+        metadata
+      );
+    }
+
+    // Filter the records based on `this.filterFunc` results.
+    const final = await this._filterEntries(data);
+    lazy.console.debug(
+      `${this.identifier} ${final.length} records after filtering.`
+    );
+    return final;
+  }
+
+  /**
+   * Synchronize the local database with the remote server.
+   *
+   * @param {Object} options See #maybeSync() options.
+   */
+  async sync(options) {
+    // We want to know which timestamp we are expected to obtain in order to leverage
+    // cache busting. We don't provide ETag because we don't want a 304.
+    const { changes } = await lazy.Utils.fetchLatestChanges(
+      lazy.Utils.SERVER_URL,
+      {
+        filters: {
+          collection: this.collectionName,
+          bucket: this.bucketName,
+        },
+      }
+    );
+    if (changes.length === 0) {
+      throw new RemoteSettingsClient.UnknownCollectionError(this.identifier);
+    }
+    // According to API, there will be one only (fail if not).
+    const [{ last_modified: expectedTimestamp }] = changes;
+
+    return this.maybeSync(expectedTimestamp, { ...options, trigger: "forced" });
+  }
+
+  /**
+   * Synchronize the local database with the remote server, **only if necessary**.
+   *
+   * @param {int}    expectedTimestamp  the lastModified date (on the server) for the remote collection.
+   *                                    This will be compared to the local timestamp, and will be used for
+   *                                    cache busting if local data is out of date.
+   * @param {Object} options            additional advanced options.
+   * @param {bool}   options.loadDump   load initial dump from disk on first sync (default: true if server is prod)
+   * @param {bool}   options.sendEvents send `"sync"` events (default: `true`)
+   * @param {string} options.trigger    label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
+   * @return {Promise}                  which rejects on sync or process failure.
+   */
+  async maybeSync(expectedTimestamp, options = {}) {
+    // Should the clients try to load JSON dump? (mainly disabled in tests)
+    const {
+      loadDump = lazy.Utils.LOAD_DUMPS,
+      trigger = "manual",
+      sendEvents = true,
+    } = options;
+
+    // Make sure we don't run several synchronizations in parallel, mainly
+    // in order to avoid race conditions in "sync" events listeners.
+    if (this._syncRunning) {
+      lazy.console.warn(`${this.identifier} sync already running`);
+      return;
+    }
+
+    // Prevent network requests and IndexedDB calls to be initiated
+    // during shutdown.
+    if (Services.startup.shuttingDown) {
+      lazy.console.warn(`${this.identifier} sync interrupted by shutdown`);
+      return;
+    }
+
+    this._syncRunning = true;
+
+    let importedFromDump = [];
+    const startedAt = new Date();
+    let reportStatus = null;
+    let thrownError = null;
+    try {
+      // If network is offline, we can't synchronize.
+      if (lazy.Utils.isOffline) {
+        throw new RemoteSettingsClient.NetworkOfflineError();
+      }
+
+      // Read last timestamp and local data before sync.
+      let collectionLastModified = await this.db.getLastModified();
+      const allData = await this.db.list();
+      // Local data can contain local fields, strip them.
+      let localRecords = allData.map(r => this._cleanLocalFields(r));
+      const localMetadata = await this.db.getMetadata();
+
+      // If there is no data currently in the collection, attempt to import
+      // initial data from the application defaults.
+      // This allows to avoid synchronizing the whole collection content on
+      // cold start.
+      if (!collectionLastModified && loadDump) {
+        try {
+          const imported = await this._importJSONDump();
+          // The worker only returns an integer. List the imported records to build the sync event.
+          if (imported > 0) {
+            lazy.console.debug(
+              `${this.identifier} ${imported} records loaded from JSON dump`
+            );
+            importedFromDump = await this.db.list();
+            // Local data is the data loaded from dump. We will need this later
+            // to compute the sync result.
+            localRecords = importedFromDump;
+          }
+          collectionLastModified = await this.db.getLastModified();
+        } catch (e) {
+          // Report but go-on.
+          console.error(e);
+        }
+      }
+      let syncResult;
+      try {
+        // Is local timestamp up to date with the server?
+        if (expectedTimestamp == collectionLastModified) {
+          lazy.console.debug(`${this.identifier} local data is up-to-date`);
+          reportStatus = lazy.UptakeTelemetry.STATUS.UP_TO_DATE;
+
+          // If the data is up-to-date but don't have metadata (records loaded from dump),
+          // we fetch them and validate the signature immediately.
+          if (this.verifySignature && lazy.ObjectUtils.isEmpty(localMetadata)) {
+            lazy.console.debug(`${this.identifier} pull collection metadata`);
+            const metadata = await this.httpClient().getData({
+              query: { _expected: expectedTimestamp },
+            });
+            await this.db.importChanges(metadata);
+            // We don't bother validating the signature if the dump was just loaded. We do
+            // if the dump was loaded at some other point (eg. from .get()).
+            if (this.verifySignature && !importedFromDump.length) {
+              lazy.console.debug(
+                `${this.identifier} verify signature of local data`
+              );
+              await this._validateCollectionSignature(
+                localRecords,
+                collectionLastModified,
+                metadata
+              );
+            }
+          }
+
+          // Since the data is up-to-date, if we didn't load any dump then we're done here.
+          if (!importedFromDump.length) {
+            return;
+          }
+          // Otherwise we want to continue with sending the sync event to notify about the created records.
+          syncResult = {
+            current: importedFromDump,
+            created: importedFromDump,
+            updated: [],
+            deleted: [],
+          };
+        } else {
+          // Local data is either outdated or tampered.
+          // In both cases we will fetch changes from server,
+          // and make sure we overwrite local data.
+          syncResult = await this._importChanges(
+            localRecords,
+            collectionLastModified,
+            localMetadata,
+            expectedTimestamp
+          );
+          if (sendEvents && this.hasListeners("sync")) {
+            // If we have listeners for the "sync" event, then compute the lists of changes.
+            // The records imported from the dump should be considered as "created" for the
+            // listeners.
+            const importedById = importedFromDump.reduce((acc, r) => {
+              acc.set(r.id, r);
+              return acc;
+            }, new Map());
+            // Deleted records should not appear as created.
+            syncResult.deleted.forEach(r => importedById.delete(r.id));
+            // Records from dump that were updated should appear in their newest form.
+            syncResult.updated.forEach(u => {
+              if (importedById.has(u.old.id)) {
+                importedById.set(u.old.id, u.new);
+              }
+            });
+            syncResult.created = syncResult.created.concat(
+              Array.from(importedById.values())
+            );
+          }
+
+          // When triggered from the daily timer, and if the sync was successful, and once
+          // all sync listeners have been executed successfully, we prune potential
+          // obsolete attachments that may have been left in the local cache.
+          if (trigger == "timer") {
+            const deleted = await this.attachments.prune(
+              this.keepAttachmentsIds
+            );
+            if (deleted > 0) {
+              lazy.console.warn(
+                `${this.identifier} Pruned ${deleted} obsolete attachments`
+              );
+            }
+          }
+        }
+      } catch (e) {
+        if (e instanceof InvalidSignatureError) {
+          // Signature verification failed during synchronization.
+          reportStatus =
+            e instanceof CorruptedDataError
+              ? lazy.UptakeTelemetry.STATUS.CORRUPTION_ERROR
+              : lazy.UptakeTelemetry.STATUS.SIGNATURE_ERROR;
+          // If sync fails with a signature error, it's likely that our
+          // local data has been modified in some way.
+          // We will attempt to fix this by retrieving the whole
+          // remote collection.
+          try {
+            lazy.console.warn(
+              `${this.identifier} Signature verified failed. Retry from scratch`
+            );
+            syncResult = await this._importChanges(
+              localRecords,
+              collectionLastModified,
+              localMetadata,
+              expectedTimestamp,
+              { retry: true }
+            );
+          } catch (e) {
+            // If the signature fails again, or if an error occured during wiping out the
+            // local data, then we report it as a *signature retry* error.
+            reportStatus = lazy.UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR;
+            throw e;
+          }
+        } else {
+          // The sync has thrown for other reason than signature verification.
+          // Obtain a more precise error than original one.
+          const adjustedError = this._adjustedError(e);
+          // Default status for errors at this step is SYNC_ERROR.
+          reportStatus = this._telemetryFromError(adjustedError, {
+            default: lazy.UptakeTelemetry.STATUS.SYNC_ERROR,
+          });
+          throw adjustedError;
+        }
+      }
+      if (sendEvents) {
+        // Filter the synchronization results using `filterFunc` (ie. JEXL).
+        const filteredSyncResult = await this._filterSyncResult(syncResult);
+        // If every changed entry is filtered, we don't even fire the event.
+        if (filteredSyncResult) {
+          try {
+            await this.emit("sync", { data: filteredSyncResult });
+          } catch (e) {
+            reportStatus = lazy.UptakeTelemetry.STATUS.APPLY_ERROR;
+            throw e;
+          }
+        } else {
+          lazy.console.info(
+            `All changes are filtered by JEXL expressions for ${this.identifier}`
+          );
+        }
+      }
+    } catch (e) {
+      thrownError = e;
+      // Obtain a more precise error than original one.
+      const adjustedError = this._adjustedError(e);
+      // If browser is shutting down, then we can report a specific status.
+      // (eg. IndexedDB will abort transactions)
+      if (Services.startup.shuttingDown) {
+        reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
+      }
+      // If no Telemetry status was determined yet (ie. outside sync step),
+      // then introspect error, default status at this step is UNKNOWN.
+      else if (reportStatus == null) {
+        reportStatus = this._telemetryFromError(adjustedError, {
+          default: lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR,
+        });
+      }
+      throw e;
+    } finally {
+      const durationMilliseconds = new Date() - startedAt;
+      // No error was reported, this is a success!
+      if (reportStatus === null) {
+        reportStatus = lazy.UptakeTelemetry.STATUS.SUCCESS;
+      }
+      // Report success/error status to Telemetry.
+      let reportArgs = {
+        source: this.identifier,
+        trigger,
+        duration: durationMilliseconds,
+      };
+      // In Bug 1617133, we will try to break down specific errors into
+      // more precise statuses by reporting the JavaScript error name
+      // ("TypeError", etc.) to Telemetry on Nightly.
+      const channel = lazy.UptakeTelemetry.Policy.getChannel();
+      if (
+        thrownError !== null &&
+        channel == "nightly" &&
+        [
+          lazy.UptakeTelemetry.STATUS.SYNC_ERROR,
+          lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR, // IndexedDB.
+          lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR,
+          lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR,
+        ].includes(reportStatus)
+      ) {
+        // List of possible error names for IndexedDB:
+        // https://searchfox.org/mozilla-central/rev/49ed791/dom/base/DOMException.cpp#28-53
+        reportArgs = { ...reportArgs, errorName: thrownError.name };
+      }
+
+      await lazy.UptakeTelemetry.report(
+        TELEMETRY_COMPONENT,
+        reportStatus,
+        reportArgs
+      );
+
+      lazy.console.debug(`${this.identifier} sync status is ${reportStatus}`);
+      this._syncRunning = false;
+    }
+  }
+
+  /**
+   * Return a more precise error instance, based on the specified
+   * error and its message.
+   * @param {Error} e the original error
+   * @returns {Error}
+   */
+  _adjustedError(e) {
+    if (/unparseable/.test(e.message)) {
+      return new RemoteSettingsClient.ServerContentParseError(e);
+    }
+    if (/NetworkError/.test(e.message)) {
+      return new RemoteSettingsClient.NetworkError(e);
+    }
+    if (/Timeout/.test(e.message)) {
+      return new RemoteSettingsClient.TimeoutError(e);
+    }
+    if (/HTTP 5??/.test(e.message)) {
+      return new RemoteSettingsClient.BackendError(e);
+    }
+    if (/Backoff/.test(e.message)) {
+      return new RemoteSettingsClient.BackoffError(e);
+    }
+    if (
+      // Errors from kinto.js IDB adapter.
+      e instanceof lazy.IDBHelpers.IndexedDBError ||
+      // Other IndexedDB errors (eg. RemoteSettingsWorker).
+      /IndexedDB/.test(e.message)
+    ) {
+      return new RemoteSettingsClient.StorageError(e);
+    }
+    return e;
+  }
+
+  /**
+   * Determine the Telemetry uptake status based on the specified
+   * error.
+   */
+  _telemetryFromError(e, options = { default: null }) {
+    let reportStatus = options.default;
+
+    if (e instanceof RemoteSettingsClient.NetworkOfflineError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR;
+    } else if (e instanceof lazy.IDBHelpers.ShutdownError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR;
+    } else if (e instanceof RemoteSettingsClient.ServerContentParseError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.PARSE_ERROR;
+    } else if (e instanceof RemoteSettingsClient.NetworkError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR;
+    } else if (e instanceof RemoteSettingsClient.TimeoutError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.TIMEOUT_ERROR;
+    } else if (e instanceof RemoteSettingsClient.BackendError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.SERVER_ERROR;
+    } else if (e instanceof RemoteSettingsClient.BackoffError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.BACKOFF;
+    } else if (e instanceof RemoteSettingsClient.StorageError) {
+      reportStatus = lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR;
+    }
+
+    return reportStatus;
+  }
+
+  /**
+   * Import the JSON files from services/settings/dump into the local DB.
+   */
+  async _importJSONDump() {
+    lazy.console.info(`${this.identifier} try to restore dump`);
+    const result = await lazy.RemoteSettingsWorker.importJSONDump(
+      this.bucketName,
+      this.collectionName
+    );
+    if (result < 0) {
+      lazy.console.debug(`${this.identifier} no dump available`);
+    } else {
+      lazy.console.info(
+        `${this.identifier} imported ${result} records from dump`
+      );
+    }
+    return result;
+  }
+
+  /**
+   * Fetch the signature info from the collection metadata and verifies that the
+   * local set of records has the same.
+   *
+   * @param {Array<Object>} records The list of records to validate.
+   * @param {int} timestamp         The timestamp associated with the list of remote records.
+   * @param {Object} metadata       The collection metadata, that contains the signature payload.
+   * @returns {Promise}
+   */
+  async _validateCollectionSignature(records, timestamp, metadata) {
+    if (!metadata?.signature) {
+      throw new MissingSignatureError(this.identifier);
+    }
+
+    if (!this._verifier) {
+      this._verifier = Cc[
+        "@mozilla.org/security/contentsignatureverifier;1"
+      ].createInstance(Ci.nsIContentSignatureVerifier);
+    }
+
+    // This is a content-signature field from an autograph response.
+    const {
+      signature: { x5u, signature },
+    } = metadata;
+    const certChain = await (await lazy.Utils.fetch(x5u)).text();
+    // Merge remote records with local ones and serialize as canonical JSON.
+    const serialized = await lazy.RemoteSettingsWorker.canonicalStringify(
+      records,
+      timestamp
+    );
+
+    lazy.console.debug(`${this.identifier} verify signature using ${x5u}`);
+    if (
+      !(await this._verifier.asyncVerifyContentSignature(
+        serialized,
+        "p384ecdsa=" + signature,
+        certChain,
+        this.signerName,
+        lazy.Utils.CERT_CHAIN_ROOT_IDENTIFIER
+      ))
+    ) {
+      throw new InvalidSignatureError(this.identifier, x5u);
+    }
+  }
+
+  /**
+   * This method is in charge of fetching data from server, applying the diff-based
+   * changes to the local DB, validating the signature, and computing a synchronization
+   * result with the list of creation, updates, and deletions.
+   *
+   * @param {Array<Object>} localRecords      Current list of records in local DB.
+   * @param {int}           localTimestamp    Current timestamp in local DB.
+   * @param {Object}        localMetadata     Current metadata in local DB.
+   * @param {int}           expectedTimestamp Cache busting of collection metadata
+   * @param {Object}        options
+   * @param {bool}          options.retry     Whether this method is called in the
+   *                                          retry situation.
+   *
+   * @returns {Promise<Object>} the computed sync result.
+   */
+  async _importChanges(
+    localRecords,
+    localTimestamp,
+    localMetadata,
+    expectedTimestamp,
+    options = {}
+  ) {
+    const { retry = false } = options;
+    const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`;
+
+    // Fetch collection metadata and list of changes from server.
+    lazy.console.debug(
+      `${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
+    );
+    const {
+      metadata,
+      remoteTimestamp,
+      remoteRecords,
+    } = await this._fetchChangeset(expectedTimestamp, since);
+
+    // We build a sync result, based on remote changes.
+    const syncResult = {
+      current: localRecords,
+      created: [],
+      updated: [],
+      deleted: [],
+    };
+    // If data wasn't changed, return empty sync result.
+    // This can happen when we update the signature but not the data.
+    lazy.console.debug(
+      `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
+    );
+    if (localTimestamp && remoteTimestamp < localTimestamp) {
+      return syncResult;
+    }
+
+    await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
+      clear: retry,
+    });
+
+    // Read the new local data, after updating.
+    const newLocal = await this.db.list();
+    const newRecords = newLocal.map(r => this._cleanLocalFields(r));
+    // And verify the signature on what is now stored.
+    if (this.verifySignature) {
+      try {
+        await this._validateCollectionSignature(
+          newRecords,
+          remoteTimestamp,
+          metadata
+        );
+      } catch (e) {
+        lazy.console.error(
+          `${this.identifier} Signature failed ${retry ? "again" : ""} ${e}`
+        );
+        if (!(e instanceof InvalidSignatureError)) {
+          // If it failed for any other kind of error (eg. shutdown)
+          // then give up quickly.
+          throw e;
+        }
+
+        // In order to distinguish signature errors that happen
+        // during sync, from hijacks of local DBs, we will verify
+        // the signature on the data that we had before syncing.
+        let localTrustworthy = false;
+        lazy.console.debug(`${this.identifier} verify data before sync`);
+        try {
+          await this._validateCollectionSignature(
+            localRecords,
+            localTimestamp,
+            localMetadata
+          );
+          localTrustworthy = true;
+        } catch (sigerr) {
+          if (!(sigerr instanceof InvalidSignatureError)) {
+            // If it fails for other reason, keep original error and give up.
+            throw sigerr;
+          }
+          lazy.console.debug(`${this.identifier} previous data was invalid`);
+        }
+
+        if (!localTrustworthy && !retry) {
+          // Signature failed, clear local DB because it contains
+          // bad data (local + remote changes).
+          lazy.console.debug(`${this.identifier} clear local data`);
+          await this.db.clear();
+          // Local data was tampered, throw and it will retry from empty DB.
+          lazy.console.error(`${this.identifier} local data was corrupted`);
+          throw new CorruptedDataError(this.identifier);
+        } else if (retry) {
+          // We retried already, we will restore the previous local data
+          // before throwing eventually.
+          if (localTrustworthy) {
+            await this.db.importChanges(
+              localMetadata,
+              localTimestamp,
+              localRecords,
+              {
+                clear: true, // clear before importing.
+              }
+            );
+          } else {
+            // Restore the dump if available (no-op if no dump)
+            const imported = await this._importJSONDump();
+            // _importJSONDump() only clears DB if dump is available,
+            // therefore do it here!
+            if (imported < 0) {
+              await this.db.clear();
+            }
+          }
+        }
+        throw e;
+      }
+    } else {
+      lazy.console.warn(`${this.identifier} has signature disabled`);
+    }
+
+    if (this.hasListeners("sync")) {
+      // If we have some listeners for the "sync" event,
+      // Compute the changes, comparing records before and after.
+      syncResult.current = newRecords;
+      const oldById = new Map(localRecords.map(e => [e.id, e]));
+      for (const r of newRecords) {
+        const old = oldById.get(r.id);
+        if (old) {
+          oldById.delete(r.id);
+          if (r.last_modified != old.last_modified) {
+            syncResult.updated.push({ old, new: r });
+          }
+        } else {
+          syncResult.created.push(r);
+        }
+      }
+      syncResult.deleted = syncResult.deleted.concat(
+        Array.from(oldById.values())
+      );
+      lazy.console.debug(
+        `${this.identifier} ${syncResult.created.length} created. ${syncResult.updated.length} updated. ${syncResult.deleted.length} deleted.`
+      );
+    }
+
+    return syncResult;
+  }
+
+  /**
+   * Fetch information from changeset endpoint.
+   *
+   * @param expectedTimestamp cache busting value
+   * @param since timestamp of last sync (optional)
+   */
+  async _fetchChangeset(expectedTimestamp, since) {
+    const client = this.httpClient();
+    const {
+      metadata,
+      timestamp: remoteTimestamp,
+      changes: remoteRecords,
+    } = await client.execute(
+      {
+        path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`,
+      },
+      {
+        query: {
+          _expected: expectedTimestamp,
+          _since: since,
+        },
+      }
+    );
+    return {
+      remoteTimestamp,
+      metadata,
+      remoteRecords,
+    };
+  }
+
+  /**
+   * Use the filter func to filter the lists of changes obtained from synchronization,
+   * and return them along with the filtered list of local records.
+   *
+   * If the filtered lists of changes are all empty, we return null (and thus don't
+   * bother listing local DB).
+   *
+   * @param {Object}     syncResult       Synchronization result without filtering.
+   *
+   * @returns {Promise<Object>} the filtered list of local records, plus the filtered
+   *                            list of created, updated and deleted records.
+   */
+  async _filterSyncResult(syncResult) {
+    // Handle the obtained records (ie. apply locally through events).
+    // Build the event data list. It should be filtered (ie. by application target)
+    const {
+      current: allData,
+      created: allCreated,
+      updated: allUpdated,
+      deleted: allDeleted,
+    } = syncResult;
+    const [created, deleted, updatedFiltered] = await Promise.all(
+      [allCreated, allDeleted, allUpdated.map(e => e.new)].map(
+        this._filterEntries.bind(this)
+      )
+    );
+    // For updates, keep entries whose updated form matches the target.
+    const updatedFilteredIds = new Set(updatedFiltered.map(e => e.id));
+    const updated = allUpdated.filter(({ new: { id } }) =>
+      updatedFilteredIds.has(id)
+    );
+
+    if (!created.length && !updated.length && !deleted.length) {
+      return null;
+    }
+    // Read local collection of records (also filtered).
+    const current = await this._filterEntries(allData);
+    return { created, updated, deleted, current };
+  }
+
+  /**
+   * Filter entries for which calls to `this.filterFunc` returns null.
+   *
+   * @param {Array<Objet>} data
+   * @returns {Array<Object>}
+   */
+  async _filterEntries(data) {
+    if (!this.filterFunc) {
+      return data;
+    }
+    const environment = cacheProxy(lazy.ClientEnvironmentBase);
+    const dataPromises = data.map(e => this.filterFunc(e, environment));
+    const results = await Promise.all(dataPromises);
+    return results.filter(Boolean);
+  }
+
+  /**
+   * Remove the fields from the specified record
+   * that are not present on server.
+   *
+   * @param {Object} record
+   */
+  _cleanLocalFields(record) {
+    const keys = ["_status"].concat(this.localFields);
+    const result = { ...record };
+    for (const key of keys) {
+      delete result[key];
+    }
+    return result;
+  }
+}