diff options
Diffstat (limited to 'services/settings/RemoteSettingsClient.sys.mjs')
| -rw-r--r-- | services/settings/RemoteSettingsClient.sys.mjs | 1281 |
1 files changed, 1281 insertions, 0 deletions
diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs new file mode 100644 index 0000000000..7e95b9baab --- /dev/null +++ b/services/settings/RemoteSettingsClient.sys.mjs @@ -0,0 +1,1281 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + +import { Downloader } from "resource://services-settings/Attachments.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + ClientEnvironmentBase: + "resource://gre/modules/components-utils/ClientEnvironment.sys.mjs", + Database: "resource://services-settings/Database.sys.mjs", + IDBHelpers: "resource://services-settings/IDBHelpers.sys.mjs", + KintoHttpClient: "resource://services-common/kinto-http-client.sys.mjs", + ObjectUtils: "resource://gre/modules/ObjectUtils.sys.mjs", + RemoteSettingsWorker: + "resource://services-settings/RemoteSettingsWorker.sys.mjs", + SharedUtils: "resource://services-settings/SharedUtils.sys.mjs", + UptakeTelemetry: "resource://services-common/uptake-telemetry.sys.mjs", + Utils: "resource://services-settings/Utils.sys.mjs", +}); + +const TELEMETRY_COMPONENT = "remotesettings"; + +ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log); + +/** + * cacheProxy returns an object Proxy that will memoize properties of the target. + * @param {Object} target the object to wrap. + * @returns {Proxy} + */ +function cacheProxy(target) { + const cache = new Map(); + return new Proxy(target, { + get(target, prop, receiver) { + if (!cache.has(prop)) { + cache.set(prop, target[prop]); + } + return cache.get(prop); + }, + }); +} + +/** + * Minimalist event emitter. + * + * Note: we don't use `toolkit/modules/EventEmitter` because **we want** to throw + * an error when a listener fails to execute. + */ +class EventEmitter { + constructor(events) { + this._listeners = new Map(); + for (const event of events) { + this._listeners.set(event, []); + } + } + + /** + * Event emitter: will execute the registered listeners in the order and + * sequentially. + * + * @param {string} event the event name + * @param {Object} payload the event payload to call the listeners with + */ + async emit(event, payload) { + const callbacks = this._listeners.get(event); + let lastError; + for (const cb of callbacks) { + try { + await cb(payload); + } catch (e) { + lastError = e; + } + } + if (lastError) { + throw lastError; + } + } + + hasListeners(event) { + return this._listeners.has(event) && !!this._listeners.get(event).length; + } + + on(event, callback) { + if (!this._listeners.has(event)) { + throw new Error(`Unknown event type ${event}`); + } + this._listeners.get(event).push(callback); + } + + off(event, callback) { + if (!this._listeners.has(event)) { + throw new Error(`Unknown event type ${event}`); + } + const callbacks = this._listeners.get(event); + const i = callbacks.indexOf(callback); + if (i < 0) { + throw new Error(`Unknown callback`); + } else { + callbacks.splice(i, 1); + } + } +} + +class APIError extends Error {} + +class NetworkError extends APIError { + constructor(e) { + super(`Network error: ${e}`, { cause: e }); + this.name = "NetworkError"; + } +} + +class NetworkOfflineError extends APIError { + constructor() { + super("Network is offline"); + this.name = "NetworkOfflineError"; + } +} + +class ServerContentParseError extends APIError { + constructor(e) { + super(`Cannot parse server content: ${e}`, { cause: e }); + this.name = "ServerContentParseError"; + } +} + +class BackendError extends APIError { + constructor(e) { + super(`Backend error: ${e}`, { cause: e }); + this.name = "BackendError"; + } +} + +class BackoffError extends APIError { + constructor(e) { + super(`Server backoff: ${e}`, { cause: e }); + this.name = "BackoffError"; + } +} + +class TimeoutError extends APIError { + constructor(e) { + super(`API timeout: ${e}`, { cause: e }); + this.name = "TimeoutError"; + } +} + +class StorageError extends Error { + constructor(e) { + super(`Storage error: ${e}`, { cause: e }); + this.name = "StorageError"; + } +} + +class InvalidSignatureError extends Error { + constructor(cid, x5u) { + let message = `Invalid content signature (${cid})`; + if (x5u) { + const chain = x5u.split("/").pop(); + message += ` using '${chain}'`; + } + super(message); + this.name = "InvalidSignatureError"; + } +} + +class MissingSignatureError extends InvalidSignatureError { + constructor(cid) { + super(cid); + this.message = `Missing signature (${cid})`; + this.name = "MissingSignatureError"; + } +} + +class CorruptedDataError extends InvalidSignatureError { + constructor(cid) { + super(cid); + this.message = `Corrupted local data (${cid})`; + this.name = "CorruptedDataError"; + } +} + +class UnknownCollectionError extends Error { + constructor(cid) { + super(`Unknown Collection "${cid}"`); + this.name = "UnknownCollectionError"; + } +} + +class AttachmentDownloader extends Downloader { + constructor(client) { + super(client.bucketName, client.collectionName); + this._client = client; + } + + get cacheImpl() { + const cacheImpl = { + get: async attachmentId => { + return this._client.db.getAttachment(attachmentId); + }, + set: async (attachmentId, attachment) => { + return this._client.db.saveAttachment(attachmentId, attachment); + }, + delete: async attachmentId => { + return this._client.db.saveAttachment(attachmentId, null); + }, + prune: async excludeIds => { + return this._client.db.pruneAttachments(excludeIds); + }, + }; + Object.defineProperty(this, "cacheImpl", { value: cacheImpl }); + return cacheImpl; + } + + /** + * Download attachment and report Telemetry on failure. + * + * @see Downloader.download + */ + async download(record, options) { + try { + // Explicitly await here to ensure we catch a network error. + return await super.download(record, options); + } catch (err) { + // Report download error. + let status = lazy.UptakeTelemetry.STATUS.DOWNLOAD_ERROR; + if (lazy.Utils.isOffline) { + status = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR; + } else if (/NetworkError/.test(err.message)) { + status = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR; + } + // If the file failed to be downloaded, report it as such in Telemetry. + await lazy.UptakeTelemetry.report(TELEMETRY_COMPONENT, status, { + source: this._client.identifier, + }); + throw err; + } + } + + /** + * Delete all downloaded records attachments. + * + * Note: the list of attachments to be deleted is based on the + * current list of records. + */ + async deleteAll() { + let allRecords = await this._client.db.list(); + return Promise.all( + allRecords + .filter(r => !!r.attachment) + .map(r => + Promise.all([this.deleteDownloaded(r), this.deleteFromDisk(r)]) + ) + ); + } +} + +export class RemoteSettingsClient extends EventEmitter { + static get APIError() { + return APIError; + } + static get NetworkError() { + return NetworkError; + } + static get NetworkOfflineError() { + return NetworkOfflineError; + } + static get ServerContentParseError() { + return ServerContentParseError; + } + static get BackendError() { + return BackendError; + } + static get BackoffError() { + return BackoffError; + } + static get TimeoutError() { + return TimeoutError; + } + static get StorageError() { + return StorageError; + } + static get InvalidSignatureError() { + return InvalidSignatureError; + } + static get MissingSignatureError() { + return MissingSignatureError; + } + static get CorruptedDataError() { + return CorruptedDataError; + } + static get UnknownCollectionError() { + return UnknownCollectionError; + } + + constructor( + collectionName, + { + bucketName = AppConstants.REMOTE_SETTINGS_DEFAULT_BUCKET, + signerName, + filterFunc, + localFields = [], + keepAttachmentsIds = [], + lastCheckTimePref, + } = {} + ) { + // Remote Settings cannot be used in child processes (no access to disk, + // easily killed, isolated observer notifications etc.). + // Since our goal here is to prevent consumers to instantiate while developing their + // feature, throwing in Nightly only is enough, and prevents unexpected crashes + // in release or beta. + if ( + !AppConstants.RELEASE_OR_BETA && + Services.appinfo.processType !== Services.appinfo.PROCESS_TYPE_DEFAULT + ) { + throw new Error( + "Cannot instantiate Remote Settings client in child processes." + ); + } + + super(["sync"]); // emitted events + + this.collectionName = collectionName; + // Client is constructed with the raw bucket name (eg. "main", "security-state", "blocklist") + // The `bucketName` will contain the `-preview` suffix if the preview mode is enabled. + this.bucketName = lazy.Utils.actualBucketName(bucketName); + this.signerName = signerName; + this.filterFunc = filterFunc; + this.localFields = localFields; + this.keepAttachmentsIds = keepAttachmentsIds; + this._lastCheckTimePref = lastCheckTimePref; + this._verifier = null; + this._syncRunning = false; + + // This attribute allows signature verification to be disabled, when running tests + // or when pulling data from a dev server. + this.verifySignature = AppConstants.REMOTE_SETTINGS_VERIFY_SIGNATURE; + + ChromeUtils.defineLazyGetter( + this, + "db", + () => new lazy.Database(this.identifier) + ); + + ChromeUtils.defineLazyGetter( + this, + "attachments", + () => new AttachmentDownloader(this) + ); + } + + /** + * Internal method to refresh the client bucket name after the preview mode + * was toggled. + * + * See `RemoteSettings.enabledPreviewMode()`. + */ + refreshBucketName() { + this.bucketName = lazy.Utils.actualBucketName(this.bucketName); + this.db.identifier = this.identifier; + } + + get identifier() { + return `${this.bucketName}/${this.collectionName}`; + } + + get lastCheckTimePref() { + return ( + this._lastCheckTimePref || + `services.settings.${this.bucketName}.${this.collectionName}.last_check` + ); + } + + httpClient() { + const api = new lazy.KintoHttpClient(lazy.Utils.SERVER_URL, { + fetchFunc: lazy.Utils.fetch, // Use fetch() wrapper. + }); + return api.bucket(this.bucketName).collection(this.collectionName); + } + + /** + * Retrieve the collection timestamp for the last synchronization. + * This is an opaque and comparable value assigned automatically by + * the server. + * + * @returns {number} + * The timestamp in milliseconds, returns -1 if retrieving + * the timestamp from the kinto collection fails. + */ + async getLastModified() { + let timestamp = -1; + try { + timestamp = await this.db.getLastModified(); + } catch (err) { + lazy.console.warn( + `Error retrieving the getLastModified timestamp from ${this.identifier} RemoteSettingsClient`, + err + ); + } + + return timestamp; + } + + /** + * Lists settings. + * + * @param {Object} options The options object. + * @param {Object} options.filters Filter the results (default: `{}`). + * @param {String} options.order The order to apply (eg. `"-last_modified"`). + * @param {boolean} options.dumpFallback Fallback to dump data if read of local DB fails (default: `true`). + * @param {boolean} options.emptyListFallback Fallback to empty list if no dump data and read of local DB fails (default: `true`). + * @param {boolean} options.loadDumpIfNewer Use dump data if it is newer than local data (default: `true`). + * @param {boolean} options.forceSync Always synchronize from server before returning results (default: `false`). + * @param {boolean} options.syncIfEmpty Synchronize from server if local data is empty (default: `true`). + * @param {boolean} options.verifySignature Verify the signature of the local data (default: `false`). + * @return {Promise} + */ + async get(options = {}) { + const { + filters = {}, + order = "", // not sorted by default. + dumpFallback = true, + emptyListFallback = true, + forceSync = false, + loadDumpIfNewer = true, + syncIfEmpty = true, + } = options; + let { verifySignature = false } = options; + + const hasParallelCall = !!this._importingPromise; + let data; + try { + let lastModified = forceSync ? null : await this.db.getLastModified(); + let hasLocalData = lastModified !== null; + + if (forceSync) { + if (!this._importingPromise) { + this._importingPromise = (async () => { + await this.sync({ sendEvents: false, trigger: "forced" }); + return true; // No need to re-verify signature after sync. + })(); + } + } else if (syncIfEmpty && !hasLocalData) { + // .get() was called before we had the chance to synchronize the local database. + // We'll try to avoid returning an empty list. + if (!this._importingPromise) { + // Prevent parallel loading when .get() is called multiple times. + this._importingPromise = (async () => { + const importedFromDump = lazy.Utils.LOAD_DUMPS + ? await this._importJSONDump() + : -1; + if (importedFromDump < 0) { + // There is no JSON dump to load, force a synchronization from the server. + // We don't want the "sync" event to be sent, since some consumers use `.get()` + // in "sync" callbacks. See Bug 1761953 + lazy.console.debug( + `${this.identifier} Local DB is empty, pull data from server` + ); + await this.sync({ loadDump: false, sendEvents: false }); + } + // Return `true` to indicate we don't need to `verifySignature`, + // since a trusted dump was loaded or a signature verification + // happened during synchronization. + return true; + })(); + } else { + lazy.console.debug(`${this.identifier} Awaiting existing import.`); + } + } else if (hasLocalData && loadDumpIfNewer) { + // Check whether the local data is older than the packaged dump. + // If it is, load the packaged dump (which overwrites the local data). + let lastModifiedDump = await lazy.Utils.getLocalDumpLastModified( + this.bucketName, + this.collectionName + ); + if (lastModified < lastModifiedDump) { + lazy.console.debug( + `${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})` + ); + if (!this._importingPromise) { + // As part of importing, any existing data is wiped. + this._importingPromise = (async () => { + const importedFromDump = await this._importJSONDump(); + // Return `true` to skip signature verification if a dump was found. + // The dump can be missing if a collection is listed in the timestamps summary file, + // because its dump is present in the source tree, but the dump was not + // included in the `package-manifest.in` file. (eg. android, thunderbird) + return importedFromDump >= 0; + })(); + } else { + lazy.console.debug(`${this.identifier} Awaiting existing import.`); + } + } + } + + if (this._importingPromise) { + try { + if (await this._importingPromise) { + // No need to verify signature, because either we've just loaded a trusted + // dump (here or in a parallel call), or it was verified during sync. + verifySignature = false; + } + } catch (e) { + if (!hasParallelCall) { + // Sync or load dump failed. Throw. + throw e; + } + // Report error, but continue because there could have been data + // loaded from a parallel call. + console.error(e); + } finally { + // then delete this promise again, as now we should have local data: + delete this._importingPromise; + } + } + + // Read from the local DB. + data = await this.db.list({ filters, order }); + } catch (e) { + // If the local DB cannot be read (for unknown reasons, Bug 1649393) + // or sync failed, we fallback to the packaged data, and filter/sort in memory. + if (!dumpFallback) { + throw e; + } + console.error(e); + let { data } = await lazy.SharedUtils.loadJSONDump( + this.bucketName, + this.collectionName + ); + if (data !== null) { + lazy.console.info(`${this.identifier} falling back to JSON dump`); + } else if (emptyListFallback) { + lazy.console.info( + `${this.identifier} no dump fallback, return empty list` + ); + data = []; + } else { + // Obtaining the records failed, there is no dump, and we don't fallback + // to an empty list. Throw the original error. + throw e; + } + if (!lazy.ObjectUtils.isEmpty(filters)) { + data = data.filter(r => lazy.Utils.filterObject(filters, r)); + } + if (order) { + data = lazy.Utils.sortObjects(order, data); + } + // No need to verify signature on JSON dumps. + // If local DB cannot be read, then we don't even try to do anything, + // we return results early. + return this._filterEntries(data); + } + + lazy.console.debug( + `${this.identifier} ${data.length} records before filtering.` + ); + + if (verifySignature) { + lazy.console.debug( + `${this.identifier} verify signature of local data on read` + ); + const allData = lazy.ObjectUtils.isEmpty(filters) + ? data + : await this.db.list(); + const localRecords = allData.map(r => this._cleanLocalFields(r)); + const timestamp = await this.db.getLastModified(); + let metadata = await this.db.getMetadata(); + if (syncIfEmpty && lazy.ObjectUtils.isEmpty(metadata)) { + // No sync occured yet, may have records from dump but no metadata. + // We don't want the "sync" event to be sent, since some consumers use `.get()` + // in "sync" callbacks. See Bug 1761953 + await this.sync({ loadDump: false, sendEvents: false }); + metadata = await this.db.getMetadata(); + } + // Will throw MissingSignatureError if no metadata and `syncIfEmpty` is false. + await this._validateCollectionSignature( + localRecords, + timestamp, + metadata + ); + } + + // Filter the records based on `this.filterFunc` results. + const final = await this._filterEntries(data); + lazy.console.debug( + `${this.identifier} ${final.length} records after filtering.` + ); + return final; + } + + /** + * Synchronize the local database with the remote server. + * + * @param {Object} options See #maybeSync() options. + */ + async sync(options) { + if (lazy.Utils.shouldSkipRemoteActivityDueToTests) { + return; + } + + // We want to know which timestamp we are expected to obtain in order to leverage + // cache busting. We don't provide ETag because we don't want a 304. + const { changes } = await lazy.Utils.fetchLatestChanges( + lazy.Utils.SERVER_URL, + { + filters: { + collection: this.collectionName, + bucket: this.bucketName, + }, + } + ); + if (changes.length === 0) { + throw new RemoteSettingsClient.UnknownCollectionError(this.identifier); + } + // According to API, there will be one only (fail if not). + const [{ last_modified: expectedTimestamp }] = changes; + + await this.maybeSync(expectedTimestamp, { ...options, trigger: "forced" }); + } + + /** + * Synchronize the local database with the remote server, **only if necessary**. + * + * @param {int} expectedTimestamp the lastModified date (on the server) for the remote collection. + * This will be compared to the local timestamp, and will be used for + * cache busting if local data is out of date. + * @param {Object} options additional advanced options. + * @param {bool} options.loadDump load initial dump from disk on first sync (default: true if server is prod) + * @param {bool} options.sendEvents send `"sync"` events (default: `true`) + * @param {string} options.trigger label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`) + * @return {Promise} which rejects on sync or process failure. + */ + async maybeSync(expectedTimestamp, options = {}) { + // Should the clients try to load JSON dump? (mainly disabled in tests) + const { + loadDump = lazy.Utils.LOAD_DUMPS, + trigger = "manual", + sendEvents = true, + } = options; + + // Make sure we don't run several synchronizations in parallel, mainly + // in order to avoid race conditions in "sync" events listeners. + if (this._syncRunning) { + lazy.console.warn(`${this.identifier} sync already running`); + return; + } + + // Prevent network requests and IndexedDB calls to be initiated + // during shutdown. + if (Services.startup.shuttingDown) { + lazy.console.warn(`${this.identifier} sync interrupted by shutdown`); + return; + } + + this._syncRunning = true; + + let importedFromDump = []; + const startedAt = new Date(); + let reportStatus = null; + let thrownError = null; + try { + // If network is offline, we can't synchronize. + if (lazy.Utils.isOffline) { + throw new RemoteSettingsClient.NetworkOfflineError(); + } + + // Read last timestamp and local data before sync. + let collectionLastModified = await this.db.getLastModified(); + const allData = await this.db.list(); + // Local data can contain local fields, strip them. + let localRecords = allData.map(r => this._cleanLocalFields(r)); + const localMetadata = await this.db.getMetadata(); + + // If there is no data currently in the collection, attempt to import + // initial data from the application defaults. + // This allows to avoid synchronizing the whole collection content on + // cold start. + if (!collectionLastModified && loadDump) { + try { + const imported = await this._importJSONDump(); + // The worker only returns an integer. List the imported records to build the sync event. + if (imported > 0) { + lazy.console.debug( + `${this.identifier} ${imported} records loaded from JSON dump` + ); + importedFromDump = await this.db.list(); + // Local data is the data loaded from dump. We will need this later + // to compute the sync result. + localRecords = importedFromDump; + } + collectionLastModified = await this.db.getLastModified(); + } catch (e) { + // Report but go-on. + console.error(e); + } + } + let syncResult; + try { + // Is local timestamp up to date with the server? + if (expectedTimestamp == collectionLastModified) { + lazy.console.debug(`${this.identifier} local data is up-to-date`); + reportStatus = lazy.UptakeTelemetry.STATUS.UP_TO_DATE; + + // If the data is up-to-date but don't have metadata (records loaded from dump), + // we fetch them and validate the signature immediately. + if (this.verifySignature && lazy.ObjectUtils.isEmpty(localMetadata)) { + lazy.console.debug(`${this.identifier} pull collection metadata`); + const metadata = await this.httpClient().getData({ + query: { _expected: expectedTimestamp }, + }); + await this.db.importChanges(metadata); + // We don't bother validating the signature if the dump was just loaded. We do + // if the dump was loaded at some other point (eg. from .get()). + if (this.verifySignature && !importedFromDump.length) { + lazy.console.debug( + `${this.identifier} verify signature of local data` + ); + await this._validateCollectionSignature( + localRecords, + collectionLastModified, + metadata + ); + } + } + + // Since the data is up-to-date, if we didn't load any dump then we're done here. + if (!importedFromDump.length) { + return; + } + // Otherwise we want to continue with sending the sync event to notify about the created records. + syncResult = { + current: importedFromDump, + created: importedFromDump, + updated: [], + deleted: [], + }; + } else { + // Local data is either outdated or tampered. + // In both cases we will fetch changes from server, + // and make sure we overwrite local data. + syncResult = await this._importChanges( + localRecords, + collectionLastModified, + localMetadata, + expectedTimestamp + ); + if (sendEvents && this.hasListeners("sync")) { + // If we have listeners for the "sync" event, then compute the lists of changes. + // The records imported from the dump should be considered as "created" for the + // listeners. + const importedById = importedFromDump.reduce((acc, r) => { + acc.set(r.id, r); + return acc; + }, new Map()); + // Deleted records should not appear as created. + syncResult.deleted.forEach(r => importedById.delete(r.id)); + // Records from dump that were updated should appear in their newest form. + syncResult.updated.forEach(u => { + if (importedById.has(u.old.id)) { + importedById.set(u.old.id, u.new); + } + }); + syncResult.created = syncResult.created.concat( + Array.from(importedById.values()) + ); + } + + // When triggered from the daily timer, and if the sync was successful, and once + // all sync listeners have been executed successfully, we prune potential + // obsolete attachments that may have been left in the local cache. + if (trigger == "timer") { + const deleted = await this.attachments.prune( + this.keepAttachmentsIds + ); + if (deleted > 0) { + lazy.console.warn( + `${this.identifier} Pruned ${deleted} obsolete attachments` + ); + } + } + } + } catch (e) { + if (e instanceof InvalidSignatureError) { + // Signature verification failed during synchronization. + reportStatus = + e instanceof CorruptedDataError + ? lazy.UptakeTelemetry.STATUS.CORRUPTION_ERROR + : lazy.UptakeTelemetry.STATUS.SIGNATURE_ERROR; + // If sync fails with a signature error, it's likely that our + // local data has been modified in some way. + // We will attempt to fix this by retrieving the whole + // remote collection. + try { + lazy.console.warn( + `${this.identifier} Signature verified failed. Retry from scratch` + ); + syncResult = await this._importChanges( + localRecords, + collectionLastModified, + localMetadata, + expectedTimestamp, + { retry: true } + ); + } catch (e) { + // If the signature fails again, or if an error occured during wiping out the + // local data, then we report it as a *signature retry* error. + reportStatus = lazy.UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR; + throw e; + } + } else { + // The sync has thrown for other reason than signature verification. + // Obtain a more precise error than original one. + const adjustedError = this._adjustedError(e); + // Default status for errors at this step is SYNC_ERROR. + reportStatus = this._telemetryFromError(adjustedError, { + default: lazy.UptakeTelemetry.STATUS.SYNC_ERROR, + }); + throw adjustedError; + } + } + if (sendEvents) { + // Filter the synchronization results using `filterFunc` (ie. JEXL). + const filteredSyncResult = await this._filterSyncResult(syncResult); + // If every changed entry is filtered, we don't even fire the event. + if (filteredSyncResult) { + try { + await this.emit("sync", { data: filteredSyncResult }); + } catch (e) { + reportStatus = lazy.UptakeTelemetry.STATUS.APPLY_ERROR; + throw e; + } + } else { + lazy.console.info( + `All changes are filtered by JEXL expressions for ${this.identifier}` + ); + } + } + } catch (e) { + thrownError = e; + // Obtain a more precise error than original one. + const adjustedError = this._adjustedError(e); + // If browser is shutting down, then we can report a specific status. + // (eg. IndexedDB will abort transactions) + if (Services.startup.shuttingDown) { + reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR; + } + // If no Telemetry status was determined yet (ie. outside sync step), + // then introspect error, default status at this step is UNKNOWN. + else if (reportStatus == null) { + reportStatus = this._telemetryFromError(adjustedError, { + default: lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR, + }); + } + throw e; + } finally { + const durationMilliseconds = new Date() - startedAt; + // No error was reported, this is a success! + if (reportStatus === null) { + reportStatus = lazy.UptakeTelemetry.STATUS.SUCCESS; + } + // Report success/error status to Telemetry. + let reportArgs = { + source: this.identifier, + trigger, + duration: durationMilliseconds, + }; + // In Bug 1617133, we will try to break down specific errors into + // more precise statuses by reporting the JavaScript error name + // ("TypeError", etc.) to Telemetry on Nightly. + const channel = lazy.UptakeTelemetry.Policy.getChannel(); + if ( + thrownError !== null && + channel == "nightly" && + [ + lazy.UptakeTelemetry.STATUS.SYNC_ERROR, + lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR, // IndexedDB. + lazy.UptakeTelemetry.STATUS.UNKNOWN_ERROR, + lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR, + ].includes(reportStatus) + ) { + // List of possible error names for IndexedDB: + // https://searchfox.org/mozilla-central/rev/49ed791/dom/base/DOMException.cpp#28-53 + reportArgs = { ...reportArgs, errorName: thrownError.name }; + } + + await lazy.UptakeTelemetry.report( + TELEMETRY_COMPONENT, + reportStatus, + reportArgs + ); + + lazy.console.debug(`${this.identifier} sync status is ${reportStatus}`); + this._syncRunning = false; + } + } + + /** + * Return a more precise error instance, based on the specified + * error and its message. + * @param {Error} e the original error + * @returns {Error} + */ + _adjustedError(e) { + if (/unparseable/.test(e.message)) { + return new RemoteSettingsClient.ServerContentParseError(e); + } + if (/NetworkError/.test(e.message)) { + return new RemoteSettingsClient.NetworkError(e); + } + if (/Timeout/.test(e.message)) { + return new RemoteSettingsClient.TimeoutError(e); + } + if (/HTTP 5??/.test(e.message)) { + return new RemoteSettingsClient.BackendError(e); + } + if (/Backoff/.test(e.message)) { + return new RemoteSettingsClient.BackoffError(e); + } + if ( + // Errors from kinto.js IDB adapter. + e instanceof lazy.IDBHelpers.IndexedDBError || + // Other IndexedDB errors (eg. RemoteSettingsWorker). + /IndexedDB/.test(e.message) + ) { + return new RemoteSettingsClient.StorageError(e); + } + return e; + } + + /** + * Determine the Telemetry uptake status based on the specified + * error. + */ + _telemetryFromError(e, options = { default: null }) { + let reportStatus = options.default; + + if (e instanceof RemoteSettingsClient.NetworkOfflineError) { + reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_OFFLINE_ERROR; + } else if (e instanceof lazy.IDBHelpers.ShutdownError) { + reportStatus = lazy.UptakeTelemetry.STATUS.SHUTDOWN_ERROR; + } else if (e instanceof RemoteSettingsClient.ServerContentParseError) { + reportStatus = lazy.UptakeTelemetry.STATUS.PARSE_ERROR; + } else if (e instanceof RemoteSettingsClient.NetworkError) { + reportStatus = lazy.UptakeTelemetry.STATUS.NETWORK_ERROR; + } else if (e instanceof RemoteSettingsClient.TimeoutError) { + reportStatus = lazy.UptakeTelemetry.STATUS.TIMEOUT_ERROR; + } else if (e instanceof RemoteSettingsClient.BackendError) { + reportStatus = lazy.UptakeTelemetry.STATUS.SERVER_ERROR; + } else if (e instanceof RemoteSettingsClient.BackoffError) { + reportStatus = lazy.UptakeTelemetry.STATUS.BACKOFF; + } else if (e instanceof RemoteSettingsClient.StorageError) { + reportStatus = lazy.UptakeTelemetry.STATUS.CUSTOM_1_ERROR; + } + + return reportStatus; + } + + /** + * Import the JSON files from services/settings/dump into the local DB. + */ + async _importJSONDump() { + lazy.console.info(`${this.identifier} try to restore dump`); + const result = await lazy.RemoteSettingsWorker.importJSONDump( + this.bucketName, + this.collectionName + ); + if (result < 0) { + lazy.console.debug(`${this.identifier} no dump available`); + } else { + lazy.console.info( + `${this.identifier} imported ${result} records from dump` + ); + } + return result; + } + + /** + * Fetch the signature info from the collection metadata and verifies that the + * local set of records has the same. + * + * @param {Array<Object>} records The list of records to validate. + * @param {int} timestamp The timestamp associated with the list of remote records. + * @param {Object} metadata The collection metadata, that contains the signature payload. + * @returns {Promise} + */ + async _validateCollectionSignature(records, timestamp, metadata) { + if (!metadata?.signature) { + throw new MissingSignatureError(this.identifier); + } + + if (!this._verifier) { + this._verifier = Cc[ + "@mozilla.org/security/contentsignatureverifier;1" + ].createInstance(Ci.nsIContentSignatureVerifier); + } + + // This is a content-signature field from an autograph response. + const { + signature: { x5u, signature }, + } = metadata; + const certChain = await (await lazy.Utils.fetch(x5u)).text(); + // Merge remote records with local ones and serialize as canonical JSON. + const serialized = await lazy.RemoteSettingsWorker.canonicalStringify( + records, + timestamp + ); + + lazy.console.debug(`${this.identifier} verify signature using ${x5u}`); + if ( + !(await this._verifier.asyncVerifyContentSignature( + serialized, + "p384ecdsa=" + signature, + certChain, + this.signerName, + lazy.Utils.CERT_CHAIN_ROOT_IDENTIFIER + )) + ) { + throw new InvalidSignatureError(this.identifier, x5u); + } + } + + /** + * This method is in charge of fetching data from server, applying the diff-based + * changes to the local DB, validating the signature, and computing a synchronization + * result with the list of creation, updates, and deletions. + * + * @param {Array<Object>} localRecords Current list of records in local DB. + * @param {int} localTimestamp Current timestamp in local DB. + * @param {Object} localMetadata Current metadata in local DB. + * @param {int} expectedTimestamp Cache busting of collection metadata + * @param {Object} options + * @param {bool} options.retry Whether this method is called in the + * retry situation. + * + * @returns {Promise<Object>} the computed sync result. + */ + async _importChanges( + localRecords, + localTimestamp, + localMetadata, + expectedTimestamp, + options = {} + ) { + const { retry = false } = options; + const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`; + + // Fetch collection metadata and list of changes from server. + lazy.console.debug( + `${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})` + ); + const { metadata, remoteTimestamp, remoteRecords } = + await this._fetchChangeset(expectedTimestamp, since); + + // We build a sync result, based on remote changes. + const syncResult = { + current: localRecords, + created: [], + updated: [], + deleted: [], + }; + // If data wasn't changed, return empty sync result. + // This can happen when we update the signature but not the data. + lazy.console.debug( + `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}` + ); + if (localTimestamp && remoteTimestamp < localTimestamp) { + return syncResult; + } + + await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, { + clear: retry, + }); + + // Read the new local data, after updating. + const newLocal = await this.db.list(); + const newRecords = newLocal.map(r => this._cleanLocalFields(r)); + // And verify the signature on what is now stored. + if (this.verifySignature) { + try { + await this._validateCollectionSignature( + newRecords, + remoteTimestamp, + metadata + ); + } catch (e) { + lazy.console.error( + `${this.identifier} Signature failed ${retry ? "again" : ""} ${e}` + ); + if (!(e instanceof InvalidSignatureError)) { + // If it failed for any other kind of error (eg. shutdown) + // then give up quickly. + throw e; + } + + // In order to distinguish signature errors that happen + // during sync, from hijacks of local DBs, we will verify + // the signature on the data that we had before syncing. + let localTrustworthy = false; + lazy.console.debug(`${this.identifier} verify data before sync`); + try { + await this._validateCollectionSignature( + localRecords, + localTimestamp, + localMetadata + ); + localTrustworthy = true; + } catch (sigerr) { + if (!(sigerr instanceof InvalidSignatureError)) { + // If it fails for other reason, keep original error and give up. + throw sigerr; + } + lazy.console.debug(`${this.identifier} previous data was invalid`); + } + + if (!localTrustworthy && !retry) { + // Signature failed, clear local DB because it contains + // bad data (local + remote changes). + lazy.console.debug(`${this.identifier} clear local data`); + await this.db.clear(); + // Local data was tampered, throw and it will retry from empty DB. + lazy.console.error(`${this.identifier} local data was corrupted`); + throw new CorruptedDataError(this.identifier); + } else if (retry) { + // We retried already, we will restore the previous local data + // before throwing eventually. + if (localTrustworthy) { + await this.db.importChanges( + localMetadata, + localTimestamp, + localRecords, + { + clear: true, // clear before importing. + } + ); + } else { + // Restore the dump if available (no-op if no dump) + const imported = await this._importJSONDump(); + // _importJSONDump() only clears DB if dump is available, + // therefore do it here! + if (imported < 0) { + await this.db.clear(); + } + } + } + throw e; + } + } else { + lazy.console.warn(`${this.identifier} has signature disabled`); + } + + if (this.hasListeners("sync")) { + // If we have some listeners for the "sync" event, + // Compute the changes, comparing records before and after. + syncResult.current = newRecords; + const oldById = new Map(localRecords.map(e => [e.id, e])); + for (const r of newRecords) { + const old = oldById.get(r.id); + if (old) { + oldById.delete(r.id); + if (r.last_modified != old.last_modified) { + syncResult.updated.push({ old, new: r }); + } + } else { + syncResult.created.push(r); + } + } + syncResult.deleted = syncResult.deleted.concat( + Array.from(oldById.values()) + ); + lazy.console.debug( + `${this.identifier} ${syncResult.created.length} created. ${syncResult.updated.length} updated. ${syncResult.deleted.length} deleted.` + ); + } + + return syncResult; + } + + /** + * Fetch information from changeset endpoint. + * + * @param expectedTimestamp cache busting value + * @param since timestamp of last sync (optional) + */ + async _fetchChangeset(expectedTimestamp, since) { + const client = this.httpClient(); + const { + metadata, + timestamp: remoteTimestamp, + changes: remoteRecords, + } = await client.execute( + { + path: `/buckets/${this.bucketName}/collections/${this.collectionName}/changeset`, + }, + { + query: { + _expected: expectedTimestamp, + _since: since, + }, + } + ); + return { + remoteTimestamp, + metadata, + remoteRecords, + }; + } + + /** + * Use the filter func to filter the lists of changes obtained from synchronization, + * and return them along with the filtered list of local records. + * + * If the filtered lists of changes are all empty, we return null (and thus don't + * bother listing local DB). + * + * @param {Object} syncResult Synchronization result without filtering. + * + * @returns {Promise<Object>} the filtered list of local records, plus the filtered + * list of created, updated and deleted records. + */ + async _filterSyncResult(syncResult) { + // Handle the obtained records (ie. apply locally through events). + // Build the event data list. It should be filtered (ie. by application target) + const { + current: allData, + created: allCreated, + updated: allUpdated, + deleted: allDeleted, + } = syncResult; + const [created, deleted, updatedFiltered] = await Promise.all( + [allCreated, allDeleted, allUpdated.map(e => e.new)].map( + this._filterEntries.bind(this) + ) + ); + // For updates, keep entries whose updated form matches the target. + const updatedFilteredIds = new Set(updatedFiltered.map(e => e.id)); + const updated = allUpdated.filter(({ new: { id } }) => + updatedFilteredIds.has(id) + ); + + if (!created.length && !updated.length && !deleted.length) { + return null; + } + // Read local collection of records (also filtered). + const current = await this._filterEntries(allData); + return { created, updated, deleted, current }; + } + + /** + * Filter entries for which calls to `this.filterFunc` returns null. + * + * @param {Array<Objet>} data + * @returns {Array<Object>} + */ + async _filterEntries(data) { + if (!this.filterFunc) { + return data; + } + const environment = cacheProxy(lazy.ClientEnvironmentBase); + const dataPromises = data.map(e => this.filterFunc(e, environment)); + const results = await Promise.all(dataPromises); + return results.filter(Boolean); + } + + /** + * Remove the fields from the specified record + * that are not present on server. + * + * @param {Object} record + */ + _cleanLocalFields(record) { + const keys = ["_status"].concat(this.localFields); + const result = { ...record }; + for (const key of keys) { + delete result[key]; + } + return result; + } +} |