diff options
Diffstat (limited to '')
| -rw-r--r-- | services/settings/Database.sys.mjs | 608 |
1 files changed, 608 insertions, 0 deletions
diff --git a/services/settings/Database.sys.mjs b/services/settings/Database.sys.mjs new file mode 100644 index 0000000000..73d61f7fce --- /dev/null +++ b/services/settings/Database.sys.mjs @@ -0,0 +1,608 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs", + CommonUtils: "resource://services-common/utils.sys.mjs", + Utils: "resource://services-settings/Utils.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + IDBHelpers: "resource://services-settings/IDBHelpers.jsm", + ObjectUtils: "resource://gre/modules/ObjectUtils.jsm", +}); +XPCOMUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log); + +/** + * Database is a tiny wrapper with the objective + * of providing major kinto-offline-client collection API. + * (with the objective of getting rid of kinto-offline-client) + */ +export class Database { + static destroy() { + return destroyIDB(); + } + + constructor(identifier) { + ensureShutdownBlocker(); + this.identifier = identifier; + } + + async list(options = {}) { + const { filters = {}, order = "" } = options; + let results = []; + try { + await executeIDB( + "records", + (store, rejectTransaction) => { + // Fast-path the (very common) no-filters case + if (lazy.ObjectUtils.isEmpty(filters)) { + const range = IDBKeyRange.only(this.identifier); + const request = store.index("cid").getAll(range); + request.onsuccess = e => { + results = e.target.result; + }; + return; + } + const request = store + .index("cid") + .openCursor(IDBKeyRange.only(this.identifier)); + const objFilters = transformSubObjectFilters(filters); + request.onsuccess = event => { + try { + const cursor = event.target.result; + if (cursor) { + const { value } = cursor; + if (lazy.Utils.filterObject(objFilters, value)) { + results.push(value); + } + cursor.continue(); + } + } catch (ex) { + rejectTransaction(ex); + } + }; + }, + { mode: "readonly" } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "list()", this.identifier); + } + // Remove IDB key field from results. + for (const result of results) { + delete result._cid; + } + return order ? lazy.Utils.sortObjects(order, results) : results; + } + + async importChanges(metadata, timestamp, records = [], options = {}) { + const { clear = false } = options; + const _cid = this.identifier; + try { + await executeIDB( + ["collections", "timestamps", "records"], + (stores, rejectTransaction) => { + const [storeMetadata, storeTimestamps, storeRecords] = stores; + + if (clear) { + // Our index is over the _cid and id fields. We want to remove + // all of the items in the collection for which the object was + // created, ie with _cid == this.identifier. + // We would like to just tell IndexedDB: + // store.index(IDBKeyRange.only(this.identifier)).delete(); + // to delete all records matching the first part of the 2-part key. + // Unfortunately such an API does not exist. + // While we could iterate over the index with a cursor, we'd do + // a roundtrip to PBackground for each item. Once you have 1000 + // items, the result is very slow because of all the overhead of + // jumping between threads and serializing/deserializing. + // So instead, we tell the store to delete everything between + // "our" _cid identifier, and what would be the next identifier + // (via lexicographical sorting). Unfortunately there does not + // seem to be a way to specify bounds for all items that share + // the same first part of the key using just that first part, hence + // the use of the hypothetical [] for the second part of the end of + // the bounds. + storeRecords.delete( + IDBKeyRange.bound([_cid], [_cid, []], false, true) + ); + } + + // Store or erase metadata. + if (metadata === null) { + storeMetadata.delete(_cid); + } else if (metadata) { + storeMetadata.put({ cid: _cid, metadata }); + } + // Store or erase timestamp. + if (timestamp === null) { + storeTimestamps.delete(_cid); + } else if (timestamp) { + storeTimestamps.put({ cid: _cid, value: timestamp }); + } + + if (!records.length) { + return; + } + + // Separate tombstones from creations/updates. + const toDelete = records.filter(r => r.deleted); + const toInsert = records.filter(r => !r.deleted); + lazy.console.debug( + `${_cid} ${toDelete.length} to delete, ${toInsert.length} to insert` + ); + // Delete local records for each tombstone. + lazy.IDBHelpers.bulkOperationHelper( + storeRecords, + { + reject: rejectTransaction, + completion() { + // Overwrite all other data. + lazy.IDBHelpers.bulkOperationHelper( + storeRecords, + { + reject: rejectTransaction, + }, + "put", + toInsert.map(item => ({ ...item, _cid })) + ); + }, + }, + "delete", + toDelete.map(item => [_cid, item.id]) + ); + }, + { desc: "importChanges() in " + _cid } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "importChanges()", _cid); + } + } + + async getLastModified() { + let entry = null; + try { + await executeIDB( + "timestamps", + store => { + store.get(this.identifier).onsuccess = e => (entry = e.target.result); + }, + { mode: "readonly" } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError( + e, + "getLastModified()", + this.identifier + ); + } + if (!entry) { + return null; + } + // Some distributions where released with a modified dump that did not + // contain timestamps for last_modified. Work around this here, and return + // the timestamp as zero, so that the entries should get updated. + if (isNaN(entry.value)) { + lazy.console.warn(`Local timestamp is NaN for ${this.identifier}`); + return 0; + } + return entry.value; + } + + async getMetadata() { + let entry = null; + try { + await executeIDB( + "collections", + store => { + store.get(this.identifier).onsuccess = e => (entry = e.target.result); + }, + { mode: "readonly" } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError( + e, + "getMetadata()", + this.identifier + ); + } + return entry ? entry.metadata : null; + } + + async getAttachment(attachmentId) { + let entry = null; + try { + await executeIDB( + "attachments", + store => { + store.get([this.identifier, attachmentId]).onsuccess = e => { + entry = e.target.result; + }; + }, + { mode: "readonly" } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError( + e, + "getAttachment()", + this.identifier + ); + } + return entry ? entry.attachment : null; + } + + async saveAttachment(attachmentId, attachment) { + try { + await executeIDB( + "attachments", + store => { + if (attachment) { + store.put({ cid: this.identifier, attachmentId, attachment }); + } else { + store.delete([this.identifier, attachmentId]); + } + }, + { desc: "saveAttachment(" + attachmentId + ") in " + this.identifier } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError( + e, + "saveAttachment()", + this.identifier + ); + } + } + + /** + * Delete all attachments which don't match any record. + * + * Attachments are linked to records, except when a fixed `attachmentId` is used. + * A record can be updated or deleted, potentially by deleting a record and restoring an updated version + * of the record with the same ID. Potentially leaving orphaned attachments in the database. + * Since we run the pruning logic after syncing, any attachment without a + * matching record can be discarded as they will be unreachable forever. + * + * @param {Array<String>} excludeIds List of attachments IDs to exclude from pruning. + */ + async pruneAttachments(excludeIds) { + const _cid = this.identifier; + let deletedCount = 0; + try { + await executeIDB( + ["attachments", "records"], + async (stores, rejectTransaction) => { + const [attachmentsStore, recordsStore] = stores; + + // List all stored attachments. + // All keys ≥ [_cid, ..] && < [_cid, []]. See comment in `importChanges()` + const rangeAllKeys = IDBKeyRange.bound( + [_cid], + [_cid, []], + false, + true + ); + const allAttachments = await new Promise((resolve, reject) => { + const request = attachmentsStore.getAll(rangeAllKeys); + request.onsuccess = e => resolve(e.target.result); + request.onerror = e => reject(e); + }); + if (!allAttachments.length) { + lazy.console.debug( + `${this.identifier} No attachments in IDB cache. Nothing to do.` + ); + return; + } + + // List all stored records. + const allRecords = await new Promise((resolve, reject) => { + const rangeAllIndexed = IDBKeyRange.only(_cid); + const request = recordsStore.index("cid").getAll(rangeAllIndexed); + request.onsuccess = e => resolve(e.target.result); + request.onerror = e => reject(e); + }); + + console.error("allRecords", allRecords); + + // Compare known records IDs to those stored along the attachments. + const currentRecordsIDs = new Set(allRecords.map(r => r.id)); + const attachmentsToDelete = allAttachments.reduce((acc, entry) => { + // Skip excluded attachments. + if (excludeIds.includes(entry.attachmentId)) { + return acc; + } + // Delete attachment if associated record does not exist. + if (!currentRecordsIDs.has(entry.attachment.record.id)) { + acc.push([_cid, entry.attachmentId]); + } + return acc; + }, []); + + // Perform a bulk delete of all obsolete attachments. + lazy.console.debug( + `${this.identifier} Bulk delete ${attachmentsToDelete.length} obsolete attachments` + ); + lazy.IDBHelpers.bulkOperationHelper( + attachmentsStore, + { + reject: rejectTransaction, + }, + "delete", + attachmentsToDelete + ); + deletedCount = attachmentsToDelete.length; + }, + { desc: "pruneAttachments() in " + this.identifier } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError( + e, + "pruneAttachments()", + this.identifier + ); + } + return deletedCount; + } + + async clear() { + try { + await this.importChanges(null, null, [], { clear: true }); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "clear()", this.identifier); + } + } + + /* + * Methods used by unit tests. + */ + + async create(record) { + if (!("id" in record)) { + record = { ...record, id: lazy.CommonUtils.generateUUID() }; + } + try { + await executeIDB( + "records", + store => { + store.add({ ...record, _cid: this.identifier }); + }, + { desc: "create() in " + this.identifier } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "create()", this.identifier); + } + return record; + } + + async update(record) { + try { + await executeIDB( + "records", + store => { + store.put({ ...record, _cid: this.identifier }); + }, + { desc: "update() in " + this.identifier } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "update()", this.identifier); + } + } + + async delete(recordId) { + try { + await executeIDB( + "records", + store => { + store.delete([this.identifier, recordId]); // [_cid, id] + }, + { desc: "delete() in " + this.identifier } + ); + } catch (e) { + throw new lazy.IDBHelpers.IndexedDBError(e, "delete()", this.identifier); + } + } +} + +let gDB = null; +let gDBPromise = null; + +/** + * This function attempts to ensure `gDB` points to a valid database value. + * If gDB is already a database, it will do no-op (but this may take a + * microtask or two). + * If opening the database fails, it will throw an IndexedDBError. + */ +async function openIDB() { + // We can be called multiple times in a race; always ensure that when + // we complete, `gDB` is no longer null, but avoid doing the actual + // IndexedDB work more than once. + if (!gDBPromise) { + // Open and initialize/upgrade if needed. + gDBPromise = lazy.IDBHelpers.openIDB(); + } + let db = await gDBPromise; + if (!gDB) { + gDB = db; + } +} + +const gPendingReadOnlyTransactions = new Set(); +const gPendingWriteOperations = new Set(); +/** + * Helper to wrap some IDBObjectStore operations into a promise. + * + * @param {IDBDatabase} db + * @param {String|String[]} storeNames - either a string or an array of strings. + * @param {function} callback + * @param {Object} options + * @param {String} options.mode + * @param {String} options.desc for shutdown tracking. + */ +async function executeIDB(storeNames, callback, options = {}) { + if (!gDB) { + // Check if we're shutting down. Services.startup.shuttingDown will + // be true sooner, but is never true in xpcshell tests, so we check + // both that and a bool we set ourselves when `profile-before-change` + // starts. + if (gShutdownStarted || Services.startup.shuttingDown) { + throw new lazy.IDBHelpers.ShutdownError( + "The application is shutting down", + "execute()" + ); + } + await openIDB(); + } else { + // Even if we have a db, wait a tick to avoid making IndexedDB sad. + // We should be able to remove this once bug 1626935 is fixed. + await Promise.resolve(); + } + + // Check for shutdown again as we've await'd something... + if (!gDB && (gShutdownStarted || Services.startup.shuttingDown)) { + throw new lazy.IDBHelpers.ShutdownError( + "The application is shutting down", + "execute()" + ); + } + + // Start the actual transaction: + const { mode = "readwrite", desc = "" } = options; + let { promise, transaction } = lazy.IDBHelpers.executeIDB( + gDB, + storeNames, + mode, + callback, + desc + ); + + // We track all readonly transactions and abort them at shutdown. + // We track all readwrite ones and await their completion at shutdown + // (to avoid dataloss when writes fail). + // We use a `.finally()` clause for this; it'll run the function irrespective + // of whether the promise resolves or rejects, and the promise it returns + // will resolve/reject with the same value. + let finishedFn; + if (mode == "readonly") { + gPendingReadOnlyTransactions.add(transaction); + finishedFn = () => gPendingReadOnlyTransactions.delete(transaction); + } else { + let obj = { promise, desc }; + gPendingWriteOperations.add(obj); + finishedFn = () => gPendingWriteOperations.delete(obj); + } + return promise.finally(finishedFn); +} + +async function destroyIDB() { + if (gDB) { + if (gShutdownStarted || Services.startup.shuttingDown) { + throw new lazy.IDBHelpers.ShutdownError( + "The application is shutting down", + "destroyIDB()" + ); + } + + // This will return immediately; the actual close will happen once + // there are no more running transactions. + gDB.close(); + const allTransactions = new Set([ + ...gPendingWriteOperations, + ...gPendingReadOnlyTransactions, + ]); + for (let transaction of Array.from(allTransactions)) { + try { + transaction.abort(); + } catch (ex) { + // Ignore errors to abort transactions, we'll destroy everything. + } + } + } + gDB = null; + gDBPromise = null; + return lazy.IDBHelpers.destroyIDB(); +} + +function makeNestedObjectFromArr(arr, val, nestedFiltersObj) { + const last = arr.length - 1; + return arr.reduce((acc, cv, i) => { + if (i === last) { + return (acc[cv] = val); + } else if (Object.prototype.hasOwnProperty.call(acc, cv)) { + return acc[cv]; + } + return (acc[cv] = {}); + }, nestedFiltersObj); +} + +function transformSubObjectFilters(filtersObj) { + const transformedFilters = {}; + for (const [key, val] of Object.entries(filtersObj)) { + const keysArr = key.split("."); + makeNestedObjectFromArr(keysArr, val, transformedFilters); + } + return transformedFilters; +} + +// We need to expose this wrapper function so we can test +// shutdown handling. +Database._executeIDB = executeIDB; + +let gShutdownStarted = false; +// Test-only helper to be able to test shutdown multiple times: +Database._cancelShutdown = () => { + gShutdownStarted = false; +}; + +let gShutdownBlocker = false; +Database._shutdownHandler = () => { + gShutdownStarted = true; + const NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR = 0x80660006; + // Duplicate the list (to avoid it being modified) and then + // abort all read-only transactions. + for (let transaction of Array.from(gPendingReadOnlyTransactions)) { + try { + transaction.abort(); + } catch (ex) { + // Ensure we don't throw/break, because either way we're in shutdown. + + // In particular, `transaction.abort` can throw if the transaction + // is complete, ie if we manage to get called in between the + // transaction completing, and our completion handler being called + // to remove the item from the set. We don't care about that. + if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) { + // Report any other errors: + console.error(ex); + } + } + } + if (gDB) { + // This will return immediately; the actual close will happen once + // there are no more running transactions. + gDB.close(); + gDB = null; + } + gDBPromise = null; + return Promise.allSettled( + Array.from(gPendingWriteOperations).map(op => op.promise) + ); +}; + +function ensureShutdownBlocker() { + if (gShutdownBlocker) { + return; + } + gShutdownBlocker = true; + lazy.AsyncShutdown.profileBeforeChange.addBlocker( + "RemoteSettingsClient - finish IDB access.", + Database._shutdownHandler, + { + fetchState() { + return Array.from(gPendingWriteOperations).map(op => op.desc); + }, + } + ); +} |