diff options
Diffstat (limited to 'services/sync/modules/engines')
| -rw-r--r-- | services/sync/modules/engines/addons.sys.mjs | 820 | ||||
| -rw-r--r-- | services/sync/modules/engines/bookmarks.sys.mjs | 953 | ||||
| -rw-r--r-- | services/sync/modules/engines/clients.sys.mjs | 1123 | ||||
| -rw-r--r-- | services/sync/modules/engines/extension-storage.sys.mjs | 303 | ||||
| -rw-r--r-- | services/sync/modules/engines/forms.sys.mjs | 298 | ||||
| -rw-r--r-- | services/sync/modules/engines/history.sys.mjs | 585 | ||||
| -rw-r--r-- | services/sync/modules/engines/passwords.sys.mjs | 565 | ||||
| -rw-r--r-- | services/sync/modules/engines/prefs.sys.mjs | 467 | ||||
| -rw-r--r-- | services/sync/modules/engines/tabs.sys.mjs | 624 |
9 files changed, 5738 insertions, 0 deletions
diff --git a/services/sync/modules/engines/addons.sys.mjs b/services/sync/modules/engines/addons.sys.mjs new file mode 100644 index 0000000000..d1b766a957 --- /dev/null +++ b/services/sync/modules/engines/addons.sys.mjs @@ -0,0 +1,820 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This file defines the add-on sync functionality. + * + * There are currently a number of known limitations: + * - We only sync XPI extensions and themes available from addons.mozilla.org. + * We hope to expand support for other add-ons eventually. + * - We only attempt syncing of add-ons between applications of the same type. + * This means add-ons will not synchronize between Firefox desktop and + * Firefox mobile, for example. This is because of significant add-on + * incompatibility between application types. + * + * Add-on records exist for each known {add-on, app-id} pair in the Sync client + * set. Each record has a randomly chosen GUID. The records then contain + * basic metadata about the add-on. + * + * We currently synchronize: + * + * - Installations + * - Uninstallations + * - User enabling and disabling + * + * Synchronization is influenced by the following preferences: + * + * - services.sync.addons.ignoreUserEnabledChanges + * - services.sync.addons.trustedSourceHostnames + * + * and also influenced by whether addons have repository caching enabled and + * whether they allow installation of addons from insecure options (both of + * which are themselves influenced by the "extensions." pref branch) + * + * See the documentation in all.js for the behavior of these prefs. + */ + +import { Preferences } from "resource://gre/modules/Preferences.sys.mjs"; + +import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs"; +import { AddonsReconciler } from "resource://services-sync/addonsreconciler.sys.mjs"; +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", + AddonRepository: "resource://gre/modules/addons/AddonRepository.sys.mjs", +}); + +// 7 days in milliseconds. +const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000; + +/** + * AddonRecord represents the state of an add-on in an application. + * + * Each add-on has its own record for each application ID it is installed + * on. + * + * The ID of add-on records is a randomly-generated GUID. It is random instead + * of deterministic so the URIs of the records cannot be guessed and so + * compromised server credentials won't result in disclosure of the specific + * add-ons present in a Sync account. + * + * The record contains the following fields: + * + * addonID + * ID of the add-on. This correlates to the "id" property on an Addon type. + * + * applicationID + * The application ID this record is associated with. + * + * enabled + * Boolean stating whether add-on is enabled or disabled by the user. + * + * source + * String indicating where an add-on is from. Currently, we only support + * the value "amo" which indicates that the add-on came from the official + * add-ons repository, addons.mozilla.org. In the future, we may support + * installing add-ons from other sources. This provides a future-compatible + * mechanism for clients to only apply records they know how to handle. + */ +function AddonRecord(collection, id) { + CryptoWrapper.call(this, collection, id); +} +AddonRecord.prototype = { + _logName: "Record.Addon", +}; +Object.setPrototypeOf(AddonRecord.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(AddonRecord, "cleartext", [ + "addonID", + "applicationID", + "enabled", + "source", +]); + +/** + * The AddonsEngine handles synchronization of add-ons between clients. + * + * The engine maintains an instance of an AddonsReconciler, which is the entity + * maintaining state for add-ons. It provides the history and tracking APIs + * that AddonManager doesn't. + * + * The engine instance overrides a handful of functions on the base class. The + * rationale for each is documented by that function. + */ +export function AddonsEngine(service) { + SyncEngine.call(this, "Addons", service); + + this._reconciler = new AddonsReconciler(this._tracker.asyncObserver); +} + +AddonsEngine.prototype = { + _storeObj: AddonsStore, + _trackerObj: AddonsTracker, + _recordObj: AddonRecord, + version: 1, + + syncPriority: 5, + + _reconciler: null, + + async initialize() { + await SyncEngine.prototype.initialize.call(this); + await this._reconciler.ensureStateLoaded(); + }, + + /** + * Override parent method to find add-ons by their public ID, not Sync GUID. + */ + async _findDupe(item) { + let id = item.addonID; + + // The reconciler should have been updated at the top of the sync, so we + // can assume it is up to date when this function is called. + let addons = this._reconciler.addons; + if (!(id in addons)) { + return null; + } + + let addon = addons[id]; + if (addon.guid != item.id) { + return addon.guid; + } + + return null; + }, + + /** + * Override getChangedIDs to pull in tracker changes plus changes from the + * reconciler log. + */ + async getChangedIDs() { + let changes = {}; + const changedIDs = await this._tracker.getChangedIDs(); + for (let [id, modified] of Object.entries(changedIDs)) { + changes[id] = modified; + } + + let lastSync = await this.getLastSync(); + let lastSyncDate = new Date(lastSync * 1000); + + // The reconciler should have been refreshed at the beginning of a sync and + // we assume this function is only called from within a sync. + let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate); + let addons = this._reconciler.addons; + for (let change of reconcilerChanges) { + let changeTime = change[0]; + let id = change[2]; + + if (!(id in addons)) { + continue; + } + + // Keep newest modified time. + if (id in changes && changeTime < changes[id]) { + continue; + } + + if (!(await this.isAddonSyncable(addons[id]))) { + continue; + } + + this._log.debug("Adding changed add-on from changes log: " + id); + let addon = addons[id]; + changes[addon.guid] = changeTime.getTime() / 1000; + } + + return changes; + }, + + /** + * Override start of sync function to refresh reconciler. + * + * Many functions in this class assume the reconciler is refreshed at the + * top of a sync. If this ever changes, those functions should be revisited. + * + * Technically speaking, we don't need to refresh the reconciler on every + * sync since it is installed as an AddonManager listener. However, add-ons + * are complicated and we force a full refresh, just in case the listeners + * missed something. + */ + async _syncStartup() { + // We refresh state before calling parent because syncStartup in the parent + // looks for changed IDs, which is dependent on add-on state being up to + // date. + await this._refreshReconcilerState(); + return SyncEngine.prototype._syncStartup.call(this); + }, + + /** + * Override end of sync to perform a little housekeeping on the reconciler. + * + * We prune changes to prevent the reconciler state from growing without + * bound. Even if it grows unbounded, there would have to be many add-on + * changes (thousands) for it to slow things down significantly. This is + * highly unlikely to occur. Still, we exercise defense just in case. + */ + async _syncCleanup() { + let lastSync = await this.getLastSync(); + let ms = 1000 * lastSync - PRUNE_ADDON_CHANGES_THRESHOLD; + this._reconciler.pruneChangesBeforeDate(new Date(ms)); + return SyncEngine.prototype._syncCleanup.call(this); + }, + + /** + * Helper function to ensure reconciler is up to date. + * + * This will load the reconciler's state from the file + * system (if needed) and refresh the state of the reconciler. + */ + async _refreshReconcilerState() { + this._log.debug("Refreshing reconciler state"); + return this._reconciler.refreshGlobalState(); + }, + + // Returns a promise + isAddonSyncable(addon, ignoreRepoCheck) { + return this._store.isAddonSyncable(addon, ignoreRepoCheck); + }, +}; +Object.setPrototypeOf(AddonsEngine.prototype, SyncEngine.prototype); + +/** + * This is the primary interface between Sync and the Addons Manager. + * + * In addition to the core store APIs, we provide convenience functions to wrap + * Add-on Manager APIs with Sync-specific semantics. + */ +function AddonsStore(name, engine) { + Store.call(this, name, engine); +} +AddonsStore.prototype = { + // Define the add-on types (.type) that we support. + _syncableTypes: ["extension", "theme"], + + _extensionsPrefs: new Preferences("extensions."), + + get reconciler() { + return this.engine._reconciler; + }, + + /** + * Override applyIncoming to filter out records we can't handle. + */ + async applyIncoming(record) { + // The fields we look at aren't present when the record is deleted. + if (!record.deleted) { + // Ignore records not belonging to our application ID because that is the + // current policy. + if (record.applicationID != Services.appinfo.ID) { + this._log.info( + "Ignoring incoming record from other App ID: " + record.id + ); + return; + } + + // Ignore records that aren't from the official add-on repository, as that + // is our current policy. + if (record.source != "amo") { + this._log.info( + "Ignoring unknown add-on source (" + + record.source + + ")" + + " for " + + record.id + ); + return; + } + } + + // Ignore incoming records for which an existing non-syncable addon + // exists. Note that we do not insist that the addon manager already have + // metadata for this addon - it's possible our reconciler previously saw the + // addon but the addon-manager cache no longer has it - which is fine for a + // new incoming addon. + // (Note that most other cases where the addon-manager cache is invalid + // doesn't get this treatment because that cache self-repairs after some + // time - but it only re-populates addons which are currently installed.) + let existingMeta = this.reconciler.addons[record.addonID]; + if ( + existingMeta && + !(await this.isAddonSyncable(existingMeta, /* ignoreRepoCheck */ true)) + ) { + this._log.info( + "Ignoring incoming record for an existing but non-syncable addon", + record.addonID + ); + return; + } + + await Store.prototype.applyIncoming.call(this, record); + }, + + /** + * Provides core Store API to create/install an add-on from a record. + */ + async create(record) { + // This will throw if there was an error. This will get caught by the sync + // engine and the record will try to be applied later. + const results = await AddonUtils.installAddons([ + { + id: record.addonID, + syncGUID: record.id, + enabled: record.enabled, + requireSecureURI: this._extensionsPrefs.get( + "install.requireSecureOrigin", + true + ), + }, + ]); + + if (results.skipped.includes(record.addonID)) { + this._log.info("Add-on skipped: " + record.addonID); + // Just early-return for skipped addons - we don't want to arrange to + // try again next time because the condition that caused up to skip + // will remain true for this addon forever. + return; + } + + let addon; + for (let a of results.addons) { + if (a.id == record.addonID) { + addon = a; + break; + } + } + + // This should never happen, but is present as a fail-safe. + if (!addon) { + throw new Error("Add-on not found after install: " + record.addonID); + } + + this._log.info("Add-on installed: " + record.addonID); + }, + + /** + * Provides core Store API to remove/uninstall an add-on from a record. + */ + async remove(record) { + // If this is called, the payload is empty, so we have to find by GUID. + let addon = await this.getAddonByGUID(record.id); + if (!addon) { + // We don't throw because if the add-on could not be found then we assume + // it has already been uninstalled and there is nothing for this function + // to do. + return; + } + + this._log.info("Uninstalling add-on: " + addon.id); + await AddonUtils.uninstallAddon(addon); + }, + + /** + * Provides core Store API to update an add-on from a record. + */ + async update(record) { + let addon = await this.getAddonByID(record.addonID); + + // update() is called if !this.itemExists. And, since itemExists consults + // the reconciler only, we need to take care of some corner cases. + // + // First, the reconciler could know about an add-on that was uninstalled + // and no longer present in the add-ons manager. + if (!addon) { + await this.create(record); + return; + } + + // It's also possible that the add-on is non-restartless and has pending + // install/uninstall activity. + // + // We wouldn't get here if the incoming record was for a deletion. So, + // check for pending uninstall and cancel if necessary. + if (addon.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL) { + addon.cancelUninstall(); + + // We continue with processing because there could be state or ID change. + } + + await this.updateUserDisabled(addon, !record.enabled); + }, + + /** + * Provide core Store API to determine if a record exists. + */ + async itemExists(guid) { + let addon = this.reconciler.getAddonStateFromSyncGUID(guid); + + return !!addon; + }, + + /** + * Create an add-on record from its GUID. + * + * @param guid + * Add-on GUID (from extensions DB) + * @param collection + * Collection to add record to. + * + * @return AddonRecord instance + */ + async createRecord(guid, collection) { + let record = new AddonRecord(collection, guid); + record.applicationID = Services.appinfo.ID; + + let addon = this.reconciler.getAddonStateFromSyncGUID(guid); + + // If we don't know about this GUID or if it has been uninstalled, we mark + // the record as deleted. + if (!addon || !addon.installed) { + record.deleted = true; + return record; + } + + record.modified = addon.modified.getTime() / 1000; + + record.addonID = addon.id; + record.enabled = addon.enabled; + + // This needs to be dynamic when add-ons don't come from AddonRepository. + record.source = "amo"; + + return record; + }, + + /** + * Changes the id of an add-on. + * + * This implements a core API of the store. + */ + async changeItemID(oldID, newID) { + // We always update the GUID in the reconciler because it will be + // referenced later in the sync process. + let state = this.reconciler.getAddonStateFromSyncGUID(oldID); + if (state) { + state.guid = newID; + await this.reconciler.saveState(); + } + + let addon = await this.getAddonByGUID(oldID); + if (!addon) { + this._log.debug( + "Cannot change item ID (" + + oldID + + ") in Add-on " + + "Manager because old add-on not present: " + + oldID + ); + return; + } + + addon.syncGUID = newID; + }, + + /** + * Obtain the set of all syncable add-on Sync GUIDs. + * + * This implements a core Store API. + */ + async getAllIDs() { + let ids = {}; + + let addons = this.reconciler.addons; + for (let id in addons) { + let addon = addons[id]; + if (await this.isAddonSyncable(addon)) { + ids[addon.guid] = true; + } + } + + return ids; + }, + + /** + * Wipe engine data. + * + * This uninstalls all syncable addons from the application. In case of + * error, it logs the error and keeps trying with other add-ons. + */ + async wipe() { + this._log.info("Processing wipe."); + + await this.engine._refreshReconcilerState(); + + // We only wipe syncable add-ons. Wipe is a Sync feature not a security + // feature. + let ids = await this.getAllIDs(); + for (let guid in ids) { + let addon = await this.getAddonByGUID(guid); + if (!addon) { + this._log.debug( + "Ignoring add-on because it couldn't be obtained: " + guid + ); + continue; + } + + this._log.info("Uninstalling add-on as part of wipe: " + addon.id); + await Utils.catch.call(this, () => addon.uninstall())(); + } + }, + + /** ************************************************************************* + * Functions below are unique to this store and not part of the Store API * + ***************************************************************************/ + + /** + * Obtain an add-on from its public ID. + * + * @param id + * Add-on ID + * @return Addon or undefined if not found + */ + async getAddonByID(id) { + return lazy.AddonManager.getAddonByID(id); + }, + + /** + * Obtain an add-on from its Sync GUID. + * + * @param guid + * Add-on Sync GUID + * @return DBAddonInternal or null + */ + async getAddonByGUID(guid) { + return lazy.AddonManager.getAddonBySyncGUID(guid); + }, + + /** + * Determines whether an add-on is suitable for Sync. + * + * @param addon + * Addon instance + * @param ignoreRepoCheck + * Should we skip checking the Addons repository (primarially useful + * for testing and validation). + * @return Boolean indicating whether it is appropriate for Sync + */ + async isAddonSyncable(addon, ignoreRepoCheck = false) { + // Currently, we limit syncable add-ons to those that are: + // 1) In a well-defined set of types + // 2) Installed in the current profile + // 3) Not installed by a foreign entity (i.e. installed by the app) + // since they act like global extensions. + // 4) Is not a hotfix. + // 5) The addons XPIProvider doesn't veto it (i.e not being installed in + // the profile directory, or any other reasons it says the addon can't + // be synced) + // 6) Are installed from AMO + + // We could represent the test as a complex boolean expression. We go the + // verbose route so the failure reason is logged. + if (!addon) { + this._log.debug("Null object passed to isAddonSyncable."); + return false; + } + + if (!this._syncableTypes.includes(addon.type)) { + this._log.debug( + addon.id + " not syncable: type not in allowed list: " + addon.type + ); + return false; + } + + if (!(addon.scope & lazy.AddonManager.SCOPE_PROFILE)) { + this._log.debug(addon.id + " not syncable: not installed in profile."); + return false; + } + + // If the addon manager says it's not syncable, we skip it. + if (!addon.isSyncable) { + this._log.debug(addon.id + " not syncable: vetoed by the addon manager."); + return false; + } + + // This may be too aggressive. If an add-on is downloaded from AMO and + // manually placed in the profile directory, foreignInstall will be set. + // Arguably, that add-on should be syncable. + // TODO Address the edge case and come up with more robust heuristics. + if (addon.foreignInstall) { + this._log.debug(addon.id + " not syncable: is foreign install."); + return false; + } + + // If the AddonRepository's cache isn't enabled (which it typically isn't + // in tests), getCachedAddonByID always returns null - so skip the check + // in that case. We also provide a way to specifically opt-out of the check + // even if the cache is enabled, which is used by the validators. + if (ignoreRepoCheck || !lazy.AddonRepository.cacheEnabled) { + return true; + } + + let result = await new Promise(res => { + lazy.AddonRepository.getCachedAddonByID(addon.id, res); + }); + + if (!result) { + this._log.debug( + addon.id + " not syncable: add-on not found in add-on repository." + ); + return false; + } + + return this.isSourceURITrusted(result.sourceURI); + }, + + /** + * Determine whether an add-on's sourceURI field is trusted and the add-on + * can be installed. + * + * This function should only ever be called from isAddonSyncable(). It is + * exposed as a separate function to make testing easier. + * + * @param uri + * nsIURI instance to validate + * @return bool + */ + isSourceURITrusted: function isSourceURITrusted(uri) { + // For security reasons, we currently limit synced add-ons to those + // installed from trusted hostname(s). We additionally require TLS with + // the add-ons site to help prevent forgeries. + let trustedHostnames = Svc.Prefs.get( + "addons.trustedSourceHostnames", + "" + ).split(","); + + if (!uri) { + this._log.debug("Undefined argument to isSourceURITrusted()."); + return false; + } + + // Scheme is validated before the hostname because uri.host may not be + // populated for certain schemes. It appears to always be populated for + // https, so we avoid the potential NS_ERROR_FAILURE on field access. + if (uri.scheme != "https") { + this._log.debug("Source URI not HTTPS: " + uri.spec); + return false; + } + + if (!trustedHostnames.includes(uri.host)) { + this._log.debug("Source hostname not trusted: " + uri.host); + return false; + } + + return true; + }, + + /** + * Update the userDisabled flag on an add-on. + * + * This will enable or disable an add-on. It has no return value and does + * not catch or handle exceptions thrown by the addon manager. If no action + * is needed it will return immediately. + * + * @param addon + * Addon instance to manipulate. + * @param value + * Boolean to which to set userDisabled on the passed Addon. + */ + async updateUserDisabled(addon, value) { + if (addon.userDisabled == value) { + return; + } + + // A pref allows changes to the enabled flag to be ignored. + if (Svc.Prefs.get("addons.ignoreUserEnabledChanges", false)) { + this._log.info( + "Ignoring enabled state change due to preference: " + addon.id + ); + return; + } + + AddonUtils.updateUserDisabled(addon, value); + // updating this flag doesn't send a notification for appDisabled addons, + // meaning the reconciler will not update its state and may resync the + // addon - so explicitly rectify the state (bug 1366994) + if (addon.appDisabled) { + await this.reconciler.rectifyStateFromAddon(addon); + } + }, +}; + +Object.setPrototypeOf(AddonsStore.prototype, Store.prototype); + +/** + * The add-ons tracker keeps track of real-time changes to add-ons. + * + * It hooks up to the reconciler and receives notifications directly from it. + */ +function AddonsTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +AddonsTracker.prototype = { + get reconciler() { + return this.engine._reconciler; + }, + + get store() { + return this.engine._store; + }, + + /** + * This callback is executed whenever the AddonsReconciler sends out a change + * notification. See AddonsReconciler.addChangeListener(). + */ + async changeListener(date, change, addon) { + this._log.debug("changeListener invoked: " + change + " " + addon.id); + // Ignore changes that occur during sync. + if (this.ignoreAll) { + return; + } + + if (!(await this.store.isAddonSyncable(addon))) { + this._log.debug( + "Ignoring change because add-on isn't syncable: " + addon.id + ); + return; + } + + const added = await this.addChangedID(addon.guid, date.getTime() / 1000); + if (added) { + this.score += SCORE_INCREMENT_XLARGE; + } + }, + + onStart() { + this.reconciler.startListening(); + this.reconciler.addChangeListener(this); + }, + + onStop() { + this.reconciler.removeChangeListener(this); + this.reconciler.stopListening(); + }, +}; + +Object.setPrototypeOf(AddonsTracker.prototype, LegacyTracker.prototype); + +export class AddonValidator extends CollectionValidator { + constructor(engine = null) { + super("addons", "id", ["addonID", "enabled", "applicationID", "source"]); + this.engine = engine; + } + + async getClientItems() { + return lazy.AddonManager.getAllAddons(); + } + + normalizeClientItem(item) { + let enabled = !item.userDisabled; + if (item.pendingOperations & lazy.AddonManager.PENDING_ENABLE) { + enabled = true; + } else if (item.pendingOperations & lazy.AddonManager.PENDING_DISABLE) { + enabled = false; + } + return { + enabled, + id: item.syncGUID, + addonID: item.id, + applicationID: Services.appinfo.ID, + source: "amo", // check item.foreignInstall? + original: item, + }; + } + + async normalizeServerItem(item) { + let guid = await this.engine._findDupe(item); + if (guid) { + item.id = guid; + } + return item; + } + + clientUnderstands(item) { + return item.applicationID === Services.appinfo.ID; + } + + async syncedByClient(item) { + return ( + !item.original.hidden && + !item.original.isSystem && + !( + item.original.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL + ) && + // No need to await the returned promise explicitely: + // |expr1 && expr2| evaluates to expr2 if expr1 is true. + this.engine.isAddonSyncable(item.original, true) + ); + } +} diff --git a/services/sync/modules/engines/bookmarks.sys.mjs b/services/sync/modules/engines/bookmarks.sys.mjs new file mode 100644 index 0000000000..8724344084 --- /dev/null +++ b/services/sync/modules/engines/bookmarks.sys.mjs @@ -0,0 +1,953 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { + Changeset, + Store, + SyncEngine, + Tracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + Observers: "resource://services-common/observers.sys.mjs", + PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs", + PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.sys.mjs", + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + Resource: "resource://services-sync/resource.sys.mjs", + SyncedBookmarksMirror: "resource://gre/modules/SyncedBookmarksMirror.sys.mjs", +}); + +const PLACES_MAINTENANCE_INTERVAL_SECONDS = 4 * 60 * 60; // 4 hours. + +const FOLDER_SORTINDEX = 1000000; + +// Roots that should be deleted from the server, instead of applied locally. +// This matches `AndroidBrowserBookmarksRepositorySession::forbiddenGUID`, +// but allows tags because we don't want to reparent tag folders or tag items +// to "unfiled". +const FORBIDDEN_INCOMING_IDS = ["pinned", "places", "readinglist"]; + +// Items with these parents should be deleted from the server. We allow +// children of the Places root, to avoid orphaning left pane queries and other +// descendants of custom roots. +const FORBIDDEN_INCOMING_PARENT_IDS = ["pinned", "readinglist"]; + +// The tracker ignores changes made by import and restore, to avoid bumping the +// score and triggering syncs during the process, as well as changes made by +// Sync. +XPCOMUtils.defineLazyGetter(lazy, "IGNORED_SOURCES", () => [ + lazy.PlacesUtils.bookmarks.SOURCES.SYNC, + lazy.PlacesUtils.bookmarks.SOURCES.IMPORT, + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE, + lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP, + lazy.PlacesUtils.bookmarks.SOURCES.SYNC_REPARENT_REMOVED_FOLDER_CHILDREN, +]); + +// The validation telemetry version for the engine. Version 1 is collected +// by `bookmark_validator.js`, and checks value as well as structure +// differences. Version 2 is collected by the engine as part of building the +// remote tree, and checks structure differences only. +const BOOKMARK_VALIDATOR_VERSION = 2; + +// The maximum time that the engine should wait before aborting a bookmark +// merge. +const BOOKMARK_APPLY_TIMEOUT_MS = 5 * 60 * 60 * 1000; // 5 minutes + +// The default frecency value to use when not known. +const FRECENCY_UNKNOWN = -1; + +// Returns the constructor for a bookmark record type. +function getTypeObject(type) { + switch (type) { + case "bookmark": + return Bookmark; + case "query": + return BookmarkQuery; + case "folder": + return BookmarkFolder; + case "livemark": + return Livemark; + case "separator": + return BookmarkSeparator; + case "item": + return PlacesItem; + } + return null; +} + +export function PlacesItem(collection, id, type) { + CryptoWrapper.call(this, collection, id); + this.type = type || "item"; +} + +PlacesItem.prototype = { + async decrypt(keyBundle) { + // Do the normal CryptoWrapper decrypt, but change types before returning + let clear = await CryptoWrapper.prototype.decrypt.call(this, keyBundle); + + // Convert the abstract places item to the actual object type + if (!this.deleted) { + Object.setPrototypeOf(this, this.getTypeObject(this.type).prototype); + } + + return clear; + }, + + getTypeObject: function PlacesItem_getTypeObject(type) { + let recordObj = getTypeObject(type); + if (!recordObj) { + throw new Error("Unknown places item object type: " + type); + } + return recordObj; + }, + + _logName: "Sync.Record.PlacesItem", + + // Converts the record to a Sync bookmark object that can be passed to + // `PlacesSyncUtils.bookmarks.{insert, update}`. + toSyncBookmark() { + let result = { + kind: this.type, + recordId: this.id, + parentRecordId: this.parentid, + }; + let dateAdded = lazy.PlacesSyncUtils.bookmarks.ratchetTimestampBackwards( + this.dateAdded, + +this.modified * 1000 + ); + if (dateAdded > 0) { + result.dateAdded = dateAdded; + } + return result; + }, + + // Populates the record from a Sync bookmark object returned from + // `PlacesSyncUtils.bookmarks.fetch`. + fromSyncBookmark(item) { + this.parentid = item.parentRecordId; + this.parentName = item.parentTitle; + if (item.dateAdded) { + this.dateAdded = item.dateAdded; + } + }, +}; + +Object.setPrototypeOf(PlacesItem.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(PlacesItem, "cleartext", [ + "hasDupe", + "parentid", + "parentName", + "type", + "dateAdded", +]); + +export function Bookmark(collection, id, type) { + PlacesItem.call(this, collection, id, type || "bookmark"); +} + +Bookmark.prototype = { + _logName: "Sync.Record.Bookmark", + + toSyncBookmark() { + let info = PlacesItem.prototype.toSyncBookmark.call(this); + info.title = this.title; + info.url = this.bmkUri; + info.description = this.description; + info.tags = this.tags; + info.keyword = this.keyword; + return info; + }, + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.title = item.title; + this.bmkUri = item.url.href; + this.description = item.description; + this.tags = item.tags; + this.keyword = item.keyword; + }, +}; + +Object.setPrototypeOf(Bookmark.prototype, PlacesItem.prototype); + +Utils.deferGetSet(Bookmark, "cleartext", [ + "title", + "bmkUri", + "description", + "tags", + "keyword", +]); + +export function BookmarkQuery(collection, id) { + Bookmark.call(this, collection, id, "query"); +} + +BookmarkQuery.prototype = { + _logName: "Sync.Record.BookmarkQuery", + + toSyncBookmark() { + let info = Bookmark.prototype.toSyncBookmark.call(this); + info.folder = this.folderName || undefined; // empty string -> undefined + info.query = this.queryId; + return info; + }, + + fromSyncBookmark(item) { + Bookmark.prototype.fromSyncBookmark.call(this, item); + this.folderName = item.folder || undefined; // empty string -> undefined + this.queryId = item.query; + }, +}; + +Object.setPrototypeOf(BookmarkQuery.prototype, Bookmark.prototype); + +Utils.deferGetSet(BookmarkQuery, "cleartext", ["folderName", "queryId"]); + +export function BookmarkFolder(collection, id, type) { + PlacesItem.call(this, collection, id, type || "folder"); +} + +BookmarkFolder.prototype = { + _logName: "Sync.Record.Folder", + + toSyncBookmark() { + let info = PlacesItem.prototype.toSyncBookmark.call(this); + info.description = this.description; + info.title = this.title; + return info; + }, + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.title = item.title; + this.description = item.description; + this.children = item.childRecordIds; + }, +}; + +Object.setPrototypeOf(BookmarkFolder.prototype, PlacesItem.prototype); + +Utils.deferGetSet(BookmarkFolder, "cleartext", [ + "description", + "title", + "children", +]); + +export function Livemark(collection, id) { + BookmarkFolder.call(this, collection, id, "livemark"); +} + +Livemark.prototype = { + _logName: "Sync.Record.Livemark", + + toSyncBookmark() { + let info = BookmarkFolder.prototype.toSyncBookmark.call(this); + info.feed = this.feedUri; + info.site = this.siteUri; + return info; + }, + + fromSyncBookmark(item) { + BookmarkFolder.prototype.fromSyncBookmark.call(this, item); + this.feedUri = item.feed.href; + if (item.site) { + this.siteUri = item.site.href; + } + }, +}; + +Object.setPrototypeOf(Livemark.prototype, BookmarkFolder.prototype); + +Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]); + +export function BookmarkSeparator(collection, id) { + PlacesItem.call(this, collection, id, "separator"); +} + +BookmarkSeparator.prototype = { + _logName: "Sync.Record.Separator", + + fromSyncBookmark(item) { + PlacesItem.prototype.fromSyncBookmark.call(this, item); + this.pos = item.index; + }, +}; + +Object.setPrototypeOf(BookmarkSeparator.prototype, PlacesItem.prototype); + +Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos"); + +/** + * The bookmarks engine uses a different store that stages downloaded bookmarks + * in a separate database, instead of writing directly to Places. The buffer + * handles reconciliation, so we stub out `_reconcile`, and wait to pull changes + * until we're ready to upload. + */ +export function BookmarksEngine(service) { + SyncEngine.call(this, "Bookmarks", service); +} + +BookmarksEngine.prototype = { + _recordObj: PlacesItem, + _trackerObj: BookmarksTracker, + _storeObj: BookmarksStore, + version: 2, + // Used to override the engine name in telemetry, so that we can distinguish + // this engine from the old, now removed non-buffered engine. + overrideTelemetryName: "bookmarks-buffered", + + // Needed to ensure we don't miss items when resuming a sync that failed or + // aborted early. + _defaultSort: "oldest", + + syncPriority: 4, + allowSkippedRecord: false, + + async _ensureCurrentSyncID(newSyncID) { + await lazy.PlacesSyncUtils.bookmarks.ensureCurrentSyncId(newSyncID); + let buf = await this._store.ensureOpenMirror(); + await buf.ensureCurrentSyncId(newSyncID); + }, + + async ensureCurrentSyncID(newSyncID) { + let shouldWipeRemote = + await lazy.PlacesSyncUtils.bookmarks.shouldWipeRemote(); + if (!shouldWipeRemote) { + this._log.debug( + "Checking if server sync ID ${newSyncID} matches existing", + { newSyncID } + ); + await this._ensureCurrentSyncID(newSyncID); + return newSyncID; + } + // We didn't take the new sync ID because we need to wipe the server + // and other clients after a restore. Send the command, wipe the + // server, and reset our sync ID to reupload everything. + this._log.debug( + "Ignoring server sync ID ${newSyncID} after restore; " + + "wiping server and resetting sync ID", + { newSyncID } + ); + await this.service.clientsEngine.sendCommand( + "wipeEngine", + [this.name], + null, + { reason: "bookmark-restore" } + ); + let assignedSyncID = await this.resetSyncID(); + return assignedSyncID; + }, + + async getSyncID() { + return lazy.PlacesSyncUtils.bookmarks.getSyncId(); + }, + + async resetSyncID() { + await this._deleteServerCollection(); + return this.resetLocalSyncID(); + }, + + async resetLocalSyncID() { + let newSyncID = await lazy.PlacesSyncUtils.bookmarks.resetSyncId(); + this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID }); + let buf = await this._store.ensureOpenMirror(); + await buf.ensureCurrentSyncId(newSyncID); + return newSyncID; + }, + + async getLastSync() { + let mirror = await this._store.ensureOpenMirror(); + return mirror.getCollectionHighWaterMark(); + }, + + async setLastSync(lastSync) { + let mirror = await this._store.ensureOpenMirror(); + await mirror.setCollectionLastModified(lastSync); + // Update the last sync time in Places so that reverting to the original + // bookmarks engine doesn't download records we've already applied. + await lazy.PlacesSyncUtils.bookmarks.setLastSync(lastSync); + }, + + async _syncStartup() { + await super._syncStartup(); + + try { + // For first syncs, back up the user's bookmarks. + let lastSync = await this.getLastSync(); + if (!lastSync) { + this._log.debug("Bookmarks backup starting"); + await lazy.PlacesBackups.create(null, true); + this._log.debug("Bookmarks backup done"); + } + } catch (ex) { + // Failure to create a backup is somewhat bad, but probably not bad + // enough to prevent syncing of bookmarks - so just log the error and + // continue. + this._log.warn( + "Error while backing up bookmarks, but continuing with sync", + ex + ); + } + }, + + async _sync() { + try { + await super._sync(); + if (this._ranMaintenanceOnLastSync) { + // If the last sync failed, we ran maintenance, and this sync succeeded, + // maintenance likely fixed the issue. + this._ranMaintenanceOnLastSync = false; + this.service.recordTelemetryEvent("maintenance", "fix", "bookmarks"); + } + } catch (ex) { + if ( + Async.isShutdownException(ex) || + ex.status > 0 || + ex.name == "InterruptedError" + ) { + // Don't run maintenance on shutdown or HTTP errors, or if we aborted + // the sync because the user changed their bookmarks during merging. + throw ex; + } + if (ex.name == "MergeConflictError") { + this._log.warn( + "Bookmark syncing ran into a merge conflict error...will retry later" + ); + return; + } + // Run Places maintenance periodically to try to recover from corruption + // that might have caused the sync to fail. We cap the interval because + // persistent failures likely indicate a problem that won't be fixed by + // running maintenance after every failed sync. + let elapsedSinceMaintenance = + Date.now() / 1000 - + Services.prefs.getIntPref("places.database.lastMaintenance", 0); + if (elapsedSinceMaintenance >= PLACES_MAINTENANCE_INTERVAL_SECONDS) { + this._log.error( + "Bookmark sync failed, ${elapsedSinceMaintenance}s " + + "elapsed since last run; running Places maintenance", + { elapsedSinceMaintenance } + ); + await lazy.PlacesDBUtils.maintenanceOnIdle(); + this._ranMaintenanceOnLastSync = true; + this.service.recordTelemetryEvent("maintenance", "run", "bookmarks"); + } else { + this._ranMaintenanceOnLastSync = false; + } + throw ex; + } + }, + + async _syncFinish() { + await SyncEngine.prototype._syncFinish.call(this); + await lazy.PlacesSyncUtils.bookmarks.ensureMobileQuery(); + }, + + async pullAllChanges() { + return this.pullNewChanges(); + }, + + async trackRemainingChanges() { + let changes = this._modified.changes; + await lazy.PlacesSyncUtils.bookmarks.pushChanges(changes); + }, + + _deleteId(id) { + this._noteDeletedId(id); + }, + + // The bookmarks engine rarely calls this method directly, except in tests or + // when handling a `reset{All, Engine}` command from another client. We + // usually reset local Sync metadata on a sync ID mismatch, which both engines + // override with logic that lives in Places and the mirror. + async _resetClient() { + await super._resetClient(); + await lazy.PlacesSyncUtils.bookmarks.reset(); + let buf = await this._store.ensureOpenMirror(); + await buf.reset(); + }, + + // Cleans up the Places root, reading list items (ignored in bug 762118, + // removed in bug 1155684), and pinned sites. + _shouldDeleteRemotely(incomingItem) { + return ( + FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) || + FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid) + ); + }, + + emptyChangeset() { + return new BookmarksChangeset(); + }, + + async _apply() { + let buf = await this._store.ensureOpenMirror(); + let watchdog = this._newWatchdog(); + watchdog.start(BOOKMARK_APPLY_TIMEOUT_MS); + + try { + let recordsToUpload = await buf.apply({ + remoteTimeSeconds: lazy.Resource.serverTime, + signal: watchdog.signal, + }); + this._modified.replace(recordsToUpload); + } finally { + watchdog.stop(); + if (watchdog.abortReason) { + this._log.warn(`Aborting bookmark merge: ${watchdog.abortReason}`); + } + } + }, + + async _processIncoming(newitems) { + await super._processIncoming(newitems); + await this._apply(); + }, + + async _reconcile(item) { + return true; + }, + + async _createRecord(id) { + let record = await this._doCreateRecord(id); + if (!record.deleted) { + // Set hasDupe on all (non-deleted) records since we don't use it and we + // want to minimize the risk of older clients corrupting records. Note + // that the SyncedBookmarksMirror sets it for all records that it created, + // but we would like to ensure that weakly uploaded records are marked as + // hasDupe as well. + record.hasDupe = true; + } + return record; + }, + + async _doCreateRecord(id) { + let change = this._modified.changes[id]; + if (!change) { + this._log.error( + "Creating record for item ${id} not in strong changeset", + { id } + ); + throw new TypeError("Can't create record for unchanged item"); + } + let record = this._recordFromCleartext(id, change.cleartext); + record.sortindex = await this._store._calculateIndex(record); + return record; + }, + + _recordFromCleartext(id, cleartext) { + let recordObj = getTypeObject(cleartext.type); + if (!recordObj) { + this._log.warn( + "Creating record for item ${id} with unknown type ${type}", + { id, type: cleartext.type } + ); + recordObj = PlacesItem; + } + let record = new recordObj(this.name, id); + record.cleartext = cleartext; + return record; + }, + + async pullChanges() { + return {}; + }, + + /** + * Writes successfully uploaded records back to the mirror, so that the + * mirror matches the server. We update the mirror before updating Places, + * which has implications for interrupted syncs. + * + * 1. Sync interrupted during upload; server doesn't support atomic uploads. + * We'll download and reapply everything that we uploaded before the + * interruption. All locally changed items retain their change counters. + * 2. Sync interrupted during upload; atomic uploads enabled. The server + * discards the batch. All changed local items retain their change + * counters, so the next sync resumes cleanly. + * 3. Sync interrupted during upload; outgoing records can't fit in a single + * batch. We'll download and reapply all records through the most recent + * committed batch. This is a variation of (1). + * 4. Sync interrupted after we update the mirror, but before cleanup. The + * mirror matches the server, but locally changed items retain their change + * counters. Reuploading them on the next sync should be idempotent, though + * unnecessary. If another client makes a conflicting remote change before + * we sync again, we may incorrectly prefer the local state. + * 5. Sync completes successfully. We'll update the mirror, and reset the + * change counters for all items. + */ + async _onRecordsWritten(succeeded, failed, serverModifiedTime) { + let records = []; + for (let id of succeeded) { + let change = this._modified.changes[id]; + if (!change) { + // TODO (Bug 1433178): Write weakly uploaded records back to the mirror. + this._log.info("Uploaded record not in strong changeset", id); + continue; + } + if (!change.synced) { + this._log.info("Record in strong changeset not uploaded", id); + continue; + } + let cleartext = change.cleartext; + if (!cleartext) { + this._log.error( + "Missing Sync record cleartext for ${id} in ${change}", + { id, change } + ); + throw new TypeError("Missing cleartext for uploaded Sync record"); + } + let record = this._recordFromCleartext(id, cleartext); + record.modified = serverModifiedTime; + records.push(record); + } + let buf = await this._store.ensureOpenMirror(); + await buf.store(records, { needsMerge: false }); + }, + + async finalize() { + await super.finalize(); + await this._store.finalize(); + }, +}; + +Object.setPrototypeOf(BookmarksEngine.prototype, SyncEngine.prototype); + +/** + * The bookmarks store delegates to the mirror for staging and applying + * records. Most `Store` methods intentionally remain abstract, so you can't use + * this store to create or update bookmarks in Places. All changes must go + * through the mirror, which takes care of merging and producing a valid tree. + */ +function BookmarksStore(name, engine) { + Store.call(this, name, engine); +} + +BookmarksStore.prototype = { + _openMirrorPromise: null, + + // For tests. + _batchChunkSize: 500, + + // Create a record starting from the weave id (places guid) + async createRecord(id, collection) { + let item = await lazy.PlacesSyncUtils.bookmarks.fetch(id); + if (!item) { + // deleted item + let record = new PlacesItem(collection, id); + record.deleted = true; + return record; + } + + let recordObj = getTypeObject(item.kind); + if (!recordObj) { + this._log.warn("Unknown item type, cannot serialize: " + item.kind); + recordObj = PlacesItem; + } + let record = new recordObj(collection, id); + record.fromSyncBookmark(item); + + record.sortindex = await this._calculateIndex(record); + + return record; + }, + + async _calculateIndex(record) { + // Ensure folders have a very high sort index so they're not synced last. + if (record.type == "folder") { + return FOLDER_SORTINDEX; + } + + // For anything directly under the toolbar, give it a boost of more than an + // unvisited bookmark + let index = 0; + if (record.parentid == "toolbar") { + index += 150; + } + + // Add in the bookmark's frecency if we have something. + if (record.bmkUri != null) { + let frecency = FRECENCY_UNKNOWN; + try { + frecency = await lazy.PlacesSyncUtils.history.fetchURLFrecency( + record.bmkUri + ); + } catch (ex) { + this._log.warn( + `Failed to fetch frecency for ${record.id}; assuming default`, + ex + ); + this._log.trace("Record {id} has invalid URL ${bmkUri}", record); + } + if (frecency != FRECENCY_UNKNOWN) { + index += frecency; + } + } + + return index; + }, + + async wipe() { + // Save a backup before clearing out all bookmarks. + await lazy.PlacesBackups.create(null, true); + await lazy.PlacesSyncUtils.bookmarks.wipe(); + }, + + ensureOpenMirror() { + if (!this._openMirrorPromise) { + this._openMirrorPromise = this._openMirror().catch(err => { + // We may have failed to open the mirror temporarily; for example, if + // the database is locked. Clear the promise so that subsequent + // `ensureOpenMirror` calls can try to open the mirror again. + this._openMirrorPromise = null; + throw err; + }); + } + return this._openMirrorPromise; + }, + + async _openMirror() { + let mirrorPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "bookmarks.sqlite" + ); + await IOUtils.makeDirectory(PathUtils.parent(mirrorPath), { + createAncestors: true, + }); + + return lazy.SyncedBookmarksMirror.open({ + path: mirrorPath, + recordStepTelemetry: (name, took, counts) => { + lazy.Observers.notify( + "weave:engine:sync:step", + { + name, + took, + counts, + }, + this.name + ); + }, + recordValidationTelemetry: (took, checked, problems) => { + lazy.Observers.notify( + "weave:engine:validate:finish", + { + version: BOOKMARK_VALIDATOR_VERSION, + took, + checked, + problems, + }, + this.name + ); + }, + }); + }, + + async applyIncomingBatch(records, countTelemetry) { + let buf = await this.ensureOpenMirror(); + for (let chunk of lazy.PlacesUtils.chunkArray( + records, + this._batchChunkSize + )) { + await buf.store(chunk); + } + // Array of failed records. + return []; + }, + + async applyIncoming(record) { + let buf = await this.ensureOpenMirror(); + await buf.store([record]); + }, + + async finalize() { + if (!this._openMirrorPromise) { + return; + } + let buf = await this._openMirrorPromise; + await buf.finalize(); + }, +}; + +Object.setPrototypeOf(BookmarksStore.prototype, Store.prototype); + +// The bookmarks tracker is a special flower. Instead of listening for changes +// via observer notifications, it queries Places for the set of items that have +// changed since the last sync. Because it's a "pull-based" tracker, it ignores +// all concepts of "add a changed ID." However, it still registers an observer +// to bump the score, so that changed bookmarks are synced immediately. +function BookmarksTracker(name, engine) { + Tracker.call(this, name, engine); +} +BookmarksTracker.prototype = { + onStart() { + this._placesListener = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + lazy.PlacesUtils.observers.addListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-guid-changed", + "bookmark-keyword-changed", + "bookmark-tags-changed", + "bookmark-time-changed", + "bookmark-title-changed", + "bookmark-url-changed", + ], + this._placesListener + ); + Svc.Obs.add("bookmarks-restore-begin", this); + Svc.Obs.add("bookmarks-restore-success", this); + Svc.Obs.add("bookmarks-restore-failed", this); + }, + + onStop() { + lazy.PlacesUtils.observers.removeListener( + [ + "bookmark-added", + "bookmark-removed", + "bookmark-moved", + "bookmark-guid-changed", + "bookmark-keyword-changed", + "bookmark-tags-changed", + "bookmark-time-changed", + "bookmark-title-changed", + "bookmark-url-changed", + ], + this._placesListener + ); + Svc.Obs.remove("bookmarks-restore-begin", this); + Svc.Obs.remove("bookmarks-restore-success", this); + Svc.Obs.remove("bookmarks-restore-failed", this); + }, + + async getChangedIDs() { + return lazy.PlacesSyncUtils.bookmarks.pullChanges(); + }, + + observe(subject, topic, data) { + switch (topic) { + case "bookmarks-restore-begin": + this._log.debug("Ignoring changes from importing bookmarks."); + break; + case "bookmarks-restore-success": + this._log.debug("Tracking all items on successful import."); + + if (data == "json") { + this._log.debug( + "Restore succeeded: wiping server and other clients." + ); + // Trigger an immediate sync. `ensureCurrentSyncID` will notice we + // restored, wipe the server and other clients, reset the sync ID, and + // upload the restored tree. + this.score += SCORE_INCREMENT_XLARGE; + } else { + // "html", "html-initial", or "json-append" + this._log.debug("Import succeeded."); + } + break; + case "bookmarks-restore-failed": + this._log.debug("Tracking all items on failed import."); + break; + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + /* Every add/remove/change will trigger a sync for MULTI_DEVICE */ + _upScore: function BMT__upScore() { + this.score += SCORE_INCREMENT_XLARGE; + }, + + handlePlacesEvents(events) { + for (let event of events) { + switch (event.type) { + case "bookmark-added": + case "bookmark-removed": + case "bookmark-moved": + case "bookmark-keyword-changed": + case "bookmark-tags-changed": + case "bookmark-time-changed": + case "bookmark-title-changed": + case "bookmark-url-changed": + if (lazy.IGNORED_SOURCES.includes(event.source)) { + continue; + } + + this._log.trace(`'${event.type}': ${event.id}`); + this._upScore(); + break; + case "bookmark-guid-changed": + if (event.source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC) { + this._log.warn( + "The source of bookmark-guid-changed event shoud be sync." + ); + continue; + } + + this._log.trace(`'${event.type}': ${event.id}`); + this._upScore(); + break; + case "purge-caches": + this._log.trace("purge-caches"); + this._upScore(); + break; + } + } + }, +}; + +Object.setPrototypeOf(BookmarksTracker.prototype, Tracker.prototype); + +/** + * A changeset that stores extra metadata in a change record for each ID. The + * engine updates this metadata when uploading Sync records, and writes it back + * to Places in `BookmarksEngine#trackRemainingChanges`. + * + * The `synced` property on a change record means its corresponding item has + * been uploaded, and we should pretend it doesn't exist in the changeset. + */ +class BookmarksChangeset extends Changeset { + // Only `_reconcile` calls `getModifiedTimestamp` and `has`, and the engine + // does its own reconciliation. + getModifiedTimestamp(id) { + throw new Error("Don't use timestamps to resolve bookmark conflicts"); + } + + has(id) { + throw new Error("Don't use the changeset to resolve bookmark conflicts"); + } + + delete(id) { + let change = this.changes[id]; + if (change) { + // Mark the change as synced without removing it from the set. We do this + // so that we can update Places in `trackRemainingChanges`. + change.synced = true; + } + } + + ids() { + let results = new Set(); + for (let id in this.changes) { + if (!this.changes[id].synced) { + results.add(id); + } + } + return [...results]; + } +} diff --git a/services/sync/modules/engines/clients.sys.mjs b/services/sync/modules/engines/clients.sys.mjs new file mode 100644 index 0000000000..f3cebc8a54 --- /dev/null +++ b/services/sync/modules/engines/clients.sys.mjs @@ -0,0 +1,1123 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * How does the clients engine work? + * + * - We use 2 files - commands.json and commands-syncing.json. + * + * - At sync upload time, we attempt a rename of commands.json to + * commands-syncing.json, and ignore errors (helps for crash during sync!). + * - We load commands-syncing.json and stash the contents in + * _currentlySyncingCommands which lives for the duration of the upload process. + * - We use _currentlySyncingCommands to build the outgoing records + * - Immediately after successful upload, we delete commands-syncing.json from + * disk (and clear _currentlySyncingCommands). We reconcile our local records + * with what we just wrote in the server, and add failed IDs commands + * back in commands.json + * - Any time we need to "save" a command for future syncs, we load + * commands.json, update it, and write it back out. + */ + +import { Async } from "resource://services-common/async.sys.mjs"; + +import { + DEVICE_TYPE_DESKTOP, + DEVICE_TYPE_MOBILE, + SINGLE_USER_THRESHOLD, + SYNC_API_VERSION, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Resource } from "resource://services-sync/resource.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +const lazy = {}; + +XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => { + return ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" + ).getFxAccountsSingleton(); +}); + +const { PREF_ACCOUNT_ROOT } = ChromeUtils.import( + "resource://gre/modules/FxAccountsCommon.js" +); + +const CLIENTS_TTL = 15552000; // 180 days +const CLIENTS_TTL_REFRESH = 604800; // 7 days +const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days + +// TTL of the message sent to another device when sending a tab +const NOTIFY_TAB_SENT_TTL_SECS = 1 * 3600; // 1 hour + +// How often we force a refresh of the FxA device list. +const REFRESH_FXA_DEVICE_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours + +// Reasons behind sending collection_changed push notifications. +const COLLECTION_MODIFIED_REASON_SENDTAB = "sendtab"; +const COLLECTION_MODIFIED_REASON_FIRSTSYNC = "firstsync"; + +const SUPPORTED_PROTOCOL_VERSIONS = [SYNC_API_VERSION]; +const LAST_MODIFIED_ON_PROCESS_COMMAND_PREF = + "services.sync.clients.lastModifiedOnProcessCommands"; + +function hasDupeCommand(commands, action) { + if (!commands) { + return false; + } + return commands.some( + other => + other.command == action.command && + Utils.deepEquals(other.args, action.args) + ); +} + +export function ClientsRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +ClientsRec.prototype = { + _logName: "Sync.Record.Clients", + ttl: CLIENTS_TTL, +}; +Object.setPrototypeOf(ClientsRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(ClientsRec, "cleartext", [ + "name", + "type", + "commands", + "version", + "protocols", + "formfactor", + "os", + "appPackage", + "application", + "device", + "fxaDeviceId", +]); + +export function ClientEngine(service) { + SyncEngine.call(this, "Clients", service); + + this.fxAccounts = lazy.fxAccounts; + this.addClientCommandQueue = Async.asyncQueueCaller(this._log); + Utils.defineLazyIDProperty(this, "localID", "services.sync.client.GUID"); +} + +ClientEngine.prototype = { + _storeObj: ClientStore, + _recordObj: ClientsRec, + _trackerObj: ClientsTracker, + allowSkippedRecord: false, + _knownStaleFxADeviceIds: null, + _lastDeviceCounts: null, + _lastFxaDeviceRefresh: 0, + + async initialize() { + // Reset the last sync timestamp on every startup so that we fetch all clients + await this.resetLastSync(); + }, + + // These two properties allow us to avoid replaying the same commands + // continuously if we cannot manage to upload our own record. + _localClientLastModified: 0, + get _lastModifiedOnProcessCommands() { + return Services.prefs.getIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, -1); + }, + + set _lastModifiedOnProcessCommands(value) { + Services.prefs.setIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, value); + }, + + get isFirstSync() { + return !this.lastRecordUpload; + }, + + // Always sync client data as it controls other sync behavior + get enabled() { + return true; + }, + + get lastRecordUpload() { + return Svc.Prefs.get(this.name + ".lastRecordUpload", 0); + }, + set lastRecordUpload(value) { + Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value)); + }, + + get remoteClients() { + // return all non-stale clients for external consumption. + return Object.values(this._store._remoteClients).filter(v => !v.stale); + }, + + remoteClient(id) { + let client = this._store._remoteClients[id]; + return client && !client.stale ? client : null; + }, + + remoteClientExists(id) { + return !!this.remoteClient(id); + }, + + // Aggregate some stats on the composition of clients on this account + get stats() { + let stats = { + hasMobile: this.localType == DEVICE_TYPE_MOBILE, + names: [this.localName], + numClients: 1, + }; + + for (let id in this._store._remoteClients) { + let { name, type, stale } = this._store._remoteClients[id]; + if (!stale) { + stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE; + stats.names.push(name); + stats.numClients++; + } + } + + return stats; + }, + + /** + * Obtain information about device types. + * + * Returns a Map of device types to integer counts. Guaranteed to include + * "desktop" (which will have at least 1 - this device) and "mobile" (which + * may have zero) counts. It almost certainly will include only these 2. + */ + get deviceTypes() { + let counts = new Map(); + + counts.set(this.localType, 1); // currently this must be DEVICE_TYPE_DESKTOP + counts.set(DEVICE_TYPE_MOBILE, 0); + + for (let id in this._store._remoteClients) { + let record = this._store._remoteClients[id]; + if (record.stale) { + continue; // pretend "stale" records don't exist. + } + let type = record.type; + if (!counts.has(type)) { + counts.set(type, 0); + } + + counts.set(type, counts.get(type) + 1); + } + + return counts; + }, + + get brandName() { + let brand = Services.strings.createBundle( + "chrome://branding/locale/brand.properties" + ); + return brand.GetStringFromName("brandShortName"); + }, + + get localName() { + return this.fxAccounts.device.getLocalName(); + }, + set localName(value) { + this.fxAccounts.device.setLocalName(value); + }, + + get localType() { + return this.fxAccounts.device.getLocalType(); + }, + + getClientName(id) { + if (id == this.localID) { + return this.localName; + } + let client = this._store._remoteClients[id]; + if (!client) { + return ""; + } + // Sometimes the sync clients don't always correctly update the device name + // However FxA always does, so try to pull the name from there first + let fxaDevice = this.fxAccounts.device.recentDeviceList?.find( + device => device.id === client.fxaDeviceId + ); + + // should be very rare, but could happen if we have yet to fetch devices, + // or the client recently disconnected + if (!fxaDevice) { + this._log.warn( + "Couldn't find associated FxA device, falling back to client name" + ); + return client.name; + } + return fxaDevice.name; + }, + + getClientFxaDeviceId(id) { + if (this._store._remoteClients[id]) { + return this._store._remoteClients[id].fxaDeviceId; + } + return null; + }, + + getClientByFxaDeviceId(fxaDeviceId) { + for (let id in this._store._remoteClients) { + let client = this._store._remoteClients[id]; + if (client.stale) { + continue; + } + if (client.fxaDeviceId == fxaDeviceId) { + return client; + } + } + return null; + }, + + getClientType(id) { + const client = this._store._remoteClients[id]; + if (client.type == DEVICE_TYPE_DESKTOP) { + return "desktop"; + } + if (client.formfactor && client.formfactor.includes("tablet")) { + return "tablet"; + } + return "phone"; + }, + + async _readCommands() { + let commands = await Utils.jsonLoad("commands", this); + return commands || {}; + }, + + /** + * Low level function, do not use directly (use _addClientCommand instead). + */ + async _saveCommands(commands) { + try { + await Utils.jsonSave("commands", this, commands); + } catch (error) { + this._log.error("Failed to save JSON outgoing commands", error); + } + }, + + async _prepareCommandsForUpload() { + try { + await Utils.jsonMove("commands", "commands-syncing", this); + } catch (e) { + // Ignore errors + } + let commands = await Utils.jsonLoad("commands-syncing", this); + return commands || {}; + }, + + async _deleteUploadedCommands() { + delete this._currentlySyncingCommands; + try { + await Utils.jsonRemove("commands-syncing", this); + } catch (err) { + this._log.error("Failed to delete syncing-commands file", err); + } + }, + + // Gets commands for a client we are yet to write to the server. Doesn't + // include commands for that client which are already on the server. + // We should rename this! + async getClientCommands(clientId) { + const allCommands = await this._readCommands(); + return allCommands[clientId] || []; + }, + + async removeLocalCommand(command) { + // the implementation of this engine is such that adding a command to + // the local client is how commands are deleted! ¯\_(ツ)_/¯ + await this._addClientCommand(this.localID, command); + }, + + async _addClientCommand(clientId, command) { + this.addClientCommandQueue.enqueueCall(async () => { + try { + const localCommands = await this._readCommands(); + const localClientCommands = localCommands[clientId] || []; + const remoteClient = this._store._remoteClients[clientId]; + let remoteClientCommands = []; + if (remoteClient && remoteClient.commands) { + remoteClientCommands = remoteClient.commands; + } + const clientCommands = localClientCommands.concat(remoteClientCommands); + if (hasDupeCommand(clientCommands, command)) { + return false; + } + localCommands[clientId] = localClientCommands.concat(command); + await this._saveCommands(localCommands); + return true; + } catch (e) { + // Failing to save a command should not "break the queue" of pending operations. + this._log.error(e); + return false; + } + }); + + return this.addClientCommandQueue.promiseCallsComplete(); + }, + + async _removeClientCommands(clientId) { + const allCommands = await this._readCommands(); + delete allCommands[clientId]; + await this._saveCommands(allCommands); + }, + + async updateKnownStaleClients() { + this._log.debug("Updating the known stale clients"); + // _fetchFxADevices side effect updates this._knownStaleFxADeviceIds. + await this._fetchFxADevices(); + let localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + // Process newer records first, so that if we hit a record with a device ID + // we've seen before, we can mark it stale immediately. + let clientList = Object.values(this._store._remoteClients).sort( + (a, b) => b.serverLastModified - a.serverLastModified + ); + let seenDeviceIds = new Set([localFxADeviceId]); + for (let client of clientList) { + // Clients might not have an `fxaDeviceId` if they fail the FxA + // registration process. + if (!client.fxaDeviceId) { + continue; + } + if (this._knownStaleFxADeviceIds.includes(client.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${client.id} - in known stale clients list` + ); + client.stale = true; + } else if (seenDeviceIds.has(client.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${client.id}` + + ` - duplicate device id ${client.fxaDeviceId}` + ); + client.stale = true; + } else { + seenDeviceIds.add(client.fxaDeviceId); + } + } + }, + + async _fetchFxADevices() { + // We only force a refresh periodically to keep the load on the servers + // down, and because we expect FxA to have received a push message in + // most cases when the FxA device list would have changed. For this reason + // we still go ahead and check the stale list even if we didn't force a + // refresh. + let now = this.fxAccounts._internal.now(); // tests mock this .now() impl. + if (now - REFRESH_FXA_DEVICE_INTERVAL_MS > this._lastFxaDeviceRefresh) { + this._lastFxaDeviceRefresh = now; + try { + await this.fxAccounts.device.refreshDeviceList(); + } catch (e) { + this._log.error("Could not refresh the FxA device list", e); + } + } + + // We assume that clients not present in the FxA Device Manager list have been + // disconnected and so are stale + this._log.debug("Refreshing the known stale clients list"); + let localClients = Object.values(this._store._remoteClients) + .filter(client => client.fxaDeviceId) // iOS client records don't have fxaDeviceId + .map(client => client.fxaDeviceId); + const fxaClients = this.fxAccounts.device.recentDeviceList + ? this.fxAccounts.device.recentDeviceList.map(device => device.id) + : []; + this._knownStaleFxADeviceIds = Utils.arraySub(localClients, fxaClients); + }, + + async _syncStartup() { + // Reupload new client record periodically. + if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) { + await this._tracker.addChangedID(this.localID); + } + return SyncEngine.prototype._syncStartup.call(this); + }, + + async _processIncoming() { + // Fetch all records from the server. + await this.resetLastSync(); + this._incomingClients = {}; + try { + await SyncEngine.prototype._processIncoming.call(this); + // Update FxA Device list. + await this._fetchFxADevices(); + // Since clients are synced unconditionally, any records in the local store + // that don't exist on the server must be for disconnected clients. Remove + // them, so that we don't upload records with commands for clients that will + // never see them. We also do this to filter out stale clients from the + // tabs collection, since showing their list of tabs is confusing. + for (let id in this._store._remoteClients) { + if (!this._incomingClients[id]) { + this._log.info(`Removing local state for deleted client ${id}`); + await this._removeRemoteClient(id); + } + } + let localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + // Bug 1264498: Mobile clients don't remove themselves from the clients + // collection when the user disconnects Sync, so we mark as stale clients + // with the same name that haven't synced in over a week. + // (Note we can't simply delete them, or we re-apply them next sync - see + // bug 1287687) + this._localClientLastModified = Math.round( + this._incomingClients[this.localID] + ); + delete this._incomingClients[this.localID]; + let names = new Set([this.localName]); + let seenDeviceIds = new Set([localFxADeviceId]); + let idToLastModifiedList = Object.entries(this._incomingClients).sort( + (a, b) => b[1] - a[1] + ); + for (let [id, serverLastModified] of idToLastModifiedList) { + let record = this._store._remoteClients[id]; + // stash the server last-modified time on the record. + record.serverLastModified = serverLastModified; + if ( + record.fxaDeviceId && + this._knownStaleFxADeviceIds.includes(record.fxaDeviceId) + ) { + this._log.info( + `Hiding stale client ${id} - in known stale clients list` + ); + record.stale = true; + } + if (!names.has(record.name)) { + if (record.fxaDeviceId) { + seenDeviceIds.add(record.fxaDeviceId); + } + names.add(record.name); + continue; + } + let remoteAge = Resource.serverTime - this._incomingClients[id]; + if (remoteAge > STALE_CLIENT_REMOTE_AGE) { + this._log.info(`Hiding stale client ${id} with age ${remoteAge}`); + record.stale = true; + continue; + } + if (record.fxaDeviceId && seenDeviceIds.has(record.fxaDeviceId)) { + this._log.info( + `Hiding stale client ${record.id}` + + ` - duplicate device id ${record.fxaDeviceId}` + ); + record.stale = true; + } else if (record.fxaDeviceId) { + seenDeviceIds.add(record.fxaDeviceId); + } + } + } finally { + this._incomingClients = null; + } + }, + + async _uploadOutgoing() { + this._currentlySyncingCommands = await this._prepareCommandsForUpload(); + const clientWithPendingCommands = Object.keys( + this._currentlySyncingCommands + ); + for (let clientId of clientWithPendingCommands) { + if (this._store._remoteClients[clientId] || this.localID == clientId) { + this._modified.set(clientId, 0); + } + } + let updatedIDs = this._modified.ids(); + await SyncEngine.prototype._uploadOutgoing.call(this); + // Record the response time as the server time for each item we uploaded. + let lastSync = await this.getLastSync(); + for (let id of updatedIDs) { + if (id == this.localID) { + this.lastRecordUpload = lastSync; + } else { + this._store._remoteClients[id].serverLastModified = lastSync; + } + } + }, + + async _onRecordsWritten(succeeded, failed) { + // Reconcile the status of the local records with what we just wrote on the + // server + for (let id of succeeded) { + const commandChanges = this._currentlySyncingCommands[id]; + if (id == this.localID) { + if (this.isFirstSync) { + this._log.info( + "Uploaded our client record for the first time, notifying other clients." + ); + this._notifyClientRecordUploaded(); + } + if (this.localCommands) { + this.localCommands = this.localCommands.filter( + command => !hasDupeCommand(commandChanges, command) + ); + } + } else { + const clientRecord = this._store._remoteClients[id]; + if (!commandChanges || !clientRecord) { + // should be impossible, else we wouldn't have been writing it. + this._log.warn( + "No command/No record changes for a client we uploaded" + ); + continue; + } + // fixup the client record, so our copy of _remoteClients matches what we uploaded. + this._store._remoteClients[id] = await this._store.createRecord(id); + // we could do better and pass the reference to the record we just uploaded, + // but this will do for now + } + } + + // Re-add failed commands + for (let id of failed) { + const commandChanges = this._currentlySyncingCommands[id]; + if (!commandChanges) { + continue; + } + await this._addClientCommand(id, commandChanges); + } + + await this._deleteUploadedCommands(); + + // Notify other devices that their own client collection changed + const idsToNotify = succeeded.reduce((acc, id) => { + if (id == this.localID) { + return acc; + } + const fxaDeviceId = this.getClientFxaDeviceId(id); + return fxaDeviceId ? acc.concat(fxaDeviceId) : acc; + }, []); + if (idsToNotify.length) { + this._notifyOtherClientsModified(idsToNotify); + } + }, + + _notifyOtherClientsModified(ids) { + // We are not waiting on this promise on purpose. + this._notifyCollectionChanged( + ids, + NOTIFY_TAB_SENT_TTL_SECS, + COLLECTION_MODIFIED_REASON_SENDTAB + ); + }, + + _notifyClientRecordUploaded() { + // We are not waiting on this promise on purpose. + this._notifyCollectionChanged( + null, + 0, + COLLECTION_MODIFIED_REASON_FIRSTSYNC + ); + }, + + /** + * @param {?string[]} ids FxA Client IDs to notify. null means everyone else. + * @param {number} ttl TTL of the push notification. + * @param {string} reason Reason for sending this push notification. + */ + async _notifyCollectionChanged(ids, ttl, reason) { + const message = { + version: 1, + command: "sync:collection_changed", + data: { + collections: ["clients"], + reason, + }, + }; + let excludedIds = null; + if (!ids) { + const localFxADeviceId = await lazy.fxAccounts.device.getLocalId(); + excludedIds = [localFxADeviceId]; + } + try { + await this.fxAccounts.notifyDevices(ids, excludedIds, message, ttl); + } catch (e) { + this._log.error("Could not notify of changes in the collection", e); + } + }, + + async _syncFinish() { + // Record histograms for our device types, and also write them to a pref + // so non-histogram telemetry (eg, UITelemetry) and the sync scheduler + // has easy access to them, and so they are accurate even before we've + // successfully synced the first time after startup. + let deviceTypeCounts = this.deviceTypes; + for (let [deviceType, count] of deviceTypeCounts) { + let hid; + let prefName = this.name + ".devices."; + switch (deviceType) { + case DEVICE_TYPE_DESKTOP: + hid = "WEAVE_DEVICE_COUNT_DESKTOP"; + prefName += "desktop"; + break; + case DEVICE_TYPE_MOBILE: + hid = "WEAVE_DEVICE_COUNT_MOBILE"; + prefName += "mobile"; + break; + default: + this._log.warn( + `Unexpected deviceType "${deviceType}" recording device telemetry.` + ); + continue; + } + Services.telemetry.getHistogramById(hid).add(count); + // Optimization: only write the pref if it changed since our last sync. + if ( + this._lastDeviceCounts == null || + this._lastDeviceCounts.get(prefName) != count + ) { + Svc.Prefs.set(prefName, count); + } + } + this._lastDeviceCounts = deviceTypeCounts; + return SyncEngine.prototype._syncFinish.call(this); + }, + + async _reconcile(item) { + // Every incoming record is reconciled, so we use this to track the + // contents of the collection on the server. + this._incomingClients[item.id] = item.modified; + + if (!(await this._store.itemExists(item.id))) { + return true; + } + // Clients are synced unconditionally, so we'll always have new records. + // Unfortunately, this will cause the scheduler to use the immediate sync + // interval for the multi-device case, instead of the active interval. We + // work around this by updating the record during reconciliation, and + // returning false to indicate that the record doesn't need to be applied + // later. + await this._store.update(item); + return false; + }, + + // Treat reset the same as wiping for locally cached clients + async _resetClient() { + await this._wipeClient(); + }, + + async _wipeClient() { + await SyncEngine.prototype._resetClient.call(this); + this._knownStaleFxADeviceIds = null; + delete this.localCommands; + await this._store.wipe(); + try { + await Utils.jsonRemove("commands", this); + } catch (err) { + this._log.warn("Could not delete commands.json", err); + } + try { + await Utils.jsonRemove("commands-syncing", this); + } catch (err) { + this._log.warn("Could not delete commands-syncing.json", err); + } + }, + + async removeClientData() { + let res = this.service.resource(this.engineURL + "/" + this.localID); + await res.delete(); + }, + + // Override the default behavior to delete bad records from the server. + async handleHMACMismatch(item, mayRetry) { + this._log.debug("Handling HMAC mismatch for " + item.id); + + let base = await SyncEngine.prototype.handleHMACMismatch.call( + this, + item, + mayRetry + ); + if (base != SyncEngine.kRecoveryStrategy.error) { + return base; + } + + // It's a bad client record. Save it to be deleted at the end of the sync. + this._log.debug("Bad client record detected. Scheduling for deletion."); + await this._deleteId(item.id); + + // Neither try again nor error; we're going to delete it. + return SyncEngine.kRecoveryStrategy.ignore; + }, + + /** + * A hash of valid commands that the client knows about. The key is a command + * and the value is a hash containing information about the command such as + * number of arguments, description, and importance (lower importance numbers + * indicate higher importance. + */ + _commands: { + resetAll: { + args: 0, + importance: 0, + desc: "Clear temporary local data for all engines", + }, + resetEngine: { + args: 1, + importance: 0, + desc: "Clear temporary local data for engine", + }, + wipeEngine: { + args: 1, + importance: 0, + desc: "Delete all client data for engine", + }, + logout: { args: 0, importance: 0, desc: "Log out client" }, + }, + + /** + * Sends a command+args pair to a specific client. + * + * @param command Command string + * @param args Array of arguments/data for command + * @param clientId Client to send command to + */ + async _sendCommandToClient(command, args, clientId, telemetryExtra) { + this._log.trace("Sending " + command + " to " + clientId); + + let client = this._store._remoteClients[clientId]; + if (!client) { + throw new Error("Unknown remote client ID: '" + clientId + "'."); + } + if (client.stale) { + throw new Error("Stale remote client ID: '" + clientId + "'."); + } + + let action = { + command, + args, + // We send the flowID to the other client so *it* can report it in its + // telemetry - we record it in ours below. + flowID: telemetryExtra.flowID, + }; + + if (await this._addClientCommand(clientId, action)) { + this._log.trace(`Client ${clientId} got a new action`, [command, args]); + await this._tracker.addChangedID(clientId); + try { + telemetryExtra.deviceID = + this.service.identity.hashedDeviceID(clientId); + } catch (_) {} + + this.service.recordTelemetryEvent( + "sendcommand", + command, + undefined, + telemetryExtra + ); + } else { + this._log.trace(`Client ${clientId} got a duplicate action`, [ + command, + args, + ]); + } + }, + + /** + * Check if the local client has any remote commands and perform them. + * + * @return false to abort sync + */ + async processIncomingCommands() { + return this._notify("clients:process-commands", "", async function () { + if ( + !this.localCommands || + (this._lastModifiedOnProcessCommands == this._localClientLastModified && + !this.ignoreLastModifiedOnProcessCommands) + ) { + return true; + } + this._lastModifiedOnProcessCommands = this._localClientLastModified; + + const clearedCommands = await this._readCommands()[this.localID]; + const commands = this.localCommands.filter( + command => !hasDupeCommand(clearedCommands, command) + ); + let didRemoveCommand = false; + // Process each command in order. + for (let rawCommand of commands) { + let shouldRemoveCommand = true; // most commands are auto-removed. + let { command, args, flowID } = rawCommand; + this._log.debug("Processing command " + command, args); + + this.service.recordTelemetryEvent( + "processcommand", + command, + undefined, + { flowID } + ); + + let engines = [args[0]]; + switch (command) { + case "resetAll": + engines = null; + // Fallthrough + case "resetEngine": + await this.service.resetClient(engines); + break; + case "wipeEngine": + await this.service.wipeClient(engines); + break; + case "logout": + this.service.logout(); + return false; + default: + this._log.warn("Received an unknown command: " + command); + break; + } + // Add the command to the "cleared" commands list + if (shouldRemoveCommand) { + await this.removeLocalCommand(rawCommand); + didRemoveCommand = true; + } + } + if (didRemoveCommand) { + await this._tracker.addChangedID(this.localID); + } + + return true; + })(); + }, + + /** + * Validates and sends a command to a client or all clients. + * + * Calling this does not actually sync the command data to the server. If the + * client already has the command/args pair, it won't receive a duplicate + * command. + * This method is async since it writes the command to a file. + * + * @param command + * Command to invoke on remote clients + * @param args + * Array of arguments to give to the command + * @param clientId + * Client ID to send command to. If undefined, send to all remote + * clients. + * @param flowID + * A unique identifier used to track success for this operation across + * devices. + */ + async sendCommand(command, args, clientId = null, telemetryExtra = {}) { + let commandData = this._commands[command]; + // Don't send commands that we don't know about. + if (!commandData) { + this._log.error("Unknown command to send: " + command); + return; + } else if (!args || args.length != commandData.args) { + // Don't send a command with the wrong number of arguments. + this._log.error( + "Expected " + + commandData.args + + " args for '" + + command + + "', but got " + + args + ); + return; + } + + // We allocate a "flowID" here, so it is used for each client. + telemetryExtra = Object.assign({}, telemetryExtra); // don't clobber the caller's object + if (!telemetryExtra.flowID) { + telemetryExtra.flowID = Utils.makeGUID(); + } + + if (clientId) { + await this._sendCommandToClient(command, args, clientId, telemetryExtra); + } else { + for (let [id, record] of Object.entries(this._store._remoteClients)) { + if (!record.stale) { + await this._sendCommandToClient(command, args, id, telemetryExtra); + } + } + } + }, + + async _removeRemoteClient(id) { + delete this._store._remoteClients[id]; + await this._tracker.removeChangedID(id); + await this._removeClientCommands(id); + this._modified.delete(id); + }, +}; +Object.setPrototypeOf(ClientEngine.prototype, SyncEngine.prototype); + +function ClientStore(name, engine) { + Store.call(this, name, engine); +} +ClientStore.prototype = { + _remoteClients: {}, + + async create(record) { + await this.update(record); + }, + + async update(record) { + if (record.id == this.engine.localID) { + // Only grab commands from the server; local name/type always wins + this.engine.localCommands = record.commands; + } else { + this._remoteClients[record.id] = record.cleartext; + } + }, + + async createRecord(id, collection) { + let record = new ClientsRec(collection, id); + + const commandsChanges = this.engine._currentlySyncingCommands + ? this.engine._currentlySyncingCommands[id] + : []; + + // Package the individual components into a record for the local client + if (id == this.engine.localID) { + try { + record.fxaDeviceId = await this.engine.fxAccounts.device.getLocalId(); + } catch (error) { + this._log.warn("failed to get fxa device id", error); + } + record.name = this.engine.localName; + record.type = this.engine.localType; + record.version = Services.appinfo.version; + record.protocols = SUPPORTED_PROTOCOL_VERSIONS; + + // Substract the commands we recorded that we've already executed + if ( + commandsChanges && + commandsChanges.length && + this.engine.localCommands && + this.engine.localCommands.length + ) { + record.commands = this.engine.localCommands.filter( + command => !hasDupeCommand(commandsChanges, command) + ); + } + + // Optional fields. + record.os = Services.appinfo.OS; // "Darwin" + record.appPackage = Services.appinfo.ID; + record.application = this.engine.brandName; // "Nightly" + + // We can't compute these yet. + // record.device = ""; // Bug 1100723 + // record.formfactor = ""; // Bug 1100722 + } else { + record.cleartext = Object.assign({}, this._remoteClients[id]); + delete record.cleartext.serverLastModified; // serverLastModified is a local only attribute. + + // Add the commands we have to send + if (commandsChanges && commandsChanges.length) { + const recordCommands = record.cleartext.commands || []; + const newCommands = commandsChanges.filter( + command => !hasDupeCommand(recordCommands, command) + ); + record.cleartext.commands = recordCommands.concat(newCommands); + } + + if (record.cleartext.stale) { + // It's almost certainly a logic error for us to upload a record we + // consider stale, so make log noise, but still remove the flag. + this._log.error( + `Preparing to upload record ${id} that we consider stale` + ); + delete record.cleartext.stale; + } + } + if (record.commands) { + const maxPayloadSize = + this.engine.service.getMemcacheMaxRecordPayloadSize(); + let origOrder = new Map(record.commands.map((c, i) => [c, i])); + // we sort first by priority, and second by age (indicated by order in the + // original list) + let commands = record.commands.slice().sort((a, b) => { + let infoA = this.engine._commands[a.command]; + let infoB = this.engine._commands[b.command]; + // Treat unknown command types as highest priority, to allow us to add + // high priority commands in the future without worrying about clients + // removing them on each-other unnecessarially. + let importA = infoA ? infoA.importance : 0; + let importB = infoB ? infoB.importance : 0; + // Higher importantance numbers indicate that we care less, so they + // go to the end of the list where they'll be popped off. + let importDelta = importA - importB; + if (importDelta != 0) { + return importDelta; + } + let origIdxA = origOrder.get(a); + let origIdxB = origOrder.get(b); + // Within equivalent priorities, we put older entries near the end + // of the list, so that they are removed first. + return origIdxB - origIdxA; + }); + let truncatedCommands = Utils.tryFitItems(commands, maxPayloadSize); + if (truncatedCommands.length != record.commands.length) { + this._log.warn( + `Removing commands from client ${id} (from ${record.commands.length} to ${truncatedCommands.length})` + ); + // Restore original order. + record.commands = truncatedCommands.sort( + (a, b) => origOrder.get(a) - origOrder.get(b) + ); + } + } + return record; + }, + + async itemExists(id) { + return id in (await this.getAllIDs()); + }, + + async getAllIDs() { + let ids = {}; + ids[this.engine.localID] = true; + for (let id in this._remoteClients) { + ids[id] = true; + } + return ids; + }, + + async wipe() { + this._remoteClients = {}; + }, +}; +Object.setPrototypeOf(ClientStore.prototype, Store.prototype); + +function ClientsTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +ClientsTracker.prototype = { + _enabled: false, + + onStart() { + Svc.Obs.add("fxaccounts:new_device_id", this.asyncObserver); + Services.prefs.addObserver( + PREF_ACCOUNT_ROOT + "device.name", + this.asyncObserver + ); + }, + onStop() { + Services.prefs.removeObserver( + PREF_ACCOUNT_ROOT + "device.name", + this.asyncObserver + ); + Svc.Obs.remove("fxaccounts:new_device_id", this.asyncObserver); + }, + + async observe(subject, topic, data) { + switch (topic) { + case "nsPref:changed": + this._log.debug("client.name preference changed"); + // Fallthrough intended. + case "fxaccounts:new_device_id": + await this.addChangedID(this.engine.localID); + this.score += SINGLE_USER_THRESHOLD + 1; // ALWAYS SYNC NOW. + break; + } + }, +}; +Object.setPrototypeOf(ClientsTracker.prototype, LegacyTracker.prototype); diff --git a/services/sync/modules/engines/extension-storage.sys.mjs b/services/sync/modules/engines/extension-storage.sys.mjs new file mode 100644 index 0000000000..f0ec21811a --- /dev/null +++ b/services/sync/modules/engines/extension-storage.sys.mjs @@ -0,0 +1,303 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; + +import { + BridgedEngine, + BridgeWrapperXPCOM, + LogAdapter, +} from "resource://services-sync/bridged_engine.sys.mjs"; +import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + MULTI_DEVICE_THRESHOLD: "resource://services-sync/constants.sys.mjs", + Observers: "resource://services-common/observers.sys.mjs", + SCORE_INCREMENT_MEDIUM: "resource://services-sync/constants.sys.mjs", + Svc: "resource://services-sync/util.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", + + extensionStorageSyncKinto: + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs", +}); + +XPCOMUtils.defineLazyServiceGetter( + lazy, + "StorageSyncService", + "@mozilla.org/extensions/storage/sync;1", + "nsIInterfaceRequestor" +); + +const PREF_FORCE_ENABLE = "engine.extension-storage.force"; + +// A helper to indicate whether extension-storage is enabled - it's based on +// the "addons" pref. The same logic is shared between both engine impls. +function getEngineEnabled() { + // By default, we sync extension storage if we sync addons. This + // lets us simplify the UX since users probably don't consider + // "extension preferences" a separate category of syncing. + // However, we also respect engine.extension-storage.force, which + // can be set to true or false, if a power user wants to customize + // the behavior despite the lack of UI. + const forced = lazy.Svc.Prefs.get(PREF_FORCE_ENABLE, undefined); + if (forced !== undefined) { + return forced; + } + return lazy.Svc.Prefs.get("engine.addons", false); +} + +function setEngineEnabled(enabled) { + // This will be called by the engine manager when declined on another device. + // Things will go a bit pear-shaped if the engine manager tries to end up + // with 'addons' and 'extension-storage' in different states - however, this + // *can* happen given we support the `engine.extension-storage.force` + // preference. So if that pref exists, we set it to this value. If that pref + // doesn't exist, we just ignore it and hope that the 'addons' engine is also + // going to be set to the same state. + if (lazy.Svc.Prefs.has(PREF_FORCE_ENABLE)) { + lazy.Svc.Prefs.set(PREF_FORCE_ENABLE, enabled); + } +} + +// A "bridged engine" to our webext-storage component. +export function ExtensionStorageEngineBridge(service) { + this.component = lazy.StorageSyncService.getInterface( + Ci.mozIBridgedSyncEngine + ); + BridgedEngine.call(this, "Extension-Storage", service); + this._bridge = new BridgeWrapperXPCOM(this.component); + + let app_services_logger = Cc["@mozilla.org/appservices/logger;1"].getService( + Ci.mozIAppServicesLogger + ); + let logger_target = "app-services:webext_storage:sync"; + app_services_logger.register(logger_target, new LogAdapter(this._log)); +} + +ExtensionStorageEngineBridge.prototype = { + syncPriority: 10, + + // Used to override the engine name in telemetry, so that we can distinguish . + overrideTelemetryName: "rust-webext-storage", + + _notifyPendingChanges() { + return new Promise(resolve => { + this.component + .QueryInterface(Ci.mozISyncedExtensionStorageArea) + .fetchPendingSyncChanges({ + QueryInterface: ChromeUtils.generateQI([ + "mozIExtensionStorageListener", + "mozIExtensionStorageCallback", + ]), + onChanged: (extId, json) => { + try { + lazy.extensionStorageSync.notifyListeners( + extId, + JSON.parse(json) + ); + } catch (ex) { + this._log.warn( + `Error notifying change listeners for ${extId}`, + ex + ); + } + }, + handleSuccess: resolve, + handleError: (code, message) => { + this._log.warn( + "Error fetching pending synced changes", + message, + code + ); + resolve(); + }, + }); + }); + }, + + _takeMigrationInfo() { + return new Promise((resolve, reject) => { + this.component + .QueryInterface(Ci.mozIExtensionStorageArea) + .takeMigrationInfo({ + QueryInterface: ChromeUtils.generateQI([ + "mozIExtensionStorageCallback", + ]), + handleSuccess: result => { + resolve(result ? JSON.parse(result) : null); + }, + handleError: (code, message) => { + this._log.warn("Error fetching migration info", message, code); + // `takeMigrationInfo` doesn't actually perform the migration, + // just reads (and clears) any data stored in the DB from the + // previous migration. + // + // Any errors here are very likely occurring a good while + // after the migration ran, so we just warn and pretend + // nothing was there. + resolve(null); + }, + }); + }); + }, + + async _syncStartup() { + let result = await super._syncStartup(); + let info = await this._takeMigrationInfo(); + if (info) { + lazy.Observers.notify( + "weave:telemetry:migration", + info, + "webext-storage" + ); + } + return result; + }, + + async _processIncoming() { + await super._processIncoming(); + try { + await this._notifyPendingChanges(); + } catch (ex) { + // Failing to notify `storage.onChanged` observers is bad, but shouldn't + // interrupt syncing. + this._log.warn("Error notifying about synced changes", ex); + } + }, + + get enabled() { + return getEngineEnabled(); + }, + set enabled(enabled) { + setEngineEnabled(enabled); + }, +}; +Object.setPrototypeOf( + ExtensionStorageEngineBridge.prototype, + BridgedEngine.prototype +); + +/** + ***************************************************************************** + * + * Deprecated support for Kinto + * + ***************************************************************************** + */ + +/** + * The Engine that manages syncing for the web extension "storage" + * API, and in particular ext.storage.sync. + * + * ext.storage.sync is implemented using Kinto, so it has mechanisms + * for syncing that we do not need to integrate in the Firefox Sync + * framework, so this is something of a stub. + */ +export function ExtensionStorageEngineKinto(service) { + SyncEngine.call(this, "Extension-Storage", service); + XPCOMUtils.defineLazyPreferenceGetter( + this, + "_skipPercentageChance", + "services.sync.extension-storage.skipPercentageChance", + 0 + ); +} + +ExtensionStorageEngineKinto.prototype = { + _trackerObj: ExtensionStorageTracker, + // we don't need these since we implement our own sync logic + _storeObj: undefined, + _recordObj: undefined, + + syncPriority: 10, + allowSkippedRecord: false, + + async _sync() { + return lazy.extensionStorageSyncKinto.syncAll(); + }, + + get enabled() { + return getEngineEnabled(); + }, + // We only need the enabled setter for the edge-case where info/collections + // has `extension-storage` - which could happen if the pref to flip the new + // engine on was once set but no longer is. + set enabled(enabled) { + setEngineEnabled(enabled); + }, + + _wipeClient() { + return lazy.extensionStorageSyncKinto.clearAll(); + }, + + shouldSkipSync(syncReason) { + if (syncReason == "user" || syncReason == "startup") { + this._log.info( + `Not skipping extension storage sync: reason == ${syncReason}` + ); + // Always sync if a user clicks the button, or if we're starting up. + return false; + } + // Ensure this wouldn't cause a resync... + if (this._tracker.score >= lazy.MULTI_DEVICE_THRESHOLD) { + this._log.info( + "Not skipping extension storage sync: Would trigger resync anyway" + ); + return false; + } + + let probability = this._skipPercentageChance / 100.0; + // Math.random() returns a value in the interval [0, 1), so `>` is correct: + // if `probability` is 1 skip every time, and if it's 0, never skip. + let shouldSkip = probability > Math.random(); + + this._log.info( + `Skipping extension-storage sync with a chance of ${probability}: ${shouldSkip}` + ); + return shouldSkip; + }, +}; +Object.setPrototypeOf( + ExtensionStorageEngineKinto.prototype, + SyncEngine.prototype +); + +function ExtensionStorageTracker(name, engine) { + Tracker.call(this, name, engine); + this._ignoreAll = false; +} +ExtensionStorageTracker.prototype = { + get ignoreAll() { + return this._ignoreAll; + }, + + set ignoreAll(value) { + this._ignoreAll = value; + }, + + onStart() { + lazy.Svc.Obs.add("ext.storage.sync-changed", this.asyncObserver); + }, + + onStop() { + lazy.Svc.Obs.remove("ext.storage.sync-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + + if (topic !== "ext.storage.sync-changed") { + return; + } + + // Single adds, removes and changes are not so important on their + // own, so let's just increment score a bit. + this.score += lazy.SCORE_INCREMENT_MEDIUM; + }, +}; +Object.setPrototypeOf(ExtensionStorageTracker.prototype, Tracker.prototype); diff --git a/services/sync/modules/engines/forms.sys.mjs b/services/sync/modules/engines/forms.sys.mjs new file mode 100644 index 0000000000..3516327659 --- /dev/null +++ b/services/sync/modules/engines/forms.sys.mjs @@ -0,0 +1,298 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; + +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { SCORE_INCREMENT_MEDIUM } from "resource://services-sync/constants.sys.mjs"; +import { + CollectionProblemData, + CollectionValidator, +} from "resource://services-sync/collection_validator.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; + +const lazy = {}; +ChromeUtils.defineESModuleGetters(lazy, { + FormHistory: "resource://gre/modules/FormHistory.sys.mjs", +}); + +const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds. + +export function FormRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +FormRec.prototype = { + _logName: "Sync.Record.Form", + ttl: FORMS_TTL, +}; +Object.setPrototypeOf(FormRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]); + +var FormWrapper = { + _log: Log.repository.getLogger("Sync.Engine.Forms"), + + _getEntryCols: ["fieldname", "value"], + _guidCols: ["guid"], + + _search(terms, searchData) { + return lazy.FormHistory.search(terms, searchData); + }, + + async _update(changes) { + if (!lazy.FormHistory.enabled) { + return; // update isn't going to do anything. + } + await lazy.FormHistory.update(changes).catch(console.error); + }, + + async getEntry(guid) { + let results = await this._search(this._getEntryCols, { guid }); + if (!results.length) { + return null; + } + return { name: results[0].fieldname, value: results[0].value }; + }, + + async getGUID(name, value) { + // Query for the provided entry. + let query = { fieldname: name, value }; + let results = await this._search(this._guidCols, query); + return results.length ? results[0].guid : null; + }, + + async hasGUID(guid) { + // We could probably use a count function here, but search exists... + let results = await this._search(this._guidCols, { guid }); + return !!results.length; + }, + + async replaceGUID(oldGUID, newGUID) { + let changes = { + op: "update", + guid: oldGUID, + newGuid: newGUID, + }; + await this._update(changes); + }, +}; + +export function FormEngine(service) { + SyncEngine.call(this, "Forms", service); +} + +FormEngine.prototype = { + _storeObj: FormStore, + _trackerObj: FormTracker, + _recordObj: FormRec, + + syncPriority: 6, + + get prefName() { + return "history"; + }, + + async _findDupe(item) { + return FormWrapper.getGUID(item.name, item.value); + }, +}; +Object.setPrototypeOf(FormEngine.prototype, SyncEngine.prototype); + +function FormStore(name, engine) { + Store.call(this, name, engine); +} +FormStore.prototype = { + async _processChange(change) { + // If this._changes is defined, then we are applying a batch, so we + // can defer it. + if (this._changes) { + this._changes.push(change); + return; + } + + // Otherwise we must handle the change right now. + await FormWrapper._update(change); + }, + + async applyIncomingBatch(records, countTelemetry) { + Async.checkAppReady(); + // We collect all the changes to be made then apply them all at once. + this._changes = []; + let failures = await Store.prototype.applyIncomingBatch.call( + this, + records, + countTelemetry + ); + if (this._changes.length) { + await FormWrapper._update(this._changes); + } + delete this._changes; + return failures; + }, + + async getAllIDs() { + let results = await FormWrapper._search(["guid"], []); + let guids = {}; + for (let result of results) { + guids[result.guid] = true; + } + return guids; + }, + + async changeItemID(oldID, newID) { + await FormWrapper.replaceGUID(oldID, newID); + }, + + async itemExists(id) { + return FormWrapper.hasGUID(id); + }, + + async createRecord(id, collection) { + let record = new FormRec(collection, id); + let entry = await FormWrapper.getEntry(id); + if (entry != null) { + record.name = entry.name; + record.value = entry.value; + } else { + record.deleted = true; + } + return record; + }, + + async create(record) { + this._log.trace("Adding form record for " + record.name); + let change = { + op: "add", + guid: record.id, + fieldname: record.name, + value: record.value, + }; + await this._processChange(change); + }, + + async remove(record) { + this._log.trace("Removing form record: " + record.id); + let change = { + op: "remove", + guid: record.id, + }; + await this._processChange(change); + }, + + async update(record) { + this._log.trace("Ignoring form record update request!"); + }, + + async wipe() { + let change = { + op: "remove", + }; + await FormWrapper._update(change); + }, +}; +Object.setPrototypeOf(FormStore.prototype, Store.prototype); + +function FormTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +FormTracker.prototype = { + QueryInterface: ChromeUtils.generateQI([ + "nsIObserver", + "nsISupportsWeakReference", + ]), + + onStart() { + Svc.Obs.add("satchel-storage-changed", this.asyncObserver); + }, + + onStop() { + Svc.Obs.remove("satchel-storage-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + switch (topic) { + case "satchel-storage-changed": + if (data == "formhistory-add" || data == "formhistory-remove") { + let guid = subject.QueryInterface(Ci.nsISupportsString).toString(); + await this.trackEntry(guid); + } + break; + } + }, + + async trackEntry(guid) { + const added = await this.addChangedID(guid); + if (added) { + this.score += SCORE_INCREMENT_MEDIUM; + } + }, +}; +Object.setPrototypeOf(FormTracker.prototype, LegacyTracker.prototype); + +class FormsProblemData extends CollectionProblemData { + getSummary() { + // We don't support syncing deleted form data, so "clientMissing" isn't a problem + return super.getSummary().filter(entry => entry.name !== "clientMissing"); + } +} + +export class FormValidator extends CollectionValidator { + constructor() { + super("forms", "id", ["name", "value"]); + this.ignoresMissingClients = true; + } + + emptyProblemData() { + return new FormsProblemData(); + } + + async getClientItems() { + return FormWrapper._search(["guid", "fieldname", "value"], {}); + } + + normalizeClientItem(item) { + return { + id: item.guid, + guid: item.guid, + name: item.fieldname, + fieldname: item.fieldname, + value: item.value, + original: item, + }; + } + + async normalizeServerItem(item) { + let res = Object.assign( + { + guid: item.id, + fieldname: item.name, + original: item, + }, + item + ); + // Missing `name` or `value` causes the getGUID call to throw + if (item.name !== undefined && item.value !== undefined) { + let guid = await FormWrapper.getGUID(item.name, item.value); + if (guid) { + res.guid = guid; + res.id = guid; + res.duped = true; + } + } + + return res; + } +} diff --git a/services/sync/modules/engines/history.sys.mjs b/services/sync/modules/engines/history.sys.mjs new file mode 100644 index 0000000000..491e7c8a89 --- /dev/null +++ b/services/sync/modules/engines/history.sys.mjs @@ -0,0 +1,585 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const HISTORY_TTL = 5184000; // 60 days in milliseconds +const THIRTY_DAYS_IN_MS = 2592000000; // 30 days in milliseconds + +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { + MAX_HISTORY_DOWNLOAD, + MAX_HISTORY_UPLOAD, + SCORE_INCREMENT_SMALL, + SCORE_INCREMENT_XLARGE, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +export function HistoryRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +HistoryRec.prototype = { + _logName: "Sync.Record.History", + ttl: HISTORY_TTL, +}; +Object.setPrototypeOf(HistoryRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]); + +export function HistoryEngine(service) { + SyncEngine.call(this, "History", service); +} + +HistoryEngine.prototype = { + _recordObj: HistoryRec, + _storeObj: HistoryStore, + _trackerObj: HistoryTracker, + downloadLimit: MAX_HISTORY_DOWNLOAD, + + syncPriority: 7, + + async getSyncID() { + return lazy.PlacesSyncUtils.history.getSyncId(); + }, + + async ensureCurrentSyncID(newSyncID) { + this._log.debug( + "Checking if server sync ID ${newSyncID} matches existing", + { newSyncID } + ); + await lazy.PlacesSyncUtils.history.ensureCurrentSyncId(newSyncID); + return newSyncID; + }, + + async resetSyncID() { + // First, delete the collection on the server. It's fine if we're + // interrupted here: on the next sync, we'll detect that our old sync ID is + // now stale, and start over as a first sync. + await this._deleteServerCollection(); + // Then, reset our local sync ID. + return this.resetLocalSyncID(); + }, + + async resetLocalSyncID() { + let newSyncID = await lazy.PlacesSyncUtils.history.resetSyncId(); + this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID }); + return newSyncID; + }, + + async getLastSync() { + let lastSync = await lazy.PlacesSyncUtils.history.getLastSync(); + return lastSync; + }, + + async setLastSync(lastSync) { + await lazy.PlacesSyncUtils.history.setLastSync(lastSync); + }, + + shouldSyncURL(url) { + return !url.startsWith("file:"); + }, + + async pullNewChanges() { + const changedIDs = await this._tracker.getChangedIDs(); + let modifiedGUIDs = Object.keys(changedIDs); + if (!modifiedGUIDs.length) { + return {}; + } + + let guidsToRemove = + await lazy.PlacesSyncUtils.history.determineNonSyncableGuids( + modifiedGUIDs + ); + await this._tracker.removeChangedID(...guidsToRemove); + return changedIDs; + }, + + async _resetClient() { + await super._resetClient(); + await lazy.PlacesSyncUtils.history.reset(); + }, +}; +Object.setPrototypeOf(HistoryEngine.prototype, SyncEngine.prototype); + +function HistoryStore(name, engine) { + Store.call(this, name, engine); +} + +HistoryStore.prototype = { + // We try and only update this many visits at one time. + MAX_VISITS_PER_INSERT: 500, + + // Some helper functions to handle GUIDs + async setGUID(uri, guid) { + if (!guid) { + guid = Utils.makeGUID(); + } + + try { + await lazy.PlacesSyncUtils.history.changeGuid(uri, guid); + } catch (e) { + this._log.error("Error setting GUID ${guid} for URI ${uri}", guid, uri); + } + + return guid; + }, + + async GUIDForUri(uri, create) { + // Use the existing GUID if it exists + let guid; + try { + guid = await lazy.PlacesSyncUtils.history.fetchGuidForURL(uri); + } catch (e) { + this._log.error("Error fetching GUID for URL ${uri}", uri); + } + + // If the URI has an existing GUID, return it. + if (guid) { + return guid; + } + + // If the URI doesn't have a GUID and we were indicated to create one. + if (create) { + return this.setGUID(uri); + } + + // If the URI doesn't have a GUID and we didn't create one for it. + return null; + }, + + async changeItemID(oldID, newID) { + let info = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(oldID); + if (!info) { + throw new Error(`Can't change ID for nonexistent history entry ${oldID}`); + } + this.setGUID(info.url, newID); + }, + + async getAllIDs() { + let urls = await lazy.PlacesSyncUtils.history.getAllURLs({ + since: new Date(Date.now() - THIRTY_DAYS_IN_MS), + limit: MAX_HISTORY_UPLOAD, + }); + + let urlsByGUID = {}; + for (let url of urls) { + if (!this.engine.shouldSyncURL(url)) { + continue; + } + let guid = await this.GUIDForUri(url, true); + urlsByGUID[guid] = url; + } + return urlsByGUID; + }, + + async applyIncomingBatch(records, countTelemetry) { + // Convert incoming records to mozIPlaceInfo objects which are applied as + // either history additions or removals. + let failed = []; + let toAdd = []; + let toRemove = []; + await Async.yieldingForEach(records, async record => { + if (record.deleted) { + toRemove.push(record); + } else { + try { + let pageInfo = await this._recordToPlaceInfo(record); + if (pageInfo) { + toAdd.push(pageInfo); + } + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to create a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + } + }); + if (toAdd.length || toRemove.length) { + if (toRemove.length) { + // PlacesUtils.history.remove takes an array of visits to remove, + // but the error semantics are tricky - a single "bad" entry will cause + // an exception before anything is removed. So we do remove them one at + // a time. + await Async.yieldingForEach(toRemove, async record => { + try { + await this.remove(record); + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to delete a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + }); + } + for (let chunk of this._generateChunks(toAdd)) { + // Per bug 1415560, we ignore any exceptions returned by insertMany + // as they are likely to be spurious. We do supply an onError handler + // and log the exceptions seen there as they are likely to be + // informative, but we still never abort the sync based on them. + try { + await lazy.PlacesUtils.history.insertMany( + chunk, + null, + failedVisit => { + this._log.info( + "Failed to insert a history record", + failedVisit.guid + ); + this._log.trace("The record that failed", failedVisit); + failed.push(failedVisit.guid); + } + ); + } catch (ex) { + this._log.info("Failed to insert history records", ex); + countTelemetry.addIncomingFailedReason(ex.message); + } + } + } + + return failed; + }, + + /** + * Returns a generator that splits records into sanely sized chunks suitable + * for passing to places to prevent places doing bad things at shutdown. + */ + *_generateChunks(records) { + // We chunk based on the number of *visits* inside each record. However, + // we do not split a single record into multiple records, because at some + // time in the future, we intend to ensure these records are ordered by + // lastModified, and advance the engine's timestamp as we process them, + // meaning we can resume exactly where we left off next sync - although + // currently that's not done, so we will retry the entire batch next sync + // if interrupted. + // ie, this means that if a single record has more than MAX_VISITS_PER_INSERT + // visits, we will call insertMany() with exactly 1 record, but with + // more than MAX_VISITS_PER_INSERT visits. + let curIndex = 0; + this._log.debug(`adding ${records.length} records to history`); + while (curIndex < records.length) { + Async.checkAppReady(); // may throw if we are shutting down. + let toAdd = []; // what we are going to insert. + let count = 0; // a counter which tells us when toAdd is full. + do { + let record = records[curIndex]; + curIndex += 1; + toAdd.push(record); + count += record.visits.length; + } while ( + curIndex < records.length && + count + records[curIndex].visits.length <= this.MAX_VISITS_PER_INSERT + ); + this._log.trace(`adding ${toAdd.length} items in this chunk`); + yield toAdd; + } + }, + + /* An internal helper to determine if we can add an entry to places. + Exists primarily so tests can override it. + */ + _canAddURI(uri) { + return lazy.PlacesUtils.history.canAddURI(uri); + }, + + /** + * Converts a Sync history record to a mozIPlaceInfo. + * + * Throws if an invalid record is encountered (invalid URI, etc.), + * returns a new PageInfo object if the record is to be applied, null + * otherwise (no visits to add, etc.), + */ + async _recordToPlaceInfo(record) { + // Sort out invalid URIs and ones Places just simply doesn't want. + record.url = lazy.PlacesUtils.normalizeToURLOrGUID(record.histUri); + record.uri = CommonUtils.makeURI(record.histUri); + + if (!Utils.checkGUID(record.id)) { + this._log.warn("Encountered record with invalid GUID: " + record.id); + return null; + } + record.guid = record.id; + + if ( + !this._canAddURI(record.uri) || + !this.engine.shouldSyncURL(record.uri.spec) + ) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": can't add this URI." + ); + return null; + } + + // We dupe visits by date and type. So an incoming visit that has + // the same timestamp and type as a local one won't get applied. + // To avoid creating new objects, we rewrite the query result so we + // can simply check for containment below. + let curVisitsAsArray = []; + let curVisits = new Set(); + try { + curVisitsAsArray = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + } + let oldestAllowed = + lazy.PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP; + if (curVisitsAsArray.length == 20) { + let oldestVisit = curVisitsAsArray[curVisitsAsArray.length - 1]; + oldestAllowed = lazy.PlacesSyncUtils.history.clampVisitDate( + lazy.PlacesUtils.toDate(oldestVisit.date).getTime() + ); + } + + let i, k; + for (i = 0; i < curVisitsAsArray.length; i++) { + // Same logic as used in the loop below to generate visitKey. + let { date, type } = curVisitsAsArray[i]; + let dateObj = lazy.PlacesUtils.toDate(date); + let millis = lazy.PlacesSyncUtils.history + .clampVisitDate(dateObj) + .getTime(); + curVisits.add(`${millis},${type}`); + } + + // Walk through the visits, make sure we have sound data, and eliminate + // dupes. The latter is done by rewriting the array in-place. + for (i = 0, k = 0; i < record.visits.length; i++) { + let visit = (record.visits[k] = record.visits[i]); + + if ( + !visit.date || + typeof visit.date != "number" || + !Number.isInteger(visit.date) + ) { + this._log.warn( + "Encountered record with invalid visit date: " + visit.date + ); + continue; + } + + if ( + !visit.type || + !Object.values(lazy.PlacesUtils.history.TRANSITIONS).includes( + visit.type + ) + ) { + this._log.warn( + "Encountered record with invalid visit type: " + + visit.type + + "; ignoring." + ); + continue; + } + + // Dates need to be integers. Future and far past dates are clamped to the + // current date and earliest sensible date, respectively. + let originalVisitDate = lazy.PlacesUtils.toDate(Math.round(visit.date)); + visit.date = + lazy.PlacesSyncUtils.history.clampVisitDate(originalVisitDate); + + if (visit.date.getTime() < oldestAllowed) { + // Visit is older than the oldest visit we have, and we have so many + // visits for this uri that we hit our limit when inserting. + continue; + } + let visitKey = `${visit.date.getTime()},${visit.type}`; + if (curVisits.has(visitKey)) { + // Visit is a dupe, don't increment 'k' so the element will be + // overwritten. + continue; + } + + // Note the visit key, so that we don't add duplicate visits with + // clamped timestamps. + curVisits.add(visitKey); + + visit.transition = visit.type; + k += 1; + } + record.visits.length = k; // truncate array + + // No update if there aren't any visits to apply. + // History wants at least one visit. + // In any case, the only thing we could change would be the title + // and that shouldn't change without a visit. + if (!record.visits.length) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": no visits to add." + ); + return null; + } + + // PageInfo is validated using validateItemProperties which does a shallow + // copy of the properties. Since record uses getters some of the properties + // are not copied over. Thus we create and return a new object. + let pageInfo = { + title: record.title, + url: record.url, + guid: record.guid, + visits: record.visits, + }; + + return pageInfo; + }, + + async remove(record) { + this._log.trace("Removing page: " + record.id); + let removed = await lazy.PlacesUtils.history.remove(record.id); + if (removed) { + this._log.trace("Removed page: " + record.id); + } else { + this._log.debug("Page already removed: " + record.id); + } + }, + + async itemExists(id) { + return !!(await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id)); + }, + + async createRecord(id, collection) { + let foo = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id); + let record = new HistoryRec(collection, id); + if (foo) { + record.histUri = foo.url; + record.title = foo.title; + record.sortindex = foo.frecency; + try { + record.visits = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + record.visits = []; + } + } else { + record.deleted = true; + } + + return record; + }, + + async wipe() { + return lazy.PlacesSyncUtils.history.wipe(); + }, +}; +Object.setPrototypeOf(HistoryStore.prototype, Store.prototype); + +function HistoryTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +HistoryTracker.prototype = { + onStart() { + this._log.info("Adding Places observer."); + this._placesObserver = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + PlacesObservers.addListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + }, + + onStop() { + this._log.info("Removing Places observer."); + if (this._placesObserver) { + PlacesObservers.removeListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + handlePlacesEvents(aEvents) { + this.asyncObserver.enqueueCall(() => this._handlePlacesEvents(aEvents)); + }, + + async _handlePlacesEvents(aEvents) { + if (this.ignoreAll) { + this._log.trace( + "ignoreAll: ignoring visits [" + + aEvents.map(v => v.guid).join(",") + + "]" + ); + return; + } + for (let event of aEvents) { + switch (event.type) { + case "page-visited": { + this._log.trace("'page-visited': " + event.url); + if ( + this.engine.shouldSyncURL(event.url) && + (await this.addChangedID(event.pageGuid)) + ) { + this.score += SCORE_INCREMENT_SMALL; + } + break; + } + case "history-cleared": { + this._log.trace("history-cleared"); + // Note that we're going to trigger a sync, but none of the cleared + // pages are tracked, so the deletions will not be propagated. + // See Bug 578694. + this.score += SCORE_INCREMENT_XLARGE; + break; + } + case "page-removed": { + if (event.reason === PlacesVisitRemoved.REASON_EXPIRED) { + return; + } + + this._log.trace( + "page-removed: " + event.url + ", reason " + event.reason + ); + const added = await this.addChangedID(event.pageGuid); + if (added) { + this.score += event.isRemovedFromStore + ? SCORE_INCREMENT_XLARGE + : SCORE_INCREMENT_SMALL; + } + break; + } + } + } + }, +}; +Object.setPrototypeOf(HistoryTracker.prototype, LegacyTracker.prototype); diff --git a/services/sync/modules/engines/passwords.sys.mjs b/services/sync/modules/engines/passwords.sys.mjs new file mode 100644 index 0000000000..8f24d7333a --- /dev/null +++ b/services/sync/modules/engines/passwords.sys.mjs @@ -0,0 +1,565 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + Collection, + CryptoWrapper, +} from "resource://services-sync/record.sys.mjs"; + +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs"; +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; + +import { Async } from "resource://services-common/async.sys.mjs"; + +// These are valid fields the server could have for a logins record +// we mainly use this to detect if there are any unknownFields and +// store (but don't process) those fields to roundtrip them back +const VALID_LOGIN_FIELDS = [ + "id", + "displayOrigin", + "formSubmitURL", + "formActionOrigin", + "httpRealm", + "hostname", + "origin", + "password", + "passwordField", + "timeCreated", + "timeLastUsed", + "timePasswordChanged", + "timesUsed", + "username", + "usernameField", + "unknownFields", +]; + +const SYNCABLE_LOGIN_FIELDS = [ + // `nsILoginInfo` fields. + "hostname", + "formSubmitURL", + "httpRealm", + "username", + "password", + "usernameField", + "passwordField", + + // `nsILoginMetaInfo` fields. + "timeCreated", + "timePasswordChanged", +]; + +// Compares two logins to determine if their syncable fields changed. The login +// manager fires `modifyLogin` for changes to all fields, including ones we +// don't sync. In particular, `timeLastUsed` changes shouldn't mark the login +// for upload; otherwise, we might overwrite changed passwords before they're +// downloaded (bug 973166). +function isSyncableChange(oldLogin, newLogin) { + oldLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo); + newLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo); + for (let property of SYNCABLE_LOGIN_FIELDS) { + if (oldLogin[property] != newLogin[property]) { + return true; + } + } + return false; +} + +export function LoginRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +LoginRec.prototype = { + _logName: "Sync.Record.Login", + + cleartextToString() { + let o = Object.assign({}, this.cleartext); + if (o.password) { + o.password = "X".repeat(o.password.length); + } + return JSON.stringify(o); + }, +}; +Object.setPrototypeOf(LoginRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(LoginRec, "cleartext", [ + "hostname", + "formSubmitURL", + "httpRealm", + "username", + "password", + "usernameField", + "passwordField", + "timeCreated", + "timePasswordChanged", +]); + +export function PasswordEngine(service) { + SyncEngine.call(this, "Passwords", service); +} + +PasswordEngine.prototype = { + _storeObj: PasswordStore, + _trackerObj: PasswordTracker, + _recordObj: LoginRec, + + syncPriority: 2, + + // Metadata for syncing is stored in the login manager + async ensureCurrentSyncID(newSyncID) { + return Services.logins.ensureCurrentSyncID(newSyncID); + }, + + async getLastSync() { + let legacyValue = await super.getLastSync(); + if (legacyValue) { + await this.setLastSync(legacyValue); + Svc.Prefs.reset(this.name + ".lastSync"); + this._log.debug( + `migrated timestamp of ${legacyValue} to the logins store` + ); + return legacyValue; + } + return Services.logins.getLastSync(); + }, + + async setLastSync(timestamp) { + await Services.logins.setLastSync(timestamp); + }, + + async _syncFinish() { + await SyncEngine.prototype._syncFinish.call(this); + + // Delete the Weave credentials from the server once. + if (!Svc.Prefs.get("deletePwdFxA", false)) { + try { + let ids = []; + for (let host of Utils.getSyncCredentialsHosts()) { + for (let info of Services.logins.findLogins(host, "", "")) { + ids.push(info.QueryInterface(Ci.nsILoginMetaInfo).guid); + } + } + if (ids.length) { + let coll = new Collection(this.engineURL, null, this.service); + coll.ids = ids; + let ret = await coll.delete(); + this._log.debug("Delete result: " + ret); + if (!ret.success && ret.status != 400) { + // A non-400 failure means try again next time. + return; + } + } else { + this._log.debug("Didn't find any passwords to delete"); + } + // If there were no ids to delete, or we succeeded, or got a 400, + // record success. + Svc.Prefs.set("deletePwdFxA", true); + Svc.Prefs.reset("deletePwd"); // The old prefname we previously used. + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.debug("Password deletes failed", ex); + } + } + }, + + async _findDupe(item) { + let login = this._store._nsLoginInfoFromRecord(item); + if (!login) { + return null; + } + + let logins = Services.logins.findLogins( + login.origin, + login.formActionOrigin, + login.httpRealm + ); + + await Async.promiseYield(); // Yield back to main thread after synchronous operation. + + // Look for existing logins that match the origin, but ignore the password. + for (let local of logins) { + if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) { + return local.guid; + } + } + + return null; + }, + + async pullAllChanges() { + let changes = {}; + let ids = await this._store.getAllIDs(); + for (let [id, info] of Object.entries(ids)) { + changes[id] = info.timePasswordChanged / 1000; + } + return changes; + }, + + getValidator() { + return new PasswordValidator(); + }, +}; +Object.setPrototypeOf(PasswordEngine.prototype, SyncEngine.prototype); + +function PasswordStore(name, engine) { + Store.call(this, name, engine); + this._nsLoginInfo = new Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" + ); +} +PasswordStore.prototype = { + _newPropertyBag() { + return Cc["@mozilla.org/hash-property-bag;1"].createInstance( + Ci.nsIWritablePropertyBag2 + ); + }, + + // Returns an stringified object of any fields not "known" by this client + // mainly used to to prevent data loss for other clients by roundtripping + // these fields without processing them + _processUnknownFields(record) { + let unknownFields = {}; + let keys = Object.keys(record); + keys + .filter(key => !VALID_LOGIN_FIELDS.includes(key)) + .forEach(key => { + unknownFields[key] = record[key]; + }); + // If we found some unknown fields, we stringify it to be able + // to properly encrypt it for roundtripping since we can't know if + // it contained sensitive fields or not + if (Object.keys(unknownFields).length) { + return JSON.stringify(unknownFields); + } + return null; + }, + + /** + * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo). + */ + _nsLoginInfoFromRecord(record) { + function nullUndefined(x) { + return x == undefined ? null : x; + } + + function stringifyNullUndefined(x) { + return x == undefined || x == null ? "" : x; + } + + if (record.formSubmitURL && record.httpRealm) { + this._log.warn( + "Record " + + record.id + + " has both formSubmitURL and httpRealm. Skipping." + ); + return null; + } + + // Passing in "undefined" results in an empty string, which later + // counts as a value. Explicitly `|| null` these fields according to JS + // truthiness. Records with empty strings or null will be unmolested. + let info = new this._nsLoginInfo( + record.hostname, + nullUndefined(record.formSubmitURL), + nullUndefined(record.httpRealm), + stringifyNullUndefined(record.username), + record.password, + record.usernameField, + record.passwordField + ); + + info.QueryInterface(Ci.nsILoginMetaInfo); + info.guid = record.id; + if (record.timeCreated && !isNaN(new Date(record.timeCreated).getTime())) { + info.timeCreated = record.timeCreated; + } + if ( + record.timePasswordChanged && + !isNaN(new Date(record.timePasswordChanged).getTime()) + ) { + info.timePasswordChanged = record.timePasswordChanged; + } + + // Check the record if there are any unknown fields from other clients + // that we want to roundtrip during sync to prevent data loss + let unknownFields = this._processUnknownFields(record.cleartext); + if (unknownFields) { + info.unknownFields = unknownFields; + } + return info; + }, + + async _getLoginFromGUID(id) { + let prop = this._newPropertyBag(); + prop.setPropertyAsAUTF8String("guid", id); + + let logins = Services.logins.searchLogins(prop); + await Async.promiseYield(); // Yield back to main thread after synchronous operation. + + if (logins.length) { + this._log.trace(logins.length + " items matching " + id + " found."); + return logins[0]; + } + + this._log.trace("No items matching " + id + " found. Ignoring"); + return null; + }, + + async getAllIDs() { + let items = {}; + let logins = Services.logins.getAllLogins(); + + for (let i = 0; i < logins.length; i++) { + // Skip over Weave password/passphrase entries. + let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo); + if (Utils.getSyncCredentialsHosts().has(metaInfo.origin)) { + continue; + } + + items[metaInfo.guid] = metaInfo; + } + + return items; + }, + + async changeItemID(oldID, newID) { + this._log.trace("Changing item ID: " + oldID + " to " + newID); + + let oldLogin = await this._getLoginFromGUID(oldID); + if (!oldLogin) { + this._log.trace("Can't change item ID: item doesn't exist"); + return; + } + if (await this._getLoginFromGUID(newID)) { + this._log.trace("Can't change item ID: new ID already in use"); + return; + } + + let prop = this._newPropertyBag(); + prop.setPropertyAsAUTF8String("guid", newID); + + Services.logins.modifyLogin(oldLogin, prop); + }, + + async itemExists(id) { + return !!(await this._getLoginFromGUID(id)); + }, + + async createRecord(id, collection) { + let record = new LoginRec(collection, id); + let login = await this._getLoginFromGUID(id); + + if (!login) { + record.deleted = true; + return record; + } + + record.hostname = login.origin; + record.formSubmitURL = login.formActionOrigin; + record.httpRealm = login.httpRealm; + record.username = login.username; + record.password = login.password; + record.usernameField = login.usernameField; + record.passwordField = login.passwordField; + + // Optional fields. + login.QueryInterface(Ci.nsILoginMetaInfo); + record.timeCreated = login.timeCreated; + record.timePasswordChanged = login.timePasswordChanged; + + // put the unknown fields back to the top-level record + // during upload + if (login.unknownFields) { + let unknownFields = JSON.parse(login.unknownFields); + if (unknownFields) { + Object.keys(unknownFields).forEach(key => { + // We have to manually add it to the cleartext since that's + // what gets processed during upload + record.cleartext[key] = unknownFields[key]; + }); + } + } + + return record; + }, + + async create(record) { + let login = this._nsLoginInfoFromRecord(record); + if (!login) { + return; + } + + this._log.trace("Adding login for " + record.hostname); + this._log.trace( + "httpRealm: " + + JSON.stringify(login.httpRealm) + + "; " + + "formSubmitURL: " + + JSON.stringify(login.formActionOrigin) + ); + await Services.logins.addLoginAsync(login); + }, + + async remove(record) { + this._log.trace("Removing login " + record.id); + + let loginItem = await this._getLoginFromGUID(record.id); + if (!loginItem) { + this._log.trace("Asked to remove record that doesn't exist, ignoring"); + return; + } + + Services.logins.removeLogin(loginItem); + }, + + async update(record) { + let loginItem = await this._getLoginFromGUID(record.id); + if (!loginItem) { + this._log.trace("Skipping update for unknown item: " + record.hostname); + return; + } + + this._log.trace("Updating " + record.hostname); + let newinfo = this._nsLoginInfoFromRecord(record); + if (!newinfo) { + return; + } + + Services.logins.modifyLogin(loginItem, newinfo); + }, + + async wipe() { + Services.logins.removeAllUserFacingLogins(); + }, +}; +Object.setPrototypeOf(PasswordStore.prototype, Store.prototype); + +function PasswordTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +PasswordTracker.prototype = { + onStart() { + Svc.Obs.add("passwordmgr-storage-changed", this.asyncObserver); + }, + + onStop() { + Svc.Obs.remove("passwordmgr-storage-changed", this.asyncObserver); + }, + + async observe(subject, topic, data) { + if (this.ignoreAll) { + return; + } + + // A single add, remove or change or removing all items + // will trigger a sync for MULTI_DEVICE. + switch (data) { + case "modifyLogin": { + subject.QueryInterface(Ci.nsIArrayExtensions); + let oldLogin = subject.GetElementAt(0); + let newLogin = subject.GetElementAt(1); + if (!isSyncableChange(oldLogin, newLogin)) { + this._log.trace(`${data}: Ignoring change for ${newLogin.guid}`); + break; + } + const tracked = await this._trackLogin(newLogin); + if (tracked) { + this._log.trace(`${data}: Tracking change for ${newLogin.guid}`); + } + break; + } + + case "addLogin": + case "removeLogin": + subject + .QueryInterface(Ci.nsILoginMetaInfo) + .QueryInterface(Ci.nsILoginInfo); + const tracked = await this._trackLogin(subject); + if (tracked) { + this._log.trace(data + ": " + subject.guid); + } + break; + + // Bug 1613620: We iterate through the removed logins and track them to ensure + // the logins are deleted across synced devices/accounts + case "removeAllLogins": + subject.QueryInterface(Ci.nsIArrayExtensions); + let count = subject.Count(); + for (let i = 0; i < count; i++) { + let currentSubject = subject.GetElementAt(i); + let tracked = await this._trackLogin(currentSubject); + if (tracked) { + this._log.trace(data + ": " + currentSubject.guid); + } + } + this.score += SCORE_INCREMENT_XLARGE; + break; + } + }, + + async _trackLogin(login) { + if (Utils.getSyncCredentialsHosts().has(login.origin)) { + // Skip over Weave password/passphrase changes. + return false; + } + const added = await this.addChangedID(login.guid); + if (!added) { + return false; + } + this.score += SCORE_INCREMENT_XLARGE; + return true; + }, +}; +Object.setPrototypeOf(PasswordTracker.prototype, LegacyTracker.prototype); + +export class PasswordValidator extends CollectionValidator { + constructor() { + super("passwords", "id", [ + "hostname", + "formSubmitURL", + "httpRealm", + "password", + "passwordField", + "username", + "usernameField", + ]); + } + + getClientItems() { + let logins = Services.logins.getAllLogins(); + let syncHosts = Utils.getSyncCredentialsHosts(); + let result = logins + .map(l => l.QueryInterface(Ci.nsILoginMetaInfo)) + .filter(l => !syncHosts.has(l.origin)); + return Promise.resolve(result); + } + + normalizeClientItem(item) { + return { + id: item.guid, + guid: item.guid, + hostname: item.hostname, + formSubmitURL: item.formSubmitURL, + httpRealm: item.httpRealm, + password: item.password, + passwordField: item.passwordField, + username: item.username, + usernameField: item.usernameField, + original: item, + }; + } + + async normalizeServerItem(item) { + return Object.assign({ guid: item.id }, item); + } +} diff --git a/services/sync/modules/engines/prefs.sys.mjs b/services/sync/modules/engines/prefs.sys.mjs new file mode 100644 index 0000000000..13d8a63070 --- /dev/null +++ b/services/sync/modules/engines/prefs.sys.mjs @@ -0,0 +1,467 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Prefs which start with this prefix are our "control" prefs - they indicate +// which preferences should be synced. +const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; + +// Prefs which have a default value are usually not synced - however, if the +// preference exists under this prefix and the value is: +// * `true`, then we do sync default values. +// * `false`, then as soon as we ever sync a non-default value out, or sync +// any value in, then we toggle the value to `true`. +// +// We never explicitly set this pref back to false, so it's one-shot. +// Some preferences which are known to have a different default value on +// different platforms have this preference with a default value of `false`, +// so they don't sync until one device changes to the non-default value, then +// that value forever syncs, even if it gets reset back to the default. +// Note that preferences handled this way *must also* have the "normal" +// control pref set. +// A possible future enhancement would be to sync these prefs so that +// other distributions can flag them if they change the default, but that +// doesn't seem worthwhile until we can be confident they'd actually create +// this special control pref at the same time they flip the default. +const PREF_SYNC_SEEN_PREFIX = "services.sync.prefs.sync-seen."; + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { Preferences } from "resource://gre/modules/Preferences.sys.mjs"; + +import { + Store, + SyncEngine, + Tracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; +import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +const lazy = {}; + +XPCOMUtils.defineLazyGetter(lazy, "PREFS_GUID", () => + CommonUtils.encodeBase64URL(Services.appinfo.ID) +); + +ChromeUtils.defineESModuleGetters(lazy, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", +}); + +// In bug 1538015, we decided that it isn't always safe to allow all "incoming" +// preferences to be applied locally. So we have introduced another preference, +// which if false (the default) will ignore all incoming preferences which don't +// already have the "control" preference locally set. If this new +// preference is set to true, then we continue our old behavior of allowing all +// preferences to be updated, even those which don't already have a local +// "control" pref. +const PREF_SYNC_PREFS_ARBITRARY = + "services.sync.prefs.dangerously_allow_arbitrary"; + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "ALLOW_ARBITRARY", + PREF_SYNC_PREFS_ARBITRARY +); + +// The SUMO supplied URL we log with more information about how custom prefs can +// continue to be synced. SUMO have told us that this URL will remain "stable". +const PREFS_DOC_URL_TEMPLATE = + "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/sync-custom-preferences"; +XPCOMUtils.defineLazyGetter(lazy, "PREFS_DOC_URL", () => + Services.urlFormatter.formatURL(PREFS_DOC_URL_TEMPLATE) +); + +// Check for a local control pref or PREF_SYNC_PREFS_ARBITRARY +function isAllowedPrefName(prefName) { + if (prefName == PREF_SYNC_PREFS_ARBITRARY) { + return false; // never allow this. + } + if (lazy.ALLOW_ARBITRARY) { + // user has set the "dangerous" pref, so everything is allowed. + return true; + } + // The pref must already have a control pref set, although it doesn't matter + // here whether that value is true or false. We can't use prefHasUserValue + // here because we also want to check prefs still with default values. + try { + Services.prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + prefName); + // pref exists! + return true; + } catch (_) { + return false; + } +} + +export function PrefRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +PrefRec.prototype = { + _logName: "Sync.Record.Pref", +}; +Object.setPrototypeOf(PrefRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(PrefRec, "cleartext", ["value"]); + +export function PrefsEngine(service) { + SyncEngine.call(this, "Prefs", service); +} + +PrefsEngine.prototype = { + _storeObj: PrefStore, + _trackerObj: PrefTracker, + _recordObj: PrefRec, + version: 2, + + syncPriority: 1, + allowSkippedRecord: false, + + async getChangedIDs() { + // No need for a proper timestamp (no conflict resolution needed). + let changedIDs = {}; + if (this._tracker.modified) { + changedIDs[lazy.PREFS_GUID] = 0; + } + return changedIDs; + }, + + async _wipeClient() { + await SyncEngine.prototype._wipeClient.call(this); + this.justWiped = true; + }, + + async _reconcile(item) { + // Apply the incoming item if we don't care about the local data + if (this.justWiped) { + this.justWiped = false; + return true; + } + return SyncEngine.prototype._reconcile.call(this, item); + }, + + async trackRemainingChanges() { + if (this._modified.count() > 0) { + this._tracker.modified = true; + } + }, +}; +Object.setPrototypeOf(PrefsEngine.prototype, SyncEngine.prototype); + +// We don't use services.sync.engine.tabs.filteredSchemes since it includes +// about: pages and the like, which we want to be syncable in preferences. +// Blob, moz-extension, data and file uris are never safe to sync, +// so we limit our check to those. +const UNSYNCABLE_URL_REGEXP = /^(moz-extension|blob|data|file):/i; +function isUnsyncableURLPref(prefName) { + if (Services.prefs.getPrefType(prefName) != Ci.nsIPrefBranch.PREF_STRING) { + return false; + } + const prefValue = Services.prefs.getStringPref(prefName, ""); + return UNSYNCABLE_URL_REGEXP.test(prefValue); +} + +function PrefStore(name, engine) { + Store.call(this, name, engine); + Svc.Obs.add( + "profile-before-change", + function () { + this.__prefs = null; + }, + this + ); +} +PrefStore.prototype = { + __prefs: null, + get _prefs() { + if (!this.__prefs) { + this.__prefs = new Preferences(); + } + return this.__prefs; + }, + + _getSyncPrefs() { + let syncPrefs = Services.prefs + .getBranch(PREF_SYNC_PREFS_PREFIX) + .getChildList("") + .filter(pref => isAllowedPrefName(pref) && !isUnsyncableURLPref(pref)); + // Also sync preferences that determine which prefs get synced. + let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref); + return controlPrefs.concat(syncPrefs); + }, + + _isSynced(pref) { + if (pref.startsWith(PREF_SYNC_PREFS_PREFIX)) { + // this is an incoming control pref, which is ignored if there's not already + // a local control pref for the preference. + let controlledPref = pref.slice(PREF_SYNC_PREFS_PREFIX.length); + return isAllowedPrefName(controlledPref); + } + + // This is the pref itself - it must be both allowed, and have a control + // pref which is true. + if (!this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false)) { + return false; + } + return isAllowedPrefName(pref); + }, + + _getAllPrefs() { + let values = {}; + for (let pref of this._getSyncPrefs()) { + // Note: _isSynced doesn't call isUnsyncableURLPref since it would cause + // us not to apply (syncable) changes to preferences that are set locally + // which have unsyncable urls. + if (this._isSynced(pref) && !isUnsyncableURLPref(pref)) { + let isSet = this._prefs.isSet(pref); + // Missing and default prefs get the null value, unless that `seen` + // pref is set, in which case it always gets the value. + let forceValue = this._prefs.get(PREF_SYNC_SEEN_PREFIX + pref, false); + values[pref] = isSet || forceValue ? this._prefs.get(pref, null) : null; + // If this is a special "sync-seen" pref, and it's not the default value, + // set the seen pref to true. + if ( + isSet && + this._prefs.get(PREF_SYNC_SEEN_PREFIX + pref, false) === false + ) { + this._log.trace(`toggling sync-seen pref for '${pref}' to true`); + this._prefs.set(PREF_SYNC_SEEN_PREFIX + pref, true); + } + } + } + return values; + }, + + _setAllPrefs(values) { + const selectedThemeIDPref = "extensions.activeThemeID"; + let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null); + let selectedThemeIDAfter = selectedThemeIDBefore; + + // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise + // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device). + let prefs = Object.keys(values).sort( + a => -a.indexOf(PREF_SYNC_PREFS_PREFIX) + ); + for (let pref of prefs) { + let value = values[pref]; + if (!this._isSynced(pref)) { + // An extra complication just so we can warn when we decline to sync a + // preference due to no local control pref. + if (!pref.startsWith(PREF_SYNC_PREFS_PREFIX)) { + // this is an incoming pref - if the incoming value is not null and + // there's no local control pref, then it means we would have previously + // applied a value, but now will decline to. + // We need to check this here rather than in _isSynced because the + // default list of prefs we sync has changed, so we don't want to report + // this message when we wouldn't have actually applied a value. + // We should probably remove all of this in ~ Firefox 80. + if (value !== null) { + // null means "use the default value" + let controlPref = PREF_SYNC_PREFS_PREFIX + pref; + let controlPrefExists; + try { + Services.prefs.getBoolPref(controlPref); + controlPrefExists = true; + } catch (ex) { + controlPrefExists = false; + } + if (!controlPrefExists) { + // This is a long message and written to both the sync log and the + // console, but note that users who have not customized the control + // prefs will never see this. + let msg = + `Not syncing the preference '${pref}' because it has no local ` + + `control preference (${PREF_SYNC_PREFS_PREFIX}${pref}) and ` + + `the preference ${PREF_SYNC_PREFS_ARBITRARY} isn't true. ` + + `See ${lazy.PREFS_DOC_URL} for more information`; + console.warn(msg); + this._log.warn(msg); + } + } + } + continue; + } + + if (typeof value == "string" && UNSYNCABLE_URL_REGEXP.test(value)) { + this._log.trace(`Skipping incoming unsyncable url for pref: ${pref}`); + continue; + } + + switch (pref) { + // Some special prefs we don't want to set directly. + case selectedThemeIDPref: + selectedThemeIDAfter = value; + break; + + // default is to just set the pref + default: + if (value == null) { + // Pref has gone missing. The best we can do is reset it. + this._prefs.reset(pref); + } else { + try { + this._prefs.set(pref, value); + } catch (ex) { + this._log.trace(`Failed to set pref: ${pref}`, ex); + } + } + // If there's a "sync-seen" pref for this it gets toggled to true + // regardless of the value. + let seenPref = PREF_SYNC_SEEN_PREFIX + pref; + if (this._prefs.get(seenPref, undefined) === false) { + this._prefs.set(PREF_SYNC_SEEN_PREFIX + pref, true); + } + } + } + // Themes are a little messy. Themes which have been installed are handled + // by the addons engine - but default themes aren't seen by that engine. + // So if there's a new default theme ID and that ID corresponds to a + // system addon, then we arrange to enable that addon here. + if (selectedThemeIDBefore != selectedThemeIDAfter) { + this._maybeEnableBuiltinTheme(selectedThemeIDAfter).catch(e => { + this._log.error("Failed to maybe update the default theme", e); + }); + } + }, + + async _maybeEnableBuiltinTheme(themeId) { + let addon = null; + try { + addon = await lazy.AddonManager.getAddonByID(themeId); + } catch (ex) { + this._log.trace( + `There's no addon with ID '${themeId} - it can't be a builtin theme` + ); + return; + } + if (addon && addon.isBuiltin && addon.type == "theme") { + this._log.trace(`Enabling builtin theme '${themeId}'`); + await addon.enable(); + } else { + this._log.trace( + `Have incoming theme ID of '${themeId}' but it's not a builtin theme` + ); + } + }, + + async getAllIDs() { + /* We store all prefs in just one WBO, with just one GUID */ + let allprefs = {}; + allprefs[lazy.PREFS_GUID] = true; + return allprefs; + }, + + async changeItemID(oldID, newID) { + this._log.trace("PrefStore GUID is constant!"); + }, + + async itemExists(id) { + return id === lazy.PREFS_GUID; + }, + + async createRecord(id, collection) { + let record = new PrefRec(collection, id); + + if (id == lazy.PREFS_GUID) { + record.value = this._getAllPrefs(); + } else { + record.deleted = true; + } + + return record; + }, + + async create(record) { + this._log.trace("Ignoring create request"); + }, + + async remove(record) { + this._log.trace("Ignoring remove request"); + }, + + async update(record) { + // Silently ignore pref updates that are for other apps. + if (record.id != lazy.PREFS_GUID) { + return; + } + + this._log.trace("Received pref updates, applying..."); + this._setAllPrefs(record.value); + }, + + async wipe() { + this._log.trace("Ignoring wipe request"); + }, +}; +Object.setPrototypeOf(PrefStore.prototype, Store.prototype); + +function PrefTracker(name, engine) { + Tracker.call(this, name, engine); + this._ignoreAll = false; + Svc.Obs.add("profile-before-change", this.asyncObserver); +} +PrefTracker.prototype = { + get ignoreAll() { + return this._ignoreAll; + }, + + set ignoreAll(value) { + this._ignoreAll = value; + }, + + get modified() { + return Svc.Prefs.get("engine.prefs.modified", false); + }, + set modified(value) { + Svc.Prefs.set("engine.prefs.modified", value); + }, + + clearChangedIDs: function clearChangedIDs() { + this.modified = false; + }, + + __prefs: null, + get _prefs() { + if (!this.__prefs) { + this.__prefs = new Preferences(); + } + return this.__prefs; + }, + + onStart() { + Services.prefs.addObserver("", this.asyncObserver); + }, + + onStop() { + this.__prefs = null; + Services.prefs.removeObserver("", this.asyncObserver); + }, + + async observe(subject, topic, data) { + switch (topic) { + case "profile-before-change": + await this.stop(); + break; + case "nsPref:changed": + if (this.ignoreAll) { + break; + } + // Trigger a sync for MULTI-DEVICE for a change that determines + // which prefs are synced or a regular pref change. + if ( + data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 || + this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false) + ) { + this.score += SCORE_INCREMENT_XLARGE; + this.modified = true; + this._log.trace("Preference " + data + " changed"); + } + break; + } + }, +}; +Object.setPrototypeOf(PrefTracker.prototype, Tracker.prototype); + +export function getPrefsGUIDForTest() { + return lazy.PREFS_GUID; +} diff --git a/services/sync/modules/engines/tabs.sys.mjs b/services/sync/modules/engines/tabs.sys.mjs new file mode 100644 index 0000000000..45242b71a7 --- /dev/null +++ b/services/sync/modules/engines/tabs.sys.mjs @@ -0,0 +1,624 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const STORAGE_VERSION = 1; // This needs to be kept in-sync with the rust storage version + +import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; +import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs"; +import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; +import { Log } from "resource://gre/modules/Log.sys.mjs"; +import { + SCORE_INCREMENT_SMALL, + STATUS_OK, + URI_LENGTH_MAX, +} from "resource://services-sync/constants.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; +import { Async } from "resource://services-common/async.sys.mjs"; +import { + SyncRecord, + SyncTelemetry, +} from "resource://services-sync/telemetry.sys.mjs"; +import { BridgedEngine } from "resource://services-sync/bridged_engine.sys.mjs"; + +const FAR_FUTURE = 4102405200000; // 2100/01/01 + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs", + ReaderMode: "resource://gre/modules/ReaderMode.sys.mjs", + TabsStore: "resource://gre/modules/RustTabs.sys.mjs", +}); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "TABS_FILTERED_SCHEMES", + "services.sync.engine.tabs.filteredSchemes", + "", + null, + val => { + return new Set(val.split("|")); + } +); + +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "SYNC_AFTER_DELAY_MS", + "services.sync.syncedTabs.syncDelayAfterTabChange", + 0 +); + +// A "bridged engine" to our tabs component. +export function TabEngine(service) { + BridgedEngine.call(this, "Tabs", service); +} + +TabEngine.prototype = { + _trackerObj: TabTracker, + syncPriority: 3, + + async prepareTheBridge(isQuickWrite) { + let clientsEngine = this.service.clientsEngine; + // Tell the bridged engine about clients. + // This is the same shape as ClientData in app-services. + // schema: https://github.com/mozilla/application-services/blob/a1168751231ed4e88c44d85f6dccc09c3b412bd2/components/sync15/src/client_types.rs#L14 + let clientData = { + local_client_id: clientsEngine.localID, + recent_clients: {}, + }; + + // We shouldn't upload tabs past what the server will accept + let tabs = await this.getTabsWithinPayloadSize(); + await this._rustStore.setLocalTabs( + tabs.map(tab => { + // rust wants lastUsed in MS but the provider gives it in seconds + tab.lastUsed = tab.lastUsed * 1000; + return tab; + }) + ); + + for (let remoteClient of clientsEngine.remoteClients) { + let id = remoteClient.id; + if (!id) { + throw new Error("Remote client somehow did not have an id"); + } + let client = { + fxa_device_id: remoteClient.fxaDeviceId, + // device_name and device_type are soft-deprecated - every client + // prefers what's in the FxA record. But fill them correctly anyway. + device_name: clientsEngine.getClientName(id) ?? "", + device_type: clientsEngine.getClientType(id), + }; + clientData.recent_clients[id] = client; + } + + // put ourself in there too so we record the correct device info in our sync record. + clientData.recent_clients[clientsEngine.localID] = { + fxa_device_id: await clientsEngine.fxAccounts.device.getLocalId(), + device_name: clientsEngine.localName, + device_type: clientsEngine.localType, + }; + + // Quick write needs to adjust the lastSync so we can POST to the server + // see quickWrite() for details + if (isQuickWrite) { + await this.setLastSync(FAR_FUTURE); + await this._bridge.prepareForSync(JSON.stringify(clientData)); + return; + } + + // Just incase we crashed while the lastSync timestamp was FAR_FUTURE, we + // reset it to zero + if ((await this.getLastSync()) === FAR_FUTURE) { + await this._bridge.setLastSync(0); + } + await this._bridge.prepareForSync(JSON.stringify(clientData)); + }, + + async _syncStartup() { + await super._syncStartup(); + await this.prepareTheBridge(); + }, + + async initialize() { + await SyncEngine.prototype.initialize.call(this); + + let path = PathUtils.join(PathUtils.profileDir, "synced-tabs.db"); + this._rustStore = await lazy.TabsStore.init(path); + this._bridge = await this._rustStore.bridgedEngine(); + + // Uniffi doesn't currently only support async methods, so we'll need to hardcode + // these values for now (which is fine for now as these hardly ever change) + this._bridge.storageVersion = STORAGE_VERSION; + this._bridge.allowSkippedRecord = true; + + this._log.info("Got a bridged engine!"); + this._tracker.modified = true; + }, + + async getChangedIDs() { + // No need for a proper timestamp (no conflict resolution needed). + let changedIDs = {}; + if (this._tracker.modified) { + changedIDs[this.service.clientsEngine.localID] = 0; + } + return changedIDs; + }, + + // API for use by Sync UI code to give user choices of tabs to open. + async getAllClients() { + let remoteTabs = await this._rustStore.getAll(); + let remoteClientTabs = []; + for (let remoteClient of this.service.clientsEngine.remoteClients) { + // We get the some client info from the rust tabs engine and some from + // the clients engine. + let rustClient = remoteTabs.find( + x => x.clientId === remoteClient.fxaDeviceId + ); + if (!rustClient) { + continue; + } + let client = { + // rust gives us ms but js uses seconds, so fix them up. + tabs: rustClient.remoteTabs.map(tab => { + tab.lastUsed = tab.lastUsed / 1000; + return tab; + }), + lastModified: rustClient.lastModified / 1000, + ...remoteClient, + }; + remoteClientTabs.push(client); + } + return remoteClientTabs; + }, + + async removeClientData() { + let url = this.engineURL + "/" + this.service.clientsEngine.localID; + await this.service.resource(url).delete(); + }, + + async trackRemainingChanges() { + if (this._modified.count() > 0) { + this._tracker.modified = true; + } + }, + + async getTabsWithinPayloadSize() { + const maxPayloadSize = this.service.getMaxRecordPayloadSize(); + // See bug 535326 comment 8 for an explanation of the estimation + const maxSerializedSize = (maxPayloadSize / 4) * 3 - 1500; + return TabProvider.getAllTabsWithEstimatedMax(true, maxSerializedSize); + }, + + // Support for "quick writes" + _engineLock: Utils.lock, + _engineLocked: false, + + // Tabs has a special lock to help support its "quick write" + get locked() { + return this._engineLocked; + }, + lock() { + if (this._engineLocked) { + return false; + } + this._engineLocked = true; + return true; + }, + unlock() { + this._engineLocked = false; + }, + + // Quickly do a POST of our current tabs if possible. + // This does things that would be dangerous for other engines - eg, posting + // without checking what's on the server could cause data-loss for other + // engines, but because each device exclusively owns exactly 1 tabs record + // with a known ID, it's safe here. + // Returns true if we successfully synced, false otherwise (either on error + // or because we declined to sync for any reason.) The return value is + // primarily for tests. + async quickWrite() { + if (!this.enabled) { + // this should be very rare, and only if tabs are disabled after the + // timer is created. + this._log.info("Can't do a quick-sync as tabs is disabled"); + return false; + } + // This quick-sync doesn't drive the login state correctly, so just + // decline to sync if out status is bad + if (this.service.status.checkSetup() != STATUS_OK) { + this._log.info( + "Can't do a quick-sync due to the service status", + this.service.status.toString() + ); + return false; + } + if (!this.service.serverConfiguration) { + this._log.info("Can't do a quick sync before the first full sync"); + return false; + } + try { + return await this._engineLock("tabs.js: quickWrite", async () => { + // We want to restore the lastSync timestamp when complete so next sync + // takes tabs written by other devices since our last real sync. + // And for this POST we don't want the protections offered by + // X-If-Unmodified-Since - we want the POST to work even if the remote + // has moved on and we will catch back up next full sync. + const origLastSync = await this.getLastSync(); + try { + return this._doQuickWrite(); + } finally { + // set the lastSync to it's original value for regular sync + await this.setLastSync(origLastSync); + } + })(); + } catch (ex) { + if (!Utils.isLockException(ex)) { + throw ex; + } + this._log.info( + "Can't do a quick-write as another tab sync is in progress" + ); + return false; + } + }, + + // The guts of the quick-write sync, after we've taken the lock, checked + // the service status etc. + async _doQuickWrite() { + // We need to track telemetry for these syncs too! + const name = "tabs"; + let telemetryRecord = new SyncRecord( + SyncTelemetry.allowedEngines, + "quick-write" + ); + telemetryRecord.onEngineStart(name); + try { + Async.checkAppReady(); + // We need to prep the bridge before we try to POST since it grabs + // the most recent local client id and properly sets a lastSync + // which is needed for a proper POST request + await this.prepareTheBridge(true); + this._tracker.clearChangedIDs(); + this._tracker.resetScore(); + + Async.checkAppReady(); + // now just the "upload" part of a sync, + // which for a rust engine is not obvious. + // We need to do is ask the rust engine for the changes. Although + // this is kinda abusing the bridged-engine interface, we know the tabs + // implementation of it works ok + let outgoing = await this._bridge.apply(); + // We know we always have exactly 1 record. + let mine = outgoing[0]; + this._log.trace("outgoing bso", mine); + // `this._recordObj` is a `BridgedRecord`, which isn't exported. + let record = this._recordObj.fromOutgoingBso(this.name, JSON.parse(mine)); + let changeset = {}; + changeset[record.id] = { synced: false, record }; + this._modified.replace(changeset); + + Async.checkAppReady(); + await this._uploadOutgoing(); + telemetryRecord.onEngineStop(name, null); + return true; + } catch (ex) { + this._log.warn("quicksync sync failed", ex); + telemetryRecord.onEngineStop(name, ex); + return false; + } finally { + // The top-level sync is never considered to fail here, just the engine + telemetryRecord.finished(null); + SyncTelemetry.takeTelemetryRecord(telemetryRecord); + } + }, + + async _sync() { + try { + await this._engineLock("tabs.js: fullSync", async () => { + await super._sync(); + })(); + } catch (ex) { + if (!Utils.isLockException(ex)) { + throw ex; + } + this._log.info( + "Can't do full tabs sync as a quick-write is currently running" + ); + } + }, +}; +Object.setPrototypeOf(TabEngine.prototype, BridgedEngine.prototype); + +export const TabProvider = { + getWindowEnumerator() { + return Services.wm.getEnumerator("navigator:browser"); + }, + + shouldSkipWindow(win) { + return win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win); + }, + + getAllBrowserTabs() { + let tabs = []; + for (let win of this.getWindowEnumerator()) { + if (this.shouldSkipWindow(win)) { + continue; + } + // Get all the tabs from the browser + for (let tab of win.gBrowser.tabs) { + tabs.push(tab); + } + } + + return tabs.sort(function (a, b) { + return b.lastAccessed - a.lastAccessed; + }); + }, + + // This function creates tabs records up to a specified amount of bytes + // It is an "estimation" since we don't accurately calculate how much the + // favicon and JSON overhead is and give a rough estimate (for optimization purposes) + async getAllTabsWithEstimatedMax(filter, bytesMax) { + let log = Log.repository.getLogger(`Sync.Engine.Tabs.Provider`); + let tabRecords = []; + let iconPromises = []; + let runningByteLength = 0; + let encoder = new TextEncoder(); + + // Fetch all the tabs the user has open + let winTabs = this.getAllBrowserTabs(); + + for (let tab of winTabs) { + // We don't want to process any more tabs than we can sync + if (runningByteLength >= bytesMax) { + log.warn( + `Can't fit all tabs in sync payload: have ${winTabs.length}, + but can only fit ${tabRecords.length}.` + ); + break; + } + + // Note that we used to sync "tab history" (ie, the "back button") state, + // but in practice this hasn't been used - only the current URI is of + // interest to clients. + // We stopped recording this in bug 1783991. + if (!tab?.linkedBrowser) { + continue; + } + let acceptable = !filter + ? url => url + : url => + url && + !lazy.TABS_FILTERED_SCHEMES.has(Services.io.extractScheme(url)); + + let url = tab.linkedBrowser.currentURI?.spec; + // Special case for reader mode. + if (url && url.startsWith("about:reader?")) { + url = lazy.ReaderMode.getOriginalUrl(url); + } + // We ignore the tab completely if the current entry url is + // not acceptable (we need something accurate to open). + if (!acceptable(url)) { + continue; + } + + if (url.length > URI_LENGTH_MAX) { + log.trace("Skipping over-long URL."); + continue; + } + + let thisTab = { + title: tab.linkedBrowser.contentTitle || "", + urlHistory: [url], + icon: "", + lastUsed: Math.floor((tab.lastAccessed || 0) / 1000), + }; + tabRecords.push(thisTab); + + // we don't want to wait for each favicon to resolve to get the bytes + // so we estimate a conservative 100 chars for the favicon and json overhead + // Rust will further optimize and trim if we happened to be wildly off + runningByteLength += + encoder.encode(thisTab.title + thisTab.lastUsed + url).byteLength + 100; + + // Use the favicon service for the icon url - we can wait for the promises at the end. + let iconPromise = lazy.PlacesUtils.promiseFaviconData(url) + .then(iconData => { + thisTab.icon = iconData.uri.spec; + }) + .catch(ex => { + log.trace( + `Failed to fetch favicon for ${url}`, + thisTab.urlHistory[0] + ); + }); + iconPromises.push(iconPromise); + } + + await Promise.allSettled(iconPromises); + return tabRecords; + }, +}; + +function TabTracker(name, engine) { + Tracker.call(this, name, engine); + + // Make sure "this" pointer is always set correctly for event listeners. + this.onTab = Utils.bind2(this, this.onTab); + this._unregisterListeners = Utils.bind2(this, this._unregisterListeners); +} +TabTracker.prototype = { + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), + + clearChangedIDs() { + this.modified = false; + }, + + // We do not track TabSelect because that almost always triggers + // the web progress listeners (onLocationChange), which we already track + _topics: ["TabOpen", "TabClose"], + + _registerListenersForWindow(window) { + this._log.trace("Registering tab listeners in window"); + for (let topic of this._topics) { + window.addEventListener(topic, this.onTab); + } + window.addEventListener("unload", this._unregisterListeners); + // If it's got a tab browser we can listen for things like navigation. + if (window.gBrowser) { + window.gBrowser.addProgressListener(this); + } + }, + + _unregisterListeners(event) { + this._unregisterListenersForWindow(event.target); + }, + + _unregisterListenersForWindow(window) { + this._log.trace("Removing tab listeners in window"); + window.removeEventListener("unload", this._unregisterListeners); + for (let topic of this._topics) { + window.removeEventListener(topic, this.onTab); + } + if (window.gBrowser) { + window.gBrowser.removeProgressListener(this); + } + }, + + onStart() { + Svc.Obs.add("domwindowopened", this.asyncObserver); + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._registerListenersForWindow(win); + } + }, + + onStop() { + Svc.Obs.remove("domwindowopened", this.asyncObserver); + for (let win of Services.wm.getEnumerator("navigator:browser")) { + this._unregisterListenersForWindow(win); + } + }, + + async observe(subject, topic, data) { + switch (topic) { + case "domwindowopened": + let onLoad = () => { + subject.removeEventListener("load", onLoad); + // Only register after the window is done loading to avoid unloads. + this._registerListenersForWindow(subject); + }; + + // Add tab listeners now that a window has opened. + subject.addEventListener("load", onLoad); + break; + } + }, + + onTab(event) { + if (event.originalTarget.linkedBrowser) { + let browser = event.originalTarget.linkedBrowser; + if ( + lazy.PrivateBrowsingUtils.isBrowserPrivate(browser) && + !lazy.PrivateBrowsingUtils.permanentPrivateBrowsing + ) { + this._log.trace("Ignoring tab event from private browsing."); + return; + } + } + this._log.trace("onTab event: " + event.type); + + switch (event.type) { + case "TabOpen": + /* We do not have a reliable way of checking the URI on the TabOpen + * so we will rely on the other methods (onLocationChange, getAllTabsWithEstimatedMax) + * to filter these when going through sync + */ + this.callScheduleSync(SCORE_INCREMENT_SMALL); + break; + case "TabClose": + // If event target has `linkedBrowser`, the event target can be assumed <tab> element. + // Else, event target is assumed <browser> element, use the target as it is. + const tab = event.target.linkedBrowser || event.target; + + // TabClose means the tab has already loaded and we can check the URI + // and ignore if it's a scheme we don't care about + if (lazy.TABS_FILTERED_SCHEMES.has(tab.currentURI.scheme)) { + return; + } + this.callScheduleSync(SCORE_INCREMENT_SMALL); + break; + } + }, + + // web progress listeners. + onLocationChange(webProgress, request, locationURI, flags) { + // We only care about top-level location changes. We do want location changes in the + // same document because if a page uses the `pushState()` API, they *appear* as though + // they are in the same document even if the URL changes. It also doesn't hurt to accurately + // reflect the fragment changing - so we allow LOCATION_CHANGE_SAME_DOCUMENT + if ( + flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_RELOAD || + !webProgress.isTopLevel || + !locationURI + ) { + return; + } + + // We can't filter out tabs that we don't sync here, because we might be + // navigating from a tab that we *did* sync to one we do not, and that + // tab we *did* sync should no longer be synced. + this.callScheduleSync(); + }, + + callScheduleSync(scoreIncrement) { + this.modified = true; + let { scheduler } = this.engine.service; + let delayInMs = lazy.SYNC_AFTER_DELAY_MS; + + // Schedule a sync once we detect a tab change + // to ensure the server always has the most up to date tabs + if ( + delayInMs > 0 && + scheduler.numClients > 1 // Only schedule quick syncs for multi client users + ) { + if (this.tabsQuickWriteTimer) { + this._log.debug( + "Detected a tab change, but a quick-write is already scheduled" + ); + return; + } + this._log.debug( + "Detected a tab change: scheduling a quick-write in " + delayInMs + "ms" + ); + CommonUtils.namedTimer( + () => { + this._log.trace("tab quick-sync timer fired."); + this.engine + .quickWrite() + .then(() => { + this._log.trace("tab quick-sync done."); + }) + .catch(ex => { + this._log.error("tab quick-sync failed.", ex); + }); + }, + delayInMs, + this, + "tabsQuickWriteTimer" + ); + } else if (scoreIncrement) { + this._log.debug( + "Detected a tab change, but conditions aren't met for a quick write - bumping score" + ); + this.score += scoreIncrement; + } else { + this._log.debug( + "Detected a tab change, but conditions aren't met for a quick write or a score bump" + ); + } + }, +}; +Object.setPrototypeOf(TabTracker.prototype, Tracker.prototype); |