summaryrefslogtreecommitdiffstats
path: root/services/sync/modules/engines
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 14:29:10 +0000
commit2aa4a82499d4becd2284cdb482213d541b8804dd (patch)
treeb80bf8bf13c3766139fbacc530efd0dd9d54394c /services/sync/modules/engines
parentInitial commit. (diff)
downloadfirefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz
firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'services/sync/modules/engines')
-rw-r--r--services/sync/modules/engines/addons.js833
-rw-r--r--services/sync/modules/engines/bookmarks.js1000
-rw-r--r--services/sync/modules/engines/clients.js1186
-rw-r--r--services/sync/modules/engines/extension-storage.js295
-rw-r--r--services/sync/modules/engines/forms.js312
-rw-r--r--services/sync/modules/engines/history.js625
-rw-r--r--services/sync/modules/engines/passwords.js537
-rw-r--r--services/sync/modules/engines/prefs.js436
-rw-r--r--services/sync/modules/engines/tabs.js395
9 files changed, 5619 insertions, 0 deletions
diff --git a/services/sync/modules/engines/addons.js b/services/sync/modules/engines/addons.js
new file mode 100644
index 0000000000..d3c79967b2
--- /dev/null
+++ b/services/sync/modules/engines/addons.js
@@ -0,0 +1,833 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file defines the add-on sync functionality.
+ *
+ * There are currently a number of known limitations:
+ * - We only sync XPI extensions and themes available from addons.mozilla.org.
+ * We hope to expand support for other add-ons eventually.
+ * - We only attempt syncing of add-ons between applications of the same type.
+ * This means add-ons will not synchronize between Firefox desktop and
+ * Firefox mobile, for example. This is because of significant add-on
+ * incompatibility between application types.
+ *
+ * Add-on records exist for each known {add-on, app-id} pair in the Sync client
+ * set. Each record has a randomly chosen GUID. The records then contain
+ * basic metadata about the add-on.
+ *
+ * We currently synchronize:
+ *
+ * - Installations
+ * - Uninstallations
+ * - User enabling and disabling
+ *
+ * Synchronization is influenced by the following preferences:
+ *
+ * - services.sync.addons.ignoreUserEnabledChanges
+ * - services.sync.addons.trustedSourceHostnames
+ *
+ * and also influenced by whether addons have repository caching enabled and
+ * whether they allow installation of addons from insecure options (both of
+ * which are themselves influenced by the "extensions." pref branch)
+ *
+ * See the documentation in all.js for the behavior of these prefs.
+ */
+"use strict";
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Preferences } = ChromeUtils.import(
+ "resource://gre/modules/Preferences.jsm"
+);
+const { AddonUtils } = ChromeUtils.import(
+ "resource://services-sync/addonutils.js"
+);
+const { AddonsReconciler } = ChromeUtils.import(
+ "resource://services-sync/addonsreconciler.js"
+);
+const { Store, SyncEngine, LegacyTracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+const { SCORE_INCREMENT_XLARGE } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+const { CollectionValidator } = ChromeUtils.import(
+ "resource://services-sync/collection_validator.js"
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "AddonManager",
+ "resource://gre/modules/AddonManager.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "AddonRepository",
+ "resource://gre/modules/addons/AddonRepository.jsm"
+);
+
+var EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"];
+
+// 7 days in milliseconds.
+const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
+
+/**
+ * AddonRecord represents the state of an add-on in an application.
+ *
+ * Each add-on has its own record for each application ID it is installed
+ * on.
+ *
+ * The ID of add-on records is a randomly-generated GUID. It is random instead
+ * of deterministic so the URIs of the records cannot be guessed and so
+ * compromised server credentials won't result in disclosure of the specific
+ * add-ons present in a Sync account.
+ *
+ * The record contains the following fields:
+ *
+ * addonID
+ * ID of the add-on. This correlates to the "id" property on an Addon type.
+ *
+ * applicationID
+ * The application ID this record is associated with.
+ *
+ * enabled
+ * Boolean stating whether add-on is enabled or disabled by the user.
+ *
+ * source
+ * String indicating where an add-on is from. Currently, we only support
+ * the value "amo" which indicates that the add-on came from the official
+ * add-ons repository, addons.mozilla.org. In the future, we may support
+ * installing add-ons from other sources. This provides a future-compatible
+ * mechanism for clients to only apply records they know how to handle.
+ */
+function AddonRecord(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+AddonRecord.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Record.Addon",
+};
+
+Utils.deferGetSet(AddonRecord, "cleartext", [
+ "addonID",
+ "applicationID",
+ "enabled",
+ "source",
+]);
+
+/**
+ * The AddonsEngine handles synchronization of add-ons between clients.
+ *
+ * The engine maintains an instance of an AddonsReconciler, which is the entity
+ * maintaining state for add-ons. It provides the history and tracking APIs
+ * that AddonManager doesn't.
+ *
+ * The engine instance overrides a handful of functions on the base class. The
+ * rationale for each is documented by that function.
+ */
+function AddonsEngine(service) {
+ SyncEngine.call(this, "Addons", service);
+
+ this._reconciler = new AddonsReconciler(this._tracker.asyncObserver);
+}
+AddonsEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: AddonsStore,
+ _trackerObj: AddonsTracker,
+ _recordObj: AddonRecord,
+ version: 1,
+
+ syncPriority: 5,
+
+ _reconciler: null,
+
+ async initialize() {
+ await SyncEngine.prototype.initialize.call(this);
+ await this._reconciler.ensureStateLoaded();
+ },
+
+ /**
+ * Override parent method to find add-ons by their public ID, not Sync GUID.
+ */
+ async _findDupe(item) {
+ let id = item.addonID;
+
+ // The reconciler should have been updated at the top of the sync, so we
+ // can assume it is up to date when this function is called.
+ let addons = this._reconciler.addons;
+ if (!(id in addons)) {
+ return null;
+ }
+
+ let addon = addons[id];
+ if (addon.guid != item.id) {
+ return addon.guid;
+ }
+
+ return null;
+ },
+
+ /**
+ * Override getChangedIDs to pull in tracker changes plus changes from the
+ * reconciler log.
+ */
+ async getChangedIDs() {
+ let changes = {};
+ const changedIDs = await this._tracker.getChangedIDs();
+ for (let [id, modified] of Object.entries(changedIDs)) {
+ changes[id] = modified;
+ }
+
+ let lastSync = await this.getLastSync();
+ let lastSyncDate = new Date(lastSync * 1000);
+
+ // The reconciler should have been refreshed at the beginning of a sync and
+ // we assume this function is only called from within a sync.
+ let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
+ let addons = this._reconciler.addons;
+ for (let change of reconcilerChanges) {
+ let changeTime = change[0];
+ let id = change[2];
+
+ if (!(id in addons)) {
+ continue;
+ }
+
+ // Keep newest modified time.
+ if (id in changes && changeTime < changes[id]) {
+ continue;
+ }
+
+ if (!(await this.isAddonSyncable(addons[id]))) {
+ continue;
+ }
+
+ this._log.debug("Adding changed add-on from changes log: " + id);
+ let addon = addons[id];
+ changes[addon.guid] = changeTime.getTime() / 1000;
+ }
+
+ return changes;
+ },
+
+ /**
+ * Override start of sync function to refresh reconciler.
+ *
+ * Many functions in this class assume the reconciler is refreshed at the
+ * top of a sync. If this ever changes, those functions should be revisited.
+ *
+ * Technically speaking, we don't need to refresh the reconciler on every
+ * sync since it is installed as an AddonManager listener. However, add-ons
+ * are complicated and we force a full refresh, just in case the listeners
+ * missed something.
+ */
+ async _syncStartup() {
+ // We refresh state before calling parent because syncStartup in the parent
+ // looks for changed IDs, which is dependent on add-on state being up to
+ // date.
+ await this._refreshReconcilerState();
+ return SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ /**
+ * Override end of sync to perform a little housekeeping on the reconciler.
+ *
+ * We prune changes to prevent the reconciler state from growing without
+ * bound. Even if it grows unbounded, there would have to be many add-on
+ * changes (thousands) for it to slow things down significantly. This is
+ * highly unlikely to occur. Still, we exercise defense just in case.
+ */
+ async _syncCleanup() {
+ let lastSync = await this.getLastSync();
+ let ms = 1000 * lastSync - PRUNE_ADDON_CHANGES_THRESHOLD;
+ this._reconciler.pruneChangesBeforeDate(new Date(ms));
+ return SyncEngine.prototype._syncCleanup.call(this);
+ },
+
+ /**
+ * Helper function to ensure reconciler is up to date.
+ *
+ * This will load the reconciler's state from the file
+ * system (if needed) and refresh the state of the reconciler.
+ */
+ async _refreshReconcilerState() {
+ this._log.debug("Refreshing reconciler state");
+ return this._reconciler.refreshGlobalState();
+ },
+
+ // Returns a promise
+ isAddonSyncable(addon, ignoreRepoCheck) {
+ return this._store.isAddonSyncable(addon, ignoreRepoCheck);
+ },
+};
+
+/**
+ * This is the primary interface between Sync and the Addons Manager.
+ *
+ * In addition to the core store APIs, we provide convenience functions to wrap
+ * Add-on Manager APIs with Sync-specific semantics.
+ */
+function AddonsStore(name, engine) {
+ Store.call(this, name, engine);
+}
+AddonsStore.prototype = {
+ __proto__: Store.prototype,
+
+ // Define the add-on types (.type) that we support.
+ _syncableTypes: ["extension", "theme"],
+
+ _extensionsPrefs: new Preferences("extensions."),
+
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ /**
+ * Override applyIncoming to filter out records we can't handle.
+ */
+ async applyIncoming(record) {
+ // The fields we look at aren't present when the record is deleted.
+ if (!record.deleted) {
+ // Ignore records not belonging to our application ID because that is the
+ // current policy.
+ if (record.applicationID != Services.appinfo.ID) {
+ this._log.info(
+ "Ignoring incoming record from other App ID: " + record.id
+ );
+ return;
+ }
+
+ // Ignore records that aren't from the official add-on repository, as that
+ // is our current policy.
+ if (record.source != "amo") {
+ this._log.info(
+ "Ignoring unknown add-on source (" +
+ record.source +
+ ")" +
+ " for " +
+ record.id
+ );
+ return;
+ }
+ }
+
+ // Ignore incoming records for which an existing non-syncable addon
+ // exists. Note that we do not insist that the addon manager already have
+ // metadata for this addon - it's possible our reconciler previously saw the
+ // addon but the addon-manager cache no longer has it - which is fine for a
+ // new incoming addon.
+ // (Note that most other cases where the addon-manager cache is invalid
+ // doesn't get this treatment because that cache self-repairs after some
+ // time - but it only re-populates addons which are currently installed.)
+ let existingMeta = this.reconciler.addons[record.addonID];
+ if (
+ existingMeta &&
+ !(await this.isAddonSyncable(existingMeta, /* ignoreRepoCheck */ true))
+ ) {
+ this._log.info(
+ "Ignoring incoming record for an existing but non-syncable addon",
+ record.addonID
+ );
+ return;
+ }
+
+ await Store.prototype.applyIncoming.call(this, record);
+ },
+
+ /**
+ * Provides core Store API to create/install an add-on from a record.
+ */
+ async create(record) {
+ // This will throw if there was an error. This will get caught by the sync
+ // engine and the record will try to be applied later.
+ const results = await AddonUtils.installAddons([
+ {
+ id: record.addonID,
+ syncGUID: record.id,
+ enabled: record.enabled,
+ requireSecureURI: this._extensionsPrefs.get(
+ "install.requireSecureOrigin",
+ true
+ ),
+ },
+ ]);
+
+ if (results.skipped.includes(record.addonID)) {
+ this._log.info("Add-on skipped: " + record.addonID);
+ // Just early-return for skipped addons - we don't want to arrange to
+ // try again next time because the condition that caused up to skip
+ // will remain true for this addon forever.
+ return;
+ }
+
+ let addon;
+ for (let a of results.addons) {
+ if (a.id == record.addonID) {
+ addon = a;
+ break;
+ }
+ }
+
+ // This should never happen, but is present as a fail-safe.
+ if (!addon) {
+ throw new Error("Add-on not found after install: " + record.addonID);
+ }
+
+ this._log.info("Add-on installed: " + record.addonID);
+ },
+
+ /**
+ * Provides core Store API to remove/uninstall an add-on from a record.
+ */
+ async remove(record) {
+ // If this is called, the payload is empty, so we have to find by GUID.
+ let addon = await this.getAddonByGUID(record.id);
+ if (!addon) {
+ // We don't throw because if the add-on could not be found then we assume
+ // it has already been uninstalled and there is nothing for this function
+ // to do.
+ return;
+ }
+
+ this._log.info("Uninstalling add-on: " + addon.id);
+ await AddonUtils.uninstallAddon(addon);
+ },
+
+ /**
+ * Provides core Store API to update an add-on from a record.
+ */
+ async update(record) {
+ let addon = await this.getAddonByID(record.addonID);
+
+ // update() is called if !this.itemExists. And, since itemExists consults
+ // the reconciler only, we need to take care of some corner cases.
+ //
+ // First, the reconciler could know about an add-on that was uninstalled
+ // and no longer present in the add-ons manager.
+ if (!addon) {
+ await this.create(record);
+ return;
+ }
+
+ // It's also possible that the add-on is non-restartless and has pending
+ // install/uninstall activity.
+ //
+ // We wouldn't get here if the incoming record was for a deletion. So,
+ // check for pending uninstall and cancel if necessary.
+ if (addon.pendingOperations & AddonManager.PENDING_UNINSTALL) {
+ addon.cancelUninstall();
+
+ // We continue with processing because there could be state or ID change.
+ }
+
+ await this.updateUserDisabled(addon, !record.enabled);
+ },
+
+ /**
+ * Provide core Store API to determine if a record exists.
+ */
+ async itemExists(guid) {
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ return !!addon;
+ },
+
+ /**
+ * Create an add-on record from its GUID.
+ *
+ * @param guid
+ * Add-on GUID (from extensions DB)
+ * @param collection
+ * Collection to add record to.
+ *
+ * @return AddonRecord instance
+ */
+ async createRecord(guid, collection) {
+ let record = new AddonRecord(collection, guid);
+ record.applicationID = Services.appinfo.ID;
+
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ // If we don't know about this GUID or if it has been uninstalled, we mark
+ // the record as deleted.
+ if (!addon || !addon.installed) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.modified = addon.modified.getTime() / 1000;
+
+ record.addonID = addon.id;
+ record.enabled = addon.enabled;
+
+ // This needs to be dynamic when add-ons don't come from AddonRepository.
+ record.source = "amo";
+
+ return record;
+ },
+
+ /**
+ * Changes the id of an add-on.
+ *
+ * This implements a core API of the store.
+ */
+ async changeItemID(oldID, newID) {
+ // We always update the GUID in the reconciler because it will be
+ // referenced later in the sync process.
+ let state = this.reconciler.getAddonStateFromSyncGUID(oldID);
+ if (state) {
+ state.guid = newID;
+ await this.reconciler.saveState();
+ }
+
+ let addon = await this.getAddonByGUID(oldID);
+ if (!addon) {
+ this._log.debug(
+ "Cannot change item ID (" +
+ oldID +
+ ") in Add-on " +
+ "Manager because old add-on not present: " +
+ oldID
+ );
+ return;
+ }
+
+ addon.syncGUID = newID;
+ },
+
+ /**
+ * Obtain the set of all syncable add-on Sync GUIDs.
+ *
+ * This implements a core Store API.
+ */
+ async getAllIDs() {
+ let ids = {};
+
+ let addons = this.reconciler.addons;
+ for (let id in addons) {
+ let addon = addons[id];
+ if (await this.isAddonSyncable(addon)) {
+ ids[addon.guid] = true;
+ }
+ }
+
+ return ids;
+ },
+
+ /**
+ * Wipe engine data.
+ *
+ * This uninstalls all syncable addons from the application. In case of
+ * error, it logs the error and keeps trying with other add-ons.
+ */
+ async wipe() {
+ this._log.info("Processing wipe.");
+
+ await this.engine._refreshReconcilerState();
+
+ // We only wipe syncable add-ons. Wipe is a Sync feature not a security
+ // feature.
+ let ids = await this.getAllIDs();
+ for (let guid in ids) {
+ let addon = await this.getAddonByGUID(guid);
+ if (!addon) {
+ this._log.debug(
+ "Ignoring add-on because it couldn't be obtained: " + guid
+ );
+ continue;
+ }
+
+ this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
+ await Utils.catch.call(this, () => addon.uninstall())();
+ }
+ },
+
+ /** *************************************************************************
+ * Functions below are unique to this store and not part of the Store API *
+ ***************************************************************************/
+
+ /**
+ * Obtain an add-on from its public ID.
+ *
+ * @param id
+ * Add-on ID
+ * @return Addon or undefined if not found
+ */
+ async getAddonByID(id) {
+ return AddonManager.getAddonByID(id);
+ },
+
+ /**
+ * Obtain an add-on from its Sync GUID.
+ *
+ * @param guid
+ * Add-on Sync GUID
+ * @return DBAddonInternal or null
+ */
+ async getAddonByGUID(guid) {
+ return AddonManager.getAddonBySyncGUID(guid);
+ },
+
+ /**
+ * Determines whether an add-on is suitable for Sync.
+ *
+ * @param addon
+ * Addon instance
+ * @param ignoreRepoCheck
+ * Should we skip checking the Addons repository (primarially useful
+ * for testing and validation).
+ * @return Boolean indicating whether it is appropriate for Sync
+ */
+ async isAddonSyncable(addon, ignoreRepoCheck = false) {
+ // Currently, we limit syncable add-ons to those that are:
+ // 1) In a well-defined set of types
+ // 2) Installed in the current profile
+ // 3) Not installed by a foreign entity (i.e. installed by the app)
+ // since they act like global extensions.
+ // 4) Is not a hotfix.
+ // 5) The addons XPIProvider doesn't veto it (i.e not being installed in
+ // the profile directory, or any other reasons it says the addon can't
+ // be synced)
+ // 6) Are installed from AMO
+
+ // We could represent the test as a complex boolean expression. We go the
+ // verbose route so the failure reason is logged.
+ if (!addon) {
+ this._log.debug("Null object passed to isAddonSyncable.");
+ return false;
+ }
+
+ if (!this._syncableTypes.includes(addon.type)) {
+ this._log.debug(
+ addon.id + " not syncable: type not in whitelist: " + addon.type
+ );
+ return false;
+ }
+
+ if (!(addon.scope & AddonManager.SCOPE_PROFILE)) {
+ this._log.debug(addon.id + " not syncable: not installed in profile.");
+ return false;
+ }
+
+ // If the addon manager says it's not syncable, we skip it.
+ if (!addon.isSyncable) {
+ this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
+ return false;
+ }
+
+ // This may be too aggressive. If an add-on is downloaded from AMO and
+ // manually placed in the profile directory, foreignInstall will be set.
+ // Arguably, that add-on should be syncable.
+ // TODO Address the edge case and come up with more robust heuristics.
+ if (addon.foreignInstall) {
+ this._log.debug(addon.id + " not syncable: is foreign install.");
+ return false;
+ }
+
+ // If the AddonRepository's cache isn't enabled (which it typically isn't
+ // in tests), getCachedAddonByID always returns null - so skip the check
+ // in that case. We also provide a way to specifically opt-out of the check
+ // even if the cache is enabled, which is used by the validators.
+ if (ignoreRepoCheck || !AddonRepository.cacheEnabled) {
+ return true;
+ }
+
+ let result = await new Promise(res => {
+ AddonRepository.getCachedAddonByID(addon.id, res);
+ });
+
+ if (!result) {
+ this._log.debug(
+ addon.id + " not syncable: add-on not found in add-on repository."
+ );
+ return false;
+ }
+
+ return this.isSourceURITrusted(result.sourceURI);
+ },
+
+ /**
+ * Determine whether an add-on's sourceURI field is trusted and the add-on
+ * can be installed.
+ *
+ * This function should only ever be called from isAddonSyncable(). It is
+ * exposed as a separate function to make testing easier.
+ *
+ * @param uri
+ * nsIURI instance to validate
+ * @return bool
+ */
+ isSourceURITrusted: function isSourceURITrusted(uri) {
+ // For security reasons, we currently limit synced add-ons to those
+ // installed from trusted hostname(s). We additionally require TLS with
+ // the add-ons site to help prevent forgeries.
+ let trustedHostnames = Svc.Prefs.get(
+ "addons.trustedSourceHostnames",
+ ""
+ ).split(",");
+
+ if (!uri) {
+ this._log.debug("Undefined argument to isSourceURITrusted().");
+ return false;
+ }
+
+ // Scheme is validated before the hostname because uri.host may not be
+ // populated for certain schemes. It appears to always be populated for
+ // https, so we avoid the potential NS_ERROR_FAILURE on field access.
+ if (uri.scheme != "https") {
+ this._log.debug("Source URI not HTTPS: " + uri.spec);
+ return false;
+ }
+
+ if (!trustedHostnames.includes(uri.host)) {
+ this._log.debug("Source hostname not trusted: " + uri.host);
+ return false;
+ }
+
+ return true;
+ },
+
+ /**
+ * Update the userDisabled flag on an add-on.
+ *
+ * This will enable or disable an add-on. It has no return value and does
+ * not catch or handle exceptions thrown by the addon manager. If no action
+ * is needed it will return immediately.
+ *
+ * @param addon
+ * Addon instance to manipulate.
+ * @param value
+ * Boolean to which to set userDisabled on the passed Addon.
+ */
+ async updateUserDisabled(addon, value) {
+ if (addon.userDisabled == value) {
+ return;
+ }
+
+ // A pref allows changes to the enabled flag to be ignored.
+ if (Svc.Prefs.get("addons.ignoreUserEnabledChanges", false)) {
+ this._log.info(
+ "Ignoring enabled state change due to preference: " + addon.id
+ );
+ return;
+ }
+
+ AddonUtils.updateUserDisabled(addon, value);
+ // updating this flag doesn't send a notification for appDisabled addons,
+ // meaning the reconciler will not update its state and may resync the
+ // addon - so explicitly rectify the state (bug 1366994)
+ if (addon.appDisabled) {
+ await this.reconciler.rectifyStateFromAddon(addon);
+ }
+ },
+};
+
+/**
+ * The add-ons tracker keeps track of real-time changes to add-ons.
+ *
+ * It hooks up to the reconciler and receives notifications directly from it.
+ */
+function AddonsTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+AddonsTracker.prototype = {
+ __proto__: LegacyTracker.prototype,
+
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ get store() {
+ return this.engine._store;
+ },
+
+ /**
+ * This callback is executed whenever the AddonsReconciler sends out a change
+ * notification. See AddonsReconciler.addChangeListener().
+ */
+ async changeListener(date, change, addon) {
+ this._log.debug("changeListener invoked: " + change + " " + addon.id);
+ // Ignore changes that occur during sync.
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (!(await this.store.isAddonSyncable(addon))) {
+ this._log.debug(
+ "Ignoring change because add-on isn't syncable: " + addon.id
+ );
+ return;
+ }
+
+ const added = await this.addChangedID(addon.guid, date.getTime() / 1000);
+ if (added) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ }
+ },
+
+ onStart() {
+ this.reconciler.startListening();
+ this.reconciler.addChangeListener(this);
+ },
+
+ onStop() {
+ this.reconciler.removeChangeListener(this);
+ this.reconciler.stopListening();
+ },
+};
+
+class AddonValidator extends CollectionValidator {
+ constructor(engine = null) {
+ super("addons", "id", ["addonID", "enabled", "applicationID", "source"]);
+ this.engine = engine;
+ }
+
+ async getClientItems() {
+ return AddonManager.getAllAddons();
+ }
+
+ normalizeClientItem(item) {
+ let enabled = !item.userDisabled;
+ if (item.pendingOperations & AddonManager.PENDING_ENABLE) {
+ enabled = true;
+ } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) {
+ enabled = false;
+ }
+ return {
+ enabled,
+ id: item.syncGUID,
+ addonID: item.id,
+ applicationID: Services.appinfo.ID,
+ source: "amo", // check item.foreignInstall?
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ let guid = await this.engine._findDupe(item);
+ if (guid) {
+ item.id = guid;
+ }
+ return item;
+ }
+
+ clientUnderstands(item) {
+ return item.applicationID === Services.appinfo.ID;
+ }
+
+ async syncedByClient(item) {
+ return (
+ !item.original.hidden &&
+ !item.original.isSystem &&
+ !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) &&
+ // No need to await the returned promise explicitely:
+ // |expr1 && expr2| evaluates to expr2 if expr1 is true.
+ this.engine.isAddonSyncable(item.original, true)
+ );
+ }
+}
diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js
new file mode 100644
index 0000000000..2c28c9a7a0
--- /dev/null
+++ b/services/sync/modules/engines/bookmarks.js
@@ -0,0 +1,1000 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = [
+ "BookmarksEngine",
+ "PlacesItem",
+ "Bookmark",
+ "BookmarkFolder",
+ "BookmarkQuery",
+ "Livemark",
+ "BookmarkSeparator",
+];
+
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Async } = ChromeUtils.import("resource://services-common/async.js");
+const { SCORE_INCREMENT_XLARGE } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+const { Changeset, Store, SyncEngine, Tracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ BookmarkValidator: "resource://services-sync/bookmark_validator.js",
+ Observers: "resource://services-common/observers.js",
+ OS: "resource://gre/modules/osfile.jsm",
+ PlacesBackups: "resource://gre/modules/PlacesBackups.jsm",
+ PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.jsm",
+ PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.jsm",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+ Resource: "resource://services-sync/resource.js",
+ SyncedBookmarksMirror: "resource://gre/modules/SyncedBookmarksMirror.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => {
+ return Services.strings.createBundle(
+ "chrome://places/locale/places.properties"
+ );
+});
+
+XPCOMUtils.defineLazyGetter(this, "ANNOS_TO_TRACK", () => [
+ PlacesUtils.LMANNO_FEEDURI,
+ PlacesUtils.LMANNO_SITEURI,
+]);
+
+const PLACES_MAINTENANCE_INTERVAL_SECONDS = 4 * 60 * 60; // 4 hours.
+
+const FOLDER_SORTINDEX = 1000000;
+
+// Roots that should be deleted from the server, instead of applied locally.
+// This matches `AndroidBrowserBookmarksRepositorySession::forbiddenGUID`,
+// but allows tags because we don't want to reparent tag folders or tag items
+// to "unfiled".
+const FORBIDDEN_INCOMING_IDS = ["pinned", "places", "readinglist"];
+
+// Items with these parents should be deleted from the server. We allow
+// children of the Places root, to avoid orphaning left pane queries and other
+// descendants of custom roots.
+const FORBIDDEN_INCOMING_PARENT_IDS = ["pinned", "readinglist"];
+
+// The tracker ignores changes made by import and restore, to avoid bumping the
+// score and triggering syncs during the process, as well as changes made by
+// Sync.
+XPCOMUtils.defineLazyGetter(this, "IGNORED_SOURCES", () => [
+ PlacesUtils.bookmarks.SOURCES.SYNC,
+ PlacesUtils.bookmarks.SOURCES.IMPORT,
+ PlacesUtils.bookmarks.SOURCES.RESTORE,
+ PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+ PlacesUtils.bookmarks.SOURCES.SYNC_REPARENT_REMOVED_FOLDER_CHILDREN,
+]);
+
+// The validation telemetry version for the engine. Version 1 is collected
+// by `bookmark_validator.js`, and checks value as well as structure
+// differences. Version 2 is collected by the engine as part of building the
+// remote tree, and checks structure differences only.
+const BOOKMARK_VALIDATOR_VERSION = 2;
+
+// The maximum time that the engine should wait before aborting a bookmark
+// merge.
+const BOOKMARK_APPLY_TIMEOUT_MS = 5 * 60 * 60 * 1000; // 5 minutes
+
+// The default frecency value to use when not known.
+const FRECENCY_UNKNOWN = -1;
+
+// Returns the constructor for a bookmark record type.
+function getTypeObject(type) {
+ switch (type) {
+ case "bookmark":
+ return Bookmark;
+ case "query":
+ return BookmarkQuery;
+ case "folder":
+ return BookmarkFolder;
+ case "livemark":
+ return Livemark;
+ case "separator":
+ return BookmarkSeparator;
+ case "item":
+ return PlacesItem;
+ }
+ return null;
+}
+
+function PlacesItem(collection, id, type) {
+ CryptoWrapper.call(this, collection, id);
+ this.type = type || "item";
+}
+PlacesItem.prototype = {
+ async decrypt(keyBundle) {
+ // Do the normal CryptoWrapper decrypt, but change types before returning
+ let clear = await CryptoWrapper.prototype.decrypt.call(this, keyBundle);
+
+ // Convert the abstract places item to the actual object type
+ if (!this.deleted) {
+ this.__proto__ = this.getTypeObject(this.type).prototype;
+ }
+
+ return clear;
+ },
+
+ getTypeObject: function PlacesItem_getTypeObject(type) {
+ let recordObj = getTypeObject(type);
+ if (!recordObj) {
+ throw new Error("Unknown places item object type: " + type);
+ }
+ return recordObj;
+ },
+
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.PlacesItem",
+
+ // Converts the record to a Sync bookmark object that can be passed to
+ // `PlacesSyncUtils.bookmarks.{insert, update}`.
+ toSyncBookmark() {
+ let result = {
+ kind: this.type,
+ recordId: this.id,
+ parentRecordId: this.parentid,
+ };
+ let dateAdded = PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
+ this.dateAdded,
+ +this.modified * 1000
+ );
+ if (dateAdded > 0) {
+ result.dateAdded = dateAdded;
+ }
+ return result;
+ },
+
+ // Populates the record from a Sync bookmark object returned from
+ // `PlacesSyncUtils.bookmarks.fetch`.
+ fromSyncBookmark(item) {
+ this.parentid = item.parentRecordId;
+ this.parentName = item.parentTitle;
+ if (item.dateAdded) {
+ this.dateAdded = item.dateAdded;
+ }
+ },
+};
+
+Utils.deferGetSet(PlacesItem, "cleartext", [
+ "hasDupe",
+ "parentid",
+ "parentName",
+ "type",
+ "dateAdded",
+]);
+
+function Bookmark(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "bookmark");
+}
+Bookmark.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Bookmark",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.title = this.title;
+ info.url = this.bmkUri;
+ info.description = this.description;
+ info.tags = this.tags;
+ info.keyword = this.keyword;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.bmkUri = item.url.href;
+ this.description = item.description;
+ this.tags = item.tags;
+ this.keyword = item.keyword;
+ },
+};
+
+Utils.deferGetSet(Bookmark, "cleartext", [
+ "title",
+ "bmkUri",
+ "description",
+ "tags",
+ "keyword",
+]);
+
+function BookmarkQuery(collection, id) {
+ Bookmark.call(this, collection, id, "query");
+}
+BookmarkQuery.prototype = {
+ __proto__: Bookmark.prototype,
+ _logName: "Sync.Record.BookmarkQuery",
+
+ toSyncBookmark() {
+ let info = Bookmark.prototype.toSyncBookmark.call(this);
+ info.folder = this.folderName || undefined; // empty string -> undefined
+ info.query = this.queryId;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ Bookmark.prototype.fromSyncBookmark.call(this, item);
+ this.folderName = item.folder || undefined; // empty string -> undefined
+ this.queryId = item.query;
+ },
+};
+
+Utils.deferGetSet(BookmarkQuery, "cleartext", ["folderName", "queryId"]);
+
+function BookmarkFolder(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "folder");
+}
+BookmarkFolder.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Folder",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.description = this.description;
+ info.title = this.title;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.description = item.description;
+ this.children = item.childRecordIds;
+ },
+};
+
+Utils.deferGetSet(BookmarkFolder, "cleartext", [
+ "description",
+ "title",
+ "children",
+]);
+
+function Livemark(collection, id) {
+ BookmarkFolder.call(this, collection, id, "livemark");
+}
+Livemark.prototype = {
+ __proto__: BookmarkFolder.prototype,
+ _logName: "Sync.Record.Livemark",
+
+ toSyncBookmark() {
+ let info = BookmarkFolder.prototype.toSyncBookmark.call(this);
+ info.feed = this.feedUri;
+ info.site = this.siteUri;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ BookmarkFolder.prototype.fromSyncBookmark.call(this, item);
+ this.feedUri = item.feed.href;
+ if (item.site) {
+ this.siteUri = item.site.href;
+ }
+ },
+};
+
+Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]);
+
+function BookmarkSeparator(collection, id) {
+ PlacesItem.call(this, collection, id, "separator");
+}
+BookmarkSeparator.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Separator",
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.pos = item.index;
+ },
+};
+
+Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos");
+
+/**
+ * The bookmarks engine uses a different store that stages downloaded bookmarks
+ * in a separate database, instead of writing directly to Places. The buffer
+ * handles reconciliation, so we stub out `_reconcile`, and wait to pull changes
+ * until we're ready to upload.
+ */
+function BookmarksEngine(service) {
+ SyncEngine.call(this, "Bookmarks", service);
+}
+BookmarksEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _recordObj: PlacesItem,
+ _trackerObj: BookmarksTracker,
+ _storeObj: BookmarksStore,
+ version: 2,
+ // Used to override the engine name in telemetry, so that we can distinguish
+ // this engine from the old, now removed non-buffered engine.
+ overrideTelemetryName: "bookmarks-buffered",
+
+ // Needed to ensure we don't miss items when resuming a sync that failed or
+ // aborted early.
+ _defaultSort: "oldest",
+
+ syncPriority: 4,
+ allowSkippedRecord: false,
+
+ async _ensureCurrentSyncID(newSyncID) {
+ await PlacesSyncUtils.bookmarks.ensureCurrentSyncId(newSyncID);
+ let buf = await this._store.ensureOpenMirror();
+ await buf.ensureCurrentSyncId(newSyncID);
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ let shouldWipeRemote = await PlacesSyncUtils.bookmarks.shouldWipeRemote();
+ if (!shouldWipeRemote) {
+ this._log.debug(
+ "Checking if server sync ID ${newSyncID} matches existing",
+ { newSyncID }
+ );
+ await this._ensureCurrentSyncID(newSyncID);
+ return newSyncID;
+ }
+ // We didn't take the new sync ID because we need to wipe the server
+ // and other clients after a restore. Send the command, wipe the
+ // server, and reset our sync ID to reupload everything.
+ this._log.debug(
+ "Ignoring server sync ID ${newSyncID} after restore; " +
+ "wiping server and resetting sync ID",
+ { newSyncID }
+ );
+ await this.service.clientsEngine.sendCommand(
+ "wipeEngine",
+ [this.name],
+ null,
+ { reason: "bookmark-restore" }
+ );
+ let assignedSyncID = await this.resetSyncID();
+ return assignedSyncID;
+ },
+
+ async getSyncID() {
+ return PlacesSyncUtils.bookmarks.getSyncId();
+ },
+
+ async resetSyncID() {
+ await this._deleteServerCollection();
+ return this.resetLocalSyncID();
+ },
+
+ async resetLocalSyncID() {
+ let newSyncID = await PlacesSyncUtils.bookmarks.resetSyncId();
+ this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID });
+ let buf = await this._store.ensureOpenMirror();
+ await buf.ensureCurrentSyncId(newSyncID);
+ return newSyncID;
+ },
+
+ async getLastSync() {
+ let mirror = await this._store.ensureOpenMirror();
+ return mirror.getCollectionHighWaterMark();
+ },
+
+ async setLastSync(lastSync) {
+ let mirror = await this._store.ensureOpenMirror();
+ await mirror.setCollectionLastModified(lastSync);
+ // Update the last sync time in Places so that reverting to the original
+ // bookmarks engine doesn't download records we've already applied.
+ await PlacesSyncUtils.bookmarks.setLastSync(lastSync);
+ },
+
+ async _syncStartup() {
+ await super._syncStartup();
+
+ try {
+ // For first syncs, back up the user's bookmarks.
+ let lastSync = await this.getLastSync();
+ if (!lastSync) {
+ this._log.debug("Bookmarks backup starting");
+ await PlacesBackups.create(null, true);
+ this._log.debug("Bookmarks backup done");
+ }
+ } catch (ex) {
+ // Failure to create a backup is somewhat bad, but probably not bad
+ // enough to prevent syncing of bookmarks - so just log the error and
+ // continue.
+ this._log.warn(
+ "Error while backing up bookmarks, but continuing with sync",
+ ex
+ );
+ }
+ },
+
+ async _sync() {
+ try {
+ await super._sync();
+ if (this._ranMaintenanceOnLastSync) {
+ // If the last sync failed, we ran maintenance, and this sync succeeded,
+ // maintenance likely fixed the issue.
+ this._ranMaintenanceOnLastSync = false;
+ this.service.recordTelemetryEvent("maintenance", "fix", "bookmarks");
+ }
+ } catch (ex) {
+ if (
+ Async.isShutdownException(ex) ||
+ ex.status > 0 ||
+ ex.name == "MergeConflictError" ||
+ ex.name == "InterruptedError"
+ ) {
+ // Don't run maintenance on shutdown or HTTP errors, or if we aborted
+ // the sync because the user changed their bookmarks during merging.
+ throw ex;
+ }
+ // Run Places maintenance periodically to try to recover from corruption
+ // that might have caused the sync to fail. We cap the interval because
+ // persistent failures likely indicate a problem that won't be fixed by
+ // running maintenance after every failed sync.
+ let elapsedSinceMaintenance =
+ Date.now() / 1000 -
+ Services.prefs.getIntPref("places.database.lastMaintenance", 0);
+ if (elapsedSinceMaintenance >= PLACES_MAINTENANCE_INTERVAL_SECONDS) {
+ this._log.error(
+ "Bookmark sync failed, ${elapsedSinceMaintenance}s " +
+ "elapsed since last run; running Places maintenance",
+ { elapsedSinceMaintenance }
+ );
+ await PlacesDBUtils.maintenanceOnIdle();
+ this._ranMaintenanceOnLastSync = true;
+ this.service.recordTelemetryEvent("maintenance", "run", "bookmarks");
+ } else {
+ this._ranMaintenanceOnLastSync = false;
+ }
+ throw ex;
+ }
+ },
+
+ async _syncFinish() {
+ await SyncEngine.prototype._syncFinish.call(this);
+ await PlacesSyncUtils.bookmarks.ensureMobileQuery();
+ },
+
+ async pullAllChanges() {
+ return this.pullNewChanges();
+ },
+
+ async trackRemainingChanges() {
+ let changes = this._modified.changes;
+ await PlacesSyncUtils.bookmarks.pushChanges(changes);
+ },
+
+ _deleteId(id) {
+ this._noteDeletedId(id);
+ },
+
+ // The bookmarks engine rarely calls this method directly, except in tests or
+ // when handling a `reset{All, Engine}` command from another client. We
+ // usually reset local Sync metadata on a sync ID mismatch, which both engines
+ // override with logic that lives in Places and the mirror.
+ async _resetClient() {
+ await super._resetClient();
+ await PlacesSyncUtils.bookmarks.reset();
+ let buf = await this._store.ensureOpenMirror();
+ await buf.reset();
+ },
+
+ // Cleans up the Places root, reading list items (ignored in bug 762118,
+ // removed in bug 1155684), and pinned sites.
+ _shouldDeleteRemotely(incomingItem) {
+ return (
+ FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) ||
+ FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid)
+ );
+ },
+
+ emptyChangeset() {
+ return new BookmarksChangeset();
+ },
+
+ async _apply() {
+ let buf = await this._store.ensureOpenMirror();
+ let watchdog = this._newWatchdog();
+ watchdog.start(BOOKMARK_APPLY_TIMEOUT_MS);
+
+ try {
+ let recordsToUpload = await buf.apply({
+ remoteTimeSeconds: Resource.serverTime,
+ weakUpload: [...this._needWeakUpload.keys()],
+ signal: watchdog.signal,
+ });
+ this._modified.replace(recordsToUpload);
+ } finally {
+ watchdog.stop();
+ if (watchdog.abortReason) {
+ this._log.warn(`Aborting bookmark merge: ${watchdog.abortReason}`);
+ }
+ this._needWeakUpload.clear();
+ }
+ },
+
+ async _processIncoming(newitems) {
+ await super._processIncoming(newitems);
+ await this._apply();
+ },
+
+ async _reconcile(item) {
+ return true;
+ },
+
+ async _createRecord(id) {
+ let record = await this._doCreateRecord(id);
+ if (!record.deleted) {
+ // Set hasDupe on all (non-deleted) records since we don't use it and we
+ // want to minimize the risk of older clients corrupting records. Note
+ // that the SyncedBookmarksMirror sets it for all records that it created,
+ // but we would like to ensure that weakly uploaded records are marked as
+ // hasDupe as well.
+ record.hasDupe = true;
+ }
+ return record;
+ },
+
+ async _doCreateRecord(id) {
+ if (this._needWeakUpload.has(id)) {
+ return this._store.createRecord(id, this.name);
+ }
+ let change = this._modified.changes[id];
+ if (!change) {
+ this._log.error(
+ "Creating record for item ${id} not in strong changeset",
+ { id }
+ );
+ throw new TypeError("Can't create record for unchanged item");
+ }
+ let record = this._recordFromCleartext(id, change.cleartext);
+ record.sortindex = await this._store._calculateIndex(record);
+ return record;
+ },
+
+ _recordFromCleartext(id, cleartext) {
+ let recordObj = getTypeObject(cleartext.type);
+ if (!recordObj) {
+ this._log.warn(
+ "Creating record for item ${id} with unknown type ${type}",
+ { id, type: cleartext.type }
+ );
+ recordObj = PlacesItem;
+ }
+ let record = new recordObj(this.name, id);
+ record.cleartext = cleartext;
+ return record;
+ },
+
+ async pullChanges() {
+ return {};
+ },
+
+ /**
+ * Writes successfully uploaded records back to the mirror, so that the
+ * mirror matches the server. We update the mirror before updating Places,
+ * which has implications for interrupted syncs.
+ *
+ * 1. Sync interrupted during upload; server doesn't support atomic uploads.
+ * We'll download and reapply everything that we uploaded before the
+ * interruption. All locally changed items retain their change counters.
+ * 2. Sync interrupted during upload; atomic uploads enabled. The server
+ * discards the batch. All changed local items retain their change
+ * counters, so the next sync resumes cleanly.
+ * 3. Sync interrupted during upload; outgoing records can't fit in a single
+ * batch. We'll download and reapply all records through the most recent
+ * committed batch. This is a variation of (1).
+ * 4. Sync interrupted after we update the mirror, but before cleanup. The
+ * mirror matches the server, but locally changed items retain their change
+ * counters. Reuploading them on the next sync should be idempotent, though
+ * unnecessary. If another client makes a conflicting remote change before
+ * we sync again, we may incorrectly prefer the local state.
+ * 5. Sync completes successfully. We'll update the mirror, and reset the
+ * change counters for all items.
+ */
+ async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
+ let records = [];
+ for (let id of succeeded) {
+ let change = this._modified.changes[id];
+ if (!change) {
+ // TODO (Bug 1433178): Write weakly uploaded records back to the mirror.
+ this._log.info("Uploaded record not in strong changeset", id);
+ continue;
+ }
+ if (!change.synced) {
+ this._log.info("Record in strong changeset not uploaded", id);
+ continue;
+ }
+ let cleartext = change.cleartext;
+ if (!cleartext) {
+ this._log.error(
+ "Missing Sync record cleartext for ${id} in ${change}",
+ { id, change }
+ );
+ throw new TypeError("Missing cleartext for uploaded Sync record");
+ }
+ let record = this._recordFromCleartext(id, cleartext);
+ record.modified = serverModifiedTime;
+ records.push(record);
+ }
+ let buf = await this._store.ensureOpenMirror();
+ await buf.store(records, { needsMerge: false });
+ },
+
+ async finalize() {
+ await super.finalize();
+ await this._store.finalize();
+ },
+};
+
+/**
+ * The bookmarks store delegates to the mirror for staging and applying
+ * records. Most `Store` methods intentionally remain abstract, so you can't use
+ * this store to create or update bookmarks in Places. All changes must go
+ * through the mirror, which takes care of merging and producing a valid tree.
+ */
+function BookmarksStore(name, engine) {
+ Store.call(this, name, engine);
+}
+
+BookmarksStore.prototype = {
+ __proto__: Store.prototype,
+
+ _openMirrorPromise: null,
+
+ // For tests.
+ _batchChunkSize: 500,
+
+ // Create a record starting from the weave id (places guid)
+ async createRecord(id, collection) {
+ let item = await PlacesSyncUtils.bookmarks.fetch(id);
+ if (!item) {
+ // deleted item
+ let record = new PlacesItem(collection, id);
+ record.deleted = true;
+ return record;
+ }
+
+ let recordObj = getTypeObject(item.kind);
+ if (!recordObj) {
+ this._log.warn("Unknown item type, cannot serialize: " + item.kind);
+ recordObj = PlacesItem;
+ }
+ let record = new recordObj(collection, id);
+ record.fromSyncBookmark(item);
+
+ record.sortindex = await this._calculateIndex(record);
+
+ return record;
+ },
+
+ async _calculateIndex(record) {
+ // Ensure folders have a very high sort index so they're not synced last.
+ if (record.type == "folder") {
+ return FOLDER_SORTINDEX;
+ }
+
+ // For anything directly under the toolbar, give it a boost of more than an
+ // unvisited bookmark
+ let index = 0;
+ if (record.parentid == "toolbar") {
+ index += 150;
+ }
+
+ // Add in the bookmark's frecency if we have something.
+ if (record.bmkUri != null) {
+ let frecency = FRECENCY_UNKNOWN;
+ try {
+ frecency = await PlacesSyncUtils.history.fetchURLFrecency(
+ record.bmkUri
+ );
+ } catch (ex) {
+ this._log.warn(
+ `Failed to fetch frecency for ${record.id}; assuming default`,
+ ex
+ );
+ this._log.trace("Record {id} has invalid URL ${bmkUri}", record);
+ }
+ if (frecency != FRECENCY_UNKNOWN) {
+ index += frecency;
+ }
+ }
+
+ return index;
+ },
+
+ async wipe() {
+ // Save a backup before clearing out all bookmarks.
+ await PlacesBackups.create(null, true);
+ await PlacesSyncUtils.bookmarks.wipe();
+ },
+
+ ensureOpenMirror() {
+ if (!this._openMirrorPromise) {
+ this._openMirrorPromise = this._openMirror().catch(err => {
+ // We may have failed to open the mirror temporarily; for example, if
+ // the database is locked. Clear the promise so that subsequent
+ // `ensureOpenMirror` calls can try to open the mirror again.
+ this._openMirrorPromise = null;
+ throw err;
+ });
+ }
+ return this._openMirrorPromise;
+ },
+
+ async _openMirror() {
+ let mirrorPath = OS.Path.join(
+ OS.Constants.Path.profileDir,
+ "weave",
+ "bookmarks.sqlite"
+ );
+ await OS.File.makeDir(OS.Path.dirname(mirrorPath), {
+ from: OS.Constants.Path.profileDir,
+ });
+
+ return SyncedBookmarksMirror.open({
+ path: mirrorPath,
+ recordStepTelemetry: (name, took, counts) => {
+ Observers.notify(
+ "weave:engine:sync:step",
+ {
+ name,
+ took,
+ counts,
+ },
+ this.name
+ );
+ },
+ recordValidationTelemetry: (took, checked, problems) => {
+ Observers.notify(
+ "weave:engine:validate:finish",
+ {
+ version: BOOKMARK_VALIDATOR_VERSION,
+ took,
+ checked,
+ problems,
+ },
+ this.name
+ );
+ },
+ });
+ },
+
+ async applyIncomingBatch(records) {
+ let buf = await this.ensureOpenMirror();
+ for (let chunk of PlacesUtils.chunkArray(records, this._batchChunkSize)) {
+ await buf.store(chunk);
+ }
+ // Array of failed records.
+ return [];
+ },
+
+ async applyIncoming(record) {
+ let buf = await this.ensureOpenMirror();
+ await buf.store([record]);
+ },
+
+ async finalize() {
+ if (!this._openMirrorPromise) {
+ return;
+ }
+ let buf = await this._openMirrorPromise;
+ await buf.finalize();
+ },
+};
+
+// The bookmarks tracker is a special flower. Instead of listening for changes
+// via observer notifications, it queries Places for the set of items that have
+// changed since the last sync. Because it's a "pull-based" tracker, it ignores
+// all concepts of "add a changed ID." However, it still registers an observer
+// to bump the score, so that changed bookmarks are synced immediately.
+function BookmarksTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ this._batchDepth = 0;
+ this._batchSawScoreIncrement = false;
+}
+BookmarksTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ onStart() {
+ PlacesUtils.bookmarks.addObserver(this, true);
+ this._placesListener = new PlacesWeakCallbackWrapper(
+ this.handlePlacesEvents.bind(this)
+ );
+ PlacesUtils.observers.addListener(
+ ["bookmark-added", "bookmark-removed"],
+ this._placesListener
+ );
+ Svc.Obs.add("bookmarks-restore-begin", this);
+ Svc.Obs.add("bookmarks-restore-success", this);
+ Svc.Obs.add("bookmarks-restore-failed", this);
+ },
+
+ onStop() {
+ PlacesUtils.bookmarks.removeObserver(this);
+ PlacesUtils.observers.removeListener(
+ ["bookmark-added", "bookmark-removed"],
+ this._placesListener
+ );
+ Svc.Obs.remove("bookmarks-restore-begin", this);
+ Svc.Obs.remove("bookmarks-restore-success", this);
+ Svc.Obs.remove("bookmarks-restore-failed", this);
+ },
+
+ async getChangedIDs() {
+ return PlacesSyncUtils.bookmarks.pullChanges();
+ },
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "bookmarks-restore-begin":
+ this._log.debug("Ignoring changes from importing bookmarks.");
+ break;
+ case "bookmarks-restore-success":
+ this._log.debug("Tracking all items on successful import.");
+
+ if (data == "json") {
+ this._log.debug(
+ "Restore succeeded: wiping server and other clients."
+ );
+ // Trigger an immediate sync. `ensureCurrentSyncID` will notice we
+ // restored, wipe the server and other clients, reset the sync ID, and
+ // upload the restored tree.
+ this.score += SCORE_INCREMENT_XLARGE;
+ } else {
+ // "html", "html-initial", or "json-append"
+ this._log.debug("Import succeeded.");
+ }
+ break;
+ case "bookmarks-restore-failed":
+ this._log.debug("Tracking all items on failed import.");
+ break;
+ }
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsINavBookmarkObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ /* Every add/remove/change will trigger a sync for MULTI_DEVICE (except in
+ a batch operation, where we do it at the end of the batch) */
+ _upScore: function BMT__upScore() {
+ if (this._batchDepth == 0) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ } else {
+ this._batchSawScoreIncrement = true;
+ }
+ },
+
+ handlePlacesEvents(events) {
+ for (let event of events) {
+ switch (event.type) {
+ case "bookmark-added":
+ if (IGNORED_SOURCES.includes(event.source)) {
+ continue;
+ }
+
+ this._log.trace("'bookmark-added': " + event.id);
+ this._upScore();
+ break;
+ case "bookmark-removed":
+ if (IGNORED_SOURCES.includes(event.source)) {
+ continue;
+ }
+
+ this._log.trace("'bookmark-removed': " + event.id);
+ this._upScore();
+ break;
+ }
+ }
+ },
+
+ // This method is oddly structured, but the idea is to return as quickly as
+ // possible -- this handler gets called *every time* a bookmark changes, for
+ // *each change*.
+ onItemChanged: function BMT_onItemChanged(
+ itemId,
+ property,
+ isAnno,
+ value,
+ lastModified,
+ itemType,
+ parentId,
+ guid,
+ parentGuid,
+ oldValue,
+ source
+ ) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ if (isAnno && !ANNOS_TO_TRACK.includes(property)) {
+ // Ignore annotations except for the ones that we sync.
+ return;
+ }
+
+ this._log.trace(
+ "onItemChanged: " +
+ itemId +
+ (", " + property + (isAnno ? " (anno)" : "")) +
+ (value ? ' = "' + value + '"' : "")
+ );
+ this._upScore();
+ },
+
+ onItemMoved: function BMT_onItemMoved(
+ itemId,
+ oldParent,
+ oldIndex,
+ newParent,
+ newIndex,
+ itemType,
+ guid,
+ oldParentGuid,
+ newParentGuid,
+ source
+ ) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ this._log.trace("onItemMoved: " + itemId);
+ this._upScore();
+ },
+
+ onBeginUpdateBatch() {
+ ++this._batchDepth;
+ },
+ onEndUpdateBatch() {
+ if (--this._batchDepth === 0 && this._batchSawScoreIncrement) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ this._batchSawScoreIncrement = false;
+ }
+ },
+};
+
+/**
+ * A changeset that stores extra metadata in a change record for each ID. The
+ * engine updates this metadata when uploading Sync records, and writes it back
+ * to Places in `BookmarksEngine#trackRemainingChanges`.
+ *
+ * The `synced` property on a change record means its corresponding item has
+ * been uploaded, and we should pretend it doesn't exist in the changeset.
+ */
+class BookmarksChangeset extends Changeset {
+ // Only `_reconcile` calls `getModifiedTimestamp` and `has`, and the engine
+ // does its own reconciliation.
+ getModifiedTimestamp(id) {
+ throw new Error("Don't use timestamps to resolve bookmark conflicts");
+ }
+
+ has(id) {
+ throw new Error("Don't use the changeset to resolve bookmark conflicts");
+ }
+
+ delete(id) {
+ let change = this.changes[id];
+ if (change) {
+ // Mark the change as synced without removing it from the set. We do this
+ // so that we can update Places in `trackRemainingChanges`.
+ change.synced = true;
+ }
+ }
+
+ ids() {
+ let results = new Set();
+ for (let id in this.changes) {
+ if (!this.changes[id].synced) {
+ results.add(id);
+ }
+ }
+ return [...results];
+ }
+}
diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js
new file mode 100644
index 0000000000..60715d73bd
--- /dev/null
+++ b/services/sync/modules/engines/clients.js
@@ -0,0 +1,1186 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * How does the clients engine work?
+ *
+ * - We use 2 files - commands.json and commands-syncing.json.
+ *
+ * - At sync upload time, we attempt a rename of commands.json to
+ * commands-syncing.json, and ignore errors (helps for crash during sync!).
+ * - We load commands-syncing.json and stash the contents in
+ * _currentlySyncingCommands which lives for the duration of the upload process.
+ * - We use _currentlySyncingCommands to build the outgoing records
+ * - Immediately after successful upload, we delete commands-syncing.json from
+ * disk (and clear _currentlySyncingCommands). We reconcile our local records
+ * with what we just wrote in the server, and add failed IDs commands
+ * back in commands.json
+ * - Any time we need to "save" a command for future syncs, we load
+ * commands.json, update it, and write it back out.
+ */
+
+var EXPORTED_SYMBOLS = ["ClientEngine", "ClientsRec"];
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Async } = ChromeUtils.import("resource://services-common/async.js");
+const {
+ DEVICE_TYPE_DESKTOP,
+ DEVICE_TYPE_MOBILE,
+ SCORE_INCREMENT_XLARGE,
+ SINGLE_USER_THRESHOLD,
+ SYNC_API_VERSION,
+} = ChromeUtils.import("resource://services-sync/constants.js");
+const { Store, SyncEngine, LegacyTracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Resource } = ChromeUtils.import("resource://services-sync/resource.js");
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "fxAccounts",
+ "resource://gre/modules/FxAccounts.jsm"
+);
+
+const { PREF_ACCOUNT_ROOT } = ChromeUtils.import(
+ "resource://gre/modules/FxAccountsCommon.js"
+);
+
+const CLIENTS_TTL = 1814400; // 21 days
+const CLIENTS_TTL_REFRESH = 604800; // 7 days
+const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
+
+// TTL of the message sent to another device when sending a tab
+const NOTIFY_TAB_SENT_TTL_SECS = 1 * 3600; // 1 hour
+
+// How often we force a refresh of the FxA device list.
+const REFRESH_FXA_DEVICE_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
+
+// Reasons behind sending collection_changed push notifications.
+const COLLECTION_MODIFIED_REASON_SENDTAB = "sendtab";
+const COLLECTION_MODIFIED_REASON_FIRSTSYNC = "firstsync";
+
+const SUPPORTED_PROTOCOL_VERSIONS = [SYNC_API_VERSION];
+const LAST_MODIFIED_ON_PROCESS_COMMAND_PREF =
+ "services.sync.clients.lastModifiedOnProcessCommands";
+
+function hasDupeCommand(commands, action) {
+ if (!commands) {
+ return false;
+ }
+ return commands.some(
+ other =>
+ other.command == action.command &&
+ Utils.deepEquals(other.args, action.args)
+ );
+}
+
+function ClientsRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+ClientsRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Clients",
+ ttl: CLIENTS_TTL,
+};
+
+Utils.deferGetSet(ClientsRec, "cleartext", [
+ "name",
+ "type",
+ "commands",
+ "version",
+ "protocols",
+ "formfactor",
+ "os",
+ "appPackage",
+ "application",
+ "device",
+ "fxaDeviceId",
+]);
+
+function ClientEngine(service) {
+ SyncEngine.call(this, "Clients", service);
+
+ this.fxAccounts = fxAccounts;
+ this.addClientCommandQueue = Async.asyncQueueCaller(this._log);
+ Utils.defineLazyIDProperty(this, "localID", "services.sync.client.GUID");
+}
+ClientEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: ClientStore,
+ _recordObj: ClientsRec,
+ _trackerObj: ClientsTracker,
+ allowSkippedRecord: false,
+ _knownStaleFxADeviceIds: null,
+ _lastDeviceCounts: null,
+ _lastFxaDeviceRefresh: 0,
+
+ async initialize() {
+ // Reset the last sync timestamp on every startup so that we fetch all clients
+ await this.resetLastSync();
+ },
+
+ // These two properties allow us to avoid replaying the same commands
+ // continuously if we cannot manage to upload our own record.
+ _localClientLastModified: 0,
+ get _lastModifiedOnProcessCommands() {
+ return Services.prefs.getIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, -1);
+ },
+
+ set _lastModifiedOnProcessCommands(value) {
+ Services.prefs.setIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, value);
+ },
+
+ get isFirstSync() {
+ return !this.lastRecordUpload;
+ },
+
+ // Always sync client data as it controls other sync behavior
+ get enabled() {
+ return true;
+ },
+
+ get lastRecordUpload() {
+ return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
+ },
+ set lastRecordUpload(value) {
+ Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
+ },
+
+ get remoteClients() {
+ // return all non-stale clients for external consumption.
+ return Object.values(this._store._remoteClients).filter(v => !v.stale);
+ },
+
+ remoteClient(id) {
+ let client = this._store._remoteClients[id];
+ return client && !client.stale ? client : null;
+ },
+
+ remoteClientExists(id) {
+ return !!this.remoteClient(id);
+ },
+
+ // Aggregate some stats on the composition of clients on this account
+ get stats() {
+ let stats = {
+ hasMobile: this.localType == DEVICE_TYPE_MOBILE,
+ names: [this.localName],
+ numClients: 1,
+ };
+
+ for (let id in this._store._remoteClients) {
+ let { name, type, stale } = this._store._remoteClients[id];
+ if (!stale) {
+ stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
+ stats.names.push(name);
+ stats.numClients++;
+ }
+ }
+
+ return stats;
+ },
+
+ /**
+ * Obtain information about device types.
+ *
+ * Returns a Map of device types to integer counts. Guaranteed to include
+ * "desktop" (which will have at least 1 - this device) and "mobile" (which
+ * may have zero) counts. It almost certainly will include only these 2.
+ */
+ get deviceTypes() {
+ let counts = new Map();
+
+ counts.set(this.localType, 1); // currently this must be DEVICE_TYPE_DESKTOP
+ counts.set(DEVICE_TYPE_MOBILE, 0);
+
+ for (let id in this._store._remoteClients) {
+ let record = this._store._remoteClients[id];
+ if (record.stale) {
+ continue; // pretend "stale" records don't exist.
+ }
+ let type = record.type;
+ if (!counts.has(type)) {
+ counts.set(type, 0);
+ }
+
+ counts.set(type, counts.get(type) + 1);
+ }
+
+ return counts;
+ },
+
+ get brandName() {
+ let brand = Services.strings.createBundle(
+ "chrome://branding/locale/brand.properties"
+ );
+ return brand.GetStringFromName("brandShortName");
+ },
+
+ get localName() {
+ return this.fxAccounts.device.getLocalName();
+ },
+ set localName(value) {
+ this.fxAccounts.device.setLocalName(value);
+ },
+
+ get localType() {
+ return this.fxAccounts.device.getLocalType();
+ },
+
+ getClientName(id) {
+ if (id == this.localID) {
+ return this.localName;
+ }
+ let client = this._store._remoteClients[id];
+ return client ? client.name : "";
+ },
+
+ getClientFxaDeviceId(id) {
+ if (this._store._remoteClients[id]) {
+ return this._store._remoteClients[id].fxaDeviceId;
+ }
+ return null;
+ },
+
+ getClientByFxaDeviceId(fxaDeviceId) {
+ for (let id in this._store._remoteClients) {
+ let client = this._store._remoteClients[id];
+ if (client.stale) {
+ continue;
+ }
+ if (client.fxaDeviceId == fxaDeviceId) {
+ return client;
+ }
+ }
+ return null;
+ },
+
+ getClientType(id) {
+ const client = this._store._remoteClients[id];
+ if (client.type == DEVICE_TYPE_DESKTOP) {
+ return "desktop";
+ }
+ if (client.formfactor && client.formfactor.includes("tablet")) {
+ return "tablet";
+ }
+ return "phone";
+ },
+
+ async _readCommands() {
+ let commands = await Utils.jsonLoad("commands", this);
+ return commands || {};
+ },
+
+ /**
+ * Low level function, do not use directly (use _addClientCommand instead).
+ */
+ async _saveCommands(commands) {
+ try {
+ await Utils.jsonSave("commands", this, commands);
+ } catch (error) {
+ this._log.error("Failed to save JSON outgoing commands", error);
+ }
+ },
+
+ async _prepareCommandsForUpload() {
+ try {
+ await Utils.jsonMove("commands", "commands-syncing", this);
+ } catch (e) {
+ // Ignore errors
+ }
+ let commands = await Utils.jsonLoad("commands-syncing", this);
+ return commands || {};
+ },
+
+ async _deleteUploadedCommands() {
+ delete this._currentlySyncingCommands;
+ try {
+ await Utils.jsonRemove("commands-syncing", this);
+ } catch (err) {
+ this._log.error("Failed to delete syncing-commands file", err);
+ }
+ },
+
+ // Gets commands for a client we are yet to write to the server. Doesn't
+ // include commands for that client which are already on the server.
+ // We should rename this!
+ async getClientCommands(clientId) {
+ const allCommands = await this._readCommands();
+ return allCommands[clientId] || [];
+ },
+
+ async removeLocalCommand(command) {
+ // the implementation of this engine is such that adding a command to
+ // the local client is how commands are deleted! ¯\_(ツ)_/¯
+ await this._addClientCommand(this.localID, command);
+ },
+
+ async _addClientCommand(clientId, command) {
+ this.addClientCommandQueue.enqueueCall(async () => {
+ try {
+ const localCommands = await this._readCommands();
+ const localClientCommands = localCommands[clientId] || [];
+ const remoteClient = this._store._remoteClients[clientId];
+ let remoteClientCommands = [];
+ if (remoteClient && remoteClient.commands) {
+ remoteClientCommands = remoteClient.commands;
+ }
+ const clientCommands = localClientCommands.concat(remoteClientCommands);
+ if (hasDupeCommand(clientCommands, command)) {
+ return false;
+ }
+ localCommands[clientId] = localClientCommands.concat(command);
+ await this._saveCommands(localCommands);
+ return true;
+ } catch (e) {
+ // Failing to save a command should not "break the queue" of pending operations.
+ this._log.error(e);
+ return false;
+ }
+ });
+
+ return this.addClientCommandQueue.promiseCallsComplete();
+ },
+
+ async _removeClientCommands(clientId) {
+ const allCommands = await this._readCommands();
+ delete allCommands[clientId];
+ await this._saveCommands(allCommands);
+ },
+
+ async updateKnownStaleClients() {
+ this._log.debug("Updating the known stale clients");
+ // _fetchFxADevices side effect updates this._knownStaleFxADeviceIds.
+ await this._fetchFxADevices();
+ let localFxADeviceId = await fxAccounts.device.getLocalId();
+ // Process newer records first, so that if we hit a record with a device ID
+ // we've seen before, we can mark it stale immediately.
+ let clientList = Object.values(this._store._remoteClients).sort(
+ (a, b) => b.serverLastModified - a.serverLastModified
+ );
+ let seenDeviceIds = new Set([localFxADeviceId]);
+ for (let client of clientList) {
+ // Clients might not have an `fxaDeviceId` if they fail the FxA
+ // registration process.
+ if (!client.fxaDeviceId) {
+ continue;
+ }
+ if (this._knownStaleFxADeviceIds.includes(client.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${client.id} - in known stale clients list`
+ );
+ client.stale = true;
+ } else if (seenDeviceIds.has(client.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${client.id}` +
+ ` - duplicate device id ${client.fxaDeviceId}`
+ );
+ client.stale = true;
+ } else {
+ seenDeviceIds.add(client.fxaDeviceId);
+ }
+ }
+ },
+
+ async _fetchFxADevices() {
+ // We only force a refresh periodically to keep the load on the servers
+ // down, and because we expect FxA to have received a push message in
+ // most cases when the FxA device list would have changed. For this reason
+ // we still go ahead and check the stale list even if we didn't force a
+ // refresh.
+ let now = this.fxAccounts._internal.now(); // tests mock this .now() impl.
+ if (now - REFRESH_FXA_DEVICE_INTERVAL_MS > this._lastFxaDeviceRefresh) {
+ this._lastFxaDeviceRefresh = now;
+ try {
+ await this.fxAccounts.device.refreshDeviceList();
+ } catch (e) {
+ this._log.error("Could not refresh the FxA device list", e);
+ }
+ }
+
+ // We assume that clients not present in the FxA Device Manager list have been
+ // disconnected and so are stale
+ this._log.debug("Refreshing the known stale clients list");
+ let localClients = Object.values(this._store._remoteClients)
+ .filter(client => client.fxaDeviceId) // iOS client records don't have fxaDeviceId
+ .map(client => client.fxaDeviceId);
+ const fxaClients = this.fxAccounts.device.recentDeviceList
+ ? this.fxAccounts.device.recentDeviceList.map(device => device.id)
+ : [];
+ this._knownStaleFxADeviceIds = Utils.arraySub(localClients, fxaClients);
+ },
+
+ async _syncStartup() {
+ // Reupload new client record periodically.
+ if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
+ await this._tracker.addChangedID(this.localID);
+ }
+ return SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ async _processIncoming() {
+ // Fetch all records from the server.
+ await this.resetLastSync();
+ this._incomingClients = {};
+ try {
+ await SyncEngine.prototype._processIncoming.call(this);
+ // Update FxA Device list.
+ await this._fetchFxADevices();
+ // Since clients are synced unconditionally, any records in the local store
+ // that don't exist on the server must be for disconnected clients. Remove
+ // them, so that we don't upload records with commands for clients that will
+ // never see them. We also do this to filter out stale clients from the
+ // tabs collection, since showing their list of tabs is confusing.
+ for (let id in this._store._remoteClients) {
+ if (!this._incomingClients[id]) {
+ this._log.info(`Removing local state for deleted client ${id}`);
+ await this._removeRemoteClient(id);
+ }
+ }
+ let localFxADeviceId = await fxAccounts.device.getLocalId();
+ // Bug 1264498: Mobile clients don't remove themselves from the clients
+ // collection when the user disconnects Sync, so we mark as stale clients
+ // with the same name that haven't synced in over a week.
+ // (Note we can't simply delete them, or we re-apply them next sync - see
+ // bug 1287687)
+ this._localClientLastModified = Math.round(
+ this._incomingClients[this.localID]
+ );
+ delete this._incomingClients[this.localID];
+ let names = new Set([this.localName]);
+ let seenDeviceIds = new Set([localFxADeviceId]);
+ let idToLastModifiedList = Object.entries(this._incomingClients).sort(
+ (a, b) => b[1] - a[1]
+ );
+ for (let [id, serverLastModified] of idToLastModifiedList) {
+ let record = this._store._remoteClients[id];
+ // stash the server last-modified time on the record.
+ record.serverLastModified = serverLastModified;
+ if (
+ record.fxaDeviceId &&
+ this._knownStaleFxADeviceIds.includes(record.fxaDeviceId)
+ ) {
+ this._log.info(
+ `Hiding stale client ${id} - in known stale clients list`
+ );
+ record.stale = true;
+ }
+ if (!names.has(record.name)) {
+ if (record.fxaDeviceId) {
+ seenDeviceIds.add(record.fxaDeviceId);
+ }
+ names.add(record.name);
+ continue;
+ }
+ let remoteAge = Resource.serverTime - this._incomingClients[id];
+ if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
+ this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
+ record.stale = true;
+ continue;
+ }
+ if (record.fxaDeviceId && seenDeviceIds.has(record.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${record.id}` +
+ ` - duplicate device id ${record.fxaDeviceId}`
+ );
+ record.stale = true;
+ } else if (record.fxaDeviceId) {
+ seenDeviceIds.add(record.fxaDeviceId);
+ }
+ }
+ } finally {
+ this._incomingClients = null;
+ }
+ },
+
+ async _uploadOutgoing() {
+ this._currentlySyncingCommands = await this._prepareCommandsForUpload();
+ const clientWithPendingCommands = Object.keys(
+ this._currentlySyncingCommands
+ );
+ for (let clientId of clientWithPendingCommands) {
+ if (this._store._remoteClients[clientId] || this.localID == clientId) {
+ this._modified.set(clientId, 0);
+ }
+ }
+ let updatedIDs = this._modified.ids();
+ await SyncEngine.prototype._uploadOutgoing.call(this);
+ // Record the response time as the server time for each item we uploaded.
+ let lastSync = await this.getLastSync();
+ for (let id of updatedIDs) {
+ if (id == this.localID) {
+ this.lastRecordUpload = lastSync;
+ } else {
+ this._store._remoteClients[id].serverLastModified = lastSync;
+ }
+ }
+ },
+
+ async _onRecordsWritten(succeeded, failed) {
+ // Reconcile the status of the local records with what we just wrote on the
+ // server
+ for (let id of succeeded) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (id == this.localID) {
+ if (this.isFirstSync) {
+ this._log.info(
+ "Uploaded our client record for the first time, notifying other clients."
+ );
+ this._notifyClientRecordUploaded();
+ }
+ if (this.localCommands) {
+ this.localCommands = this.localCommands.filter(
+ command => !hasDupeCommand(commandChanges, command)
+ );
+ }
+ } else {
+ const clientRecord = this._store._remoteClients[id];
+ if (!commandChanges || !clientRecord) {
+ // should be impossible, else we wouldn't have been writing it.
+ this._log.warn(
+ "No command/No record changes for a client we uploaded"
+ );
+ continue;
+ }
+ // fixup the client record, so our copy of _remoteClients matches what we uploaded.
+ this._store._remoteClients[id] = await this._store.createRecord(id);
+ // we could do better and pass the reference to the record we just uploaded,
+ // but this will do for now
+ }
+ }
+
+ // Re-add failed commands
+ for (let id of failed) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (!commandChanges) {
+ continue;
+ }
+ await this._addClientCommand(id, commandChanges);
+ }
+
+ await this._deleteUploadedCommands();
+
+ // Notify other devices that their own client collection changed
+ const idsToNotify = succeeded.reduce((acc, id) => {
+ if (id == this.localID) {
+ return acc;
+ }
+ const fxaDeviceId = this.getClientFxaDeviceId(id);
+ return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
+ }, []);
+ if (idsToNotify.length > 0) {
+ this._notifyOtherClientsModified(idsToNotify);
+ }
+ },
+
+ _notifyOtherClientsModified(ids) {
+ // We are not waiting on this promise on purpose.
+ this._notifyCollectionChanged(
+ ids,
+ NOTIFY_TAB_SENT_TTL_SECS,
+ COLLECTION_MODIFIED_REASON_SENDTAB
+ );
+ },
+
+ _notifyClientRecordUploaded() {
+ // We are not waiting on this promise on purpose.
+ this._notifyCollectionChanged(
+ null,
+ 0,
+ COLLECTION_MODIFIED_REASON_FIRSTSYNC
+ );
+ },
+
+ /**
+ * @param {?string[]} ids FxA Client IDs to notify. null means everyone else.
+ * @param {number} ttl TTL of the push notification.
+ * @param {string} reason Reason for sending this push notification.
+ */
+ async _notifyCollectionChanged(ids, ttl, reason) {
+ const message = {
+ version: 1,
+ command: "sync:collection_changed",
+ data: {
+ collections: ["clients"],
+ reason,
+ },
+ };
+ let excludedIds = null;
+ if (!ids) {
+ const localFxADeviceId = await fxAccounts.device.getLocalId();
+ excludedIds = [localFxADeviceId];
+ }
+ try {
+ await this.fxAccounts.notifyDevices(ids, excludedIds, message, ttl);
+ } catch (e) {
+ this._log.error("Could not notify of changes in the collection", e);
+ }
+ },
+
+ async _syncFinish() {
+ // Record histograms for our device types, and also write them to a pref
+ // so non-histogram telemetry (eg, UITelemetry) and the sync scheduler
+ // has easy access to them, and so they are accurate even before we've
+ // successfully synced the first time after startup.
+ let deviceTypeCounts = this.deviceTypes;
+ for (let [deviceType, count] of deviceTypeCounts) {
+ let hid;
+ let prefName = this.name + ".devices.";
+ switch (deviceType) {
+ case DEVICE_TYPE_DESKTOP:
+ hid = "WEAVE_DEVICE_COUNT_DESKTOP";
+ prefName += "desktop";
+ break;
+ case DEVICE_TYPE_MOBILE:
+ hid = "WEAVE_DEVICE_COUNT_MOBILE";
+ prefName += "mobile";
+ break;
+ default:
+ this._log.warn(
+ `Unexpected deviceType "${deviceType}" recording device telemetry.`
+ );
+ continue;
+ }
+ Services.telemetry.getHistogramById(hid).add(count);
+ // Optimization: only write the pref if it changed since our last sync.
+ if (
+ this._lastDeviceCounts == null ||
+ this._lastDeviceCounts.get(prefName) != count
+ ) {
+ Svc.Prefs.set(prefName, count);
+ }
+ }
+ this._lastDeviceCounts = deviceTypeCounts;
+ return SyncEngine.prototype._syncFinish.call(this);
+ },
+
+ async _reconcile(item) {
+ // Every incoming record is reconciled, so we use this to track the
+ // contents of the collection on the server.
+ this._incomingClients[item.id] = item.modified;
+
+ if (!(await this._store.itemExists(item.id))) {
+ return true;
+ }
+ // Clients are synced unconditionally, so we'll always have new records.
+ // Unfortunately, this will cause the scheduler to use the immediate sync
+ // interval for the multi-device case, instead of the active interval. We
+ // work around this by updating the record during reconciliation, and
+ // returning false to indicate that the record doesn't need to be applied
+ // later.
+ await this._store.update(item);
+ return false;
+ },
+
+ // Treat reset the same as wiping for locally cached clients
+ async _resetClient() {
+ await this._wipeClient();
+ },
+
+ async _wipeClient() {
+ await SyncEngine.prototype._resetClient.call(this);
+ this._knownStaleFxADeviceIds = null;
+ delete this.localCommands;
+ await this._store.wipe();
+ try {
+ await Utils.jsonRemove("commands", this);
+ } catch (err) {
+ this._log.warn("Could not delete commands.json", err);
+ }
+ try {
+ await Utils.jsonRemove("commands-syncing", this);
+ } catch (err) {
+ this._log.warn("Could not delete commands-syncing.json", err);
+ }
+ },
+
+ async removeClientData() {
+ let res = this.service.resource(this.engineURL + "/" + this.localID);
+ await res.delete();
+ },
+
+ // Override the default behavior to delete bad records from the server.
+ async handleHMACMismatch(item, mayRetry) {
+ this._log.debug("Handling HMAC mismatch for " + item.id);
+
+ let base = await SyncEngine.prototype.handleHMACMismatch.call(
+ this,
+ item,
+ mayRetry
+ );
+ if (base != SyncEngine.kRecoveryStrategy.error) {
+ return base;
+ }
+
+ // It's a bad client record. Save it to be deleted at the end of the sync.
+ this._log.debug("Bad client record detected. Scheduling for deletion.");
+ await this._deleteId(item.id);
+
+ // Neither try again nor error; we're going to delete it.
+ return SyncEngine.kRecoveryStrategy.ignore;
+ },
+
+ /**
+ * A hash of valid commands that the client knows about. The key is a command
+ * and the value is a hash containing information about the command such as
+ * number of arguments, description, and importance (lower importance numbers
+ * indicate higher importance.
+ */
+ _commands: {
+ resetAll: {
+ args: 0,
+ importance: 0,
+ desc: "Clear temporary local data for all engines",
+ },
+ resetEngine: {
+ args: 1,
+ importance: 0,
+ desc: "Clear temporary local data for engine",
+ },
+ wipeAll: {
+ args: 0,
+ importance: 0,
+ desc: "Delete all client data for all engines",
+ },
+ wipeEngine: {
+ args: 1,
+ importance: 0,
+ desc: "Delete all client data for engine",
+ },
+ logout: { args: 0, importance: 0, desc: "Log out client" },
+ displayURI: {
+ args: 3,
+ importance: 1,
+ desc: "Instruct a client to display a URI",
+ },
+ },
+
+ /**
+ * Sends a command+args pair to a specific client.
+ *
+ * @param command Command string
+ * @param args Array of arguments/data for command
+ * @param clientId Client to send command to
+ */
+ async _sendCommandToClient(command, args, clientId, telemetryExtra) {
+ this._log.trace("Sending " + command + " to " + clientId);
+
+ let client = this._store._remoteClients[clientId];
+ if (!client) {
+ throw new Error("Unknown remote client ID: '" + clientId + "'.");
+ }
+ if (client.stale) {
+ throw new Error("Stale remote client ID: '" + clientId + "'.");
+ }
+
+ let action = {
+ command,
+ args,
+ // We send the flowID to the other client so *it* can report it in its
+ // telemetry - we record it in ours below.
+ flowID: telemetryExtra.flowID,
+ };
+
+ if (await this._addClientCommand(clientId, action)) {
+ this._log.trace(`Client ${clientId} got a new action`, [command, args]);
+ await this._tracker.addChangedID(clientId);
+ try {
+ telemetryExtra.deviceID = this.service.identity.hashedDeviceID(
+ clientId
+ );
+ } catch (_) {}
+
+ this.service.recordTelemetryEvent(
+ "sendcommand",
+ command,
+ undefined,
+ telemetryExtra
+ );
+ } else {
+ this._log.trace(`Client ${clientId} got a duplicate action`, [
+ command,
+ args,
+ ]);
+ }
+ },
+
+ /**
+ * Check if the local client has any remote commands and perform them.
+ *
+ * @return false to abort sync
+ */
+ async processIncomingCommands() {
+ return this._notify("clients:process-commands", "", async function() {
+ if (
+ !this.localCommands ||
+ (this._lastModifiedOnProcessCommands == this._localClientLastModified &&
+ !this.ignoreLastModifiedOnProcessCommands)
+ ) {
+ return true;
+ }
+ this._lastModifiedOnProcessCommands = this._localClientLastModified;
+
+ const clearedCommands = await this._readCommands()[this.localID];
+ const commands = this.localCommands.filter(
+ command => !hasDupeCommand(clearedCommands, command)
+ );
+ let didRemoveCommand = false;
+ let URIsToDisplay = [];
+ // Process each command in order.
+ for (let rawCommand of commands) {
+ let shouldRemoveCommand = true; // most commands are auto-removed.
+ let { command, args, flowID } = rawCommand;
+ this._log.debug("Processing command " + command, args);
+
+ this.service.recordTelemetryEvent(
+ "processcommand",
+ command,
+ undefined,
+ { flowID }
+ );
+
+ let engines = [args[0]];
+ switch (command) {
+ case "resetAll":
+ engines = null;
+ // Fallthrough
+ case "resetEngine":
+ await this.service.resetClient(engines);
+ break;
+ case "wipeAll":
+ engines = null;
+ // Fallthrough
+ case "wipeEngine":
+ await this.service.wipeClient(engines);
+ break;
+ case "logout":
+ this.service.logout();
+ return false;
+ case "displayURI":
+ let [uri, clientId, title] = args;
+ URIsToDisplay.push({ uri, clientId, title });
+ break;
+ default:
+ this._log.warn("Received an unknown command: " + command);
+ break;
+ }
+ // Add the command to the "cleared" commands list
+ if (shouldRemoveCommand) {
+ await this.removeLocalCommand(rawCommand);
+ didRemoveCommand = true;
+ }
+ }
+ if (didRemoveCommand) {
+ await this._tracker.addChangedID(this.localID);
+ }
+
+ if (URIsToDisplay.length) {
+ this._handleDisplayURIs(URIsToDisplay);
+ }
+
+ return true;
+ })();
+ },
+
+ /**
+ * Validates and sends a command to a client or all clients.
+ *
+ * Calling this does not actually sync the command data to the server. If the
+ * client already has the command/args pair, it won't receive a duplicate
+ * command.
+ * This method is async since it writes the command to a file.
+ *
+ * @param command
+ * Command to invoke on remote clients
+ * @param args
+ * Array of arguments to give to the command
+ * @param clientId
+ * Client ID to send command to. If undefined, send to all remote
+ * clients.
+ * @param flowID
+ * A unique identifier used to track success for this operation across
+ * devices.
+ */
+ async sendCommand(command, args, clientId = null, telemetryExtra = {}) {
+ let commandData = this._commands[command];
+ // Don't send commands that we don't know about.
+ if (!commandData) {
+ this._log.error("Unknown command to send: " + command);
+ return;
+ } else if (!args || args.length != commandData.args) {
+ // Don't send a command with the wrong number of arguments.
+ this._log.error(
+ "Expected " +
+ commandData.args +
+ " args for '" +
+ command +
+ "', but got " +
+ args
+ );
+ return;
+ }
+
+ // We allocate a "flowID" here, so it is used for each client.
+ telemetryExtra = Object.assign({}, telemetryExtra); // don't clobber the caller's object
+ if (!telemetryExtra.flowID) {
+ telemetryExtra.flowID = Utils.makeGUID();
+ }
+
+ if (clientId) {
+ await this._sendCommandToClient(command, args, clientId, telemetryExtra);
+ } else {
+ for (let [id, record] of Object.entries(this._store._remoteClients)) {
+ if (!record.stale) {
+ await this._sendCommandToClient(command, args, id, telemetryExtra);
+ }
+ }
+ }
+ },
+
+ /**
+ * Send a URI to another client for display.
+ *
+ * A side effect is the score is increased dramatically to incur an
+ * immediate sync.
+ *
+ * If an unknown client ID is specified, sendCommand() will throw an
+ * Error object.
+ *
+ * @param uri
+ * URI (as a string) to send and display on the remote client
+ * @param clientId
+ * ID of client to send the command to. If not defined, will be sent
+ * to all remote clients.
+ * @param title
+ * Title of the page being sent.
+ */
+ async sendURIToClientForDisplay(uri, clientId, title) {
+ this._log.trace(
+ "Sending URI to client: " + uri + " -> " + clientId + " (" + title + ")"
+ );
+ await this.sendCommand("displayURI", [uri, this.localID, title], clientId);
+
+ this._tracker.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ /**
+ * Handle a bunch of received 'displayURI' commands.
+ *
+ * Interested parties should observe the "weave:engine:clients:display-uris"
+ * topic. The callback will receive an array as the subject parameter
+ * containing objects with the following keys:
+ *
+ * uri URI (string) that is requested for display.
+ * sender.id ID of client that sent the command.
+ * sender.name Name of client that sent the command.
+ * title Title of page that loaded URI (likely) corresponds to.
+ *
+ * The 'data' parameter to the callback will not be defined.
+ *
+ * @param uris
+ * An array containing URI objects to display
+ * @param uris[].uri
+ * String URI that was received
+ * @param uris[].clientId
+ * ID of client that sent URI
+ * @param uris[].title
+ * String title of page that URI corresponds to. Older clients may not
+ * send this.
+ */
+ _handleDisplayURIs(uris) {
+ uris.forEach(uri => {
+ uri.sender = {
+ id: uri.clientId,
+ name: this.getClientName(uri.clientId),
+ };
+ });
+ Svc.Obs.notify("weave:engine:clients:display-uris", uris);
+ },
+
+ async _removeRemoteClient(id) {
+ delete this._store._remoteClients[id];
+ await this._tracker.removeChangedID(id);
+ await this._removeClientCommands(id);
+ this._modified.delete(id);
+ },
+};
+
+function ClientStore(name, engine) {
+ Store.call(this, name, engine);
+}
+ClientStore.prototype = {
+ __proto__: Store.prototype,
+
+ _remoteClients: {},
+
+ async create(record) {
+ await this.update(record);
+ },
+
+ async update(record) {
+ if (record.id == this.engine.localID) {
+ // Only grab commands from the server; local name/type always wins
+ this.engine.localCommands = record.commands;
+ } else {
+ this._remoteClients[record.id] = record.cleartext;
+ }
+ },
+
+ async createRecord(id, collection) {
+ let record = new ClientsRec(collection, id);
+
+ const commandsChanges = this.engine._currentlySyncingCommands
+ ? this.engine._currentlySyncingCommands[id]
+ : [];
+
+ // Package the individual components into a record for the local client
+ if (id == this.engine.localID) {
+ try {
+ record.fxaDeviceId = await this.engine.fxAccounts.device.getLocalId();
+ } catch (error) {
+ this._log.warn("failed to get fxa device id", error);
+ }
+ record.name = this.engine.localName;
+ record.type = this.engine.localType;
+ record.version = Services.appinfo.version;
+ record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
+
+ // Substract the commands we recorded that we've already executed
+ if (
+ commandsChanges &&
+ commandsChanges.length &&
+ this.engine.localCommands &&
+ this.engine.localCommands.length
+ ) {
+ record.commands = this.engine.localCommands.filter(
+ command => !hasDupeCommand(commandsChanges, command)
+ );
+ }
+
+ // Optional fields.
+ record.os = Services.appinfo.OS; // "Darwin"
+ record.appPackage = Services.appinfo.ID;
+ record.application = this.engine.brandName; // "Nightly"
+
+ // We can't compute these yet.
+ // record.device = ""; // Bug 1100723
+ // record.formfactor = ""; // Bug 1100722
+ } else {
+ record.cleartext = Object.assign({}, this._remoteClients[id]);
+ delete record.cleartext.serverLastModified; // serverLastModified is a local only attribute.
+
+ // Add the commands we have to send
+ if (commandsChanges && commandsChanges.length) {
+ const recordCommands = record.cleartext.commands || [];
+ const newCommands = commandsChanges.filter(
+ command => !hasDupeCommand(recordCommands, command)
+ );
+ record.cleartext.commands = recordCommands.concat(newCommands);
+ }
+
+ if (record.cleartext.stale) {
+ // It's almost certainly a logic error for us to upload a record we
+ // consider stale, so make log noise, but still remove the flag.
+ this._log.error(
+ `Preparing to upload record ${id} that we consider stale`
+ );
+ delete record.cleartext.stale;
+ }
+ }
+ if (record.commands) {
+ const maxPayloadSize = this.engine.service.getMemcacheMaxRecordPayloadSize();
+ let origOrder = new Map(record.commands.map((c, i) => [c, i]));
+ // we sort first by priority, and second by age (indicated by order in the
+ // original list)
+ let commands = record.commands.slice().sort((a, b) => {
+ let infoA = this.engine._commands[a.command];
+ let infoB = this.engine._commands[b.command];
+ // Treat unknown command types as highest priority, to allow us to add
+ // high priority commands in the future without worrying about clients
+ // removing them on each-other unnecessarially.
+ let importA = infoA ? infoA.importance : 0;
+ let importB = infoB ? infoB.importance : 0;
+ // Higher importantance numbers indicate that we care less, so they
+ // go to the end of the list where they'll be popped off.
+ let importDelta = importA - importB;
+ if (importDelta != 0) {
+ return importDelta;
+ }
+ let origIdxA = origOrder.get(a);
+ let origIdxB = origOrder.get(b);
+ // Within equivalent priorities, we put older entries near the end
+ // of the list, so that they are removed first.
+ return origIdxB - origIdxA;
+ });
+ let truncatedCommands = Utils.tryFitItems(commands, maxPayloadSize);
+ if (truncatedCommands.length != record.commands.length) {
+ this._log.warn(
+ `Removing commands from client ${id} (from ${record.commands.length} to ${truncatedCommands.length})`
+ );
+ // Restore original order.
+ record.commands = truncatedCommands.sort(
+ (a, b) => origOrder.get(a) - origOrder.get(b)
+ );
+ }
+ }
+ return record;
+ },
+
+ async itemExists(id) {
+ return id in (await this.getAllIDs());
+ },
+
+ async getAllIDs() {
+ let ids = {};
+ ids[this.engine.localID] = true;
+ for (let id in this._remoteClients) {
+ ids[id] = true;
+ }
+ return ids;
+ },
+
+ async wipe() {
+ this._remoteClients = {};
+ },
+};
+
+function ClientsTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+ClientsTracker.prototype = {
+ __proto__: LegacyTracker.prototype,
+
+ _enabled: false,
+
+ onStart() {
+ Svc.Obs.add("fxaccounts:new_device_id", this.asyncObserver);
+ Services.prefs.addObserver(
+ PREF_ACCOUNT_ROOT + "device.name",
+ this.asyncObserver
+ );
+ },
+ onStop() {
+ Services.prefs.removeObserver(
+ PREF_ACCOUNT_ROOT + "device.name",
+ this.asyncObserver
+ );
+ Svc.Obs.remove("fxaccounts:new_device_id", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "nsPref:changed":
+ this._log.debug("client.name preference changed");
+ // Fallthrough intended.
+ case "fxaccounts:new_device_id":
+ await this.addChangedID(this.engine.localID);
+ this.score += SINGLE_USER_THRESHOLD + 1; // ALWAYS SYNC NOW.
+ break;
+ }
+ },
+};
diff --git a/services/sync/modules/engines/extension-storage.js b/services/sync/modules/engines/extension-storage.js
new file mode 100644
index 0000000000..705091957e
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.js
@@ -0,0 +1,295 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = [
+ "ExtensionStorageEngineKinto",
+ "ExtensionStorageEngineBridge",
+];
+
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ BridgedEngine: "resource://services-sync/bridged_engine.js",
+ LogAdapter: "resource://services-sync/bridged_engine.js",
+ extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.jsm",
+ Observers: "resource://services-common/observers.js",
+ Svc: "resource://services-sync/util.js",
+ SyncEngine: "resource://services-sync/engines.js",
+ Tracker: "resource://services-sync/engines.js",
+ SCORE_INCREMENT_MEDIUM: "resource://services-sync/constants.js",
+ MULTI_DEVICE_THRESHOLD: "resource://services-sync/constants.js",
+});
+
+XPCOMUtils.defineLazyModuleGetter(
+ this,
+ "extensionStorageSyncKinto",
+ "resource://gre/modules/ExtensionStorageSyncKinto.jsm",
+ "extensionStorageSync"
+);
+
+XPCOMUtils.defineLazyServiceGetter(
+ this,
+ "StorageSyncService",
+ "@mozilla.org/extensions/storage/sync;1",
+ "nsIInterfaceRequestor"
+);
+
+const PREF_FORCE_ENABLE = "engine.extension-storage.force";
+
+// A helper to indicate whether extension-storage is enabled - it's based on
+// the "addons" pref. The same logic is shared between both engine impls.
+function getEngineEnabled() {
+ // By default, we sync extension storage if we sync addons. This
+ // lets us simplify the UX since users probably don't consider
+ // "extension preferences" a separate category of syncing.
+ // However, we also respect engine.extension-storage.force, which
+ // can be set to true or false, if a power user wants to customize
+ // the behavior despite the lack of UI.
+ const forced = Svc.Prefs.get(PREF_FORCE_ENABLE, undefined);
+ if (forced !== undefined) {
+ return forced;
+ }
+ return Svc.Prefs.get("engine.addons", false);
+}
+
+function setEngineEnabled(enabled) {
+ // This will be called by the engine manager when declined on another device.
+ // Things will go a bit pear-shaped if the engine manager tries to end up
+ // with 'addons' and 'extension-storage' in different states - however, this
+ // *can* happen given we support the `engine.extension-storage.force`
+ // preference. So if that pref exists, we set it to this value. If that pref
+ // doesn't exist, we just ignore it and hope that the 'addons' engine is also
+ // going to be set to the same state.
+ if (Svc.Prefs.has(PREF_FORCE_ENABLE)) {
+ Svc.Prefs.set(PREF_FORCE_ENABLE, enabled);
+ }
+}
+
+// A "bridged engine" to our webext-storage component.
+function ExtensionStorageEngineBridge(service) {
+ let bridge = StorageSyncService.getInterface(Ci.mozIBridgedSyncEngine);
+ BridgedEngine.call(this, bridge, "Extension-Storage", service);
+
+ let app_services_logger = Cc["@mozilla.org/appservices/logger;1"].getService(
+ Ci.mozIAppServicesLogger
+ );
+ let logger_target = "app-services:webext_storage:sync";
+ app_services_logger.register(logger_target, new LogAdapter(this._log));
+}
+
+ExtensionStorageEngineBridge.prototype = {
+ __proto__: BridgedEngine.prototype,
+ syncPriority: 10,
+
+ // Used to override the engine name in telemetry, so that we can distinguish .
+ overrideTelemetryName: "rust-webext-storage",
+
+ _notifyPendingChanges() {
+ return new Promise(resolve => {
+ this._bridge
+ .QueryInterface(Ci.mozISyncedExtensionStorageArea)
+ .fetchPendingSyncChanges({
+ QueryInterface: ChromeUtils.generateQI([
+ "mozIExtensionStorageListener",
+ "mozIExtensionStorageCallback",
+ ]),
+ onChanged: (extId, json) => {
+ try {
+ extensionStorageSync.notifyListeners(extId, JSON.parse(json));
+ } catch (ex) {
+ this._log.warn(
+ `Error notifying change listeners for ${extId}`,
+ ex
+ );
+ }
+ },
+ handleSuccess: resolve,
+ handleError: (code, message) => {
+ this._log.warn(
+ "Error fetching pending synced changes",
+ message,
+ code
+ );
+ resolve();
+ },
+ });
+ });
+ },
+
+ _takeMigrationInfo() {
+ return new Promise((resolve, reject) => {
+ this._bridge
+ .QueryInterface(Ci.mozIExtensionStorageArea)
+ .takeMigrationInfo({
+ QueryInterface: ChromeUtils.generateQI([
+ "mozIExtensionStorageCallback",
+ ]),
+ handleSuccess: result => {
+ resolve(result ? JSON.parse(result) : null);
+ },
+ handleError: (code, message) => {
+ this._log.warn("Error fetching migration info", message, code);
+ // `takeMigrationInfo` doesn't actually perform the migration,
+ // just reads (and clears) any data stored in the DB from the
+ // previous migration.
+ //
+ // Any errors here are very likely occurring a good while
+ // after the migration ran, so we just warn and pretend
+ // nothing was there.
+ resolve(null);
+ },
+ });
+ });
+ },
+
+ async _syncStartup() {
+ let result = await super._syncStartup();
+ let info = await this._takeMigrationInfo();
+ if (info) {
+ Observers.notify("weave:telemetry:migration", info, "webext-storage");
+ }
+ return result;
+ },
+
+ async _processIncoming() {
+ await super._processIncoming();
+ try {
+ await this._notifyPendingChanges();
+ } catch (ex) {
+ // Failing to notify `storage.onChanged` observers is bad, but shouldn't
+ // interrupt syncing.
+ this._log.warn("Error notifying about synced changes", ex);
+ }
+ },
+
+ get enabled() {
+ return getEngineEnabled();
+ },
+ set enabled(enabled) {
+ setEngineEnabled(enabled);
+ },
+};
+
+/**
+ *****************************************************************************
+ *
+ * Deprecated support for Kinto
+ *
+ *****************************************************************************
+ */
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+function ExtensionStorageEngineKinto(service) {
+ SyncEngine.call(this, "Extension-Storage", service);
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_skipPercentageChance",
+ "services.sync.extension-storage.skipPercentageChance",
+ 0
+ );
+}
+ExtensionStorageEngineKinto.prototype = {
+ __proto__: SyncEngine.prototype,
+ _trackerObj: ExtensionStorageTracker,
+ // we don't need these since we implement our own sync logic
+ _storeObj: undefined,
+ _recordObj: undefined,
+
+ syncPriority: 10,
+ allowSkippedRecord: false,
+
+ async _sync() {
+ return extensionStorageSyncKinto.syncAll();
+ },
+
+ get enabled() {
+ return getEngineEnabled();
+ },
+ // We only need the enabled setter for the edge-case where info/collections
+ // has `extension-storage` - which could happen if the pref to flip the new
+ // engine on was once set but no longer is.
+ set enabled(enabled) {
+ setEngineEnabled(enabled);
+ },
+
+ _wipeClient() {
+ return extensionStorageSyncKinto.clearAll();
+ },
+
+ shouldSkipSync(syncReason) {
+ if (syncReason == "user" || syncReason == "startup") {
+ this._log.info(
+ `Not skipping extension storage sync: reason == ${syncReason}`
+ );
+ // Always sync if a user clicks the button, or if we're starting up.
+ return false;
+ }
+ // Ensure this wouldn't cause a resync...
+ if (this._tracker.score >= MULTI_DEVICE_THRESHOLD) {
+ this._log.info(
+ "Not skipping extension storage sync: Would trigger resync anyway"
+ );
+ return false;
+ }
+
+ let probability = this._skipPercentageChance / 100.0;
+ // Math.random() returns a value in the interval [0, 1), so `>` is correct:
+ // if `probability` is 1 skip every time, and if it's 0, never skip.
+ let shouldSkip = probability > Math.random();
+
+ this._log.info(
+ `Skipping extension-storage sync with a chance of ${probability}: ${shouldSkip}`
+ );
+ return shouldSkip;
+ },
+};
+
+function ExtensionStorageTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ this._ignoreAll = false;
+}
+ExtensionStorageTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ get ignoreAll() {
+ return this._ignoreAll;
+ },
+
+ set ignoreAll(value) {
+ this._ignoreAll = value;
+ },
+
+ onStart() {
+ Svc.Obs.add("ext.storage.sync-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ Svc.Obs.remove("ext.storage.sync-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (topic !== "ext.storage.sync-changed") {
+ return;
+ }
+
+ // Single adds, removes and changes are not so important on their
+ // own, so let's just increment score a bit.
+ this.score += SCORE_INCREMENT_MEDIUM;
+ },
+};
diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js
new file mode 100644
index 0000000000..fac1172e00
--- /dev/null
+++ b/services/sync/modules/engines/forms.js
@@ -0,0 +1,312 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["FormEngine", "FormRec", "FormValidator"];
+
+const { Store, SyncEngine, LegacyTracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+const { SCORE_INCREMENT_MEDIUM } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+const { CollectionProblemData, CollectionValidator } = ChromeUtils.import(
+ "resource://services-sync/collection_validator.js"
+);
+const { Async } = ChromeUtils.import("resource://services-common/async.js");
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+ChromeUtils.defineModuleGetter(
+ this,
+ "FormHistory",
+ "resource://gre/modules/FormHistory.jsm"
+);
+
+const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds.
+
+function FormRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+FormRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Form",
+ ttl: FORMS_TTL,
+};
+
+Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
+
+var FormWrapper = {
+ _log: Log.repository.getLogger("Sync.Engine.Forms"),
+
+ _getEntryCols: ["fieldname", "value"],
+ _guidCols: ["guid"],
+
+ async _search(terms, searchData) {
+ return new Promise(resolve => {
+ let results = [];
+ let callbacks = {
+ handleResult(result) {
+ results.push(result);
+ },
+ handleCompletion(reason) {
+ resolve(results);
+ },
+ };
+ FormHistory.search(terms, searchData, callbacks);
+ });
+ },
+
+ async _update(changes) {
+ if (!FormHistory.enabled) {
+ return; // update isn't going to do anything.
+ }
+ await new Promise(resolve => {
+ let callbacks = {
+ handleCompletion(reason) {
+ resolve();
+ },
+ };
+ FormHistory.update(changes, callbacks);
+ });
+ },
+
+ async getEntry(guid) {
+ let results = await this._search(this._getEntryCols, { guid });
+ if (!results.length) {
+ return null;
+ }
+ return { name: results[0].fieldname, value: results[0].value };
+ },
+
+ async getGUID(name, value) {
+ // Query for the provided entry.
+ let query = { fieldname: name, value };
+ let results = await this._search(this._guidCols, query);
+ return results.length ? results[0].guid : null;
+ },
+
+ async hasGUID(guid) {
+ // We could probably use a count function here, but search exists...
+ let results = await this._search(this._guidCols, { guid });
+ return results.length != 0;
+ },
+
+ async replaceGUID(oldGUID, newGUID) {
+ let changes = {
+ op: "update",
+ guid: oldGUID,
+ newGuid: newGUID,
+ };
+ await this._update(changes);
+ },
+};
+
+function FormEngine(service) {
+ SyncEngine.call(this, "Forms", service);
+}
+FormEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: FormStore,
+ _trackerObj: FormTracker,
+ _recordObj: FormRec,
+
+ syncPriority: 6,
+
+ get prefName() {
+ return "history";
+ },
+
+ async _findDupe(item) {
+ return FormWrapper.getGUID(item.name, item.value);
+ },
+};
+
+function FormStore(name, engine) {
+ Store.call(this, name, engine);
+}
+FormStore.prototype = {
+ __proto__: Store.prototype,
+
+ async _processChange(change) {
+ // If this._changes is defined, then we are applying a batch, so we
+ // can defer it.
+ if (this._changes) {
+ this._changes.push(change);
+ return;
+ }
+
+ // Otherwise we must handle the change right now.
+ await FormWrapper._update(change);
+ },
+
+ async applyIncomingBatch(records) {
+ Async.checkAppReady();
+ // We collect all the changes to be made then apply them all at once.
+ this._changes = [];
+ let failures = await Store.prototype.applyIncomingBatch.call(this, records);
+ if (this._changes.length) {
+ await FormWrapper._update(this._changes);
+ }
+ delete this._changes;
+ return failures;
+ },
+
+ async getAllIDs() {
+ let results = await FormWrapper._search(["guid"], []);
+ let guids = {};
+ for (let result of results) {
+ guids[result.guid] = true;
+ }
+ return guids;
+ },
+
+ async changeItemID(oldID, newID) {
+ await FormWrapper.replaceGUID(oldID, newID);
+ },
+
+ async itemExists(id) {
+ return FormWrapper.hasGUID(id);
+ },
+
+ async createRecord(id, collection) {
+ let record = new FormRec(collection, id);
+ let entry = await FormWrapper.getEntry(id);
+ if (entry != null) {
+ record.name = entry.name;
+ record.value = entry.value;
+ } else {
+ record.deleted = true;
+ }
+ return record;
+ },
+
+ async create(record) {
+ this._log.trace("Adding form record for " + record.name);
+ let change = {
+ op: "add",
+ guid: record.id,
+ fieldname: record.name,
+ value: record.value,
+ };
+ await this._processChange(change);
+ },
+
+ async remove(record) {
+ this._log.trace("Removing form record: " + record.id);
+ let change = {
+ op: "remove",
+ guid: record.id,
+ };
+ await this._processChange(change);
+ },
+
+ async update(record) {
+ this._log.trace("Ignoring form record update request!");
+ },
+
+ async wipe() {
+ let change = {
+ op: "remove",
+ };
+ await FormWrapper._update(change);
+ },
+};
+
+function FormTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+FormTracker.prototype = {
+ __proto__: LegacyTracker.prototype,
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ onStart() {
+ Svc.Obs.add("satchel-storage-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ Svc.Obs.remove("satchel-storage-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+ switch (topic) {
+ case "satchel-storage-changed":
+ if (data == "formhistory-add" || data == "formhistory-remove") {
+ let guid = subject.QueryInterface(Ci.nsISupportsString).toString();
+ await this.trackEntry(guid);
+ }
+ break;
+ }
+ },
+
+ async trackEntry(guid) {
+ const added = await this.addChangedID(guid);
+ if (added) {
+ this.score += SCORE_INCREMENT_MEDIUM;
+ }
+ },
+};
+
+class FormsProblemData extends CollectionProblemData {
+ getSummary() {
+ // We don't support syncing deleted form data, so "clientMissing" isn't a problem
+ return super.getSummary().filter(entry => entry.name !== "clientMissing");
+ }
+}
+
+class FormValidator extends CollectionValidator {
+ constructor() {
+ super("forms", "id", ["name", "value"]);
+ this.ignoresMissingClients = true;
+ }
+
+ emptyProblemData() {
+ return new FormsProblemData();
+ }
+
+ async getClientItems() {
+ return FormWrapper._search(["guid", "fieldname", "value"], {});
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ name: item.fieldname,
+ fieldname: item.fieldname,
+ value: item.value,
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ let res = Object.assign(
+ {
+ guid: item.id,
+ fieldname: item.name,
+ original: item,
+ },
+ item
+ );
+ // Missing `name` or `value` causes the getGUID call to throw
+ if (item.name !== undefined && item.value !== undefined) {
+ let guid = await FormWrapper.getGUID(item.name, item.value);
+ if (guid) {
+ res.guid = guid;
+ res.id = guid;
+ res.duped = true;
+ }
+ }
+
+ return res;
+ }
+}
diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js
new file mode 100644
index 0000000000..f2821cfc83
--- /dev/null
+++ b/services/sync/modules/engines/history.js
@@ -0,0 +1,625 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["HistoryEngine", "HistoryRec"];
+
+const HISTORY_TTL = 5184000; // 60 days in milliseconds
+const THIRTY_DAYS_IN_MS = 2592000000; // 30 days in milliseconds
+
+const { Async } = ChromeUtils.import("resource://services-common/async.js");
+const { CommonUtils } = ChromeUtils.import(
+ "resource://services-common/utils.js"
+);
+const {
+ MAX_HISTORY_DOWNLOAD,
+ MAX_HISTORY_UPLOAD,
+ SCORE_INCREMENT_SMALL,
+ SCORE_INCREMENT_XLARGE,
+} = ChromeUtils.import("resource://services-sync/constants.js");
+const { Store, SyncEngine, LegacyTracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Utils } = ChromeUtils.import("resource://services-sync/util.js");
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "PlacesUtils",
+ "resource://gre/modules/PlacesUtils.jsm"
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "PlacesSyncUtils",
+ "resource://gre/modules/PlacesSyncUtils.jsm"
+);
+
+function HistoryRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+HistoryRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.History",
+ ttl: HISTORY_TTL,
+};
+
+Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]);
+
+function HistoryEngine(service) {
+ SyncEngine.call(this, "History", service);
+}
+HistoryEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _recordObj: HistoryRec,
+ _storeObj: HistoryStore,
+ _trackerObj: HistoryTracker,
+ downloadLimit: MAX_HISTORY_DOWNLOAD,
+
+ syncPriority: 7,
+
+ async getSyncID() {
+ return PlacesSyncUtils.history.getSyncId();
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ this._log.debug(
+ "Checking if server sync ID ${newSyncID} matches existing",
+ { newSyncID }
+ );
+ await PlacesSyncUtils.history.ensureCurrentSyncId(newSyncID);
+ return newSyncID;
+ },
+
+ async resetSyncID() {
+ // First, delete the collection on the server. It's fine if we're
+ // interrupted here: on the next sync, we'll detect that our old sync ID is
+ // now stale, and start over as a first sync.
+ await this._deleteServerCollection();
+ // Then, reset our local sync ID.
+ return this.resetLocalSyncID();
+ },
+
+ async resetLocalSyncID() {
+ let newSyncID = await PlacesSyncUtils.history.resetSyncId();
+ this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID });
+ return newSyncID;
+ },
+
+ async getLastSync() {
+ let lastSync = await PlacesSyncUtils.history.getLastSync();
+ return lastSync;
+ },
+
+ async setLastSync(lastSync) {
+ await PlacesSyncUtils.history.setLastSync(lastSync);
+ },
+
+ shouldSyncURL(url) {
+ return !url.startsWith("file:");
+ },
+
+ async pullNewChanges() {
+ const changedIDs = await this._tracker.getChangedIDs();
+ let modifiedGUIDs = Object.keys(changedIDs);
+ if (!modifiedGUIDs.length) {
+ return {};
+ }
+
+ let guidsToRemove = await PlacesSyncUtils.history.determineNonSyncableGuids(
+ modifiedGUIDs
+ );
+ await this._tracker.removeChangedID(...guidsToRemove);
+ return changedIDs;
+ },
+
+ async _resetClient() {
+ await super._resetClient();
+ await PlacesSyncUtils.history.reset();
+ },
+};
+
+function HistoryStore(name, engine) {
+ Store.call(this, name, engine);
+}
+
+HistoryStore.prototype = {
+ __proto__: Store.prototype,
+
+ // We try and only update this many visits at one time.
+ MAX_VISITS_PER_INSERT: 500,
+
+ // Some helper functions to handle GUIDs
+ async setGUID(uri, guid) {
+ if (!guid) {
+ guid = Utils.makeGUID();
+ }
+
+ try {
+ await PlacesSyncUtils.history.changeGuid(uri, guid);
+ } catch (e) {
+ this._log.error("Error setting GUID ${guid} for URI ${uri}", guid, uri);
+ }
+
+ return guid;
+ },
+
+ async GUIDForUri(uri, create) {
+ // Use the existing GUID if it exists
+ let guid;
+ try {
+ guid = await PlacesSyncUtils.history.fetchGuidForURL(uri);
+ } catch (e) {
+ this._log.error("Error fetching GUID for URL ${uri}", uri);
+ }
+
+ // If the URI has an existing GUID, return it.
+ if (guid) {
+ return guid;
+ }
+
+ // If the URI doesn't have a GUID and we were indicated to create one.
+ if (create) {
+ return this.setGUID(uri);
+ }
+
+ // If the URI doesn't have a GUID and we didn't create one for it.
+ return null;
+ },
+
+ async changeItemID(oldID, newID) {
+ let info = await PlacesSyncUtils.history.fetchURLInfoForGuid(oldID);
+ if (!info) {
+ throw new Error(`Can't change ID for nonexistent history entry ${oldID}`);
+ }
+ this.setGUID(info.url, newID);
+ },
+
+ async getAllIDs() {
+ let urls = await PlacesSyncUtils.history.getAllURLs({
+ since: new Date(Date.now() - THIRTY_DAYS_IN_MS),
+ limit: MAX_HISTORY_UPLOAD,
+ });
+
+ let urlsByGUID = {};
+ for (let url of urls) {
+ if (!this.engine.shouldSyncURL(url)) {
+ continue;
+ }
+ let guid = await this.GUIDForUri(url, true);
+ urlsByGUID[guid] = url;
+ }
+ return urlsByGUID;
+ },
+
+ async applyIncomingBatch(records) {
+ // Convert incoming records to mozIPlaceInfo objects which are applied as
+ // either history additions or removals.
+ let failed = [];
+ let toAdd = [];
+ let toRemove = [];
+ await Async.yieldingForEach(records, async record => {
+ if (record.deleted) {
+ toRemove.push(record);
+ } else {
+ try {
+ let pageInfo = await this._recordToPlaceInfo(record);
+ if (pageInfo) {
+ toAdd.push(pageInfo);
+ }
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.error("Failed to create a place info", ex);
+ this._log.trace("The record that failed", record);
+ failed.push(record.id);
+ }
+ }
+ });
+ if (toAdd.length || toRemove.length) {
+ // We want to notify history observers that a batch operation is underway
+ // so they don't do lots of work for each incoming record.
+ let observers = PlacesUtils.history.getObservers();
+ const notifyHistoryObservers = notification => {
+ for (let observer of observers) {
+ try {
+ observer[notification]();
+ } catch (ex) {
+ // don't log an error - it's not our code that failed and we don't
+ // want an error log written just for this.
+ this._log.info("history observer failed", ex);
+ }
+ }
+ };
+ notifyHistoryObservers("onBeginUpdateBatch");
+ try {
+ if (toRemove.length) {
+ // PlacesUtils.history.remove takes an array of visits to remove,
+ // but the error semantics are tricky - a single "bad" entry will cause
+ // an exception before anything is removed. So we do remove them one at
+ // a time.
+ await Async.yieldingForEach(toRemove, async record => {
+ try {
+ await this.remove(record);
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.error("Failed to delete a place info", ex);
+ this._log.trace("The record that failed", record);
+ failed.push(record.id);
+ }
+ });
+ }
+ for (let chunk of this._generateChunks(toAdd)) {
+ // Per bug 1415560, we ignore any exceptions returned by insertMany
+ // as they are likely to be spurious. We do supply an onError handler
+ // and log the exceptions seen there as they are likely to be
+ // informative, but we still never abort the sync based on them.
+ try {
+ await PlacesUtils.history.insertMany(chunk, null, failedVisit => {
+ this._log.info(
+ "Failed to insert a history record",
+ failedVisit.guid
+ );
+ this._log.trace("The record that failed", failedVisit);
+ failed.push(failedVisit.guid);
+ });
+ } catch (ex) {
+ this._log.info("Failed to insert history records", ex);
+ }
+ }
+ } finally {
+ notifyHistoryObservers("onEndUpdateBatch");
+ }
+ }
+
+ return failed;
+ },
+
+ /**
+ * Returns a generator that splits records into sanely sized chunks suitable
+ * for passing to places to prevent places doing bad things at shutdown.
+ */
+ *_generateChunks(records) {
+ // We chunk based on the number of *visits* inside each record. However,
+ // we do not split a single record into multiple records, because at some
+ // time in the future, we intend to ensure these records are ordered by
+ // lastModified, and advance the engine's timestamp as we process them,
+ // meaning we can resume exactly where we left off next sync - although
+ // currently that's not done, so we will retry the entire batch next sync
+ // if interrupted.
+ // ie, this means that if a single record has more than MAX_VISITS_PER_INSERT
+ // visits, we will call insertMany() with exactly 1 record, but with
+ // more than MAX_VISITS_PER_INSERT visits.
+ let curIndex = 0;
+ this._log.debug(`adding ${records.length} records to history`);
+ while (curIndex < records.length) {
+ Async.checkAppReady(); // may throw if we are shutting down.
+ let toAdd = []; // what we are going to insert.
+ let count = 0; // a counter which tells us when toAdd is full.
+ do {
+ let record = records[curIndex];
+ curIndex += 1;
+ toAdd.push(record);
+ count += record.visits.length;
+ } while (
+ curIndex < records.length &&
+ count + records[curIndex].visits.length <= this.MAX_VISITS_PER_INSERT
+ );
+ this._log.trace(`adding ${toAdd.length} items in this chunk`);
+ yield toAdd;
+ }
+ },
+
+ /* An internal helper to determine if we can add an entry to places.
+ Exists primarily so tests can override it.
+ */
+ _canAddURI(uri) {
+ return PlacesUtils.history.canAddURI(uri);
+ },
+
+ /**
+ * Converts a Sync history record to a mozIPlaceInfo.
+ *
+ * Throws if an invalid record is encountered (invalid URI, etc.),
+ * returns a new PageInfo object if the record is to be applied, null
+ * otherwise (no visits to add, etc.),
+ */
+ async _recordToPlaceInfo(record) {
+ // Sort out invalid URIs and ones Places just simply doesn't want.
+ record.url = PlacesUtils.normalizeToURLOrGUID(record.histUri);
+ record.uri = CommonUtils.makeURI(record.histUri);
+
+ if (!Utils.checkGUID(record.id)) {
+ this._log.warn("Encountered record with invalid GUID: " + record.id);
+ return null;
+ }
+ record.guid = record.id;
+
+ if (
+ !this._canAddURI(record.uri) ||
+ !this.engine.shouldSyncURL(record.uri.spec)
+ ) {
+ this._log.trace(
+ "Ignoring record " +
+ record.id +
+ " with URI " +
+ record.uri.spec +
+ ": can't add this URI."
+ );
+ return null;
+ }
+
+ // We dupe visits by date and type. So an incoming visit that has
+ // the same timestamp and type as a local one won't get applied.
+ // To avoid creating new objects, we rewrite the query result so we
+ // can simply check for containment below.
+ let curVisitsAsArray = [];
+ let curVisits = new Set();
+ try {
+ curVisitsAsArray = await PlacesSyncUtils.history.fetchVisitsForURL(
+ record.histUri
+ );
+ } catch (e) {
+ this._log.error(
+ "Error while fetching visits for URL ${record.histUri}",
+ record.histUri
+ );
+ }
+ let oldestAllowed = PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP;
+ if (curVisitsAsArray.length == 20) {
+ let oldestVisit = curVisitsAsArray[curVisitsAsArray.length - 1];
+ oldestAllowed = PlacesSyncUtils.history.clampVisitDate(
+ PlacesUtils.toDate(oldestVisit.date).getTime()
+ );
+ }
+
+ let i, k;
+ for (i = 0; i < curVisitsAsArray.length; i++) {
+ // Same logic as used in the loop below to generate visitKey.
+ let { date, type } = curVisitsAsArray[i];
+ let dateObj = PlacesUtils.toDate(date);
+ let millis = PlacesSyncUtils.history.clampVisitDate(dateObj).getTime();
+ curVisits.add(`${millis},${type}`);
+ }
+
+ // Walk through the visits, make sure we have sound data, and eliminate
+ // dupes. The latter is done by rewriting the array in-place.
+ for (i = 0, k = 0; i < record.visits.length; i++) {
+ let visit = (record.visits[k] = record.visits[i]);
+
+ if (
+ !visit.date ||
+ typeof visit.date != "number" ||
+ !Number.isInteger(visit.date)
+ ) {
+ this._log.warn(
+ "Encountered record with invalid visit date: " + visit.date
+ );
+ continue;
+ }
+
+ if (
+ !visit.type ||
+ !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)
+ ) {
+ this._log.warn(
+ "Encountered record with invalid visit type: " +
+ visit.type +
+ "; ignoring."
+ );
+ continue;
+ }
+
+ // Dates need to be integers. Future and far past dates are clamped to the
+ // current date and earliest sensible date, respectively.
+ let originalVisitDate = PlacesUtils.toDate(Math.round(visit.date));
+ visit.date = PlacesSyncUtils.history.clampVisitDate(originalVisitDate);
+
+ if (visit.date.getTime() < oldestAllowed) {
+ // Visit is older than the oldest visit we have, and we have so many
+ // visits for this uri that we hit our limit when inserting.
+ continue;
+ }
+ let visitKey = `${visit.date.getTime()},${visit.type}`;
+ if (curVisits.has(visitKey)) {
+ // Visit is a dupe, don't increment 'k' so the element will be
+ // overwritten.
+ continue;
+ }
+
+ // Note the visit key, so that we don't add duplicate visits with
+ // clamped timestamps.
+ curVisits.add(visitKey);
+
+ visit.transition = visit.type;
+ k += 1;
+ }
+ record.visits.length = k; // truncate array
+
+ // No update if there aren't any visits to apply.
+ // History wants at least one visit.
+ // In any case, the only thing we could change would be the title
+ // and that shouldn't change without a visit.
+ if (!record.visits.length) {
+ this._log.trace(
+ "Ignoring record " +
+ record.id +
+ " with URI " +
+ record.uri.spec +
+ ": no visits to add."
+ );
+ return null;
+ }
+
+ // PageInfo is validated using validateItemProperties which does a shallow
+ // copy of the properties. Since record uses getters some of the properties
+ // are not copied over. Thus we create and return a new object.
+ let pageInfo = {
+ title: record.title,
+ url: record.url,
+ guid: record.guid,
+ visits: record.visits,
+ };
+
+ return pageInfo;
+ },
+
+ async remove(record) {
+ this._log.trace("Removing page: " + record.id);
+ let removed = await PlacesUtils.history.remove(record.id);
+ if (removed) {
+ this._log.trace("Removed page: " + record.id);
+ } else {
+ this._log.debug("Page already removed: " + record.id);
+ }
+ },
+
+ async itemExists(id) {
+ return !!(await PlacesSyncUtils.history.fetchURLInfoForGuid(id));
+ },
+
+ async createRecord(id, collection) {
+ let foo = await PlacesSyncUtils.history.fetchURLInfoForGuid(id);
+ let record = new HistoryRec(collection, id);
+ if (foo) {
+ record.histUri = foo.url;
+ record.title = foo.title;
+ record.sortindex = foo.frecency;
+ try {
+ record.visits = await PlacesSyncUtils.history.fetchVisitsForURL(
+ record.histUri
+ );
+ } catch (e) {
+ this._log.error(
+ "Error while fetching visits for URL ${record.histUri}",
+ record.histUri
+ );
+ record.visits = [];
+ }
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ async wipe() {
+ return PlacesSyncUtils.history.wipe();
+ },
+};
+
+function HistoryTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+HistoryTracker.prototype = {
+ __proto__: LegacyTracker.prototype,
+
+ onStart() {
+ this._log.info("Adding Places observer.");
+ PlacesUtils.history.addObserver(this, true);
+ this._placesObserver = new PlacesWeakCallbackWrapper(
+ this.handlePlacesEvents.bind(this)
+ );
+ PlacesObservers.addListener(
+ ["page-visited", "history-cleared"],
+ this._placesObserver
+ );
+ },
+
+ onStop() {
+ this._log.info("Removing Places observer.");
+ PlacesUtils.history.removeObserver(this);
+ if (this._placesObserver) {
+ PlacesObservers.removeListener(
+ ["page-visited", "history-cleared"],
+ this._placesObserver
+ );
+ }
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsINavHistoryObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ async onDeleteAffectsGUID(uri, guid, reason, source, increment) {
+ if (this.ignoreAll || reason == Ci.nsINavHistoryObserver.REASON_EXPIRED) {
+ return;
+ }
+ this._log.trace(source + ": " + uri.spec + ", reason " + reason);
+ const added = await this.addChangedID(guid);
+ if (added) {
+ this.score += increment;
+ }
+ },
+
+ onDeleteVisits(uri, partialRemoval, guid, reason) {
+ this.asyncObserver.enqueueCall(() =>
+ this.onDeleteAffectsGUID(
+ uri,
+ guid,
+ reason,
+ "onDeleteVisits",
+ SCORE_INCREMENT_SMALL
+ )
+ );
+ },
+
+ onDeleteURI(uri, guid, reason) {
+ this.asyncObserver.enqueueCall(() =>
+ this.onDeleteAffectsGUID(
+ uri,
+ guid,
+ reason,
+ "onDeleteURI",
+ SCORE_INCREMENT_XLARGE
+ )
+ );
+ },
+
+ handlePlacesEvents(aEvents) {
+ this.asyncObserver.enqueueCall(() => this._handlePlacesEvents(aEvents));
+ },
+
+ async _handlePlacesEvents(aEvents) {
+ if (this.ignoreAll) {
+ this._log.trace(
+ "ignoreAll: ignoring visits [" +
+ aEvents.map(v => v.guid).join(",") +
+ "]"
+ );
+ return;
+ }
+ for (let event of aEvents) {
+ switch (event.type) {
+ case "page-visited": {
+ this._log.trace("'page-visited': " + event.url);
+ if (
+ this.engine.shouldSyncURL(event.url) &&
+ (await this.addChangedID(event.pageGuid))
+ ) {
+ this.score += SCORE_INCREMENT_SMALL;
+ }
+ break;
+ }
+ case "history-cleared": {
+ this._log.trace("history-cleared");
+ // Note that we're going to trigger a sync, but none of the cleared
+ // pages are tracked, so the deletions will not be propagated.
+ // See Bug 578694.
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ }
+ }
+ },
+
+ onBeginUpdateBatch() {},
+ onEndUpdateBatch() {},
+ onBeforeDeleteURI() {},
+};
diff --git a/services/sync/modules/engines/passwords.js b/services/sync/modules/engines/passwords.js
new file mode 100644
index 0000000000..5b6d2ab408
--- /dev/null
+++ b/services/sync/modules/engines/passwords.js
@@ -0,0 +1,537 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["PasswordEngine", "LoginRec", "PasswordValidator"];
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Collection, CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { SCORE_INCREMENT_XLARGE } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+const { CollectionValidator } = ChromeUtils.import(
+ "resource://services-sync/collection_validator.js"
+);
+const { Store, SyncEngine, LegacyTracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+const { Async } = ChromeUtils.import("resource://services-common/async.js");
+
+const SYNCABLE_LOGIN_FIELDS = [
+ // `nsILoginInfo` fields.
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "username",
+ "password",
+ "usernameField",
+ "passwordField",
+
+ // `nsILoginMetaInfo` fields.
+ "timeCreated",
+ "timePasswordChanged",
+];
+
+// Compares two logins to determine if their syncable fields changed. The login
+// manager fires `modifyLogin` for changes to all fields, including ones we
+// don't sync. In particular, `timeLastUsed` changes shouldn't mark the login
+// for upload; otherwise, we might overwrite changed passwords before they're
+// downloaded (bug 973166).
+function isSyncableChange(oldLogin, newLogin) {
+ oldLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+ newLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+ for (let property of SYNCABLE_LOGIN_FIELDS) {
+ if (oldLogin[property] != newLogin[property]) {
+ return true;
+ }
+ }
+ return false;
+}
+
+function LoginRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+LoginRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Login",
+
+ cleartextToString() {
+ let o = Object.assign({}, this.cleartext);
+ if (o.password) {
+ o.password = "X".repeat(o.password.length);
+ }
+ return JSON.stringify(o);
+ },
+};
+
+Utils.deferGetSet(LoginRec, "cleartext", [
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "username",
+ "password",
+ "usernameField",
+ "passwordField",
+ "timeCreated",
+ "timePasswordChanged",
+]);
+
+function PasswordEngine(service) {
+ SyncEngine.call(this, "Passwords", service);
+}
+PasswordEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: PasswordStore,
+ _trackerObj: PasswordTracker,
+ _recordObj: LoginRec,
+
+ syncPriority: 2,
+
+ // Metadata for syncing is stored in the login manager. We also migrate it
+ // from preferences which can be removed eventually via bug 1651568. Note that
+ // we don't support profile downgrades - once it's migrated, the login manager
+ // becomes the single source of truth.
+ // Note also that the syncID is stored encrypted and null is returned if it
+ // can't be decrypted - this is done for that 'return null' side-effect rather
+ // than due to privacy - we want failure to decrypt the store to be treated as
+ // an engine reset.
+ async getSyncID() {
+ let legacyValue = this._syncID; // the base preference getter.
+ if (legacyValue) {
+ await Services.logins.setSyncID(legacyValue);
+ Svc.Prefs.reset(this.name + ".syncID");
+ this._log.debug(`migrated syncID of ${legacyValue} to the logins store`);
+ return legacyValue;
+ }
+ return Services.logins.getSyncID();
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ // getSyncID above really only exists for this function - the rest of sync
+ // has already moved away from it, and even our tests barely use it.
+ // When we remove the migration code (bug 1651568) we should consider
+ // removing getSyncID() from both here and the login manager, and pushing
+ // this ensureCurrentSyncID() function down into the login manager.
+ let existingSyncID = await this.getSyncID();
+ if (existingSyncID == newSyncID) {
+ return existingSyncID;
+ }
+ this._log.debug("Engine syncIDs: " + [newSyncID, existingSyncID]);
+
+ await Services.logins.setSyncID(newSyncID);
+ await Services.logins.setLastSync(0);
+ return newSyncID;
+ },
+
+ async getLastSync() {
+ let legacyValue = await super.getLastSync();
+ if (legacyValue) {
+ await this.setLastSync(legacyValue);
+ Svc.Prefs.reset(this.name + ".lastSync");
+ this._log.debug(
+ `migrated timestamp of ${legacyValue} to the logins store`
+ );
+ return legacyValue;
+ }
+ return Services.logins.getLastSync();
+ },
+
+ async setLastSync(timestamp) {
+ await Services.logins.setLastSync(timestamp);
+ },
+
+ async _syncFinish() {
+ await SyncEngine.prototype._syncFinish.call(this);
+
+ // Delete the Weave credentials from the server once.
+ if (!Svc.Prefs.get("deletePwdFxA", false)) {
+ try {
+ let ids = [];
+ for (let host of Utils.getSyncCredentialsHosts()) {
+ for (let info of Services.logins.findLogins(host, "", "")) {
+ ids.push(info.QueryInterface(Ci.nsILoginMetaInfo).guid);
+ }
+ }
+ if (ids.length) {
+ let coll = new Collection(this.engineURL, null, this.service);
+ coll.ids = ids;
+ let ret = await coll.delete();
+ this._log.debug("Delete result: " + ret);
+ if (!ret.success && ret.status != 400) {
+ // A non-400 failure means try again next time.
+ return;
+ }
+ } else {
+ this._log.debug("Didn't find any passwords to delete");
+ }
+ // If there were no ids to delete, or we succeeded, or got a 400,
+ // record success.
+ Svc.Prefs.set("deletePwdFxA", true);
+ Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Password deletes failed", ex);
+ }
+ }
+ },
+
+ async _findDupe(item) {
+ let login = this._store._nsLoginInfoFromRecord(item);
+ if (!login) {
+ return null;
+ }
+
+ let logins = Services.logins.findLogins(
+ login.origin,
+ login.formActionOrigin,
+ login.httpRealm
+ );
+
+ await Async.promiseYield(); // Yield back to main thread after synchronous operation.
+
+ // Look for existing logins that match the origin, but ignore the password.
+ for (let local of logins) {
+ if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
+ return local.guid;
+ }
+ }
+
+ return null;
+ },
+
+ async pullAllChanges() {
+ let changes = {};
+ let ids = await this._store.getAllIDs();
+ for (let [id, info] of Object.entries(ids)) {
+ changes[id] = info.timePasswordChanged / 1000;
+ }
+ return changes;
+ },
+
+ getValidator() {
+ return new PasswordValidator();
+ },
+};
+
+function PasswordStore(name, engine) {
+ Store.call(this, name, engine);
+ this._nsLoginInfo = new Components.Constructor(
+ "@mozilla.org/login-manager/loginInfo;1",
+ Ci.nsILoginInfo,
+ "init"
+ );
+}
+PasswordStore.prototype = {
+ __proto__: Store.prototype,
+
+ _newPropertyBag() {
+ return Cc["@mozilla.org/hash-property-bag;1"].createInstance(
+ Ci.nsIWritablePropertyBag2
+ );
+ },
+
+ /**
+ * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
+ */
+ _nsLoginInfoFromRecord(record) {
+ function nullUndefined(x) {
+ return x == undefined ? null : x;
+ }
+
+ function stringifyNullUndefined(x) {
+ return x == undefined || x == null ? "" : x;
+ }
+
+ if (record.formSubmitURL && record.httpRealm) {
+ this._log.warn(
+ "Record " +
+ record.id +
+ " has both formSubmitURL and httpRealm. Skipping."
+ );
+ return null;
+ }
+
+ // Passing in "undefined" results in an empty string, which later
+ // counts as a value. Explicitly `|| null` these fields according to JS
+ // truthiness. Records with empty strings or null will be unmolested.
+ let info = new this._nsLoginInfo(
+ record.hostname,
+ nullUndefined(record.formSubmitURL),
+ nullUndefined(record.httpRealm),
+ stringifyNullUndefined(record.username),
+ record.password,
+ record.usernameField,
+ record.passwordField
+ );
+
+ info.QueryInterface(Ci.nsILoginMetaInfo);
+ info.guid = record.id;
+ if (record.timeCreated && !isNaN(new Date(record.timeCreated).getTime())) {
+ info.timeCreated = record.timeCreated;
+ }
+ if (
+ record.timePasswordChanged &&
+ !isNaN(new Date(record.timePasswordChanged).getTime())
+ ) {
+ info.timePasswordChanged = record.timePasswordChanged;
+ }
+
+ return info;
+ },
+
+ async _getLoginFromGUID(id) {
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", id);
+
+ let logins = Services.logins.searchLogins(prop);
+ await Async.promiseYield(); // Yield back to main thread after synchronous operation.
+
+ if (logins.length > 0) {
+ this._log.trace(logins.length + " items matching " + id + " found.");
+ return logins[0];
+ }
+
+ this._log.trace("No items matching " + id + " found. Ignoring");
+ return null;
+ },
+
+ async getAllIDs() {
+ let items = {};
+ let logins = Services.logins.getAllLogins();
+
+ for (let i = 0; i < logins.length; i++) {
+ // Skip over Weave password/passphrase entries.
+ let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo);
+ if (Utils.getSyncCredentialsHosts().has(metaInfo.origin)) {
+ continue;
+ }
+
+ items[metaInfo.guid] = metaInfo;
+ }
+
+ return items;
+ },
+
+ async changeItemID(oldID, newID) {
+ this._log.trace("Changing item ID: " + oldID + " to " + newID);
+
+ let oldLogin = await this._getLoginFromGUID(oldID);
+ if (!oldLogin) {
+ this._log.trace("Can't change item ID: item doesn't exist");
+ return;
+ }
+ if (await this._getLoginFromGUID(newID)) {
+ this._log.trace("Can't change item ID: new ID already in use");
+ return;
+ }
+
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", newID);
+
+ Services.logins.modifyLogin(oldLogin, prop);
+ },
+
+ async itemExists(id) {
+ return !!(await this._getLoginFromGUID(id));
+ },
+
+ async createRecord(id, collection) {
+ let record = new LoginRec(collection, id);
+ let login = await this._getLoginFromGUID(id);
+
+ if (!login) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.hostname = login.origin;
+ record.formSubmitURL = login.formActionOrigin;
+ record.httpRealm = login.httpRealm;
+ record.username = login.username;
+ record.password = login.password;
+ record.usernameField = login.usernameField;
+ record.passwordField = login.passwordField;
+
+ // Optional fields.
+ login.QueryInterface(Ci.nsILoginMetaInfo);
+ record.timeCreated = login.timeCreated;
+ record.timePasswordChanged = login.timePasswordChanged;
+
+ return record;
+ },
+
+ async create(record) {
+ let login = this._nsLoginInfoFromRecord(record);
+ if (!login) {
+ return;
+ }
+
+ this._log.trace("Adding login for " + record.hostname);
+ this._log.trace(
+ "httpRealm: " +
+ JSON.stringify(login.httpRealm) +
+ "; " +
+ "formSubmitURL: " +
+ JSON.stringify(login.formActionOrigin)
+ );
+ Services.logins.addLogin(login);
+ },
+
+ async remove(record) {
+ this._log.trace("Removing login " + record.id);
+
+ let loginItem = await this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.trace("Asked to remove record that doesn't exist, ignoring");
+ return;
+ }
+
+ Services.logins.removeLogin(loginItem);
+ },
+
+ async update(record) {
+ let loginItem = await this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.trace("Skipping update for unknown item: " + record.hostname);
+ return;
+ }
+
+ this._log.trace("Updating " + record.hostname);
+ let newinfo = this._nsLoginInfoFromRecord(record);
+ if (!newinfo) {
+ return;
+ }
+
+ Services.logins.modifyLogin(loginItem, newinfo);
+ },
+
+ async wipe() {
+ Services.logins.removeAllUserFacingLogins();
+ },
+};
+
+function PasswordTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+PasswordTracker.prototype = {
+ __proto__: LegacyTracker.prototype,
+
+ onStart() {
+ Svc.Obs.add("passwordmgr-storage-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ Svc.Obs.remove("passwordmgr-storage-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+
+ // A single add, remove or change or removing all items
+ // will trigger a sync for MULTI_DEVICE.
+ switch (data) {
+ case "modifyLogin": {
+ subject.QueryInterface(Ci.nsIArrayExtensions);
+ let oldLogin = subject.GetElementAt(0);
+ let newLogin = subject.GetElementAt(1);
+ if (!isSyncableChange(oldLogin, newLogin)) {
+ this._log.trace(`${data}: Ignoring change for ${newLogin.guid}`);
+ break;
+ }
+ const tracked = await this._trackLogin(newLogin);
+ if (tracked) {
+ this._log.trace(`${data}: Tracking change for ${newLogin.guid}`);
+ }
+ break;
+ }
+
+ case "addLogin":
+ case "removeLogin":
+ subject
+ .QueryInterface(Ci.nsILoginMetaInfo)
+ .QueryInterface(Ci.nsILoginInfo);
+ const tracked = await this._trackLogin(subject);
+ if (tracked) {
+ this._log.trace(data + ": " + subject.guid);
+ }
+ break;
+
+ // Bug 1613620: We iterate through the removed logins and track them to ensure
+ // the logins are deleted across synced devices/accounts
+ case "removeAllLogins":
+ subject.QueryInterface(Ci.nsIArrayExtensions);
+ let count = subject.Count();
+ for (let i = 0; i < count; i++) {
+ let currentSubject = subject.GetElementAt(i);
+ let tracked = await this._trackLogin(currentSubject);
+ if (tracked) {
+ this._log.trace(data + ": " + currentSubject.guid);
+ }
+ }
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ },
+
+ async _trackLogin(login) {
+ if (Utils.getSyncCredentialsHosts().has(login.origin)) {
+ // Skip over Weave password/passphrase changes.
+ return false;
+ }
+ const added = await this.addChangedID(login.guid);
+ if (!added) {
+ return false;
+ }
+ this.score += SCORE_INCREMENT_XLARGE;
+ return true;
+ },
+};
+
+class PasswordValidator extends CollectionValidator {
+ constructor() {
+ super("passwords", "id", [
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "password",
+ "passwordField",
+ "username",
+ "usernameField",
+ ]);
+ }
+
+ getClientItems() {
+ let logins = Services.logins.getAllLogins();
+ let syncHosts = Utils.getSyncCredentialsHosts();
+ let result = logins
+ .map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
+ .filter(l => !syncHosts.has(l.origin));
+ return Promise.resolve(result);
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ hostname: item.hostname,
+ formSubmitURL: item.formSubmitURL,
+ httpRealm: item.httpRealm,
+ password: item.password,
+ passwordField: item.passwordField,
+ username: item.username,
+ usernameField: item.usernameField,
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ return Object.assign({ guid: item.id }, item);
+ }
+}
diff --git a/services/sync/modules/engines/prefs.js b/services/sync/modules/engines/prefs.js
new file mode 100644
index 0000000000..e4a3b99b29
--- /dev/null
+++ b/services/sync/modules/engines/prefs.js
@@ -0,0 +1,436 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["PrefsEngine", "PrefRec", "PREFS_GUID"];
+
+const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
+
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Preferences } = ChromeUtils.import(
+ "resource://gre/modules/Preferences.jsm"
+);
+const { Store, SyncEngine, Tracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+const { SCORE_INCREMENT_XLARGE } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+const { CommonUtils } = ChromeUtils.import(
+ "resource://services-common/utils.js"
+);
+
+XPCOMUtils.defineLazyGetter(this, "PREFS_GUID", () =>
+ CommonUtils.encodeBase64URL(Services.appinfo.ID)
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "AddonManager",
+ "resource://gre/modules/AddonManager.jsm"
+);
+
+// In bug 1538015, we decided that it isn't always safe to allow all "incoming"
+// preferences to be applied locally. So we have introduced another preference,
+// which if false (the default) will ignore all incoming preferences which don't
+// already have the "control" preference locally set. If this new
+// preference is set to true, then we continue our old behavior of allowing all
+// preferences to be updated, even those which don't already have a local
+// "control" pref.
+const PREF_SYNC_PREFS_ARBITRARY =
+ "services.sync.prefs.dangerously_allow_arbitrary";
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "ALLOW_ARBITRARY",
+ PREF_SYNC_PREFS_ARBITRARY
+);
+
+// The SUMO supplied URL we log with more information about how custom prefs can
+// continue to be synced. SUMO have told us that this URL will remain "stable".
+const PREFS_DOC_URL_TEMPLATE =
+ "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/sync-custom-preferences";
+XPCOMUtils.defineLazyGetter(this, "PREFS_DOC_URL", () =>
+ Services.urlFormatter.formatURL(PREFS_DOC_URL_TEMPLATE)
+);
+
+// Check for a local control pref or PREF_SYNC_PREFS_ARBITRARY
+this.isAllowedPrefName = function(prefName) {
+ if (prefName == PREF_SYNC_PREFS_ARBITRARY) {
+ return false; // never allow this.
+ }
+ if (ALLOW_ARBITRARY) {
+ // user has set the "dangerous" pref, so everything is allowed.
+ return true;
+ }
+ // The pref must already have a control pref set, although it doesn't matter
+ // here whether that value is true or false. We can't use prefHasUserValue
+ // here because we also want to check prefs still with default values.
+ try {
+ Services.prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + prefName);
+ // pref exists!
+ return true;
+ } catch (_) {
+ return false;
+ }
+};
+
+function PrefRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+PrefRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Pref",
+};
+
+Utils.deferGetSet(PrefRec, "cleartext", ["value"]);
+
+function PrefsEngine(service) {
+ SyncEngine.call(this, "Prefs", service);
+}
+PrefsEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: PrefStore,
+ _trackerObj: PrefTracker,
+ _recordObj: PrefRec,
+ version: 2,
+
+ syncPriority: 1,
+ allowSkippedRecord: false,
+
+ async getChangedIDs() {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified) {
+ changedIDs[PREFS_GUID] = 0;
+ }
+ return changedIDs;
+ },
+
+ async _wipeClient() {
+ await SyncEngine.prototype._wipeClient.call(this);
+ this.justWiped = true;
+ },
+
+ async _reconcile(item) {
+ // Apply the incoming item if we don't care about the local data
+ if (this.justWiped) {
+ this.justWiped = false;
+ return true;
+ }
+ return SyncEngine.prototype._reconcile.call(this, item);
+ },
+
+ async trackRemainingChanges() {
+ if (this._modified.count() > 0) {
+ this._tracker.modified = true;
+ }
+ },
+};
+
+// We don't use services.sync.engine.tabs.filteredUrls since it includes
+// about: pages and the like, which we want to be syncable in preferences.
+// Blob and moz-extension uris are never safe to sync, so we limit our check
+// to those.
+const UNSYNCABLE_URL_REGEXP = /^(moz-extension|blob):/i;
+function isUnsyncableURLPref(prefName) {
+ if (Services.prefs.getPrefType(prefName) != Ci.nsIPrefBranch.PREF_STRING) {
+ return false;
+ }
+ const prefValue = Services.prefs.getStringPref(prefName, "");
+ return UNSYNCABLE_URL_REGEXP.test(prefValue);
+}
+
+function PrefStore(name, engine) {
+ Store.call(this, name, engine);
+ Svc.Obs.add(
+ "profile-before-change",
+ function() {
+ this.__prefs = null;
+ },
+ this
+ );
+}
+PrefStore.prototype = {
+ __proto__: Store.prototype,
+
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ _getSyncPrefs() {
+ let syncPrefs = Services.prefs
+ .getBranch(PREF_SYNC_PREFS_PREFIX)
+ .getChildList("")
+ .filter(pref => isAllowedPrefName(pref) && !isUnsyncableURLPref(pref));
+ // Also sync preferences that determine which prefs get synced.
+ let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
+ return controlPrefs.concat(syncPrefs);
+ },
+
+ _isSynced(pref) {
+ if (pref.startsWith(PREF_SYNC_PREFS_PREFIX)) {
+ // this is an incoming control pref, which is ignored if there's not already
+ // a local control pref for the preference.
+ let controlledPref = pref.slice(PREF_SYNC_PREFS_PREFIX.length);
+ return isAllowedPrefName(controlledPref);
+ }
+
+ // This is the pref itself - it must be both allowed, and have a control
+ // pref which is true.
+ if (!this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false)) {
+ return false;
+ }
+ return isAllowedPrefName(pref);
+ },
+
+ _getAllPrefs() {
+ let values = {};
+ for (let pref of this._getSyncPrefs()) {
+ // Note: _isSynced doesn't call isUnsyncableURLPref since it would cause
+ // us not to apply (syncable) changes to preferences that are set locally
+ // which have unsyncable urls.
+ if (this._isSynced(pref) && !isUnsyncableURLPref(pref)) {
+ // Missing and default prefs get the null value.
+ values[pref] = this._prefs.isSet(pref)
+ ? this._prefs.get(pref, null)
+ : null;
+ }
+ }
+ return values;
+ },
+
+ _setAllPrefs(values) {
+ const selectedThemeIDPref = "extensions.activeThemeID";
+ let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
+ let selectedThemeIDAfter = selectedThemeIDBefore;
+
+ // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
+ // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
+ let prefs = Object.keys(values).sort(
+ a => -a.indexOf(PREF_SYNC_PREFS_PREFIX)
+ );
+ for (let pref of prefs) {
+ let value = values[pref];
+ if (!this._isSynced(pref)) {
+ // An extra complication just so we can warn when we decline to sync a
+ // preference due to no local control pref.
+ if (!pref.startsWith(PREF_SYNC_PREFS_PREFIX)) {
+ // this is an incoming pref - if the incoming value is not null and
+ // there's no local control pref, then it means we would have previously
+ // applied a value, but now will decline to.
+ // We need to check this here rather than in _isSynced because the
+ // default list of prefs we sync has changed, so we don't want to report
+ // this message when we wouldn't have actually applied a value.
+ // We should probably remove all of this in ~ Firefox 80.
+ if (value !== null) {
+ // null means "use the default value"
+ let controlPref = PREF_SYNC_PREFS_PREFIX + pref;
+ let controlPrefExists;
+ try {
+ Services.prefs.getBoolPref(controlPref);
+ controlPrefExists = true;
+ } catch (ex) {
+ controlPrefExists = false;
+ }
+ if (!controlPrefExists) {
+ // This is a long message and written to both the sync log and the
+ // console, but note that users who have not customized the control
+ // prefs will never see this.
+ let msg =
+ `Not syncing the preference '${pref}' because it has no local ` +
+ `control preference (${PREF_SYNC_PREFS_PREFIX}${pref}) and ` +
+ `the preference ${PREF_SYNC_PREFS_ARBITRARY} isn't true. ` +
+ `See ${PREFS_DOC_URL} for more information`;
+ console.warn(msg);
+ this._log.warn(msg);
+ }
+ }
+ }
+ continue;
+ }
+
+ if (typeof value == "string" && UNSYNCABLE_URL_REGEXP.test(value)) {
+ this._log.trace(`Skipping incoming unsyncable url for pref: ${pref}`);
+ continue;
+ }
+
+ switch (pref) {
+ // Some special prefs we don't want to set directly.
+ case selectedThemeIDPref:
+ selectedThemeIDAfter = value;
+ break;
+
+ // default is to just set the pref
+ default:
+ if (value == null) {
+ // Pref has gone missing. The best we can do is reset it.
+ this._prefs.reset(pref);
+ } else {
+ try {
+ this._prefs.set(pref, value);
+ } catch (ex) {
+ this._log.trace(`Failed to set pref: ${pref}`, ex);
+ }
+ }
+ }
+ }
+ // Themes are a little messy. Themes which have been installed are handled
+ // by the addons engine - but default themes aren't seen by that engine.
+ // So if there's a new default theme ID and that ID corresponds to a
+ // system addon, then we arrange to enable that addon here.
+ if (selectedThemeIDBefore != selectedThemeIDAfter) {
+ this._maybeEnableBuiltinTheme(selectedThemeIDAfter).catch(e => {
+ this._log.error("Failed to maybe update the default theme", e);
+ });
+ }
+ },
+
+ async _maybeEnableBuiltinTheme(themeId) {
+ let addon = null;
+ try {
+ addon = await AddonManager.getAddonByID(themeId);
+ } catch (ex) {
+ this._log.trace(
+ `There's no addon with ID '${themeId} - it can't be a builtin theme`
+ );
+ return;
+ }
+ if (addon && addon.isBuiltin && addon.type == "theme") {
+ this._log.trace(`Enabling builtin theme '${themeId}'`);
+ await addon.enable();
+ } else {
+ this._log.trace(
+ `Have incoming theme ID of '${themeId}' but it's not a builtin theme`
+ );
+ }
+ },
+
+ async getAllIDs() {
+ /* We store all prefs in just one WBO, with just one GUID */
+ let allprefs = {};
+ allprefs[PREFS_GUID] = true;
+ return allprefs;
+ },
+
+ async changeItemID(oldID, newID) {
+ this._log.trace("PrefStore GUID is constant!");
+ },
+
+ async itemExists(id) {
+ return id === PREFS_GUID;
+ },
+
+ async createRecord(id, collection) {
+ let record = new PrefRec(collection, id);
+
+ if (id == PREFS_GUID) {
+ record.value = this._getAllPrefs();
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ async create(record) {
+ this._log.trace("Ignoring create request");
+ },
+
+ async remove(record) {
+ this._log.trace("Ignoring remove request");
+ },
+
+ async update(record) {
+ // Silently ignore pref updates that are for other apps.
+ if (record.id != PREFS_GUID) {
+ return;
+ }
+
+ this._log.trace("Received pref updates, applying...");
+ this._setAllPrefs(record.value);
+ },
+
+ async wipe() {
+ this._log.trace("Ignoring wipe request");
+ },
+};
+
+function PrefTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ this._ignoreAll = false;
+ Svc.Obs.add("profile-before-change", this.asyncObserver);
+}
+PrefTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ get ignoreAll() {
+ return this._ignoreAll;
+ },
+
+ set ignoreAll(value) {
+ this._ignoreAll = value;
+ },
+
+ get modified() {
+ return Svc.Prefs.get("engine.prefs.modified", false);
+ },
+ set modified(value) {
+ Svc.Prefs.set("engine.prefs.modified", value);
+ },
+
+ clearChangedIDs: function clearChangedIDs() {
+ this.modified = false;
+ },
+
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ onStart() {
+ Services.prefs.addObserver("", this.asyncObserver);
+ },
+
+ onStop() {
+ this.__prefs = null;
+ Services.prefs.removeObserver("", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "profile-before-change":
+ await this.stop();
+ break;
+ case "nsPref:changed":
+ if (this.ignoreAll) {
+ break;
+ }
+ // Trigger a sync for MULTI-DEVICE for a change that determines
+ // which prefs are synced or a regular pref change.
+ if (
+ data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
+ this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)
+ ) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ this.modified = true;
+ this._log.trace("Preference " + data + " changed");
+ }
+ break;
+ }
+ },
+};
diff --git a/services/sync/modules/engines/tabs.js b/services/sync/modules/engines/tabs.js
new file mode 100644
index 0000000000..99ff59ca8b
--- /dev/null
+++ b/services/sync/modules/engines/tabs.js
@@ -0,0 +1,395 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"];
+
+const TABS_TTL = 31622400; // 366 days (1 leap year).
+const TAB_ENTRIES_LIMIT = 5; // How many URLs to include in tab history.
+
+const { XPCOMUtils } = ChromeUtils.import(
+ "resource://gre/modules/XPCOMUtils.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
+const { Store, SyncEngine, Tracker } = ChromeUtils.import(
+ "resource://services-sync/engines.js"
+);
+const { CryptoWrapper } = ChromeUtils.import(
+ "resource://services-sync/record.js"
+);
+const { Svc, Utils } = ChromeUtils.import("resource://services-sync/util.js");
+const { SCORE_INCREMENT_SMALL, URI_LENGTH_MAX } = ChromeUtils.import(
+ "resource://services-sync/constants.js"
+);
+
+ChromeUtils.defineModuleGetter(
+ this,
+ "PrivateBrowsingUtils",
+ "resource://gre/modules/PrivateBrowsingUtils.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ this,
+ "SessionStore",
+ "resource:///modules/sessionstore/SessionStore.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+ PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+});
+
+function TabSetRecord(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+TabSetRecord.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Tabs",
+ ttl: TABS_TTL,
+};
+
+Utils.deferGetSet(TabSetRecord, "cleartext", ["clientName", "tabs"]);
+
+function TabEngine(service) {
+ SyncEngine.call(this, "Tabs", service);
+}
+TabEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: TabStore,
+ _trackerObj: TabTracker,
+ _recordObj: TabSetRecord,
+
+ syncPriority: 3,
+
+ async initialize() {
+ await SyncEngine.prototype.initialize.call(this);
+
+ // Reset the client on every startup so that we fetch recent tabs.
+ await this._resetClient();
+ },
+
+ async getChangedIDs() {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified) {
+ changedIDs[this.service.clientsEngine.localID] = 0;
+ }
+ return changedIDs;
+ },
+
+ // API for use by Sync UI code to give user choices of tabs to open.
+ getAllClients() {
+ return this._store._remoteClients;
+ },
+
+ getClientById(id) {
+ return this._store._remoteClients[id];
+ },
+
+ async _resetClient() {
+ await SyncEngine.prototype._resetClient.call(this);
+ await this._store.wipe();
+ this._tracker.modified = true;
+ },
+
+ async removeClientData() {
+ let url = this.engineURL + "/" + this.service.clientsEngine.localID;
+ await this.service.resource(url).delete();
+ },
+
+ async _reconcile(item) {
+ // Skip our own record.
+ // TabStore.itemExists tests only against our local client ID.
+ if (await this._store.itemExists(item.id)) {
+ this._log.trace(
+ "Ignoring incoming tab item because of its id: " + item.id
+ );
+ return false;
+ }
+
+ return SyncEngine.prototype._reconcile.call(this, item);
+ },
+
+ async trackRemainingChanges() {
+ if (this._modified.count() > 0) {
+ this._tracker.modified = true;
+ }
+ },
+};
+
+function TabStore(name, engine) {
+ Store.call(this, name, engine);
+}
+TabStore.prototype = {
+ __proto__: Store.prototype,
+
+ async itemExists(id) {
+ return id == this.engine.service.clientsEngine.localID;
+ },
+
+ getWindowEnumerator() {
+ return Services.wm.getEnumerator("navigator:browser");
+ },
+
+ shouldSkipWindow(win) {
+ return win.closed || PrivateBrowsingUtils.isWindowPrivate(win);
+ },
+
+ getTabState(tab) {
+ return JSON.parse(SessionStore.getTabState(tab));
+ },
+
+ async getAllTabs(filter) {
+ let filteredUrls = new RegExp(
+ Svc.Prefs.get("engine.tabs.filteredUrls"),
+ "i"
+ );
+
+ let allTabs = [];
+
+ for (let win of this.getWindowEnumerator()) {
+ if (this.shouldSkipWindow(win)) {
+ continue;
+ }
+
+ for (let tab of win.gBrowser.tabs) {
+ let tabState = this.getTabState(tab);
+
+ // Make sure there are history entries to look at.
+ if (!tabState || !tabState.entries.length) {
+ continue;
+ }
+
+ let acceptable = !filter
+ ? url => url
+ : url => url && !filteredUrls.test(url);
+
+ let entries = tabState.entries;
+ let index = tabState.index;
+ let current = entries[index - 1];
+
+ // We ignore the tab completely if the current entry url is
+ // not acceptable (we need something accurate to open).
+ if (!acceptable(current.url)) {
+ continue;
+ }
+
+ if (current.url.length > URI_LENGTH_MAX) {
+ this._log.trace("Skipping over-long URL.");
+ continue;
+ }
+
+ // The element at `index` is the current page. Previous URLs were
+ // previously visited URLs; subsequent URLs are in the 'forward' stack,
+ // which we can't represent in Sync, so we truncate here.
+ let candidates =
+ entries.length == index ? entries : entries.slice(0, index);
+
+ let urls = candidates
+ .map(entry => entry.url)
+ .filter(acceptable)
+ .reverse(); // Because Sync puts current at index 0, and history after.
+
+ // Truncate if necessary.
+ if (urls.length > TAB_ENTRIES_LIMIT) {
+ urls.length = TAB_ENTRIES_LIMIT;
+ }
+
+ // tabState has .image, but it's a large data: url. So we ask the favicon service for the url.
+ let icon = "";
+ try {
+ let iconData = await PlacesUtils.promiseFaviconData(urls[0]);
+ icon = iconData.uri.spec;
+ } catch (ex) {
+ this._log.trace(`Failed to fetch favicon for ${urls[0]}`, ex);
+ }
+ allTabs.push({
+ title: current.title || "",
+ urlHistory: urls,
+ icon,
+ lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000),
+ });
+ }
+ }
+
+ return allTabs;
+ },
+
+ async createRecord(id, collection) {
+ let record = new TabSetRecord(collection, id);
+ record.clientName = this.engine.service.clientsEngine.localName;
+
+ // Sort tabs in descending-used order to grab the most recently used
+ let tabs = (await this.getAllTabs(true)).sort(function(a, b) {
+ return b.lastUsed - a.lastUsed;
+ });
+ const maxPayloadSize = this.engine.service.getMemcacheMaxRecordPayloadSize();
+ let records = Utils.tryFitItems(tabs, maxPayloadSize);
+
+ if (records.length != tabs.length) {
+ this._log.warn(
+ `Can't fit all tabs in sync payload: have ${tabs.length}, but can only fit ${records.length}.`
+ );
+ }
+
+ if (this._log.level <= Log.Level.Trace) {
+ records.forEach(tab => {
+ this._log.trace("Wrapping tab: ", tab);
+ });
+ }
+
+ record.tabs = records;
+ return record;
+ },
+
+ async getAllIDs() {
+ // Don't report any tabs if all windows are in private browsing for
+ // first syncs.
+ let ids = {};
+ let allWindowsArePrivate = false;
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ if (PrivateBrowsingUtils.isWindowPrivate(win)) {
+ // Ensure that at least there is a private window.
+ allWindowsArePrivate = true;
+ } else {
+ // If there is a not private windown then finish and continue.
+ allWindowsArePrivate = false;
+ break;
+ }
+ }
+
+ if (
+ allWindowsArePrivate &&
+ !PrivateBrowsingUtils.permanentPrivateBrowsing
+ ) {
+ return ids;
+ }
+
+ ids[this.engine.service.clientsEngine.localID] = true;
+ return ids;
+ },
+
+ async wipe() {
+ this._remoteClients = {};
+ },
+
+ async create(record) {
+ this._log.debug("Adding remote tabs from " + record.id);
+ this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
+ lastModified: record.modified,
+ });
+ },
+
+ async update(record) {
+ this._log.trace("Ignoring tab updates as local ones win");
+ },
+};
+
+function TabTracker(name, engine) {
+ Tracker.call(this, name, engine);
+
+ // Make sure "this" pointer is always set correctly for event listeners.
+ this.onTab = Utils.bind2(this, this.onTab);
+ this._unregisterListeners = Utils.bind2(this, this._unregisterListeners);
+}
+TabTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
+
+ clearChangedIDs() {
+ this.modified = false;
+ },
+
+ _topics: ["pageshow", "TabOpen", "TabClose", "TabSelect"],
+
+ _registerListenersForWindow(window) {
+ this._log.trace("Registering tab listeners in window");
+ for (let topic of this._topics) {
+ window.addEventListener(topic, this.onTab);
+ }
+ window.addEventListener("unload", this._unregisterListeners);
+ // If it's got a tab browser we can listen for things like navigation.
+ if (window.gBrowser) {
+ window.gBrowser.addProgressListener(this);
+ }
+ },
+
+ _unregisterListeners(event) {
+ this._unregisterListenersForWindow(event.target);
+ },
+
+ _unregisterListenersForWindow(window) {
+ this._log.trace("Removing tab listeners in window");
+ window.removeEventListener("unload", this._unregisterListeners);
+ for (let topic of this._topics) {
+ window.removeEventListener(topic, this.onTab);
+ }
+ if (window.gBrowser) {
+ window.gBrowser.removeProgressListener(this);
+ }
+ },
+
+ onStart() {
+ Svc.Obs.add("domwindowopened", this.asyncObserver);
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ this._registerListenersForWindow(win);
+ }
+ },
+
+ onStop() {
+ Svc.Obs.remove("domwindowopened", this.asyncObserver);
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ this._unregisterListenersForWindow(win);
+ }
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "domwindowopened":
+ let onLoad = () => {
+ subject.removeEventListener("load", onLoad);
+ // Only register after the window is done loading to avoid unloads.
+ this._registerListenersForWindow(subject);
+ };
+
+ // Add tab listeners now that a window has opened.
+ subject.addEventListener("load", onLoad);
+ break;
+ }
+ },
+
+ onTab(event) {
+ if (event.originalTarget.linkedBrowser) {
+ let browser = event.originalTarget.linkedBrowser;
+ if (
+ PrivateBrowsingUtils.isBrowserPrivate(browser) &&
+ !PrivateBrowsingUtils.permanentPrivateBrowsing
+ ) {
+ this._log.trace("Ignoring tab event from private browsing.");
+ return;
+ }
+ }
+
+ this._log.trace("onTab event: " + event.type);
+ this.modified = true;
+
+ // For page shows, bump the score 10% of the time, emulating a partial
+ // score. We don't want to sync too frequently. For all other page
+ // events, always bump the score.
+ if (event.type != "pageshow" || Math.random() < 0.1) {
+ this.score += SCORE_INCREMENT_SMALL;
+ }
+ },
+
+ // web progress listeners.
+ onLocationChange(webProgress, request, location, flags) {
+ // We only care about top-level location changes which are not in the same
+ // document.
+ if (
+ webProgress.isTopLevel &&
+ (flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0
+ ) {
+ this.modified = true;
+ }
+ },
+};