summaryrefslogtreecommitdiffstats
path: root/services/sync/modules
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:47:29 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:47:29 +0000
commit0ebf5bdf043a27fd3dfb7f92e0cb63d88954c44d (patch)
treea31f07c9bcca9d56ce61e9a1ffd30ef350d513aa /services/sync/modules
parentInitial commit. (diff)
downloadfirefox-esr-upstream/115.8.0esr.tar.xz
firefox-esr-upstream/115.8.0esr.zip
Adding upstream version 115.8.0esr.upstream/115.8.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'services/sync/modules')
-rw-r--r--services/sync/modules/SyncDisconnect.sys.mjs240
-rw-r--r--services/sync/modules/SyncedTabs.sys.mjs332
-rw-r--r--services/sync/modules/UIState.sys.mjs287
-rw-r--r--services/sync/modules/addonsreconciler.sys.mjs584
-rw-r--r--services/sync/modules/addonutils.sys.mjs390
-rw-r--r--services/sync/modules/bridged_engine.sys.mjs499
-rw-r--r--services/sync/modules/collection_validator.sys.mjs267
-rw-r--r--services/sync/modules/constants.sys.mjs133
-rw-r--r--services/sync/modules/doctor.sys.mjs188
-rw-r--r--services/sync/modules/engines.sys.mjs2260
-rw-r--r--services/sync/modules/engines/addons.sys.mjs820
-rw-r--r--services/sync/modules/engines/bookmarks.sys.mjs953
-rw-r--r--services/sync/modules/engines/clients.sys.mjs1123
-rw-r--r--services/sync/modules/engines/extension-storage.sys.mjs303
-rw-r--r--services/sync/modules/engines/forms.sys.mjs298
-rw-r--r--services/sync/modules/engines/history.sys.mjs585
-rw-r--r--services/sync/modules/engines/passwords.sys.mjs565
-rw-r--r--services/sync/modules/engines/prefs.sys.mjs467
-rw-r--r--services/sync/modules/engines/tabs.sys.mjs624
-rw-r--r--services/sync/modules/keys.sys.mjs166
-rw-r--r--services/sync/modules/main.sys.mjs25
-rw-r--r--services/sync/modules/policies.sys.mjs1057
-rw-r--r--services/sync/modules/record.sys.mjs1335
-rw-r--r--services/sync/modules/resource.sys.mjs292
-rw-r--r--services/sync/modules/service.sys.mjs1630
-rw-r--r--services/sync/modules/stages/declined.sys.mjs78
-rw-r--r--services/sync/modules/stages/enginesync.sys.mjs400
-rw-r--r--services/sync/modules/status.sys.mjs135
-rw-r--r--services/sync/modules/sync_auth.sys.mjs656
-rw-r--r--services/sync/modules/telemetry.sys.mjs1313
-rw-r--r--services/sync/modules/util.sys.mjs783
31 files changed, 18788 insertions, 0 deletions
diff --git a/services/sync/modules/SyncDisconnect.sys.mjs b/services/sync/modules/SyncDisconnect.sys.mjs
new file mode 100644
index 0000000000..5829085084
--- /dev/null
+++ b/services/sync/modules/SyncDisconnect.sys.mjs
@@ -0,0 +1,240 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// This module provides a facility for disconnecting Sync and FxA, optionally
+// sanitizing profile data as part of the process.
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs",
+ Log: "resource://gre/modules/Log.sys.mjs",
+ Sanitizer: "resource:///modules/Sanitizer.sys.mjs",
+ Utils: "resource://services-sync/util.sys.mjs",
+ setTimeout: "resource://gre/modules/Timer.sys.mjs",
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "FxAccountsCommon", function () {
+ return ChromeUtils.import("resource://gre/modules/FxAccountsCommon.js");
+});
+
+export const SyncDisconnectInternal = {
+ lockRetryInterval: 1000, // wait 1 seconds before trying for the lock again.
+ lockRetryCount: 120, // Try 120 times (==2 mins) before giving up in disgust.
+ promiseDisconnectFinished: null, // If we are sanitizing, a promise for completion.
+
+ // mocked by tests.
+ getWeave() {
+ return ChromeUtils.importESModule("resource://services-sync/main.sys.mjs")
+ .Weave;
+ },
+
+ // Returns a promise that resolves when we are not syncing, waiting until
+ // a current Sync completes if necessary. Resolves with true if we
+ // successfully waited, in which case the sync lock will have been taken to
+ // ensure future syncs don't state, or resolves with false if we gave up
+ // waiting for the sync to complete (in which case we didn't take a lock -
+ // but note that Sync probably remains locked in this case regardless.)
+ async promiseNotSyncing(abortController) {
+ let weave = this.getWeave();
+ let log = lazy.Log.repository.getLogger("Sync.Service");
+ // We might be syncing - poll for up to 2 minutes waiting for the lock.
+ // (2 minutes seems extreme, but should be very rare.)
+ return new Promise(resolve => {
+ abortController.signal.onabort = () => {
+ resolve(false);
+ };
+
+ let attempts = 0;
+ let checkLock = () => {
+ if (abortController.signal.aborted) {
+ // We've already resolved, so don't want a new timer to ever start.
+ return;
+ }
+ if (weave.Service.lock()) {
+ resolve(true);
+ return;
+ }
+ attempts += 1;
+ if (attempts >= this.lockRetryCount) {
+ log.error(
+ "Gave up waiting for the sync lock - going ahead with sanitize anyway"
+ );
+ resolve(false);
+ return;
+ }
+ log.debug("Waiting a couple of seconds to get the sync lock");
+ lazy.setTimeout(checkLock, this.lockRetryInterval);
+ };
+ checkLock();
+ });
+ },
+
+ // Sanitize Sync-related data.
+ async doSanitizeSyncData() {
+ let weave = this.getWeave();
+ // Get the sync logger - if stuff goes wrong it can be useful to have that
+ // recorded in the sync logs.
+ let log = lazy.Log.repository.getLogger("Sync.Service");
+ log.info("Starting santitize of Sync data");
+ try {
+ // We clobber data for all Sync engines that are enabled.
+ await weave.Service.promiseInitialized;
+ weave.Service.enabled = false;
+
+ log.info("starting actual sanitization");
+ for (let engine of weave.Service.engineManager.getAll()) {
+ if (engine.enabled) {
+ try {
+ log.info("Wiping engine", engine.name);
+ await engine.wipeClient();
+ } catch (ex) {
+ log.error("Failed to wipe engine", ex);
+ }
+ }
+ }
+ // Reset the pref which is used to show a warning when a different user
+ // signs in - this is no longer a concern now that we've removed the
+ // data from the profile.
+ Services.prefs.clearUserPref(lazy.FxAccountsCommon.PREF_LAST_FXA_USER);
+
+ log.info("Finished wiping sync data");
+ } catch (ex) {
+ log.error("Failed to sanitize Sync data", ex);
+ console.error("Failed to sanitize Sync data", ex);
+ }
+ try {
+ // ensure any logs we wrote are flushed to disk.
+ await weave.Service.errorHandler.resetFileLog();
+ } catch (ex) {
+ console.log("Failed to flush the Sync log", ex);
+ }
+ },
+
+ // Sanitize all Browser data.
+ async doSanitizeBrowserData() {
+ try {
+ // sanitize everything other than "open windows" (and we don't do that
+ // because it may confuse the user - they probably want to see
+ // about:prefs with the disconnection reflected.
+ let itemsToClear = Object.keys(lazy.Sanitizer.items).filter(
+ k => k != "openWindows"
+ );
+ await lazy.Sanitizer.sanitize(itemsToClear);
+ } catch (ex) {
+ console.error("Failed to sanitize other data", ex);
+ }
+ },
+
+ async doSyncAndAccountDisconnect(shouldUnlock) {
+ // We do a startOver of Sync first - if we do the account first we end
+ // up with Sync configured but FxA not configured, which causes the browser
+ // UI to briefly enter a "needs reauth" state.
+ let Weave = this.getWeave();
+ await Weave.Service.promiseInitialized;
+ await Weave.Service.startOver();
+ await lazy.fxAccounts.signOut();
+ // Sync may have been disabled if we santized, so re-enable it now or
+ // else the user will be unable to resync should they sign in before a
+ // restart.
+ Weave.Service.enabled = true;
+
+ // and finally, if we managed to get the lock before, we should unlock it
+ // now.
+ if (shouldUnlock) {
+ Weave.Service.unlock();
+ }
+ },
+
+ // Start the sanitization process. Returns a promise that resolves when
+ // the sanitize is complete, and an AbortController which can be used to
+ // abort the process of waiting for a sync to complete.
+ async _startDisconnect(abortController, sanitizeData = false) {
+ // This is a bit convoluted - we want to wait for a sync to finish before
+ // sanitizing, but want to abort that wait if the browser shuts down while
+ // we are waiting (in which case we'll charge ahead anyway).
+ // So we do this by using an AbortController and passing that to the
+ // function that waits for the sync lock - it will immediately resolve
+ // if the abort controller is aborted.
+ let log = lazy.Log.repository.getLogger("Sync.Service");
+
+ // If the master-password is locked then we will fail to fully sanitize,
+ // so prompt for that now. If canceled, we just abort now.
+ log.info("checking master-password state");
+ if (!lazy.Utils.ensureMPUnlocked()) {
+ log.warn(
+ "The master-password needs to be unlocked to fully disconnect from sync"
+ );
+ return;
+ }
+
+ log.info("waiting for any existing syncs to complete");
+ let locked = await this.promiseNotSyncing(abortController);
+
+ if (sanitizeData) {
+ await this.doSanitizeSyncData();
+
+ // We disconnect before sanitizing the browser data - in a worst-case
+ // scenario where the sanitize takes so long that even the shutdown
+ // blocker doesn't allow it to finish, we should still at least be in
+ // a disconnected state on the next startup.
+ log.info("disconnecting account");
+ await this.doSyncAndAccountDisconnect(locked);
+
+ await this.doSanitizeBrowserData();
+ } else {
+ log.info("disconnecting account");
+ await this.doSyncAndAccountDisconnect(locked);
+ }
+ },
+
+ async disconnect(sanitizeData) {
+ if (this.promiseDisconnectFinished) {
+ throw new Error("A disconnect is already in progress");
+ }
+ let abortController = new AbortController();
+ let promiseDisconnectFinished = this._startDisconnect(
+ abortController,
+ sanitizeData
+ );
+ this.promiseDisconnectFinished = promiseDisconnectFinished;
+ let shutdownBlocker = () => {
+ // oh dear - we are sanitizing (probably stuck waiting for a sync to
+ // complete) and the browser is shutting down. Let's avoid the wait
+ // for sync to complete and continue the process anyway.
+ abortController.abort();
+ return promiseDisconnectFinished;
+ };
+ lazy.AsyncShutdown.quitApplicationGranted.addBlocker(
+ "SyncDisconnect: removing requested data",
+ shutdownBlocker
+ );
+
+ // wait for it to finish - hopefully without the blocker being called.
+ await promiseDisconnectFinished;
+ this.promiseDisconnectFinished = null;
+
+ // sanitize worked so remove our blocker - it's a noop if the blocker
+ // did call us.
+ lazy.AsyncShutdown.quitApplicationGranted.removeBlocker(shutdownBlocker);
+ },
+};
+
+export const SyncDisconnect = {
+ get promiseDisconnectFinished() {
+ return SyncDisconnectInternal.promiseDisconnectFinished;
+ },
+
+ disconnect(sanitizeData) {
+ return SyncDisconnectInternal.disconnect(sanitizeData);
+ },
+};
diff --git a/services/sync/modules/SyncedTabs.sys.mjs b/services/sync/modules/SyncedTabs.sys.mjs
new file mode 100644
index 0000000000..9af388da8a
--- /dev/null
+++ b/services/sync/modules/SyncedTabs.sys.mjs
@@ -0,0 +1,332 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ CLIENT_NOT_CONFIGURED: "resource://services-sync/constants.sys.mjs",
+ Preferences: "resource://gre/modules/Preferences.sys.mjs",
+ Weave: "resource://services-sync/main.sys.mjs",
+});
+
+// The Sync XPCOM service
+XPCOMUtils.defineLazyGetter(lazy, "weaveXPCService", function () {
+ return Cc["@mozilla.org/weave/service;1"].getService(
+ Ci.nsISupports
+ ).wrappedJSObject;
+});
+
+// from MDN...
+function escapeRegExp(string) {
+ return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// A topic we fire whenever we have new tabs available. This might be due
+// to a request made by this module to refresh the tab list, or as the result
+// of a regularly scheduled sync. The intent is that consumers just listen
+// for this notification and update their UI in response.
+const TOPIC_TABS_CHANGED = "services.sync.tabs.changed";
+
+// The interval, in seconds, before which we consider the existing list
+// of tabs "fresh enough" and don't force a new sync.
+const TABS_FRESH_ENOUGH_INTERVAL_SECONDS = 30;
+
+XPCOMUtils.defineLazyGetter(lazy, "log", function () {
+ let log = Log.repository.getLogger("Sync.RemoteTabs");
+ log.manageLevelFromPref("services.sync.log.logger.tabs");
+ return log;
+});
+
+// A private singleton that does the work.
+let SyncedTabsInternal = {
+ /* Make a "tab" record. Returns a promise */
+ async _makeTab(client, tab, url, showRemoteIcons) {
+ let icon;
+ if (showRemoteIcons) {
+ icon = tab.icon;
+ }
+ if (!icon) {
+ // By not specifying a size the favicon service will pick the default,
+ // that is usually set through setDefaultIconURIPreferredSize by the
+ // first browser window. Commonly it's 16px at current dpi.
+ icon = "page-icon:" + url;
+ }
+ return {
+ type: "tab",
+ title: tab.title || url,
+ url,
+ icon,
+ client: client.id,
+ lastUsed: tab.lastUsed,
+ };
+ },
+
+ /* Make a "client" record. Returns a promise for consistency with _makeTab */
+ async _makeClient(client) {
+ return {
+ id: client.id,
+ type: "client",
+ name: lazy.Weave.Service.clientsEngine.getClientName(client.id),
+ clientType: lazy.Weave.Service.clientsEngine.getClientType(client.id),
+ lastModified: client.lastModified * 1000, // sec to ms
+ tabs: [],
+ };
+ },
+
+ _tabMatchesFilter(tab, filter) {
+ let reFilter = new RegExp(escapeRegExp(filter), "i");
+ return reFilter.test(tab.url) || reFilter.test(tab.title);
+ },
+
+ _createRecentTabsList(clients, maxCount) {
+ let tabs = [];
+
+ for (let client of clients) {
+ for (let tab of client.tabs) {
+ tab.device = client.name;
+ tab.deviceType = client.clientType;
+ }
+ tabs = [...tabs, ...client.tabs.reverse()];
+ }
+ tabs = this._filterRecentTabsDupes(tabs);
+ tabs = tabs.sort((a, b) => b.lastUsed - a.lastUsed).slice(0, maxCount);
+ return tabs;
+ },
+
+ _filterRecentTabsDupes(tabs) {
+ // Filter out any tabs with duplicate URLs preserving
+ // the duplicate with the most recent lastUsed value
+ return tabs.filter(tab => {
+ return !tabs.some(t => {
+ return t.url === tab.url && tab.lastUsed < t.lastUsed;
+ });
+ });
+ },
+
+ async getTabClients(filter) {
+ lazy.log.info("Generating tab list with filter", filter);
+ let result = [];
+
+ // If Sync isn't ready, don't try and get anything.
+ if (!lazy.weaveXPCService.ready) {
+ lazy.log.debug("Sync isn't yet ready, so returning an empty tab list");
+ return result;
+ }
+
+ // A boolean that controls whether we should show the icon from the remote tab.
+ const showRemoteIcons = lazy.Preferences.get(
+ "services.sync.syncedTabs.showRemoteIcons",
+ true
+ );
+
+ let engine = lazy.Weave.Service.engineManager.get("tabs");
+
+ let ntabs = 0;
+ let clientTabList = await engine.getAllClients();
+ for (let client of clientTabList) {
+ if (!lazy.Weave.Service.clientsEngine.remoteClientExists(client.id)) {
+ continue;
+ }
+ let clientRepr = await this._makeClient(client);
+ lazy.log.debug("Processing client", clientRepr);
+
+ for (let tab of client.tabs) {
+ let url = tab.urlHistory[0];
+ lazy.log.trace("remote tab", url);
+
+ if (!url) {
+ continue;
+ }
+ let tabRepr = await this._makeTab(client, tab, url, showRemoteIcons);
+ if (filter && !this._tabMatchesFilter(tabRepr, filter)) {
+ continue;
+ }
+ clientRepr.tabs.push(tabRepr);
+ }
+ // We return all clients, even those without tabs - the consumer should
+ // filter it if they care.
+ ntabs += clientRepr.tabs.length;
+ result.push(clientRepr);
+ }
+ lazy.log.info(
+ `Final tab list has ${result.length} clients with ${ntabs} tabs.`
+ );
+ return result;
+ },
+
+ async syncTabs(force) {
+ if (!force) {
+ // Don't bother refetching tabs if we already did so recently
+ let lastFetch = lazy.Preferences.get("services.sync.lastTabFetch", 0);
+ let now = Math.floor(Date.now() / 1000);
+ if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL_SECONDS) {
+ lazy.log.info("_refetchTabs was done recently, do not doing it again");
+ return false;
+ }
+ }
+
+ // If Sync isn't configured don't try and sync, else we will get reports
+ // of a login failure.
+ if (lazy.Weave.Status.checkSetup() === lazy.CLIENT_NOT_CONFIGURED) {
+ lazy.log.info(
+ "Sync client is not configured, so not attempting a tab sync"
+ );
+ return false;
+ }
+ // If the primary pass is locked, we should not try to sync
+ if (lazy.Weave.Utils.mpLocked()) {
+ lazy.log.info(
+ "Can't sync tabs due to the primary password being locked",
+ lazy.Weave.Status.login
+ );
+ return false;
+ }
+ // Ask Sync to just do the tabs engine if it can.
+ try {
+ lazy.log.info("Doing a tab sync.");
+ await lazy.Weave.Service.sync({ why: "tabs", engines: ["tabs"] });
+ return true;
+ } catch (ex) {
+ lazy.log.error("Sync failed", ex);
+ throw ex;
+ }
+ },
+
+ observe(subject, topic, data) {
+ lazy.log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`);
+ switch (topic) {
+ case "weave:engine:sync:finish":
+ if (data != "tabs") {
+ return;
+ }
+ // The tabs engine just finished syncing
+ // Set our lastTabFetch pref here so it tracks both explicit sync calls
+ // and normally scheduled ones.
+ lazy.Preferences.set(
+ "services.sync.lastTabFetch",
+ Math.floor(Date.now() / 1000)
+ );
+ Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED);
+ break;
+ case "weave:service:start-over":
+ // start-over needs to notify so consumers find no tabs.
+ lazy.Preferences.reset("services.sync.lastTabFetch");
+ Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED);
+ break;
+ case "nsPref:changed":
+ Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED);
+ break;
+ default:
+ break;
+ }
+ },
+
+ // Returns true if Sync is configured to Sync tabs, false otherwise
+ get isConfiguredToSyncTabs() {
+ if (!lazy.weaveXPCService.ready) {
+ lazy.log.debug("Sync isn't yet ready; assuming tab engine is enabled");
+ return true;
+ }
+
+ let engine = lazy.Weave.Service.engineManager.get("tabs");
+ return engine && engine.enabled;
+ },
+
+ get hasSyncedThisSession() {
+ let engine = lazy.Weave.Service.engineManager.get("tabs");
+ return engine && engine.hasSyncedThisSession;
+ },
+};
+
+Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish");
+Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over");
+// Observe the pref the indicates the state of the tabs engine has changed.
+// This will force consumers to re-evaluate the state of sync and update
+// accordingly.
+Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal);
+
+// The public interface.
+export var SyncedTabs = {
+ // A mock-point for tests.
+ _internal: SyncedTabsInternal,
+
+ // We make the topic for the observer notification public.
+ TOPIC_TABS_CHANGED,
+
+ // Expose the interval used to determine if synced tabs data needs a new sync
+ TABS_FRESH_ENOUGH_INTERVAL_SECONDS,
+
+ // Returns true if Sync is configured to Sync tabs, false otherwise
+ get isConfiguredToSyncTabs() {
+ return this._internal.isConfiguredToSyncTabs;
+ },
+
+ // Returns true if a tab sync has completed once this session. If this
+ // returns false, then getting back no clients/tabs possibly just means we
+ // are waiting for that first sync to complete.
+ get hasSyncedThisSession() {
+ return this._internal.hasSyncedThisSession;
+ },
+
+ // Return a promise that resolves with an array of client records, each with
+ // a .tabs array. Note that part of the contract for this module is that the
+ // returned objects are not shared between invocations, so callers are free
+ // to mutate the returned objects (eg, sort, truncate) however they see fit.
+ getTabClients(query) {
+ return this._internal.getTabClients(query);
+ },
+
+ // Starts a background request to start syncing tabs. Returns a promise that
+ // resolves when the sync is complete, but there's no resolved value -
+ // callers should be listening for TOPIC_TABS_CHANGED.
+ // If |force| is true we always sync. If false, we only sync if the most
+ // recent sync wasn't "recently".
+ syncTabs(force) {
+ return this._internal.syncTabs(force);
+ },
+
+ sortTabClientsByLastUsed(clients) {
+ // First sort the list of tabs for each client. Note that
+ // this module promises that the objects it returns are never
+ // shared, so we are free to mutate those objects directly.
+ for (let client of clients) {
+ let tabs = client.tabs;
+ tabs.sort((a, b) => b.lastUsed - a.lastUsed);
+ }
+ // Now sort the clients - the clients are sorted in the order of the
+ // most recent tab for that client (ie, it is important the tabs for
+ // each client are already sorted.)
+ clients.sort((a, b) => {
+ if (!a.tabs.length) {
+ return 1; // b comes first.
+ }
+ if (!b.tabs.length) {
+ return -1; // a comes first.
+ }
+ return b.tabs[0].lastUsed - a.tabs[0].lastUsed;
+ });
+ },
+
+ recordSyncedTabsTelemetry(object, tabEvent, extraOptions) {
+ Services.telemetry.setEventRecordingEnabled("synced_tabs", true);
+ Services.telemetry.recordEvent(
+ "synced_tabs",
+ tabEvent,
+ object,
+ null,
+ extraOptions
+ );
+ },
+
+ // Get list of synced tabs across all devices/clients
+ // truncated by value of maxCount param, sorted by
+ // lastUsed value, and filtered for duplicate URLs
+ async getRecentTabs(maxCount) {
+ let clients = await this.getTabClients();
+ return this._internal._createRecentTabsList(clients, maxCount);
+ },
+};
diff --git a/services/sync/modules/UIState.sys.mjs b/services/sync/modules/UIState.sys.mjs
new file mode 100644
index 0000000000..c169712523
--- /dev/null
+++ b/services/sync/modules/UIState.sys.mjs
@@ -0,0 +1,287 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * @typedef {Object} UIState
+ * @property {string} status The Sync/FxA status, see STATUS_* constants.
+ * @property {string} [email] The FxA email configured to log-in with Sync.
+ * @property {string} [displayName] The user's FxA display name.
+ * @property {string} [avatarURL] The user's FxA avatar URL.
+ * @property {Date} [lastSync] The last sync time.
+ * @property {boolean} [syncing] Whether or not we are currently syncing.
+ */
+
+const lazy = {};
+ChromeUtils.defineESModuleGetters(lazy, {
+ LOGIN_FAILED_LOGIN_REJECTED: "resource://services-sync/constants.sys.mjs",
+ Weave: "resource://services-sync/main.sys.mjs",
+});
+
+const TOPICS = [
+ "weave:connected",
+ "weave:service:login:got-hashed-id",
+ "weave:service:login:error",
+ "weave:service:ready",
+ "weave:service:sync:start",
+ "weave:service:sync:finish",
+ "weave:service:sync:error",
+ "weave:service:start-over:finish",
+ "fxaccounts:onverified",
+ "fxaccounts:onlogin", // Defined in FxAccountsCommon, pulling it is expensive.
+ "fxaccounts:onlogout",
+ "fxaccounts:profilechange",
+ "fxaccounts:statechange",
+];
+
+const ON_UPDATE = "sync-ui-state:update";
+
+const STATUS_NOT_CONFIGURED = "not_configured";
+const STATUS_LOGIN_FAILED = "login_failed";
+const STATUS_NOT_VERIFIED = "not_verified";
+const STATUS_SIGNED_IN = "signed_in";
+
+const DEFAULT_STATE = {
+ status: STATUS_NOT_CONFIGURED,
+};
+
+const UIStateInternal = {
+ _initialized: false,
+ _state: null,
+
+ // We keep _syncing out of the state object because we can only track it
+ // using sync events and we can't determine it at any point in time.
+ _syncing: false,
+
+ get state() {
+ if (!this._state) {
+ return DEFAULT_STATE;
+ }
+ return Object.assign({}, this._state, { syncing: this._syncing });
+ },
+
+ isReady() {
+ if (!this._initialized) {
+ this.init();
+ return false;
+ }
+ return true;
+ },
+
+ init() {
+ this._initialized = true;
+ // Because the FxA toolbar is usually visible, this module gets loaded at
+ // browser startup, and we want to avoid pulling in all of FxA or Sync at
+ // that time, so we refresh the state after the browser has settled.
+ Services.tm.idleDispatchToMainThread(() => {
+ this.refreshState().catch(e => {
+ console.error(e);
+ });
+ }, 2000);
+ },
+
+ // Used for testing.
+ reset() {
+ this._state = null;
+ this._syncing = false;
+ this._initialized = false;
+ },
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "weave:service:sync:start":
+ this.toggleSyncActivity(true);
+ break;
+ case "weave:service:sync:finish":
+ case "weave:service:sync:error":
+ this.toggleSyncActivity(false);
+ break;
+ default:
+ this.refreshState().catch(e => {
+ console.error(e);
+ });
+ break;
+ }
+ },
+
+ // Builds a new state from scratch.
+ async refreshState() {
+ const newState = {};
+ await this._refreshFxAState(newState);
+ // Optimize the "not signed in" case to avoid refreshing twice just after
+ // startup - if there's currently no _state, and we still aren't configured,
+ // just early exit.
+ if (this._state == null && newState.status == DEFAULT_STATE.status) {
+ return this.state;
+ }
+ if (newState.syncEnabled) {
+ this._setLastSyncTime(newState); // We want this in case we change accounts.
+ }
+ this._state = newState;
+
+ this.notifyStateUpdated();
+ return this.state;
+ },
+
+ // Update the current state with the last sync time/currently syncing status.
+ toggleSyncActivity(syncing) {
+ this._syncing = syncing;
+ this._setLastSyncTime(this._state);
+
+ this.notifyStateUpdated();
+ },
+
+ notifyStateUpdated() {
+ Services.obs.notifyObservers(null, ON_UPDATE);
+ },
+
+ async _refreshFxAState(newState) {
+ let userData = await this._getUserData();
+ await this._populateWithUserData(newState, userData);
+ },
+
+ async _populateWithUserData(state, userData) {
+ let status;
+ let syncUserName = Services.prefs.getStringPref(
+ "services.sync.username",
+ ""
+ );
+ if (!userData) {
+ // If Sync thinks it is configured but there's no FxA user, then we
+ // want to enter the "login failed" state so the user can get
+ // reconfigured.
+ if (syncUserName) {
+ state.email = syncUserName;
+ status = STATUS_LOGIN_FAILED;
+ } else {
+ // everyone agrees nothing is configured.
+ status = STATUS_NOT_CONFIGURED;
+ }
+ } else {
+ let loginFailed = await this._loginFailed();
+ if (loginFailed) {
+ status = STATUS_LOGIN_FAILED;
+ } else if (!userData.verified) {
+ status = STATUS_NOT_VERIFIED;
+ } else {
+ status = STATUS_SIGNED_IN;
+ }
+ state.uid = userData.uid;
+ state.email = userData.email;
+ state.displayName = userData.displayName;
+ // for better or worse, this module renames these attribues.
+ state.avatarURL = userData.avatar;
+ state.avatarIsDefault = userData.avatarDefault;
+ state.syncEnabled = !!syncUserName;
+ }
+ state.status = status;
+ },
+
+ async _getUserData() {
+ try {
+ return await this.fxAccounts.getSignedInUser();
+ } catch (e) {
+ // This is most likely in tests, where we quickly log users in and out.
+ // The most likely scenario is a user logged out, so reflect that.
+ // Bug 995134 calls for better errors so we could retry if we were
+ // sure this was the failure reason.
+ console.error("Error updating FxA account info: " + e);
+ return null;
+ }
+ },
+
+ _setLastSyncTime(state) {
+ if (state?.status == UIState.STATUS_SIGNED_IN) {
+ const lastSync = Services.prefs.getCharPref(
+ "services.sync.lastSync",
+ null
+ );
+ state.lastSync = lastSync ? new Date(lastSync) : null;
+ }
+ },
+
+ async _loginFailed() {
+ // First ask FxA if it thinks the user needs re-authentication. In practice,
+ // this check is probably canonical (ie, we probably don't really need
+ // the check below at all as we drop local session info on the first sign
+ // of a problem) - but we keep it for now to keep the risk down.
+ let hasLocalSession = await this.fxAccounts.hasLocalSession();
+ if (!hasLocalSession) {
+ return true;
+ }
+
+ // Referencing Weave.Service will implicitly initialize sync, and we don't
+ // want to force that - so first check if it is ready.
+ let service = Cc["@mozilla.org/weave/service;1"].getService(
+ Ci.nsISupports
+ ).wrappedJSObject;
+ if (!service.ready) {
+ return false;
+ }
+ // LOGIN_FAILED_LOGIN_REJECTED explicitly means "you must log back in".
+ // All other login failures are assumed to be transient and should go
+ // away by themselves, so aren't reflected here.
+ return lazy.Weave.Status.login == lazy.LOGIN_FAILED_LOGIN_REJECTED;
+ },
+
+ set fxAccounts(mockFxAccounts) {
+ delete this.fxAccounts;
+ this.fxAccounts = mockFxAccounts;
+ },
+};
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+XPCOMUtils.defineLazyGetter(UIStateInternal, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+
+for (let topic of TOPICS) {
+ Services.obs.addObserver(UIStateInternal, topic);
+}
+
+export var UIState = {
+ _internal: UIStateInternal,
+
+ ON_UPDATE,
+
+ STATUS_NOT_CONFIGURED,
+ STATUS_LOGIN_FAILED,
+ STATUS_NOT_VERIFIED,
+ STATUS_SIGNED_IN,
+
+ /**
+ * Returns true if the module has been initialized and the state set.
+ * If not, return false and trigger an init in the background.
+ */
+ isReady() {
+ return this._internal.isReady();
+ },
+
+ /**
+ * @returns {UIState} The current Sync/FxA UI State.
+ */
+ get() {
+ return this._internal.state;
+ },
+
+ /**
+ * Refresh the state. Used for testing, don't call this directly since
+ * UIState already listens to Sync/FxA notifications to determine if the state
+ * needs to be refreshed. ON_UPDATE will be fired once the state is refreshed.
+ *
+ * @returns {Promise<UIState>} Resolved once the state is refreshed.
+ */
+ refresh() {
+ return this._internal.refreshState();
+ },
+
+ /**
+ * Reset the state of the whole module. Used for testing.
+ */
+ reset() {
+ this._internal.reset();
+ },
+};
diff --git a/services/sync/modules/addonsreconciler.sys.mjs b/services/sync/modules/addonsreconciler.sys.mjs
new file mode 100644
index 0000000000..902f57348e
--- /dev/null
+++ b/services/sync/modules/addonsreconciler.sys.mjs
@@ -0,0 +1,584 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains middleware to reconcile state of AddonManager for
+ * purposes of tracking events for Sync. The content in this file exists
+ * because AddonManager does not have a getChangesSinceX() API and adding
+ * that functionality properly was deemed too time-consuming at the time
+ * add-on sync was originally written. If/when AddonManager adds this API,
+ * this file can go away and the add-ons engine can be rewritten to use it.
+ *
+ * It was decided to have this tracking functionality exist in a separate
+ * standalone file so it could be more easily understood, tested, and
+ * hopefully ported.
+ */
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { AddonManager } from "resource://gre/modules/AddonManager.sys.mjs";
+
+const DEFAULT_STATE_FILE = "addonsreconciler";
+
+export var CHANGE_INSTALLED = 1;
+export var CHANGE_UNINSTALLED = 2;
+export var CHANGE_ENABLED = 3;
+export var CHANGE_DISABLED = 4;
+
+/**
+ * Maintains state of add-ons.
+ *
+ * State is maintained in 2 data structures, an object mapping add-on IDs
+ * to metadata and an array of changes over time. The object mapping can be
+ * thought of as a minimal copy of data from AddonManager which is needed for
+ * Sync. The array is effectively a log of changes over time.
+ *
+ * The data structures are persisted to disk by serializing to a JSON file in
+ * the current profile. The data structures are updated by 2 mechanisms. First,
+ * they can be refreshed from the global state of the AddonManager. This is a
+ * sure-fire way of ensuring the reconciler is up to date. Second, the
+ * reconciler adds itself as an AddonManager listener. When it receives change
+ * notifications, it updates its internal state incrementally.
+ *
+ * The internal state is persisted to a JSON file in the profile directory.
+ *
+ * An instance of this is bound to an AddonsEngine instance. In reality, it
+ * likely exists as a singleton. To AddonsEngine, it functions as a store and
+ * an entity which emits events for tracking.
+ *
+ * The usage pattern for instances of this class is:
+ *
+ * let reconciler = new AddonsReconciler(...);
+ * await reconciler.ensureStateLoaded();
+ *
+ * // At this point, your instance should be ready to use.
+ *
+ * When you are finished with the instance, please call:
+ *
+ * reconciler.stopListening();
+ * await reconciler.saveState(...);
+ *
+ * This class uses the AddonManager AddonListener interface.
+ * When an add-on is installed, listeners are called in the following order:
+ * AL.onInstalling, AL.onInstalled
+ *
+ * For uninstalls, we see AL.onUninstalling then AL.onUninstalled.
+ *
+ * Enabling and disabling work by sending:
+ *
+ * AL.onEnabling, AL.onEnabled
+ * AL.onDisabling, AL.onDisabled
+ *
+ * Actions can be undone. All undoable actions notify the same
+ * AL.onOperationCancelled event. We treat this event like any other.
+ *
+ * When an add-on is uninstalled from about:addons, the user is offered an
+ * "Undo" option, which leads to the following sequence of events as
+ * observed by an AddonListener:
+ * Add-ons are first disabled then they are actually uninstalled. So, we will
+ * see AL.onDisabling and AL.onDisabled. The onUninstalling and onUninstalled
+ * events only come after the Addon Manager is closed or another view is
+ * switched to. In the case of Sync performing the uninstall, the uninstall
+ * events will occur immediately. However, we still see disabling events and
+ * heed them like they were normal. In the end, the state is proper.
+ */
+export function AddonsReconciler(queueCaller) {
+ this._log = Log.repository.getLogger("Sync.AddonsReconciler");
+ this._log.manageLevelFromPref("services.sync.log.logger.addonsreconciler");
+ this.queueCaller = queueCaller;
+
+ Svc.Obs.add("xpcom-shutdown", this.stopListening, this);
+}
+
+AddonsReconciler.prototype = {
+ /** Flag indicating whether we are listening to AddonManager events. */
+ _listening: false,
+
+ /**
+ * Define this as false if the reconciler should not persist state
+ * to disk when handling events.
+ *
+ * This allows test code to avoid spinning to write during observer
+ * notifications and xpcom shutdown, which appears to cause hangs on WinXP
+ * (Bug 873861).
+ */
+ _shouldPersist: true,
+
+ /** Log logger instance */
+ _log: null,
+
+ /**
+ * Container for add-on metadata.
+ *
+ * Keys are add-on IDs. Values are objects which describe the state of the
+ * add-on. This is a minimal mirror of data that can be queried from
+ * AddonManager. In some cases, we retain data longer than AddonManager.
+ */
+ _addons: {},
+
+ /**
+ * List of add-on changes over time.
+ *
+ * Each element is an array of [time, change, id].
+ */
+ _changes: [],
+
+ /**
+ * Objects subscribed to changes made to this instance.
+ */
+ _listeners: [],
+
+ /**
+ * Accessor for add-ons in this object.
+ *
+ * Returns an object mapping add-on IDs to objects containing metadata.
+ */
+ get addons() {
+ return this._addons;
+ },
+
+ async ensureStateLoaded() {
+ if (!this._promiseStateLoaded) {
+ this._promiseStateLoaded = this.loadState();
+ }
+ return this._promiseStateLoaded;
+ },
+
+ /**
+ * Load reconciler state from a file.
+ *
+ * The path is relative to the weave directory in the profile. If no
+ * path is given, the default one is used.
+ *
+ * If the file does not exist or there was an error parsing the file, the
+ * state will be transparently defined as empty.
+ *
+ * @param file
+ * Path to load. ".json" is appended automatically. If not defined,
+ * a default path will be consulted.
+ */
+ async loadState(file = DEFAULT_STATE_FILE) {
+ let json = await Utils.jsonLoad(file, this);
+ this._addons = {};
+ this._changes = [];
+
+ if (!json) {
+ this._log.debug("No data seen in loaded file: " + file);
+ return false;
+ }
+
+ let version = json.version;
+ if (!version || version != 1) {
+ this._log.error(
+ "Could not load JSON file because version not " +
+ "supported: " +
+ version
+ );
+ return false;
+ }
+
+ this._addons = json.addons;
+ for (let id in this._addons) {
+ let record = this._addons[id];
+ record.modified = new Date(record.modified);
+ }
+
+ for (let [time, change, id] of json.changes) {
+ this._changes.push([new Date(time), change, id]);
+ }
+
+ return true;
+ },
+
+ /**
+ * Saves the current state to a file in the local profile.
+ *
+ * @param file
+ * String path in profile to save to. If not defined, the default
+ * will be used.
+ */
+ async saveState(file = DEFAULT_STATE_FILE) {
+ let state = { version: 1, addons: {}, changes: [] };
+
+ for (let [id, record] of Object.entries(this._addons)) {
+ state.addons[id] = {};
+ for (let [k, v] of Object.entries(record)) {
+ if (k == "modified") {
+ state.addons[id][k] = v.getTime();
+ } else {
+ state.addons[id][k] = v;
+ }
+ }
+ }
+
+ for (let [time, change, id] of this._changes) {
+ state.changes.push([time.getTime(), change, id]);
+ }
+
+ this._log.info("Saving reconciler state to file: " + file);
+ await Utils.jsonSave(file, this, state);
+ },
+
+ /**
+ * Registers a change listener with this instance.
+ *
+ * Change listeners are called every time a change is recorded. The listener
+ * is an object with the function "changeListener" that takes 3 arguments,
+ * the Date at which the change happened, the type of change (a CHANGE_*
+ * constant), and the add-on state object reflecting the current state of
+ * the add-on at the time of the change.
+ *
+ * @param listener
+ * Object containing changeListener function.
+ */
+ addChangeListener: function addChangeListener(listener) {
+ if (!this._listeners.includes(listener)) {
+ this._log.debug("Adding change listener.");
+ this._listeners.push(listener);
+ }
+ },
+
+ /**
+ * Removes a previously-installed change listener from the instance.
+ *
+ * @param listener
+ * Listener instance to remove.
+ */
+ removeChangeListener: function removeChangeListener(listener) {
+ this._listeners = this._listeners.filter(element => {
+ if (element == listener) {
+ this._log.debug("Removing change listener.");
+ return false;
+ }
+ return true;
+ });
+ },
+
+ /**
+ * Tells the instance to start listening for AddonManager changes.
+ *
+ * This is typically called automatically when Sync is loaded.
+ */
+ startListening: function startListening() {
+ if (this._listening) {
+ return;
+ }
+
+ this._log.info("Registering as Add-on Manager listener.");
+ AddonManager.addAddonListener(this);
+ this._listening = true;
+ },
+
+ /**
+ * Tells the instance to stop listening for AddonManager changes.
+ *
+ * The reconciler should always be listening. This should only be called when
+ * the instance is being destroyed.
+ *
+ * This function will get called automatically on XPCOM shutdown. However, it
+ * is a best practice to call it yourself.
+ */
+ stopListening: function stopListening() {
+ if (!this._listening) {
+ return;
+ }
+
+ this._log.debug("Stopping listening and removing AddonManager listener.");
+ AddonManager.removeAddonListener(this);
+ this._listening = false;
+ },
+
+ /**
+ * Refreshes the global state of add-ons by querying the AddonManager.
+ */
+ async refreshGlobalState() {
+ this._log.info("Refreshing global state from AddonManager.");
+
+ let installs;
+ let addons = await AddonManager.getAllAddons();
+
+ let ids = {};
+
+ for (let addon of addons) {
+ ids[addon.id] = true;
+ await this.rectifyStateFromAddon(addon);
+ }
+
+ // Look for locally-defined add-ons that no longer exist and update their
+ // record.
+ for (let [id, addon] of Object.entries(this._addons)) {
+ if (id in ids) {
+ continue;
+ }
+
+ // If the id isn't in ids, it means that the add-on has been deleted or
+ // the add-on is in the process of being installed. We detect the
+ // latter by seeing if an AddonInstall is found for this add-on.
+
+ if (!installs) {
+ installs = await AddonManager.getAllInstalls();
+ }
+
+ let installFound = false;
+ for (let install of installs) {
+ if (
+ install.addon &&
+ install.addon.id == id &&
+ install.state == AddonManager.STATE_INSTALLED
+ ) {
+ installFound = true;
+ break;
+ }
+ }
+
+ if (installFound) {
+ continue;
+ }
+
+ if (addon.installed) {
+ addon.installed = false;
+ this._log.debug(
+ "Adding change because add-on not present in " +
+ "Add-on Manager: " +
+ id
+ );
+ await this._addChange(new Date(), CHANGE_UNINSTALLED, addon);
+ }
+ }
+
+ // See note for _shouldPersist.
+ if (this._shouldPersist) {
+ await this.saveState();
+ }
+ },
+
+ /**
+ * Rectifies the state of an add-on from an Addon instance.
+ *
+ * This basically says "given an Addon instance, assume it is truth and
+ * apply changes to the local state to reflect it."
+ *
+ * This function could result in change listeners being called if the local
+ * state differs from the passed add-on's state.
+ *
+ * @param addon
+ * Addon instance being updated.
+ */
+ async rectifyStateFromAddon(addon) {
+ this._log.debug(
+ `Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`
+ );
+
+ let id = addon.id;
+ let enabled = !addon.userDisabled;
+ let guid = addon.syncGUID;
+ let now = new Date();
+
+ if (!(id in this._addons)) {
+ let record = {
+ id,
+ guid,
+ enabled,
+ installed: true,
+ modified: now,
+ type: addon.type,
+ scope: addon.scope,
+ foreignInstall: addon.foreignInstall,
+ isSyncable: addon.isSyncable,
+ };
+ this._addons[id] = record;
+ this._log.debug(
+ "Adding change because add-on not present locally: " + id
+ );
+ await this._addChange(now, CHANGE_INSTALLED, record);
+ return;
+ }
+
+ let record = this._addons[id];
+ record.isSyncable = addon.isSyncable;
+
+ if (!record.installed) {
+ // It is possible the record is marked as uninstalled because an
+ // uninstall is pending.
+ if (!(addon.pendingOperations & AddonManager.PENDING_UNINSTALL)) {
+ record.installed = true;
+ record.modified = now;
+ }
+ }
+
+ if (record.enabled != enabled) {
+ record.enabled = enabled;
+ record.modified = now;
+ let change = enabled ? CHANGE_ENABLED : CHANGE_DISABLED;
+ this._log.debug("Adding change because enabled state changed: " + id);
+ await this._addChange(new Date(), change, record);
+ }
+
+ if (record.guid != guid) {
+ record.guid = guid;
+ // We don't record a change because the Sync engine rectifies this on its
+ // own. This is tightly coupled with Sync. If this code is ever lifted
+ // outside of Sync, this exception should likely be removed.
+ }
+ },
+
+ /**
+ * Record a change in add-on state.
+ *
+ * @param date
+ * Date at which the change occurred.
+ * @param change
+ * The type of the change. A CHANGE_* constant.
+ * @param state
+ * The new state of the add-on. From this.addons.
+ */
+ async _addChange(date, change, state) {
+ this._log.info("Change recorded for " + state.id);
+ this._changes.push([date, change, state.id]);
+
+ for (let listener of this._listeners) {
+ try {
+ await listener.changeListener(date, change, state);
+ } catch (ex) {
+ this._log.error("Exception calling change listener", ex);
+ }
+ }
+ },
+
+ /**
+ * Obtain the set of changes to add-ons since the date passed.
+ *
+ * This will return an array of arrays. Each entry in the array has the
+ * elements [date, change_type, id], where
+ *
+ * date - Date instance representing when the change occurred.
+ * change_type - One of CHANGE_* constants.
+ * id - ID of add-on that changed.
+ */
+ getChangesSinceDate(date) {
+ let length = this._changes.length;
+ for (let i = 0; i < length; i++) {
+ if (this._changes[i][0] >= date) {
+ return this._changes.slice(i);
+ }
+ }
+
+ return [];
+ },
+
+ /**
+ * Prunes all recorded changes from before the specified Date.
+ *
+ * @param date
+ * Entries older than this Date will be removed.
+ */
+ pruneChangesBeforeDate(date) {
+ this._changes = this._changes.filter(function test_age(change) {
+ return change[0] >= date;
+ });
+ },
+
+ /**
+ * Obtains the set of all known Sync GUIDs for add-ons.
+ */
+ getAllSyncGUIDs() {
+ let result = {};
+ for (let id in this.addons) {
+ result[id] = true;
+ }
+
+ return result;
+ },
+
+ /**
+ * Obtain the add-on state record for an add-on by Sync GUID.
+ *
+ * If the add-on could not be found, returns null.
+ *
+ * @param guid
+ * Sync GUID of add-on to retrieve.
+ */
+ getAddonStateFromSyncGUID(guid) {
+ for (let id in this.addons) {
+ let addon = this.addons[id];
+ if (addon.guid == guid) {
+ return addon;
+ }
+ }
+
+ return null;
+ },
+
+ /**
+ * Handler that is invoked as part of the AddonManager listeners.
+ */
+ async _handleListener(action, addon) {
+ // Since this is called as an observer, we explicitly trap errors and
+ // log them to ourselves so we don't see errors reported elsewhere.
+ try {
+ let id = addon.id;
+ this._log.debug("Add-on change: " + action + " to " + id);
+
+ switch (action) {
+ case "onEnabled":
+ case "onDisabled":
+ case "onInstalled":
+ case "onInstallEnded":
+ case "onOperationCancelled":
+ await this.rectifyStateFromAddon(addon);
+ break;
+
+ case "onUninstalled":
+ let id = addon.id;
+ let addons = this.addons;
+ if (id in addons) {
+ let now = new Date();
+ let record = addons[id];
+ record.installed = false;
+ record.modified = now;
+ this._log.debug(
+ "Adding change because of uninstall listener: " + id
+ );
+ await this._addChange(now, CHANGE_UNINSTALLED, record);
+ }
+ }
+
+ // See note for _shouldPersist.
+ if (this._shouldPersist) {
+ await this.saveState();
+ }
+ } catch (ex) {
+ this._log.warn("Exception", ex);
+ }
+ },
+
+ // AddonListeners
+ onEnabled: function onEnabled(addon) {
+ this.queueCaller.enqueueCall(() =>
+ this._handleListener("onEnabled", addon)
+ );
+ },
+ onDisabled: function onDisabled(addon) {
+ this.queueCaller.enqueueCall(() =>
+ this._handleListener("onDisabled", addon)
+ );
+ },
+ onInstalled: function onInstalled(addon) {
+ this.queueCaller.enqueueCall(() =>
+ this._handleListener("onInstalled", addon)
+ );
+ },
+ onUninstalled: function onUninstalled(addon) {
+ this.queueCaller.enqueueCall(() =>
+ this._handleListener("onUninstalled", addon)
+ );
+ },
+ onOperationCancelled: function onOperationCancelled(addon) {
+ this.queueCaller.enqueueCall(() =>
+ this._handleListener("onOperationCancelled", addon)
+ );
+ },
+};
diff --git a/services/sync/modules/addonutils.sys.mjs b/services/sync/modules/addonutils.sys.mjs
new file mode 100644
index 0000000000..46167611e1
--- /dev/null
+++ b/services/sync/modules/addonutils.sys.mjs
@@ -0,0 +1,390 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Svc } from "resource://services-sync/util.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
+ AddonRepository: "resource://gre/modules/addons/AddonRepository.sys.mjs",
+});
+
+function AddonUtilsInternal() {
+ this._log = Log.repository.getLogger("Sync.AddonUtils");
+ this._log.Level = Log.Level[Svc.Prefs.get("log.logger.addonutils")];
+}
+AddonUtilsInternal.prototype = {
+ /**
+ * Obtain an AddonInstall object from an AddonSearchResult instance.
+ *
+ * The returned promise will be an AddonInstall on success or null (failure or
+ * addon not found)
+ *
+ * @param addon
+ * AddonSearchResult to obtain install from.
+ */
+ getInstallFromSearchResult(addon) {
+ this._log.debug("Obtaining install for " + addon.id);
+
+ // We should theoretically be able to obtain (and use) addon.install if
+ // it is available. However, the addon.sourceURI rewriting won't be
+ // reflected in the AddonInstall, so we can't use it. If we ever get rid
+ // of sourceURI rewriting, we can avoid having to reconstruct the
+ // AddonInstall.
+ return lazy.AddonManager.getInstallForURL(addon.sourceURI.spec, {
+ name: addon.name,
+ icons: addon.iconURL,
+ version: addon.version,
+ telemetryInfo: { source: "sync" },
+ });
+ },
+
+ /**
+ * Installs an add-on from an AddonSearchResult instance.
+ *
+ * The options argument defines extra options to control the install.
+ * Recognized keys in this map are:
+ *
+ * syncGUID - Sync GUID to use for the new add-on.
+ * enabled - Boolean indicating whether the add-on should be enabled upon
+ * install.
+ *
+ * The result object has the following keys:
+ *
+ * id ID of add-on that was installed.
+ * install AddonInstall that was installed.
+ * addon Addon that was installed.
+ *
+ * @param addon
+ * AddonSearchResult to install add-on from.
+ * @param options
+ * Object with additional metadata describing how to install add-on.
+ */
+ async installAddonFromSearchResult(addon, options) {
+ this._log.info("Trying to install add-on from search result: " + addon.id);
+
+ const install = await this.getInstallFromSearchResult(addon);
+ if (!install) {
+ throw new Error("AddonInstall not available: " + addon.id);
+ }
+
+ try {
+ this._log.info("Installing " + addon.id);
+ let log = this._log;
+
+ return new Promise((res, rej) => {
+ let listener = {
+ onInstallStarted: function onInstallStarted(install) {
+ if (!options) {
+ return;
+ }
+
+ if (options.syncGUID) {
+ log.info(
+ "Setting syncGUID of " + install.name + ": " + options.syncGUID
+ );
+ install.addon.syncGUID = options.syncGUID;
+ }
+
+ // We only need to change userDisabled if it is disabled because
+ // enabled is the default.
+ if ("enabled" in options && !options.enabled) {
+ log.info(
+ "Marking add-on as disabled for install: " + install.name
+ );
+ install.addon.disable();
+ }
+ },
+ onInstallEnded(install, addon) {
+ install.removeListener(listener);
+
+ res({ id: addon.id, install, addon });
+ },
+ onInstallFailed(install) {
+ install.removeListener(listener);
+
+ rej(new Error("Install failed: " + install.error));
+ },
+ onDownloadFailed(install) {
+ install.removeListener(listener);
+
+ rej(new Error("Download failed: " + install.error));
+ },
+ };
+ install.addListener(listener);
+ install.install();
+ });
+ } catch (ex) {
+ this._log.error("Error installing add-on", ex);
+ throw ex;
+ }
+ },
+
+ /**
+ * Uninstalls the addon instance.
+ *
+ * @param addon
+ * Addon instance to uninstall.
+ */
+ async uninstallAddon(addon) {
+ return new Promise(res => {
+ let listener = {
+ onUninstalling(uninstalling, needsRestart) {
+ if (addon.id != uninstalling.id) {
+ return;
+ }
+
+ // We assume restartless add-ons will send the onUninstalled event
+ // soon.
+ if (!needsRestart) {
+ return;
+ }
+
+ // For non-restartless add-ons, we issue the callback on uninstalling
+ // because we will likely never see the uninstalled event.
+ lazy.AddonManager.removeAddonListener(listener);
+ res(addon);
+ },
+ onUninstalled(uninstalled) {
+ if (addon.id != uninstalled.id) {
+ return;
+ }
+
+ lazy.AddonManager.removeAddonListener(listener);
+ res(addon);
+ },
+ };
+ lazy.AddonManager.addAddonListener(listener);
+ addon.uninstall();
+ });
+ },
+
+ /**
+ * Installs multiple add-ons specified by metadata.
+ *
+ * The first argument is an array of objects. Each object must have the
+ * following keys:
+ *
+ * id - public ID of the add-on to install.
+ * syncGUID - syncGUID for new add-on.
+ * enabled - boolean indicating whether the add-on should be enabled.
+ * requireSecureURI - Boolean indicating whether to require a secure
+ * URI when installing from a remote location. This defaults to
+ * true.
+ *
+ * The callback will be called when activity on all add-ons is complete. The
+ * callback receives 2 arguments, error and result.
+ *
+ * If error is truthy, it contains a string describing the overall error.
+ *
+ * The 2nd argument to the callback is always an object with details on the
+ * overall execution state. It contains the following keys:
+ *
+ * installedIDs Array of add-on IDs that were installed.
+ * installs Array of AddonInstall instances that were installed.
+ * addons Array of Addon instances that were installed.
+ * errors Array of errors encountered. Only has elements if error is
+ * truthy.
+ *
+ * @param installs
+ * Array of objects describing add-ons to install.
+ */
+ async installAddons(installs) {
+ let ids = [];
+ for (let addon of installs) {
+ ids.push(addon.id);
+ }
+
+ let addons = await lazy.AddonRepository.getAddonsByIDs(ids);
+ this._log.info(
+ `Found ${addons.length} / ${ids.length}` +
+ " add-ons during repository search."
+ );
+
+ let ourResult = {
+ installedIDs: [],
+ installs: [],
+ addons: [],
+ skipped: [],
+ errors: [],
+ };
+
+ let toInstall = [];
+
+ // Rewrite the "src" query string parameter of the source URI to note
+ // that the add-on was installed by Sync and not something else so
+ // server-side metrics aren't skewed (bug 708134). The server should
+ // ideally send proper URLs, but this solution was deemed too
+ // complicated at the time the functionality was implemented.
+ for (let addon of addons) {
+ // Find the specified options for this addon.
+ let options;
+ for (let install of installs) {
+ if (install.id == addon.id) {
+ options = install;
+ break;
+ }
+ }
+ if (!this.canInstallAddon(addon, options)) {
+ ourResult.skipped.push(addon.id);
+ continue;
+ }
+
+ // We can go ahead and attempt to install it.
+ toInstall.push(addon);
+
+ // We should always be able to QI the nsIURI to nsIURL. If not, we
+ // still try to install the add-on, but we don't rewrite the URL,
+ // potentially skewing metrics.
+ try {
+ addon.sourceURI.QueryInterface(Ci.nsIURL);
+ } catch (ex) {
+ this._log.warn(
+ "Unable to QI sourceURI to nsIURL: " + addon.sourceURI.spec
+ );
+ continue;
+ }
+
+ let params = addon.sourceURI.query
+ .split("&")
+ .map(function rewrite(param) {
+ if (param.indexOf("src=") == 0) {
+ return "src=sync";
+ }
+ return param;
+ });
+
+ addon.sourceURI = addon.sourceURI
+ .mutate()
+ .setQuery(params.join("&"))
+ .finalize();
+ }
+
+ if (!toInstall.length) {
+ return ourResult;
+ }
+
+ const installPromises = [];
+ // Start all the installs asynchronously. They will report back to us
+ // as they finish, eventually triggering the global callback.
+ for (let addon of toInstall) {
+ let options = {};
+ for (let install of installs) {
+ if (install.id == addon.id) {
+ options = install;
+ break;
+ }
+ }
+
+ installPromises.push(
+ (async () => {
+ try {
+ const result = await this.installAddonFromSearchResult(
+ addon,
+ options
+ );
+ ourResult.installedIDs.push(result.id);
+ ourResult.installs.push(result.install);
+ ourResult.addons.push(result.addon);
+ } catch (error) {
+ ourResult.errors.push(error);
+ }
+ })()
+ );
+ }
+
+ await Promise.all(installPromises);
+
+ if (ourResult.errors.length) {
+ throw new Error("1 or more add-ons failed to install");
+ }
+ return ourResult;
+ },
+
+ /**
+ * Returns true if we are able to install the specified addon, false
+ * otherwise. It is expected that this will log the reason if it returns
+ * false.
+ *
+ * @param addon
+ * (Addon) Add-on instance to check.
+ * @param options
+ * (object) The options specified for this addon. See installAddons()
+ * for the valid elements.
+ */
+ canInstallAddon(addon, options) {
+ // sourceURI presence isn't enforced by AddonRepository. So, we skip
+ // add-ons without a sourceURI.
+ if (!addon.sourceURI) {
+ this._log.info(
+ "Skipping install of add-on because missing sourceURI: " + addon.id
+ );
+ return false;
+ }
+ // Verify that the source URI uses TLS. We don't allow installs from
+ // insecure sources for security reasons. The Addon Manager ensures
+ // that cert validation etc is performed.
+ // (We should also consider just dropping this entirely and calling
+ // XPIProvider.isInstallAllowed, but that has additional semantics we might
+ // need to think through...)
+ let requireSecureURI = true;
+ if (options && options.requireSecureURI !== undefined) {
+ requireSecureURI = options.requireSecureURI;
+ }
+
+ if (requireSecureURI) {
+ let scheme = addon.sourceURI.scheme;
+ if (scheme != "https") {
+ this._log.info(
+ `Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted`
+ );
+ return false;
+ }
+ }
+
+ // Policy prevents either installing this addon or any addon
+ if (
+ Services.policies &&
+ (!Services.policies.mayInstallAddon(addon) ||
+ !Services.policies.isAllowed("xpinstall"))
+ ) {
+ this._log.info(
+ `Skipping install of "${addon.id}" due to enterprise policy`
+ );
+ return false;
+ }
+
+ this._log.info(`Add-on "${addon.id}" is able to be installed`);
+ return true;
+ },
+
+ /**
+ * Update the user disabled flag for an add-on.
+ *
+ * If the new flag matches the existing or if the add-on
+ * isn't currently active, the function will return immediately.
+ *
+ * @param addon
+ * (Addon) Add-on instance to operate on.
+ * @param value
+ * (bool) New value for add-on's userDisabled property.
+ */
+ updateUserDisabled(addon, value) {
+ if (addon.userDisabled == value) {
+ return;
+ }
+
+ this._log.info("Updating userDisabled flag: " + addon.id + " -> " + value);
+ if (value) {
+ addon.disable();
+ } else {
+ addon.enable();
+ }
+ },
+};
+
+export const AddonUtils = new AddonUtilsInternal();
diff --git a/services/sync/modules/bridged_engine.sys.mjs b/services/sync/modules/bridged_engine.sys.mjs
new file mode 100644
index 0000000000..45e5f685cd
--- /dev/null
+++ b/services/sync/modules/bridged_engine.sys.mjs
@@ -0,0 +1,499 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file has all the machinery for hooking up bridged engines implemented
+ * in Rust. It's the JavaScript side of the Golden Gate bridge that connects
+ * Desktop Sync to a Rust `BridgedEngine`, via the `mozIBridgedSyncEngine`
+ * XPCOM interface.
+ *
+ * Creating a bridged engine only takes a few lines of code, since most of the
+ * hard work is done on the Rust side. On the JS side, you'll need to subclass
+ * `BridgedEngine` (instead of `SyncEngine`), supply a `mozIBridgedSyncEngine`
+ * for your subclass to wrap, and optionally implement and override the tracker.
+ */
+
+import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs";
+import { RawCryptoWrapper } from "resource://services-sync/record.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Log: "resource://gre/modules/Log.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+/**
+ * A stub store that converts between raw decrypted incoming records and
+ * envelopes. Since the interface we need is so minimal, this class doesn't
+ * inherit from the base `Store` implementation...it would take more code to
+ * override all those behaviors!
+ *
+ * This class isn't meant to be subclassed, because bridged engines shouldn't
+ * override their store classes in `_storeObj`.
+ */
+class BridgedStore {
+ constructor(name, engine) {
+ if (!engine) {
+ throw new Error("Store must be associated with an Engine instance.");
+ }
+ this.engine = engine;
+ this._log = lazy.Log.repository.getLogger(`Sync.Engine.${name}.Store`);
+ this._batchChunkSize = 500;
+ }
+
+ async applyIncomingBatch(records, countTelemetry) {
+ for (let chunk of lazy.PlacesUtils.chunkArray(
+ records,
+ this._batchChunkSize
+ )) {
+ let incomingEnvelopesAsJSON = chunk.map(record =>
+ JSON.stringify(record.toIncomingBso())
+ );
+ this._log.trace("incoming envelopes", incomingEnvelopesAsJSON);
+ await this.engine._bridge.storeIncoming(incomingEnvelopesAsJSON);
+ }
+ // Array of failed records.
+ return [];
+ }
+
+ async wipe() {
+ await this.engine._bridge.wipe();
+ }
+}
+
+/**
+ * A wrapper class to convert between BSOs on the JS side, and envelopes on the
+ * Rust side. This class intentionally subclasses `RawCryptoWrapper`, because we
+ * don't want the stringification and parsing machinery in `CryptoWrapper`.
+ *
+ * This class isn't meant to be subclassed, because bridged engines shouldn't
+ * override their record classes in `_recordObj`.
+ */
+class BridgedRecord extends RawCryptoWrapper {
+ /**
+ * Creates an outgoing record from a BSO returned by a bridged engine.
+ *
+ * @param {String} collection The collection name.
+ * @param {Object} bso The outgoing bso (ie, a sync15::bso::OutgoingBso) returned from
+ * `mozIBridgedSyncEngine::apply`.
+ * @return {BridgedRecord} A Sync record ready to encrypt and upload.
+ */
+ static fromOutgoingBso(collection, bso) {
+ // The BSO has already been JSON serialized coming out of Rust, so the
+ // envelope has been flattened.
+ if (typeof bso.id != "string") {
+ throw new TypeError("Outgoing BSO missing ID");
+ }
+ if (typeof bso.payload != "string") {
+ throw new TypeError("Outgoing BSO missing payload");
+ }
+ let record = new BridgedRecord(collection, bso.id);
+ record.cleartext = bso.payload;
+ return record;
+ }
+
+ transformBeforeEncrypt(cleartext) {
+ if (typeof cleartext != "string") {
+ throw new TypeError("Outgoing bridged engine records must be strings");
+ }
+ return cleartext;
+ }
+
+ transformAfterDecrypt(cleartext) {
+ if (typeof cleartext != "string") {
+ throw new TypeError("Incoming bridged engine records must be strings");
+ }
+ return cleartext;
+ }
+
+ /*
+ * Converts this incoming record into an envelope to pass to a bridged engine.
+ * This object must be kept in sync with `sync15::IncomingBso`.
+ *
+ * @return {Object} The incoming envelope, to pass to
+ * `mozIBridgedSyncEngine::storeIncoming`.
+ */
+ toIncomingBso() {
+ return {
+ id: this.data.id,
+ modified: this.data.modified,
+ payload: this.cleartext,
+ };
+ }
+}
+
+class BridgeError extends Error {
+ constructor(code, message) {
+ super(message);
+ this.name = "BridgeError";
+ // TODO: We may want to use a different name for this, since errors with
+ // a `result` property are treated specially by telemetry, discarding the
+ // message...but, unlike other `nserror`s, the message is actually useful,
+ // and we still want to capture it.
+ this.result = code;
+ }
+}
+
+class InterruptedError extends Error {
+ constructor(message) {
+ super(message);
+ this.name = "InterruptedError";
+ }
+}
+
+/**
+ * Adapts a `Log.sys.mjs` logger to a `mozIServicesLogSink`. This class is copied
+ * from `SyncedBookmarksMirror.jsm`.
+ */
+export class LogAdapter {
+ constructor(log) {
+ this.log = log;
+ }
+
+ get maxLevel() {
+ let level = this.log.level;
+ if (level <= lazy.Log.Level.All) {
+ return Ci.mozIServicesLogSink.LEVEL_TRACE;
+ }
+ if (level <= lazy.Log.Level.Info) {
+ return Ci.mozIServicesLogSink.LEVEL_DEBUG;
+ }
+ if (level <= lazy.Log.Level.Warn) {
+ return Ci.mozIServicesLogSink.LEVEL_WARN;
+ }
+ if (level <= lazy.Log.Level.Error) {
+ return Ci.mozIServicesLogSink.LEVEL_ERROR;
+ }
+ return Ci.mozIServicesLogSink.LEVEL_OFF;
+ }
+
+ trace(message) {
+ this.log.trace(message);
+ }
+
+ debug(message) {
+ this.log.debug(message);
+ }
+
+ warn(message) {
+ this.log.warn(message);
+ }
+
+ error(message) {
+ this.log.error(message);
+ }
+}
+
+// This converts the XPCOM-defined, callback-based mozIBridgedSyncEngine to
+// a promise-based implementation.
+export class BridgeWrapperXPCOM {
+ constructor(component) {
+ this.comp = component;
+ }
+
+ // A few sync, non-callback based attributes.
+ get storageVersion() {
+ return this.comp.storageVersion;
+ }
+
+ get allowSkippedRecord() {
+ return this.comp.allowSkippedRecord;
+ }
+
+ get logger() {
+ return this.comp.logger;
+ }
+
+ // And the async functions we promisify.
+ // Note this is `lastSync` via uniffi but `getLastSync` via xpcom
+ lastSync() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.getLastSync);
+ }
+
+ setLastSync(lastSyncMillis) {
+ return BridgeWrapperXPCOM.#promisify(this.comp.setLastSync, lastSyncMillis);
+ }
+
+ getSyncId() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.getSyncId);
+ }
+
+ resetSyncId() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.resetSyncId);
+ }
+
+ ensureCurrentSyncId(newSyncId) {
+ return BridgeWrapperXPCOM.#promisify(
+ this.comp.ensureCurrentSyncId,
+ newSyncId
+ );
+ }
+
+ syncStarted() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.syncStarted);
+ }
+
+ storeIncoming(incomingEnvelopesAsJSON) {
+ return BridgeWrapperXPCOM.#promisify(
+ this.comp.storeIncoming,
+ incomingEnvelopesAsJSON
+ );
+ }
+
+ apply() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.apply);
+ }
+
+ setUploaded(newTimestampMillis, uploadedIds) {
+ return BridgeWrapperXPCOM.#promisify(
+ this.comp.setUploaded,
+ newTimestampMillis,
+ uploadedIds
+ );
+ }
+
+ syncFinished() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.syncFinished);
+ }
+
+ reset() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.reset);
+ }
+
+ wipe() {
+ return BridgeWrapperXPCOM.#promisify(this.comp.wipe);
+ }
+
+ // Converts a XPCOM bridged function that takes a callback into one that returns a
+ // promise.
+ static #promisify(func, ...params) {
+ return new Promise((resolve, reject) => {
+ func(...params, {
+ // This object implicitly implements all three callback interfaces
+ // (`mozIBridgedSyncEngine{Apply, Result}Callback`), because they have
+ // the same methods. The only difference is the type of the argument
+ // passed to `handleSuccess`, which doesn't matter in JS.
+ handleSuccess: resolve,
+ handleError(code, message) {
+ reject(transformError(code, message));
+ },
+ });
+ });
+ }
+}
+
+/**
+ * A base class used to plug a Rust engine into Sync, and have it work like any
+ * other engine. The constructor takes a bridge as its first argument, which is
+ * a "bridged sync engine", as defined by UniFFI in the application-services
+ * crate.
+ * For backwards compatibility, this can also be an instance of an XPCOM
+ * component class that implements `mozIBridgedSyncEngine`, wrapped in
+ * a `BridgeWrapperXPCOM` wrapper.
+ * (Note that at time of writing, the above is slightly aspirational; the
+ * actual definition of the UniFFI shared bridged engine is still in flux.)
+ *
+ * This class inherits from `SyncEngine`, which has a lot of machinery that we
+ * don't need, but that's fairly easy to override. It would be harder to
+ * reimplement the machinery that we _do_ need here. However, because of that,
+ * this class has lots of methods that do nothing, or return empty data. The
+ * docs above each method explain what it's overriding, and why.
+ *
+ * This class is designed to be subclassed, but the only part that your engine
+ * may want to override is `_trackerObj`. Even then, using the default (no-op)
+ * tracker is fine, because the shape of the `Tracker` interface may not make
+ * sense for all engines.
+ */
+export function BridgedEngine(name, service) {
+ SyncEngine.call(this, name, service);
+}
+
+BridgedEngine.prototype = {
+ /**
+ * The Rust implemented bridge. Must be set by the engine which subclasses us.
+ */
+ _bridge: null,
+ /**
+ * The tracker class for this engine. Subclasses may want to override this
+ * with their own tracker, though using the default `Tracker` is fine.
+ */
+ _trackerObj: Tracker,
+
+ /** Returns the record class for all bridged engines. */
+ get _recordObj() {
+ return BridgedRecord;
+ },
+
+ set _recordObj(obj) {
+ throw new TypeError("Don't override the record class for bridged engines");
+ },
+
+ /** Returns the store class for all bridged engines. */
+ get _storeObj() {
+ return BridgedStore;
+ },
+
+ set _storeObj(obj) {
+ throw new TypeError("Don't override the store class for bridged engines");
+ },
+
+ /** Returns the storage version for this engine. */
+ get version() {
+ return this._bridge.storageVersion;
+ },
+
+ // Legacy engines allow sync to proceed if some records are too large to
+ // upload (eg, a payload that's bigger than the server's published limits).
+ // If this returns true, we will just skip the record without even attempting
+ // to upload. If this is false, we'll abort the entire batch.
+ // If the engine allows this, it will need to detect this scenario by noticing
+ // the ID is not in the 'success' records reported to `setUploaded`.
+ // (Note that this is not to be confused with the fact server's can currently
+ // reject records as part of a POST - but we hope to remove this ability from
+ // the server API. Note also that this is not bullet-proof - if the count of
+ // records is high, it's possible that we will have committed a previous
+ // batch before we hit the relevant limits, so things might have been written.
+ // We hope to fix this by ensuring batch limits are such that this is
+ // impossible)
+ get allowSkippedRecord() {
+ return this._bridge.allowSkippedRecord;
+ },
+
+ /**
+ * Returns the sync ID for this engine. This is exposed for tests, but
+ * Sync code always calls `resetSyncID()` and `ensureCurrentSyncID()`,
+ * not this.
+ *
+ * @returns {String?} The sync ID, or `null` if one isn't set.
+ */
+ async getSyncID() {
+ // Note that all methods on an XPCOM class instance are automatically bound,
+ // so we don't need to write `this._bridge.getSyncId.bind(this._bridge)`.
+ let syncID = await this._bridge.getSyncId();
+ return syncID;
+ },
+
+ async resetSyncID() {
+ await this._deleteServerCollection();
+ let newSyncID = await this.resetLocalSyncID();
+ return newSyncID;
+ },
+
+ async resetLocalSyncID() {
+ let newSyncID = await this._bridge.resetSyncId();
+ return newSyncID;
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ let assignedSyncID = await this._bridge.ensureCurrentSyncId(newSyncID);
+ return assignedSyncID;
+ },
+
+ async getLastSync() {
+ // The bridge defines lastSync as integer ms, but sync itself wants to work
+ // in a float seconds with 2 decimal places.
+ let lastSyncMS = await this._bridge.lastSync();
+ return Math.round(lastSyncMS / 10) / 100;
+ },
+
+ async setLastSync(lastSyncSeconds) {
+ await this._bridge.setLastSync(Math.round(lastSyncSeconds * 1000));
+ },
+
+ /**
+ * Returns the initial changeset for the sync. Bridged engines handle
+ * reconciliation internally, so we don't know what changed until after we've
+ * stored and applied all incoming records. So we return an empty changeset
+ * here, and replace it with the real one in `_processIncoming`.
+ */
+ async pullChanges() {
+ return {};
+ },
+
+ async trackRemainingChanges() {
+ await this._bridge.syncFinished();
+ },
+
+ /**
+ * Marks a record for a hard-`DELETE` at the end of the sync. The base method
+ * also removes it from the tracker, but we don't use the tracker for that,
+ * so we override the method to just mark.
+ */
+ _deleteId(id) {
+ this._noteDeletedId(id);
+ },
+
+ /**
+ * Always stage incoming records, bypassing the base engine's reconciliation
+ * machinery.
+ */
+ async _reconcile() {
+ return true;
+ },
+
+ async _syncStartup() {
+ await super._syncStartup();
+ await this._bridge.syncStarted();
+ },
+
+ async _processIncoming(newitems) {
+ await super._processIncoming(newitems);
+
+ let outgoingBsosAsJSON = await this._bridge.apply();
+ let changeset = {};
+ for (let bsoAsJSON of outgoingBsosAsJSON) {
+ this._log.trace("outgoing bso", bsoAsJSON);
+ let record = BridgedRecord.fromOutgoingBso(
+ this.name,
+ JSON.parse(bsoAsJSON)
+ );
+ changeset[record.id] = {
+ synced: false,
+ record,
+ };
+ }
+ this._modified.replace(changeset);
+ },
+
+ /**
+ * Notify the bridged engine that we've successfully uploaded a batch, so
+ * that it can update its local state. For example, if the engine uses a
+ * mirror and a temp table for outgoing records, it can write the uploaded
+ * records from the outgoing table back to the mirror.
+ */
+ async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
+ // JS uses seconds but Rust uses milliseconds so we'll need to convert
+ let serverModifiedMS = Math.round(serverModifiedTime * 1000);
+ await this._bridge.setUploaded(Math.floor(serverModifiedMS), succeeded);
+ },
+
+ async _createTombstone() {
+ throw new Error("Bridged engines don't support weak uploads");
+ },
+
+ async _createRecord(id) {
+ let change = this._modified.changes[id];
+ if (!change) {
+ throw new TypeError("Can't create record for unchanged item");
+ }
+ return change.record;
+ },
+
+ async _resetClient() {
+ await super._resetClient();
+ await this._bridge.reset();
+ },
+};
+Object.setPrototypeOf(BridgedEngine.prototype, SyncEngine.prototype);
+
+function transformError(code, message) {
+ switch (code) {
+ case Cr.NS_ERROR_ABORT:
+ return new InterruptedError(message);
+
+ default:
+ return new BridgeError(code, message);
+ }
+}
diff --git a/services/sync/modules/collection_validator.sys.mjs b/services/sync/modules/collection_validator.sys.mjs
new file mode 100644
index 0000000000..a64ede10e9
--- /dev/null
+++ b/services/sync/modules/collection_validator.sys.mjs
@@ -0,0 +1,267 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Async: "resource://services-common/async.sys.mjs",
+});
+
+export class CollectionProblemData {
+ constructor() {
+ this.missingIDs = 0;
+ this.clientDuplicates = [];
+ this.duplicates = [];
+ this.clientMissing = [];
+ this.serverMissing = [];
+ this.serverDeleted = [];
+ this.serverUnexpected = [];
+ this.differences = [];
+ }
+
+ /**
+ * Produce a list summarizing problems found. Each entry contains {name, count},
+ * where name is the field name for the problem, and count is the number of times
+ * the problem was encountered.
+ *
+ * Validation has failed if all counts are not 0.
+ */
+ getSummary() {
+ return [
+ { name: "clientMissing", count: this.clientMissing.length },
+ { name: "serverMissing", count: this.serverMissing.length },
+ { name: "serverDeleted", count: this.serverDeleted.length },
+ { name: "serverUnexpected", count: this.serverUnexpected.length },
+ { name: "differences", count: this.differences.length },
+ { name: "missingIDs", count: this.missingIDs },
+ { name: "clientDuplicates", count: this.clientDuplicates.length },
+ { name: "duplicates", count: this.duplicates.length },
+ ];
+ }
+}
+
+export class CollectionValidator {
+ // Construct a generic collection validator. This is intended to be called by
+ // subclasses.
+ // - name: Name of the engine
+ // - idProp: Property that identifies a record. That is, if a client and server
+ // record have the same value for the idProp property, they should be
+ // compared against eachother.
+ // - props: Array of properties that should be compared
+ constructor(name, idProp, props) {
+ this.name = name;
+ this.props = props;
+ this.idProp = idProp;
+
+ // This property deals with the fact that form history records are never
+ // deleted from the server. The FormValidator subclass needs to ignore the
+ // client missing records, and it uses this property to achieve it -
+ // (Bug 1354016).
+ this.ignoresMissingClients = false;
+ }
+
+ // Should a custom ProblemData type be needed, return it here.
+ emptyProblemData() {
+ return new CollectionProblemData();
+ }
+
+ async getServerItems(engine) {
+ let collection = engine.itemSource();
+ let collectionKey = engine.service.collectionKeys.keyForCollection(
+ engine.name
+ );
+ collection.full = true;
+ let result = await collection.getBatched();
+ if (!result.response.success) {
+ throw result.response;
+ }
+ let cleartexts = [];
+
+ await lazy.Async.yieldingForEach(result.records, async record => {
+ await record.decrypt(collectionKey);
+ cleartexts.push(record.cleartext);
+ });
+
+ return cleartexts;
+ }
+
+ // Should return a promise that resolves to an array of client items.
+ getClientItems() {
+ return Promise.reject("Must implement");
+ }
+
+ /**
+ * Can we guarantee validation will fail with a reason that isn't actually a
+ * problem? For example, if we know there are pending changes left over from
+ * the last sync, this should resolve to false. By default resolves to true.
+ */
+ async canValidate() {
+ return true;
+ }
+
+ // Turn the client item into something that can be compared with the server item,
+ // and is also safe to mutate.
+ normalizeClientItem(item) {
+ return Cu.cloneInto(item, {});
+ }
+
+ // Turn the server item into something that can be easily compared with the client
+ // items.
+ async normalizeServerItem(item) {
+ return item;
+ }
+
+ // Return whether or not a server item should be present on the client. Expected
+ // to be overridden.
+ clientUnderstands(item) {
+ return true;
+ }
+
+ // Return whether or not a client item should be present on the server. Expected
+ // to be overridden
+ async syncedByClient(item) {
+ return true;
+ }
+
+ // Compare the server item and the client item, and return a list of property
+ // names that are different. Can be overridden if needed.
+ getDifferences(client, server) {
+ let differences = [];
+ for (let prop of this.props) {
+ let clientProp = client[prop];
+ let serverProp = server[prop];
+ if ((clientProp || "") !== (serverProp || "")) {
+ differences.push(prop);
+ }
+ }
+ return differences;
+ }
+
+ // Returns an object containing
+ // problemData: an instance of the class returned by emptyProblemData(),
+ // clientRecords: Normalized client records
+ // records: Normalized server records,
+ // deletedRecords: Array of ids that were marked as deleted by the server.
+ async compareClientWithServer(clientItems, serverItems) {
+ const yieldState = lazy.Async.yieldState();
+
+ const clientRecords = [];
+
+ await lazy.Async.yieldingForEach(
+ clientItems,
+ item => {
+ clientRecords.push(this.normalizeClientItem(item));
+ },
+ yieldState
+ );
+
+ const serverRecords = [];
+ await lazy.Async.yieldingForEach(
+ serverItems,
+ async item => {
+ serverRecords.push(await this.normalizeServerItem(item));
+ },
+ yieldState
+ );
+
+ let problems = this.emptyProblemData();
+ let seenServer = new Map();
+ let serverDeleted = new Set();
+ let allRecords = new Map();
+
+ for (let record of serverRecords) {
+ let id = record[this.idProp];
+ if (!id) {
+ ++problems.missingIDs;
+ continue;
+ }
+ if (record.deleted) {
+ serverDeleted.add(record);
+ } else {
+ let serverHasPossibleDupe = seenServer.has(id);
+ if (serverHasPossibleDupe) {
+ problems.duplicates.push(id);
+ } else {
+ seenServer.set(id, record);
+ allRecords.set(id, { server: record, client: null });
+ }
+ record.understood = this.clientUnderstands(record);
+ }
+ }
+
+ let seenClient = new Map();
+ for (let record of clientRecords) {
+ let id = record[this.idProp];
+ record.shouldSync = await this.syncedByClient(record);
+ let clientHasPossibleDupe = seenClient.has(id);
+ if (clientHasPossibleDupe && record.shouldSync) {
+ // Only report duplicate client IDs for syncable records.
+ problems.clientDuplicates.push(id);
+ continue;
+ }
+ seenClient.set(id, record);
+ let combined = allRecords.get(id);
+ if (combined) {
+ combined.client = record;
+ } else {
+ allRecords.set(id, { client: record, server: null });
+ }
+ }
+
+ for (let [id, { server, client }] of allRecords) {
+ if (!client && !server) {
+ throw new Error("Impossible: no client or server record for " + id);
+ } else if (server && !client) {
+ if (!this.ignoresMissingClients && server.understood) {
+ problems.clientMissing.push(id);
+ }
+ } else if (client && !server) {
+ if (client.shouldSync) {
+ problems.serverMissing.push(id);
+ }
+ } else {
+ if (!client.shouldSync) {
+ if (!problems.serverUnexpected.includes(id)) {
+ problems.serverUnexpected.push(id);
+ }
+ continue;
+ }
+ let differences = this.getDifferences(client, server);
+ if (differences && differences.length) {
+ problems.differences.push({ id, differences });
+ }
+ }
+ }
+ return {
+ problemData: problems,
+ clientRecords,
+ records: serverRecords,
+ deletedRecords: [...serverDeleted],
+ };
+ }
+
+ async validate(engine) {
+ let start = Cu.now();
+ let clientItems = await this.getClientItems();
+ let serverItems = await this.getServerItems(engine);
+ let serverRecordCount = serverItems.length;
+ let result = await this.compareClientWithServer(clientItems, serverItems);
+ let end = Cu.now();
+ let duration = end - start;
+ engine._log.debug(`Validated ${this.name} in ${duration}ms`);
+ engine._log.debug(`Problem summary`);
+ for (let { name, count } of result.problemData.getSummary()) {
+ engine._log.debug(` ${name}: ${count}`);
+ }
+ return {
+ duration,
+ version: this.version,
+ problems: result.problemData,
+ recordCount: serverRecordCount,
+ };
+ }
+}
+
+// Default to 0, some engines may override.
+CollectionValidator.prototype.version = 0;
diff --git a/services/sync/modules/constants.sys.mjs b/services/sync/modules/constants.sys.mjs
new file mode 100644
index 0000000000..5db9f53a24
--- /dev/null
+++ b/services/sync/modules/constants.sys.mjs
@@ -0,0 +1,133 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Don't manually modify this line, as it is automatically replaced on merge day
+// by the gecko_migration.py script.
+export const WEAVE_VERSION = "1.117.0";
+
+// Sync Server API version that the client supports.
+export const SYNC_API_VERSION = "1.5";
+
+// Version of the data format this client supports. The data format describes
+// how records are packaged; this is separate from the Server API version and
+// the per-engine cleartext formats.
+export const STORAGE_VERSION = 5;
+export const PREFS_BRANCH = "services.sync.";
+
+// Put in [] because those aren't allowed in a collection name.
+export const DEFAULT_KEYBUNDLE_NAME = "[default]";
+
+// Key dimensions.
+export const SYNC_KEY_ENCODED_LENGTH = 26;
+export const SYNC_KEY_DECODED_LENGTH = 16;
+
+export const NO_SYNC_NODE_INTERVAL = 10 * 60 * 1000; // 10 minutes
+
+export const MAX_ERROR_COUNT_BEFORE_BACKOFF = 3;
+
+// Backoff intervals
+export const MINIMUM_BACKOFF_INTERVAL = 15 * 60 * 1000; // 15 minutes
+export const MAXIMUM_BACKOFF_INTERVAL = 8 * 60 * 60 * 1000; // 8 hours
+
+// HMAC event handling timeout.
+// 10 minutes = a compromise between the multi-desktop sync interval
+// and the mobile sync interval.
+export const HMAC_EVENT_INTERVAL = 600000;
+
+// How long to wait between sync attempts if the Master Password is locked.
+export const MASTER_PASSWORD_LOCKED_RETRY_INTERVAL = 15 * 60 * 1000; // 15 minutes
+
+// 50 is hardcoded here because of URL length restrictions.
+// (GUIDs can be up to 64 chars long.)
+// Individual engines can set different values for their limit if their
+// identifiers are shorter.
+export const DEFAULT_GUID_FETCH_BATCH_SIZE = 50;
+
+// Default batch size for download batching
+// (how many records are fetched at a time from the server when batching is used).
+export const DEFAULT_DOWNLOAD_BATCH_SIZE = 1000;
+
+// score thresholds for early syncs
+export const SINGLE_USER_THRESHOLD = 1000;
+export const MULTI_DEVICE_THRESHOLD = 300;
+
+// Other score increment constants
+export const SCORE_INCREMENT_SMALL = 1;
+export const SCORE_INCREMENT_MEDIUM = 10;
+
+// Instant sync score increment
+export const SCORE_INCREMENT_XLARGE = 300 + 1; //MULTI_DEVICE_THRESHOLD + 1
+
+// Delay before incrementing global score
+export const SCORE_UPDATE_DELAY = 100;
+
+// Delay for the back observer debouncer. This is chosen to be longer than any
+// observed spurious idle/back events and short enough to pre-empt user activity.
+export const IDLE_OBSERVER_BACK_DELAY = 100;
+
+// Duplicate URI_LENGTH_MAX from Places (from nsNavHistory.h), used to discard
+// tabs with huge uris during tab sync.
+export const URI_LENGTH_MAX = 65536;
+
+export const MAX_HISTORY_UPLOAD = 5000;
+export const MAX_HISTORY_DOWNLOAD = 5000;
+
+// Top-level statuses
+export const STATUS_OK = "success.status_ok";
+export const SYNC_FAILED = "error.sync.failed";
+export const LOGIN_FAILED = "error.login.failed";
+export const SYNC_FAILED_PARTIAL = "error.sync.failed_partial";
+export const CLIENT_NOT_CONFIGURED = "service.client_not_configured";
+export const STATUS_DISABLED = "service.disabled";
+export const MASTER_PASSWORD_LOCKED = "service.master_password_locked";
+
+// success states
+export const LOGIN_SUCCEEDED = "success.login";
+export const SYNC_SUCCEEDED = "success.sync";
+export const ENGINE_SUCCEEDED = "success.engine";
+
+// login failure status codes
+export const LOGIN_FAILED_NO_USERNAME = "error.login.reason.no_username";
+export const LOGIN_FAILED_NO_PASSPHRASE = "error.login.reason.no_recoverykey";
+export const LOGIN_FAILED_NETWORK_ERROR = "error.login.reason.network";
+export const LOGIN_FAILED_SERVER_ERROR = "error.login.reason.server";
+export const LOGIN_FAILED_INVALID_PASSPHRASE = "error.login.reason.recoverykey";
+export const LOGIN_FAILED_LOGIN_REJECTED = "error.login.reason.account";
+
+// sync failure status codes
+export const METARECORD_DOWNLOAD_FAIL =
+ "error.sync.reason.metarecord_download_fail";
+export const VERSION_OUT_OF_DATE = "error.sync.reason.version_out_of_date";
+export const CREDENTIALS_CHANGED = "error.sync.reason.credentials_changed";
+export const ABORT_SYNC_COMMAND = "aborting sync, process commands said so";
+export const NO_SYNC_NODE_FOUND = "error.sync.reason.no_node_found";
+export const OVER_QUOTA = "error.sync.reason.over_quota";
+export const SERVER_MAINTENANCE = "error.sync.reason.serverMaintenance";
+
+export const RESPONSE_OVER_QUOTA = "14";
+
+// engine failure status codes
+export const ENGINE_UPLOAD_FAIL = "error.engine.reason.record_upload_fail";
+export const ENGINE_DOWNLOAD_FAIL = "error.engine.reason.record_download_fail";
+export const ENGINE_UNKNOWN_FAIL = "error.engine.reason.unknown_fail";
+export const ENGINE_APPLY_FAIL = "error.engine.reason.apply_fail";
+// an upload failure where the batch was interrupted with a 412
+export const ENGINE_BATCH_INTERRUPTED = "error.engine.reason.batch_interrupted";
+
+// Ways that a sync can be disabled (messages only to be printed in debug log)
+export const kSyncMasterPasswordLocked =
+ "User elected to leave Primary Password locked";
+export const kSyncWeaveDisabled = "Weave is disabled";
+export const kSyncNetworkOffline = "Network is offline";
+export const kSyncBackoffNotMet =
+ "Trying to sync before the server said it's okay";
+export const kFirstSyncChoiceNotMade =
+ "User has not selected an action for first sync";
+export const kSyncNotConfigured = "Sync is not configured";
+export const kFirefoxShuttingDown = "Firefox is about to shut down";
+
+export const DEVICE_TYPE_DESKTOP = "desktop";
+export const DEVICE_TYPE_MOBILE = "mobile";
+
+export const SQLITE_MAX_VARIABLE_NUMBER = 999;
diff --git a/services/sync/modules/doctor.sys.mjs b/services/sync/modules/doctor.sys.mjs
new file mode 100644
index 0000000000..fe9385c3ff
--- /dev/null
+++ b/services/sync/modules/doctor.sys.mjs
@@ -0,0 +1,188 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A doctor for our collections. She can be asked to make a consultation, and
+// may just diagnose an issue without attempting to cure it, may diagnose and
+// attempt to cure, or may decide she is overworked and underpaid.
+// Or something - naming is hard :)
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { Observers } from "resource://services-common/observers.sys.mjs";
+import { Service } from "resource://services-sync/service.sys.mjs";
+import { Resource } from "resource://services-sync/resource.sys.mjs";
+import { Svc } from "resource://services-sync/util.sys.mjs";
+
+const log = Log.repository.getLogger("Sync.Doctor");
+
+export var Doctor = {
+ async consult(recentlySyncedEngines) {
+ if (!Services.telemetry.canRecordBase) {
+ log.info("Skipping consultation: telemetry reporting is disabled");
+ return;
+ }
+
+ let engineInfos = this._getEnginesToValidate(recentlySyncedEngines);
+
+ await this._runValidators(engineInfos);
+ },
+
+ _getEnginesToValidate(recentlySyncedEngines) {
+ let result = {};
+ for (let e of recentlySyncedEngines) {
+ let prefPrefix = `engine.${e.name}.`;
+ if (!Svc.Prefs.get(prefPrefix + "validation.enabled", false)) {
+ log.info(`Skipping check of ${e.name} - disabled via preferences`);
+ continue;
+ }
+ // Check the last validation time for the engine.
+ let lastValidation = Svc.Prefs.get(prefPrefix + "validation.lastTime", 0);
+ let validationInterval = Svc.Prefs.get(
+ prefPrefix + "validation.interval"
+ );
+ let nowSeconds = this._now();
+
+ if (nowSeconds - lastValidation < validationInterval) {
+ log.info(
+ `Skipping validation of ${e.name}: too recent since last validation attempt`
+ );
+ continue;
+ }
+ // Update the time now, even if we decline to actually perform a
+ // validation. We don't want to check the rest of these more frequently
+ // than once a day.
+ Svc.Prefs.set(prefPrefix + "validation.lastTime", Math.floor(nowSeconds));
+
+ // Validation only occurs a certain percentage of the time.
+ let validationProbability =
+ Svc.Prefs.get(prefPrefix + "validation.percentageChance", 0) / 100.0;
+ if (validationProbability < Math.random()) {
+ log.info(
+ `Skipping validation of ${e.name}: Probability threshold not met`
+ );
+ continue;
+ }
+
+ let maxRecords = Svc.Prefs.get(prefPrefix + "validation.maxRecords");
+ if (!maxRecords) {
+ log.info(`Skipping validation of ${e.name}: No maxRecords specified`);
+ continue;
+ }
+ // OK, so this is a candidate - the final decision will be based on the
+ // number of records actually found.
+ result[e.name] = { engine: e, maxRecords };
+ }
+ return result;
+ },
+
+ async _runValidators(engineInfos) {
+ if (!Object.keys(engineInfos).length) {
+ log.info("Skipping validation: no engines qualify");
+ return;
+ }
+
+ if (Object.values(engineInfos).filter(i => i.maxRecords != -1).length) {
+ // at least some of the engines have maxRecord restrictions which require
+ // us to ask the server for the counts.
+ let countInfo = await this._fetchCollectionCounts();
+ for (let [engineName, recordCount] of Object.entries(countInfo)) {
+ if (engineName in engineInfos) {
+ engineInfos[engineName].recordCount = recordCount;
+ }
+ }
+ }
+
+ for (let [
+ engineName,
+ { engine, maxRecords, recordCount },
+ ] of Object.entries(engineInfos)) {
+ // maxRecords of -1 means "any number", so we can skip asking the server.
+ // Used for tests.
+ if (maxRecords >= 0 && recordCount > maxRecords) {
+ log.debug(
+ `Skipping validation for ${engineName} because ` +
+ `the number of records (${recordCount}) is greater ` +
+ `than the maximum allowed (${maxRecords}).`
+ );
+ continue;
+ }
+ let validator = engine.getValidator();
+ if (!validator) {
+ // This is probably only possible in profile downgrade cases.
+ log.warn(
+ `engine.getValidator returned null for ${engineName} but the pref that controls validation is enabled.`
+ );
+ continue;
+ }
+
+ if (!(await validator.canValidate())) {
+ log.debug(
+ `Skipping validation for ${engineName} because validator.canValidate() is false`
+ );
+ continue;
+ }
+
+ // Let's do it!
+ Services.console.logStringMessage(
+ `Sync is about to run a consistency check of ${engine.name}. This may be slow, and ` +
+ `can be controlled using the pref "services.sync.${engine.name}.validation.enabled".\n` +
+ `If you encounter any problems because of this, please file a bug.`
+ );
+
+ try {
+ log.info(`Running validator for ${engine.name}`);
+ let result = await validator.validate(engine);
+ let { problems, version, duration, recordCount } = result;
+ Observers.notify(
+ "weave:engine:validate:finish",
+ {
+ version,
+ checked: recordCount,
+ took: duration,
+ problems: problems ? problems.getSummary(true) : null,
+ },
+ engine.name
+ );
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ log.error(`Failed to run validation on ${engine.name}!`, ex);
+ Observers.notify("weave:engine:validate:error", ex, engine.name);
+ // Keep validating -- there's no reason to think that a failure for one
+ // validator would mean the others will fail.
+ }
+ }
+ },
+
+ // mainly for mocking.
+ async _fetchCollectionCounts() {
+ let collectionCountsURL = Service.userBaseURL + "info/collection_counts";
+ try {
+ let infoResp = await Service._fetchInfo(collectionCountsURL);
+ if (!infoResp.success) {
+ log.error(
+ "Can't fetch collection counts: request to info/collection_counts responded with " +
+ infoResp.status
+ );
+ return {};
+ }
+ return infoResp.obj; // might throw because obj is a getter which parses json.
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ // Not running validation is totally fine, so we just write an error log and return.
+ log.error("Caught error when fetching counts", ex);
+ return {};
+ }
+ },
+
+ // functions used so tests can mock them
+ _now() {
+ // We use the server time, which is SECONDS
+ return Resource.serverTime;
+ },
+};
diff --git a/services/sync/modules/engines.sys.mjs b/services/sync/modules/engines.sys.mjs
new file mode 100644
index 0000000000..dc3fa185b2
--- /dev/null
+++ b/services/sync/modules/engines.sys.mjs
@@ -0,0 +1,2260 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { JSONFile } from "resource://gre/modules/JSONFile.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { Observers } from "resource://services-common/observers.sys.mjs";
+
+import {
+ DEFAULT_DOWNLOAD_BATCH_SIZE,
+ DEFAULT_GUID_FETCH_BATCH_SIZE,
+ ENGINE_BATCH_INTERRUPTED,
+ ENGINE_DOWNLOAD_FAIL,
+ ENGINE_UPLOAD_FAIL,
+ VERSION_OUT_OF_DATE,
+} from "resource://services-sync/constants.sys.mjs";
+
+import {
+ Collection,
+ CryptoWrapper,
+} from "resource://services-sync/record.sys.mjs";
+import { Resource } from "resource://services-sync/resource.sys.mjs";
+import {
+ SerializableSet,
+ Svc,
+ Utils,
+} from "resource://services-sync/util.sys.mjs";
+import { SyncedRecordsTelemetry } from "resource://services-sync/telemetry.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+function ensureDirectory(path) {
+ return IOUtils.makeDirectory(PathUtils.parent(path), {
+ createAncestors: true,
+ });
+}
+
+/**
+ * Trackers are associated with a single engine and deal with
+ * listening for changes to their particular data type.
+ *
+ * The base `Tracker` only supports listening for changes, and bumping the score
+ * to indicate how urgently the engine wants to sync. It does not persist any
+ * data. Engines that track changes directly in the storage layer (like
+ * bookmarks, bridged engines, addresses, and credit cards) or only upload a
+ * single record (tabs and preferences) should subclass `Tracker`.
+ */
+export function Tracker(name, engine) {
+ if (!engine) {
+ throw new Error("Tracker must be associated with an Engine instance.");
+ }
+
+ name = name || "Unnamed";
+ this.name = name.toLowerCase();
+ this.engine = engine;
+
+ this._log = Log.repository.getLogger(`Sync.Engine.${name}.Tracker`);
+
+ this._score = 0;
+
+ this.asyncObserver = Async.asyncObserver(this, this._log);
+}
+
+Tracker.prototype = {
+ // New-style trackers use change sources to filter out changes made by Sync in
+ // observer notifications, so we don't want to let the engine ignore all
+ // changes during a sync.
+ get ignoreAll() {
+ return false;
+ },
+
+ // Define an empty setter so that the engine doesn't throw a `TypeError`
+ // setting a read-only property.
+ set ignoreAll(value) {},
+
+ /*
+ * Score can be called as often as desired to decide which engines to sync
+ *
+ * Valid values for score:
+ * -1: Do not sync unless the user specifically requests it (almost disabled)
+ * 0: Nothing has changed
+ * 100: Please sync me ASAP!
+ *
+ * Setting it to other values should (but doesn't currently) throw an exception
+ */
+ get score() {
+ return this._score;
+ },
+
+ set score(value) {
+ this._score = value;
+ Observers.notify("weave:engine:score:updated", this.name);
+ },
+
+ // Should be called by service everytime a sync has been done for an engine
+ resetScore() {
+ this._score = 0;
+ },
+
+ // Unsupported, and throws a more descriptive error to ensure callers aren't
+ // accidentally using persistence.
+ async getChangedIDs() {
+ throw new TypeError("This tracker doesn't store changed IDs");
+ },
+
+ // Also unsupported.
+ async addChangedID(id, when) {
+ throw new TypeError("Can't add changed ID to this tracker");
+ },
+
+ // Ditto.
+ async removeChangedID(...ids) {
+ throw new TypeError("Can't remove changed IDs from this tracker");
+ },
+
+ // This method is called at various times, so we override with a no-op
+ // instead of throwing.
+ clearChangedIDs() {},
+
+ _now() {
+ return Date.now() / 1000;
+ },
+
+ _isTracking: false,
+
+ start() {
+ if (!this.engineIsEnabled()) {
+ return;
+ }
+ this._log.trace("start().");
+ if (!this._isTracking) {
+ this.onStart();
+ this._isTracking = true;
+ }
+ },
+
+ async stop() {
+ this._log.trace("stop().");
+ if (this._isTracking) {
+ await this.asyncObserver.promiseObserversComplete();
+ this.onStop();
+ this._isTracking = false;
+ }
+ },
+
+ // Override these in your subclasses.
+ onStart() {},
+ onStop() {},
+ async observe(subject, topic, data) {},
+
+ engineIsEnabled() {
+ if (!this.engine) {
+ // Can't tell -- we must be running in a test!
+ return true;
+ }
+ return this.engine.enabled;
+ },
+
+ /**
+ * Starts or stops listening for changes depending on the associated engine's
+ * enabled state.
+ *
+ * @param {Boolean} engineEnabled Whether the engine was enabled.
+ */
+ async onEngineEnabledChanged(engineEnabled) {
+ if (engineEnabled == this._isTracking) {
+ return;
+ }
+
+ if (engineEnabled) {
+ this.start();
+ } else {
+ await this.stop();
+ this.clearChangedIDs();
+ }
+ },
+
+ async finalize() {
+ await this.stop();
+ },
+};
+
+/*
+ * A tracker that persists a list of IDs for all changed items that need to be
+ * synced. This is 🚨 _extremely deprecated_ 🚨 and only kept around for current
+ * engines. ⚠️ Please **don't use it** for new engines! ⚠️
+ *
+ * Why is this kind of external change tracking deprecated? Because it causes
+ * consistency issues due to missed notifications, interrupted syncs, and the
+ * tracker's view of what changed diverging from the data store's.
+ */
+export function LegacyTracker(name, engine) {
+ Tracker.call(this, name, engine);
+
+ this._ignored = [];
+ this.file = this.name;
+ this._storage = new JSONFile({
+ path: Utils.jsonFilePath("changes", this.file),
+ dataPostProcessor: json => this._dataPostProcessor(json),
+ beforeSave: () => this._beforeSave(),
+ });
+ this._ignoreAll = false;
+}
+
+LegacyTracker.prototype = {
+ get ignoreAll() {
+ return this._ignoreAll;
+ },
+
+ set ignoreAll(value) {
+ this._ignoreAll = value;
+ },
+
+ // Default to an empty object if the file doesn't exist.
+ _dataPostProcessor(json) {
+ return (typeof json == "object" && json) || {};
+ },
+
+ // Ensure the Weave storage directory exists before writing the file.
+ _beforeSave() {
+ return ensureDirectory(this._storage.path);
+ },
+
+ async getChangedIDs() {
+ await this._storage.load();
+ return this._storage.data;
+ },
+
+ _saveChangedIDs() {
+ this._storage.saveSoon();
+ },
+
+ // ignore/unignore specific IDs. Useful for ignoring items that are
+ // being processed, or that shouldn't be synced.
+ // But note: not persisted to disk
+
+ ignoreID(id) {
+ this.unignoreID(id);
+ this._ignored.push(id);
+ },
+
+ unignoreID(id) {
+ let index = this._ignored.indexOf(id);
+ if (index != -1) {
+ this._ignored.splice(index, 1);
+ }
+ },
+
+ async _saveChangedID(id, when) {
+ this._log.trace(`Adding changed ID: ${id}, ${JSON.stringify(when)}`);
+ const changedIDs = await this.getChangedIDs();
+ changedIDs[id] = when;
+ this._saveChangedIDs();
+ },
+
+ async addChangedID(id, when) {
+ if (!id) {
+ this._log.warn("Attempted to add undefined ID to tracker");
+ return false;
+ }
+
+ if (this.ignoreAll || this._ignored.includes(id)) {
+ return false;
+ }
+
+ // Default to the current time in seconds if no time is provided.
+ if (when == null) {
+ when = this._now();
+ }
+
+ const changedIDs = await this.getChangedIDs();
+ // Add/update the entry if we have a newer time.
+ if ((changedIDs[id] || -Infinity) < when) {
+ await this._saveChangedID(id, when);
+ }
+
+ return true;
+ },
+
+ async removeChangedID(...ids) {
+ if (!ids.length || this.ignoreAll) {
+ return false;
+ }
+ for (let id of ids) {
+ if (!id) {
+ this._log.warn("Attempted to remove undefined ID from tracker");
+ continue;
+ }
+ if (this._ignored.includes(id)) {
+ this._log.debug(`Not removing ignored ID ${id} from tracker`);
+ continue;
+ }
+ const changedIDs = await this.getChangedIDs();
+ if (changedIDs[id] != null) {
+ this._log.trace("Removing changed ID " + id);
+ delete changedIDs[id];
+ }
+ }
+ this._saveChangedIDs();
+ return true;
+ },
+
+ clearChangedIDs() {
+ this._log.trace("Clearing changed ID list");
+ this._storage.data = {};
+ this._saveChangedIDs();
+ },
+
+ async finalize() {
+ // Persist all pending tracked changes to disk, and wait for the final write
+ // to finish.
+ await super.finalize();
+ this._saveChangedIDs();
+ await this._storage.finalize();
+ },
+};
+Object.setPrototypeOf(LegacyTracker.prototype, Tracker.prototype);
+
+/**
+ * The Store serves as the interface between Sync and stored data.
+ *
+ * The name "store" is slightly a misnomer because it doesn't actually "store"
+ * anything. Instead, it serves as a gateway to something that actually does
+ * the "storing."
+ *
+ * The store is responsible for record management inside an engine. It tells
+ * Sync what items are available for Sync, converts items to and from Sync's
+ * record format, and applies records from Sync into changes on the underlying
+ * store.
+ *
+ * Store implementations require a number of functions to be implemented. These
+ * are all documented below.
+ *
+ * For stores that deal with many records or which have expensive store access
+ * routines, it is highly recommended to implement a custom applyIncomingBatch
+ * and/or applyIncoming function on top of the basic APIs.
+ */
+
+export function Store(name, engine) {
+ if (!engine) {
+ throw new Error("Store must be associated with an Engine instance.");
+ }
+
+ name = name || "Unnamed";
+ this.name = name.toLowerCase();
+ this.engine = engine;
+
+ this._log = Log.repository.getLogger(`Sync.Engine.${name}.Store`);
+
+ XPCOMUtils.defineLazyGetter(this, "_timer", function () {
+ return Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+ });
+}
+
+Store.prototype = {
+ /**
+ * Apply multiple incoming records against the store.
+ *
+ * This is called with a set of incoming records to process. The function
+ * should look at each record, reconcile with the current local state, and
+ * make the local changes required to bring its state in alignment with the
+ * record.
+ *
+ * The default implementation simply iterates over all records and calls
+ * applyIncoming(). Store implementations may overwrite this function
+ * if desired.
+ *
+ * @param records Array of records to apply
+ * @param a SyncedRecordsTelemetry obj that will keep track of failed reasons
+ * @return Array of record IDs which did not apply cleanly
+ */
+ async applyIncomingBatch(records, countTelemetry) {
+ let failed = [];
+
+ await Async.yieldingForEach(records, async record => {
+ try {
+ await this.applyIncoming(record);
+ } catch (ex) {
+ if (ex.code == SyncEngine.prototype.eEngineAbortApplyIncoming) {
+ // This kind of exception should have a 'cause' attribute, which is an
+ // originating exception.
+ // ex.cause will carry its stack with it when rethrown.
+ throw ex.cause;
+ }
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.warn("Failed to apply incoming record " + record.id, ex);
+ failed.push(record.id);
+ countTelemetry.addIncomingFailedReason(ex.message);
+ }
+ });
+
+ return failed;
+ },
+
+ /**
+ * Apply a single record against the store.
+ *
+ * This takes a single record and makes the local changes required so the
+ * local state matches what's in the record.
+ *
+ * The default implementation calls one of remove(), create(), or update()
+ * depending on the state obtained from the store itself. Store
+ * implementations may overwrite this function if desired.
+ *
+ * @param record
+ * Record to apply
+ */
+ async applyIncoming(record) {
+ if (record.deleted) {
+ await this.remove(record);
+ } else if (!(await this.itemExists(record.id))) {
+ await this.create(record);
+ } else {
+ await this.update(record);
+ }
+ },
+
+ // override these in derived objects
+
+ /**
+ * Create an item in the store from a record.
+ *
+ * This is called by the default implementation of applyIncoming(). If using
+ * applyIncomingBatch(), this won't be called unless your store calls it.
+ *
+ * @param record
+ * The store record to create an item from
+ */
+ async create(record) {
+ throw new Error("override create in a subclass");
+ },
+
+ /**
+ * Remove an item in the store from a record.
+ *
+ * This is called by the default implementation of applyIncoming(). If using
+ * applyIncomingBatch(), this won't be called unless your store calls it.
+ *
+ * @param record
+ * The store record to delete an item from
+ */
+ async remove(record) {
+ throw new Error("override remove in a subclass");
+ },
+
+ /**
+ * Update an item from a record.
+ *
+ * This is called by the default implementation of applyIncoming(). If using
+ * applyIncomingBatch(), this won't be called unless your store calls it.
+ *
+ * @param record
+ * The record to use to update an item from
+ */
+ async update(record) {
+ throw new Error("override update in a subclass");
+ },
+
+ /**
+ * Determine whether a record with the specified ID exists.
+ *
+ * Takes a string record ID and returns a booleans saying whether the record
+ * exists.
+ *
+ * @param id
+ * string record ID
+ * @return boolean indicating whether record exists locally
+ */
+ async itemExists(id) {
+ throw new Error("override itemExists in a subclass");
+ },
+
+ /**
+ * Create a record from the specified ID.
+ *
+ * If the ID is known, the record should be populated with metadata from
+ * the store. If the ID is not known, the record should be created with the
+ * delete field set to true.
+ *
+ * @param id
+ * string record ID
+ * @param collection
+ * Collection to add record to. This is typically passed into the
+ * constructor for the newly-created record.
+ * @return record type for this engine
+ */
+ async createRecord(id, collection) {
+ throw new Error("override createRecord in a subclass");
+ },
+
+ /**
+ * Change the ID of a record.
+ *
+ * @param oldID
+ * string old/current record ID
+ * @param newID
+ * string new record ID
+ */
+ async changeItemID(oldID, newID) {
+ throw new Error("override changeItemID in a subclass");
+ },
+
+ /**
+ * Obtain the set of all known record IDs.
+ *
+ * @return Object with ID strings as keys and values of true. The values
+ * are ignored.
+ */
+ async getAllIDs() {
+ throw new Error("override getAllIDs in a subclass");
+ },
+
+ /**
+ * Wipe all data in the store.
+ *
+ * This function is called during remote wipes or when replacing local data
+ * with remote data.
+ *
+ * This function should delete all local data that the store is managing. It
+ * can be thought of as clearing out all state and restoring the "new
+ * browser" state.
+ */
+ async wipe() {
+ throw new Error("override wipe in a subclass");
+ },
+};
+
+export function EngineManager(service) {
+ this.service = service;
+
+ this._engines = {};
+
+ this._altEngineInfo = {};
+
+ // This will be populated by Service on startup.
+ this._declined = new Set();
+ this._log = Log.repository.getLogger("Sync.EngineManager");
+ this._log.manageLevelFromPref("services.sync.log.logger.service.engines");
+ // define the default level for all engine logs here (although each engine
+ // allows its level to be controlled via a specific, non-default pref)
+ Log.repository
+ .getLogger(`Sync.Engine`)
+ .manageLevelFromPref("services.sync.log.logger.engine");
+}
+
+EngineManager.prototype = {
+ get(name) {
+ // Return an array of engines if we have an array of names
+ if (Array.isArray(name)) {
+ let engines = [];
+ name.forEach(function (name) {
+ let engine = this.get(name);
+ if (engine) {
+ engines.push(engine);
+ }
+ }, this);
+ return engines;
+ }
+
+ return this._engines[name]; // Silently returns undefined for unknown names.
+ },
+
+ getAll() {
+ let engines = [];
+ for (let [, engine] of Object.entries(this._engines)) {
+ engines.push(engine);
+ }
+ return engines;
+ },
+
+ /**
+ * If a user has changed a pref that controls which variant of a sync engine
+ * for a given collection we use, unregister the old engine and register the
+ * new one.
+ *
+ * This is called by EngineSynchronizer before every sync.
+ */
+ async switchAlternatives() {
+ for (let [name, info] of Object.entries(this._altEngineInfo)) {
+ let prefValue = info.prefValue;
+ if (prefValue === info.lastValue) {
+ this._log.trace(
+ `No change for engine ${name} (${info.pref} is still ${prefValue})`
+ );
+ continue;
+ }
+ // Unregister the old engine, register the new one.
+ this._log.info(
+ `Switching ${name} engine ("${info.pref}" went from ${info.lastValue} => ${prefValue})`
+ );
+ try {
+ await this._removeAndFinalize(name);
+ } catch (e) {
+ this._log.warn(`Failed to remove previous ${name} engine...`, e);
+ }
+ let engineType = prefValue ? info.whenTrue : info.whenFalse;
+ try {
+ // If register throws, we'll try again next sync, but until then there
+ // won't be an engine registered for this collection.
+ await this.register(engineType);
+ info.lastValue = prefValue;
+ // Note: engineType.name is using Function.prototype.name.
+ this._log.info(`Switched the ${name} engine to use ${engineType.name}`);
+ } catch (e) {
+ this._log.warn(
+ `Switching the ${name} engine to use ${engineType.name} failed (couldn't register)`,
+ e
+ );
+ }
+ }
+ },
+
+ async registerAlternatives(name, pref, whenTrue, whenFalse) {
+ let info = { name, pref, whenTrue, whenFalse };
+
+ XPCOMUtils.defineLazyPreferenceGetter(info, "prefValue", pref, false);
+
+ let chosen = info.prefValue ? info.whenTrue : info.whenFalse;
+ info.lastValue = info.prefValue;
+ this._altEngineInfo[name] = info;
+
+ await this.register(chosen);
+ },
+
+ /**
+ * N.B., does not pay attention to the declined list.
+ */
+ getEnabled() {
+ return this.getAll()
+ .filter(engine => engine.enabled)
+ .sort((a, b) => a.syncPriority - b.syncPriority);
+ },
+
+ get enabledEngineNames() {
+ return this.getEnabled().map(e => e.name);
+ },
+
+ persistDeclined() {
+ Svc.Prefs.set("declinedEngines", [...this._declined].join(","));
+ },
+
+ /**
+ * Returns an array.
+ */
+ getDeclined() {
+ return [...this._declined];
+ },
+
+ setDeclined(engines) {
+ this._declined = new Set(engines);
+ this.persistDeclined();
+ },
+
+ isDeclined(engineName) {
+ return this._declined.has(engineName);
+ },
+
+ /**
+ * Accepts a Set or an array.
+ */
+ decline(engines) {
+ for (let e of engines) {
+ this._declined.add(e);
+ }
+ this.persistDeclined();
+ },
+
+ undecline(engines) {
+ for (let e of engines) {
+ this._declined.delete(e);
+ }
+ this.persistDeclined();
+ },
+
+ /**
+ * Register an Engine to the service. Alternatively, give an array of engine
+ * objects to register.
+ *
+ * @param engineObject
+ * Engine object used to get an instance of the engine
+ * @return The engine object if anything failed
+ */
+ async register(engineObject) {
+ if (Array.isArray(engineObject)) {
+ for (const e of engineObject) {
+ await this.register(e);
+ }
+ return;
+ }
+
+ try {
+ let engine = new engineObject(this.service);
+ let name = engine.name;
+ if (name in this._engines) {
+ this._log.error("Engine '" + name + "' is already registered!");
+ } else {
+ if (engine.initialize) {
+ await engine.initialize();
+ }
+ this._engines[name] = engine;
+ }
+ } catch (ex) {
+ let name = engineObject || "";
+ name = name.prototype || "";
+ name = name.name || "";
+
+ this._log.error(`Could not initialize engine ${name}`, ex);
+ }
+ },
+
+ async unregister(val) {
+ let name = val;
+ if (val instanceof SyncEngine) {
+ name = val.name;
+ }
+ await this._removeAndFinalize(name);
+ delete this._altEngineInfo[name];
+ },
+
+ // Common code for disabling an engine by name, that doesn't complain if the
+ // engine doesn't exist. Doesn't touch the engine's alternative info (if any
+ // exists).
+ async _removeAndFinalize(name) {
+ if (name in this._engines) {
+ let engine = this._engines[name];
+ delete this._engines[name];
+ await engine.finalize();
+ }
+ },
+
+ async clear() {
+ for (let name in this._engines) {
+ let engine = this._engines[name];
+ delete this._engines[name];
+ await engine.finalize();
+ }
+ this._altEngineInfo = {};
+ },
+};
+
+export function SyncEngine(name, service) {
+ if (!service) {
+ throw new Error("SyncEngine must be associated with a Service instance.");
+ }
+
+ this.Name = name || "Unnamed";
+ this.name = name.toLowerCase();
+ this.service = service;
+
+ this._notify = Utils.notify("weave:engine:");
+ this._log = Log.repository.getLogger("Sync.Engine." + this.Name);
+ this._log.manageLevelFromPref(`services.sync.log.logger.engine.${this.name}`);
+
+ this._modified = this.emptyChangeset();
+ this._tracker; // initialize tracker to load previously changed IDs
+ this._log.debug("Engine constructed");
+
+ this._toFetchStorage = new JSONFile({
+ path: Utils.jsonFilePath("toFetch", this.name),
+ dataPostProcessor: json => this._metadataPostProcessor(json),
+ beforeSave: () => this._beforeSaveMetadata(),
+ });
+
+ this._previousFailedStorage = new JSONFile({
+ path: Utils.jsonFilePath("failed", this.name),
+ dataPostProcessor: json => this._metadataPostProcessor(json),
+ beforeSave: () => this._beforeSaveMetadata(),
+ });
+
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_enabled",
+ `services.sync.engine.${this.prefName}`,
+ false
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_syncID",
+ `services.sync.${this.name}.syncID`,
+ ""
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_lastSync",
+ `services.sync.${this.name}.lastSync`,
+ "0",
+ null,
+ v => parseFloat(v)
+ );
+ // Async initializations can be made in the initialize() method.
+
+ this.asyncObserver = Async.asyncObserver(this, this._log);
+}
+
+// Enumeration to define approaches to handling bad records.
+// Attached to the constructor to allow use as a kind of static enumeration.
+SyncEngine.kRecoveryStrategy = {
+ ignore: "ignore",
+ retry: "retry",
+ error: "error",
+};
+
+SyncEngine.prototype = {
+ _recordObj: CryptoWrapper,
+ // _storeObj, and _trackerObj should to be overridden in subclasses
+ _storeObj: Store,
+ _trackerObj: Tracker,
+ version: 1,
+
+ // Local 'constant'.
+ // Signal to the engine that processing further records is pointless.
+ eEngineAbortApplyIncoming: "error.engine.abort.applyincoming",
+
+ // Should we keep syncing if we find a record that cannot be uploaded (ever)?
+ // If this is false, we'll throw, otherwise, we'll ignore the record and
+ // continue. This currently can only happen due to the record being larger
+ // than the record upload limit.
+ allowSkippedRecord: true,
+
+ // Which sortindex to use when retrieving records for this engine.
+ _defaultSort: undefined,
+
+ _hasSyncedThisSession: false,
+
+ _metadataPostProcessor(json) {
+ if (Array.isArray(json)) {
+ // Pre-`JSONFile` storage stored an array, but `JSONFile` defaults to
+ // an object, so we wrap the array for consistency.
+ json = { ids: json };
+ }
+ if (!json.ids) {
+ json.ids = [];
+ }
+ // The set serializes the same way as an array, but offers more efficient
+ // methods of manipulation.
+ json.ids = new SerializableSet(json.ids);
+ return json;
+ },
+
+ async _beforeSaveMetadata() {
+ await ensureDirectory(this._toFetchStorage.path);
+ await ensureDirectory(this._previousFailedStorage.path);
+ },
+
+ // A relative priority to use when computing an order
+ // for engines to be synced. Higher-priority engines
+ // (lower numbers) are synced first.
+ // It is recommended that a unique value be used for each engine,
+ // in order to guarantee a stable sequence.
+ syncPriority: 0,
+
+ // How many records to pull in a single sync. This is primarily to avoid very
+ // long first syncs against profiles with many history records.
+ downloadLimit: null,
+
+ // How many records to pull at one time when specifying IDs. This is to avoid
+ // URI length limitations.
+ guidFetchBatchSize: DEFAULT_GUID_FETCH_BATCH_SIZE,
+
+ downloadBatchSize: DEFAULT_DOWNLOAD_BATCH_SIZE,
+
+ async initialize() {
+ await this._toFetchStorage.load();
+ await this._previousFailedStorage.load();
+ Svc.Prefs.observe(`engine.${this.prefName}`, this.asyncObserver);
+ this._log.debug("SyncEngine initialized", this.name);
+ },
+
+ get prefName() {
+ return this.name;
+ },
+
+ get enabled() {
+ return this._enabled;
+ },
+
+ set enabled(val) {
+ if (!!val != this._enabled) {
+ Svc.Prefs.set("engine." + this.prefName, !!val);
+ }
+ },
+
+ get score() {
+ return this._tracker.score;
+ },
+
+ get _store() {
+ let store = new this._storeObj(this.Name, this);
+ this.__defineGetter__("_store", () => store);
+ return store;
+ },
+
+ get _tracker() {
+ let tracker = new this._trackerObj(this.Name, this);
+ this.__defineGetter__("_tracker", () => tracker);
+ return tracker;
+ },
+
+ get storageURL() {
+ return this.service.storageURL;
+ },
+
+ get engineURL() {
+ return this.storageURL + this.name;
+ },
+
+ get cryptoKeysURL() {
+ return this.storageURL + "crypto/keys";
+ },
+
+ get metaURL() {
+ return this.storageURL + "meta/global";
+ },
+
+ startTracking() {
+ this._tracker.start();
+ },
+
+ // Returns a promise
+ stopTracking() {
+ return this._tracker.stop();
+ },
+
+ // Listens for engine enabled state changes, and updates the tracker's state.
+ // This is an async observer because the tracker waits on all its async
+ // observers to finish when it's stopped.
+ async observe(subject, topic, data) {
+ if (
+ topic == "nsPref:changed" &&
+ data == `services.sync.engine.${this.prefName}`
+ ) {
+ await this._tracker.onEngineEnabledChanged(this._enabled);
+ }
+ },
+
+ async sync() {
+ if (!this.enabled) {
+ return false;
+ }
+
+ if (!this._sync) {
+ throw new Error("engine does not implement _sync method");
+ }
+
+ return this._notify("sync", this.name, this._sync)();
+ },
+
+ // Override this method to return a new changeset type.
+ emptyChangeset() {
+ return new Changeset();
+ },
+
+ /**
+ * Returns the local sync ID for this engine, or `""` if the engine hasn't
+ * synced for the first time. This is exposed for tests.
+ *
+ * @return the current sync ID.
+ */
+ async getSyncID() {
+ return this._syncID;
+ },
+
+ /**
+ * Ensures that the local sync ID for the engine matches the sync ID for the
+ * collection on the server. A mismatch indicates that another client wiped
+ * the collection; we're syncing after a node reassignment, and another
+ * client synced before us; or the store was replaced since the last sync.
+ * In case of a mismatch, we need to reset all local Sync state and start
+ * over as a first sync.
+ *
+ * In most cases, this method should return the new sync ID as-is. However, an
+ * engine may ignore the given ID and assign a different one, if it determines
+ * that the sync ID on the server is out of date. The bookmarks engine uses
+ * this to wipe the server and other clients on the first sync after the user
+ * restores from a backup.
+ *
+ * @param newSyncID
+ * The new sync ID for the collection from `meta/global`.
+ * @return The assigned sync ID. If this doesn't match `newSyncID`, we'll
+ * replace the sync ID in `meta/global` with the assigned ID.
+ */
+ async ensureCurrentSyncID(newSyncID) {
+ let existingSyncID = this._syncID;
+ if (existingSyncID == newSyncID) {
+ return existingSyncID;
+ }
+ this._log.debug("Engine syncIDs: " + [newSyncID, existingSyncID]);
+ Svc.Prefs.set(this.name + ".syncID", newSyncID);
+ Svc.Prefs.set(this.name + ".lastSync", "0");
+ return newSyncID;
+ },
+
+ /**
+ * Resets the local sync ID for the engine, wipes the server, and resets all
+ * local Sync state to start over as a first sync.
+ *
+ * @return the new sync ID.
+ */
+ async resetSyncID() {
+ let newSyncID = await this.resetLocalSyncID();
+ await this.wipeServer();
+ return newSyncID;
+ },
+
+ /**
+ * Resets the local sync ID for the engine, signaling that we're starting over
+ * as a first sync.
+ *
+ * @return the new sync ID.
+ */
+ async resetLocalSyncID() {
+ return this.ensureCurrentSyncID(Utils.makeGUID());
+ },
+
+ /**
+ * Allows overriding scheduler logic -- added to help reduce kinto server
+ * getting hammered because our scheduler never got tuned for it.
+ *
+ * Note: Overriding engines must take resyncs into account -- score will not
+ * be cleared.
+ */
+ shouldSkipSync(syncReason) {
+ return false;
+ },
+
+ /*
+ * lastSync is a timestamp in server time.
+ */
+ async getLastSync() {
+ return this._lastSync;
+ },
+ async setLastSync(lastSync) {
+ // Store the value as a string to keep floating point precision
+ Svc.Prefs.set(this.name + ".lastSync", lastSync.toString());
+ },
+ async resetLastSync() {
+ this._log.debug("Resetting " + this.name + " last sync time");
+ await this.setLastSync(0);
+ },
+
+ get hasSyncedThisSession() {
+ return this._hasSyncedThisSession;
+ },
+
+ set hasSyncedThisSession(hasSynced) {
+ this._hasSyncedThisSession = hasSynced;
+ },
+
+ get toFetch() {
+ this._toFetchStorage.ensureDataReady();
+ return this._toFetchStorage.data.ids;
+ },
+
+ set toFetch(ids) {
+ if (ids.constructor.name != "SerializableSet") {
+ throw new Error(
+ "Bug: Attempted to set toFetch to something that isn't a SerializableSet"
+ );
+ }
+ this._toFetchStorage.data = { ids };
+ this._toFetchStorage.saveSoon();
+ },
+
+ get previousFailed() {
+ this._previousFailedStorage.ensureDataReady();
+ return this._previousFailedStorage.data.ids;
+ },
+
+ set previousFailed(ids) {
+ if (ids.constructor.name != "SerializableSet") {
+ throw new Error(
+ "Bug: Attempted to set previousFailed to something that isn't a SerializableSet"
+ );
+ }
+ this._previousFailedStorage.data = { ids };
+ this._previousFailedStorage.saveSoon();
+ },
+
+ /*
+ * Returns a changeset for this sync. Engine implementations can override this
+ * method to bypass the tracker for certain or all changed items.
+ */
+ async getChangedIDs() {
+ return this._tracker.getChangedIDs();
+ },
+
+ // Create a new record using the store and add in metadata.
+ async _createRecord(id) {
+ let record = await this._store.createRecord(id, this.name);
+ record.id = id;
+ record.collection = this.name;
+ return record;
+ },
+
+ // Creates a tombstone Sync record with additional metadata.
+ _createTombstone(id) {
+ let tombstone = new this._recordObj(this.name, id);
+ tombstone.id = id;
+ tombstone.collection = this.name;
+ tombstone.deleted = true;
+ return tombstone;
+ },
+
+ // Any setup that needs to happen at the beginning of each sync.
+ async _syncStartup() {
+ // Determine if we need to wipe on outdated versions
+ let metaGlobal = await this.service.recordManager.get(this.metaURL);
+ let engines = metaGlobal.payload.engines || {};
+ let engineData = engines[this.name] || {};
+
+ // Assume missing versions are 0 and wipe the server
+ if ((engineData.version || 0) < this.version) {
+ this._log.debug("Old engine data: " + [engineData.version, this.version]);
+
+ // Clear the server and reupload everything on bad version or missing
+ // meta. Note that we don't regenerate per-collection keys here.
+ let newSyncID = await this.resetSyncID();
+
+ // Set the newer version and newly generated syncID
+ engineData.version = this.version;
+ engineData.syncID = newSyncID;
+
+ // Put the new data back into meta/global and mark for upload
+ engines[this.name] = engineData;
+ metaGlobal.payload.engines = engines;
+ metaGlobal.changed = true;
+ } else if (engineData.version > this.version) {
+ // Don't sync this engine if the server has newer data
+
+ let error = new Error("New data: " + [engineData.version, this.version]);
+ error.failureCode = VERSION_OUT_OF_DATE;
+ throw error;
+ } else {
+ // Changes to syncID mean we'll need to upload everything
+ let assignedSyncID = await this.ensureCurrentSyncID(engineData.syncID);
+ if (assignedSyncID != engineData.syncID) {
+ engineData.syncID = assignedSyncID;
+ metaGlobal.changed = true;
+ }
+ }
+
+ // Save objects that need to be uploaded in this._modified. As we
+ // successfully upload objects we remove them from this._modified. If an
+ // error occurs or any objects fail to upload, they will remain in
+ // this._modified. At the end of a sync, or after an error, we add all
+ // objects remaining in this._modified to the tracker.
+ let initialChanges = await this.pullChanges();
+ this._modified.replace(initialChanges);
+ // Clear the tracker now. If the sync fails we'll add the ones we failed
+ // to upload back.
+ this._tracker.clearChangedIDs();
+ this._tracker.resetScore();
+
+ // Keep track of what to delete at the end of sync
+ this._delete = {};
+ },
+
+ async pullChanges() {
+ let lastSync = await this.getLastSync();
+ if (lastSync) {
+ return this.pullNewChanges();
+ }
+ this._log.debug("First sync, uploading all items");
+ return this.pullAllChanges();
+ },
+
+ /**
+ * A tiny abstraction to make it easier to test incoming record
+ * application.
+ */
+ itemSource() {
+ return new Collection(this.engineURL, this._recordObj, this.service);
+ },
+
+ /**
+ * Download and apply remote records changed since the last sync. This
+ * happens in three stages.
+ *
+ * In the first stage, we fetch full records for all changed items, newest
+ * first, up to the download limit. The limit lets us make progress for large
+ * collections, where the sync is likely to be interrupted before we
+ * can fetch everything.
+ *
+ * In the second stage, we fetch the IDs of any remaining records changed
+ * since the last sync, add them to our backlog, and fast-forward our last
+ * sync time.
+ *
+ * In the third stage, we fetch and apply records for all backlogged IDs,
+ * as well as any records that failed to apply during the last sync. We
+ * request records for the IDs in chunks, to avoid exceeding URL length
+ * limits, then remove successfully applied records from the backlog, and
+ * record IDs of any records that failed to apply to retry on the next sync.
+ */
+ async _processIncoming() {
+ this._log.trace("Downloading & applying server changes");
+
+ let newitems = this.itemSource();
+ let lastSync = await this.getLastSync();
+
+ newitems.newer = lastSync;
+ newitems.full = true;
+
+ let downloadLimit = Infinity;
+ if (this.downloadLimit) {
+ // Fetch new records up to the download limit. Currently, only the history
+ // engine sets a limit, since the history collection has the highest volume
+ // of changed records between syncs. The other engines fetch all records
+ // changed since the last sync.
+ if (this._defaultSort) {
+ // A download limit with a sort order doesn't make sense: we won't know
+ // which records to backfill.
+ throw new Error("Can't specify download limit with default sort order");
+ }
+ newitems.sort = "newest";
+ downloadLimit = newitems.limit = this.downloadLimit;
+ } else if (this._defaultSort) {
+ // The bookmarks engine fetches records by sort index; other engines leave
+ // the order unspecified. We can remove `_defaultSort` entirely after bug
+ // 1305563: the sort index won't matter because we'll buffer all bookmarks
+ // before applying.
+ newitems.sort = this._defaultSort;
+ }
+
+ // applied => number of items that should be applied.
+ // failed => number of items that failed in this sync.
+ // newFailed => number of items that failed for the first time in this sync.
+ // reconciled => number of items that were reconciled.
+ // failedReasons => {name, count} of reasons a record failed
+ let countTelemetry = new SyncedRecordsTelemetry();
+ let count = countTelemetry.incomingCounts;
+ let recordsToApply = [];
+ let failedInCurrentSync = new SerializableSet();
+
+ let oldestModified = this.lastModified;
+ let downloadedIDs = new Set();
+
+ // Stage 1: Fetch new records from the server, up to the download limit.
+ if (this.lastModified == null || this.lastModified > lastSync) {
+ let { response, records } = await newitems.getBatched(
+ this.downloadBatchSize
+ );
+ if (!response.success) {
+ response.failureCode = ENGINE_DOWNLOAD_FAIL;
+ throw response;
+ }
+
+ await Async.yieldingForEach(records, async record => {
+ downloadedIDs.add(record.id);
+
+ if (record.modified < oldestModified) {
+ oldestModified = record.modified;
+ }
+
+ let { shouldApply, error } = await this._maybeReconcile(record);
+ if (error) {
+ failedInCurrentSync.add(record.id);
+ count.failed++;
+ countTelemetry.addIncomingFailedReason(error.message);
+ return;
+ }
+ if (!shouldApply) {
+ count.reconciled++;
+ return;
+ }
+ recordsToApply.push(record);
+ });
+
+ let failedToApply = await this._applyRecords(
+ recordsToApply,
+ countTelemetry
+ );
+ Utils.setAddAll(failedInCurrentSync, failedToApply);
+
+ // `applied` is a bit of a misnomer: it counts records that *should* be
+ // applied, so it also includes records that we tried to apply and failed.
+ // `recordsToApply.length - failedToApply.length` is the number of records
+ // that we *successfully* applied.
+ count.failed += failedToApply.length;
+ count.applied += recordsToApply.length;
+ }
+
+ // Stage 2: If we reached our download limit, we might still have records
+ // on the server that changed since the last sync. Fetch the IDs for the
+ // remaining records, and add them to the backlog. Note that this stage
+ // only runs for engines that set a download limit.
+ if (downloadedIDs.size == downloadLimit) {
+ let guidColl = this.itemSource();
+
+ guidColl.newer = lastSync;
+ guidColl.older = oldestModified;
+ guidColl.sort = "oldest";
+
+ let guids = await guidColl.get();
+ if (!guids.success) {
+ throw guids;
+ }
+
+ // Filtering out already downloaded IDs here isn't necessary. We only do
+ // that in case the Sync server doesn't support `older` (bug 1316110).
+ let remainingIDs = guids.obj.filter(id => !downloadedIDs.has(id));
+ if (remainingIDs.length) {
+ this.toFetch = Utils.setAddAll(this.toFetch, remainingIDs);
+ }
+ }
+
+ // Fast-foward the lastSync timestamp since we have backlogged the
+ // remaining items.
+ if (lastSync < this.lastModified) {
+ lastSync = this.lastModified;
+ await this.setLastSync(lastSync);
+ }
+
+ // Stage 3: Backfill records from the backlog, and those that failed to
+ // decrypt or apply during the last sync. We only backfill up to the
+ // download limit, to prevent a large backlog for one engine from blocking
+ // the others. We'll keep processing the backlog on subsequent engine syncs.
+ let failedInPreviousSync = this.previousFailed;
+ let idsToBackfill = Array.from(
+ Utils.setAddAll(
+ Utils.subsetOfSize(this.toFetch, downloadLimit),
+ failedInPreviousSync
+ )
+ );
+
+ // Note that we intentionally overwrite the previously failed list here.
+ // Records that fail to decrypt or apply in two consecutive syncs are likely
+ // corrupt; we remove them from the list because retrying and failing on
+ // every subsequent sync just adds noise.
+ this.previousFailed = failedInCurrentSync;
+
+ let backfilledItems = this.itemSource();
+
+ backfilledItems.sort = "newest";
+ backfilledItems.full = true;
+
+ // `getBatched` includes the list of IDs as a query parameter, so we need to fetch
+ // records in chunks to avoid exceeding URI length limits.
+ if (this.guidFetchBatchSize) {
+ for (let ids of lazy.PlacesUtils.chunkArray(
+ idsToBackfill,
+ this.guidFetchBatchSize
+ )) {
+ backfilledItems.ids = ids;
+
+ let { response, records } = await backfilledItems.getBatched(
+ this.downloadBatchSize
+ );
+ if (!response.success) {
+ response.failureCode = ENGINE_DOWNLOAD_FAIL;
+ throw response;
+ }
+
+ let backfilledRecordsToApply = [];
+ let failedInBackfill = [];
+
+ await Async.yieldingForEach(records, async record => {
+ let { shouldApply, error } = await this._maybeReconcile(record);
+ if (error) {
+ failedInBackfill.push(record.id);
+ count.failed++;
+ countTelemetry.addIncomingFailedReason(error.message);
+ return;
+ }
+ if (!shouldApply) {
+ count.reconciled++;
+ return;
+ }
+ backfilledRecordsToApply.push(record);
+ });
+
+ let failedToApply = await this._applyRecords(
+ backfilledRecordsToApply,
+ countTelemetry
+ );
+ failedInBackfill.push(...failedToApply);
+
+ count.failed += failedToApply.length;
+ count.applied += backfilledRecordsToApply.length;
+
+ this.toFetch = Utils.setDeleteAll(this.toFetch, ids);
+ this.previousFailed = Utils.setAddAll(
+ this.previousFailed,
+ failedInBackfill
+ );
+
+ if (lastSync < this.lastModified) {
+ lastSync = this.lastModified;
+ await this.setLastSync(lastSync);
+ }
+ }
+ }
+
+ count.newFailed = 0;
+ for (let item of this.previousFailed) {
+ // Anything that failed in the current sync that also failed in
+ // the previous sync means there is likely something wrong with
+ // the record, we remove it from trying again to prevent
+ // infinitely syncing corrupted records
+ if (failedInPreviousSync.has(item)) {
+ this.previousFailed.delete(item);
+ } else {
+ // otherwise it's a new failed and we count it as so
+ ++count.newFailed;
+ }
+ }
+
+ count.succeeded = Math.max(0, count.applied - count.failed);
+ this._log.info(
+ [
+ "Records:",
+ count.applied,
+ "applied,",
+ count.succeeded,
+ "successfully,",
+ count.failed,
+ "failed to apply,",
+ count.newFailed,
+ "newly failed to apply,",
+ count.reconciled,
+ "reconciled.",
+ ].join(" ")
+ );
+ Observers.notify("weave:engine:sync:applied", count, this.name);
+ },
+
+ async _maybeReconcile(item) {
+ let key = this.service.collectionKeys.keyForCollection(this.name);
+
+ // Grab a later last modified if possible
+ if (this.lastModified == null || item.modified > this.lastModified) {
+ this.lastModified = item.modified;
+ }
+
+ try {
+ try {
+ await item.decrypt(key);
+ } catch (ex) {
+ if (!Utils.isHMACMismatch(ex)) {
+ throw ex;
+ }
+ let strategy = await this.handleHMACMismatch(item, true);
+ if (strategy == SyncEngine.kRecoveryStrategy.retry) {
+ // You only get one retry.
+ try {
+ // Try decrypting again, typically because we've got new keys.
+ this._log.info("Trying decrypt again...");
+ key = this.service.collectionKeys.keyForCollection(this.name);
+ await item.decrypt(key);
+ strategy = null;
+ } catch (ex) {
+ if (!Utils.isHMACMismatch(ex)) {
+ throw ex;
+ }
+ strategy = await this.handleHMACMismatch(item, false);
+ }
+ }
+
+ switch (strategy) {
+ case null:
+ // Retry succeeded! No further handling.
+ break;
+ case SyncEngine.kRecoveryStrategy.retry:
+ this._log.debug("Ignoring second retry suggestion.");
+ // Fall through to error case.
+ case SyncEngine.kRecoveryStrategy.error:
+ this._log.warn("Error decrypting record", ex);
+ return { shouldApply: false, error: ex };
+ case SyncEngine.kRecoveryStrategy.ignore:
+ this._log.debug(
+ "Ignoring record " + item.id + " with bad HMAC: already handled."
+ );
+ return { shouldApply: false, error: null };
+ }
+ }
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.warn("Error decrypting record", ex);
+ return { shouldApply: false, error: ex };
+ }
+
+ if (this._shouldDeleteRemotely(item)) {
+ this._log.trace("Deleting item from server without applying", item);
+ await this._deleteId(item.id);
+ return { shouldApply: false, error: null };
+ }
+
+ let shouldApply;
+ try {
+ shouldApply = await this._reconcile(item);
+ } catch (ex) {
+ if (ex.code == SyncEngine.prototype.eEngineAbortApplyIncoming) {
+ this._log.warn("Reconciliation failed: aborting incoming processing.");
+ throw ex.cause;
+ } else if (!Async.isShutdownException(ex)) {
+ this._log.warn("Failed to reconcile incoming record " + item.id, ex);
+ return { shouldApply: false, error: ex };
+ } else {
+ throw ex;
+ }
+ }
+
+ if (!shouldApply) {
+ this._log.trace("Skipping reconciled incoming item " + item.id);
+ }
+
+ return { shouldApply, error: null };
+ },
+
+ async _applyRecords(records, countTelemetry) {
+ this._tracker.ignoreAll = true;
+ try {
+ let failedIDs = await this._store.applyIncomingBatch(
+ records,
+ countTelemetry
+ );
+ return failedIDs;
+ } catch (ex) {
+ // Catch any error that escapes from applyIncomingBatch. At present
+ // those will all be abort events.
+ this._log.warn("Got exception, aborting processIncoming", ex);
+ throw ex;
+ } finally {
+ this._tracker.ignoreAll = false;
+ }
+ },
+
+ // Indicates whether an incoming item should be deleted from the server at
+ // the end of the sync. Engines can override this method to clean up records
+ // that shouldn't be on the server.
+ _shouldDeleteRemotely(remoteItem) {
+ return false;
+ },
+
+ /**
+ * Find a GUID of an item that is a duplicate of the incoming item but happens
+ * to have a different GUID
+ *
+ * @return GUID of the similar item; falsy otherwise
+ */
+ async _findDupe(item) {
+ // By default, assume there's no dupe items for the engine
+ },
+
+ /**
+ * Called before a remote record is discarded due to failed reconciliation.
+ * Used by bookmark sync to merge folder child orders.
+ */
+ beforeRecordDiscard(localRecord, remoteRecord, remoteIsNewer) {},
+
+ // Called when the server has a record marked as deleted, but locally we've
+ // changed it more recently than the deletion. If we return false, the
+ // record will be deleted locally. If we return true, we'll reupload the
+ // record to the server -- any extra work that's needed as part of this
+ // process should be done at this point (such as mark the record's parent
+ // for reuploading in the case of bookmarks).
+ async _shouldReviveRemotelyDeletedRecord(remoteItem) {
+ return true;
+ },
+
+ async _deleteId(id) {
+ await this._tracker.removeChangedID(id);
+ this._noteDeletedId(id);
+ },
+
+ // Marks an ID for deletion at the end of the sync.
+ _noteDeletedId(id) {
+ if (this._delete.ids == null) {
+ this._delete.ids = [id];
+ } else {
+ this._delete.ids.push(id);
+ }
+ },
+
+ async _switchItemToDupe(localDupeGUID, incomingItem) {
+ // The local, duplicate ID is always deleted on the server.
+ await this._deleteId(localDupeGUID);
+
+ // We unconditionally change the item's ID in case the engine knows of
+ // an item but doesn't expose it through itemExists. If the API
+ // contract were stronger, this could be changed.
+ this._log.debug(
+ "Switching local ID to incoming: " +
+ localDupeGUID +
+ " -> " +
+ incomingItem.id
+ );
+ return this._store.changeItemID(localDupeGUID, incomingItem.id);
+ },
+
+ /**
+ * Reconcile incoming record with local state.
+ *
+ * This function essentially determines whether to apply an incoming record.
+ *
+ * @param item
+ * Record from server to be tested for application.
+ * @return boolean
+ * Truthy if incoming record should be applied. False if not.
+ */
+ async _reconcile(item) {
+ if (this._log.level <= Log.Level.Trace) {
+ this._log.trace("Incoming: " + item);
+ }
+
+ // We start reconciling by collecting a bunch of state. We do this here
+ // because some state may change during the course of this function and we
+ // need to operate on the original values.
+ let existsLocally = await this._store.itemExists(item.id);
+ let locallyModified = this._modified.has(item.id);
+
+ // TODO Handle clock drift better. Tracked in bug 721181.
+ let remoteAge = Resource.serverTime - item.modified;
+ let localAge = locallyModified
+ ? Date.now() / 1000 - this._modified.getModifiedTimestamp(item.id)
+ : null;
+ let remoteIsNewer = remoteAge < localAge;
+
+ this._log.trace(
+ "Reconciling " +
+ item.id +
+ ". exists=" +
+ existsLocally +
+ "; modified=" +
+ locallyModified +
+ "; local age=" +
+ localAge +
+ "; incoming age=" +
+ remoteAge
+ );
+
+ // We handle deletions first so subsequent logic doesn't have to check
+ // deleted flags.
+ if (item.deleted) {
+ // If the item doesn't exist locally, there is nothing for us to do. We
+ // can't check for duplicates because the incoming record has no data
+ // which can be used for duplicate detection.
+ if (!existsLocally) {
+ this._log.trace(
+ "Ignoring incoming item because it was deleted and " +
+ "the item does not exist locally."
+ );
+ return false;
+ }
+
+ // We decide whether to process the deletion by comparing the record
+ // ages. If the item is not modified locally, the remote side wins and
+ // the deletion is processed. If it is modified locally, we take the
+ // newer record.
+ if (!locallyModified) {
+ this._log.trace(
+ "Applying incoming delete because the local item " +
+ "exists and isn't modified."
+ );
+ return true;
+ }
+ this._log.trace("Incoming record is deleted but we had local changes.");
+
+ if (remoteIsNewer) {
+ this._log.trace("Remote record is newer -- deleting local record.");
+ return true;
+ }
+ // If the local record is newer, we defer to individual engines for
+ // how to handle this. By default, we revive the record.
+ let willRevive = await this._shouldReviveRemotelyDeletedRecord(item);
+ this._log.trace("Local record is newer -- reviving? " + willRevive);
+
+ return !willRevive;
+ }
+
+ // At this point the incoming record is not for a deletion and must have
+ // data. If the incoming record does not exist locally, we check for a local
+ // duplicate existing under a different ID. The default implementation of
+ // _findDupe() is empty, so engines have to opt in to this functionality.
+ //
+ // If we find a duplicate, we change the local ID to the incoming ID and we
+ // refresh the metadata collected above. See bug 710448 for the history
+ // of this logic.
+ if (!existsLocally) {
+ let localDupeGUID = await this._findDupe(item);
+ if (localDupeGUID) {
+ this._log.trace(
+ "Local item " +
+ localDupeGUID +
+ " is a duplicate for " +
+ "incoming item " +
+ item.id
+ );
+
+ // The current API contract does not mandate that the ID returned by
+ // _findDupe() actually exists. Therefore, we have to perform this
+ // check.
+ existsLocally = await this._store.itemExists(localDupeGUID);
+
+ // If the local item was modified, we carry its metadata forward so
+ // appropriate reconciling can be performed.
+ if (this._modified.has(localDupeGUID)) {
+ locallyModified = true;
+ localAge =
+ this._tracker._now() -
+ this._modified.getModifiedTimestamp(localDupeGUID);
+ remoteIsNewer = remoteAge < localAge;
+
+ this._modified.changeID(localDupeGUID, item.id);
+ } else {
+ locallyModified = false;
+ localAge = null;
+ }
+
+ // Tell the engine to do whatever it needs to switch the items.
+ await this._switchItemToDupe(localDupeGUID, item);
+
+ this._log.debug(
+ "Local item after duplication: age=" +
+ localAge +
+ "; modified=" +
+ locallyModified +
+ "; exists=" +
+ existsLocally
+ );
+ } else {
+ this._log.trace("No duplicate found for incoming item: " + item.id);
+ }
+ }
+
+ // At this point we've performed duplicate detection. But, nothing here
+ // should depend on duplicate detection as the above should have updated
+ // state seamlessly.
+
+ if (!existsLocally) {
+ // If the item doesn't exist locally and we have no local modifications
+ // to the item (implying that it was not deleted), always apply the remote
+ // item.
+ if (!locallyModified) {
+ this._log.trace(
+ "Applying incoming because local item does not exist " +
+ "and was not deleted."
+ );
+ return true;
+ }
+
+ // If the item was modified locally but isn't present, it must have
+ // been deleted. If the incoming record is younger, we restore from
+ // that record.
+ if (remoteIsNewer) {
+ this._log.trace(
+ "Applying incoming because local item was deleted " +
+ "before the incoming item was changed."
+ );
+ this._modified.delete(item.id);
+ return true;
+ }
+
+ this._log.trace(
+ "Ignoring incoming item because the local item's " +
+ "deletion is newer."
+ );
+ return false;
+ }
+
+ // If the remote and local records are the same, there is nothing to be
+ // done, so we don't do anything. In the ideal world, this logic wouldn't
+ // be here and the engine would take a record and apply it. The reason we
+ // want to defer this logic is because it would avoid a redundant and
+ // possibly expensive dip into the storage layer to query item state.
+ // This should get addressed in the async rewrite, so we ignore it for now.
+ let localRecord = await this._createRecord(item.id);
+ let recordsEqual = Utils.deepEquals(item.cleartext, localRecord.cleartext);
+
+ // If the records are the same, we don't need to do anything. This does
+ // potentially throw away a local modification time. But, if the records
+ // are the same, does it matter?
+ if (recordsEqual) {
+ this._log.trace(
+ "Ignoring incoming item because the local item is identical."
+ );
+
+ this._modified.delete(item.id);
+ return false;
+ }
+
+ // At this point the records are different.
+
+ // If we have no local modifications, always take the server record.
+ if (!locallyModified) {
+ this._log.trace("Applying incoming record because no local conflicts.");
+ return true;
+ }
+
+ // At this point, records are different and the local record is modified.
+ // We resolve conflicts by record age, where the newest one wins. This does
+ // result in data loss and should be handled by giving the engine an
+ // opportunity to merge the records. Bug 720592 tracks this feature.
+ this._log.warn(
+ "DATA LOSS: Both local and remote changes to record: " + item.id
+ );
+ if (!remoteIsNewer) {
+ this.beforeRecordDiscard(localRecord, item, remoteIsNewer);
+ }
+ return remoteIsNewer;
+ },
+
+ // Upload outgoing records.
+ async _uploadOutgoing() {
+ this._log.trace("Uploading local changes to server.");
+
+ // collection we'll upload
+ let up = new Collection(this.engineURL, null, this.service);
+ let modifiedIDs = new Set(this._modified.ids());
+ let countTelemetry = new SyncedRecordsTelemetry();
+ let counts = countTelemetry.outgoingCounts;
+ this._log.info(`Uploading ${modifiedIDs.size} outgoing records`);
+ if (modifiedIDs.size) {
+ counts.sent = modifiedIDs.size;
+
+ let failed = [];
+ let successful = [];
+ let lastSync = await this.getLastSync();
+ let handleResponse = async (postQueue, resp, batchOngoing) => {
+ // Note: We don't want to update this.lastSync, or this._modified until
+ // the batch is complete, however we want to remember success/failure
+ // indicators for when that happens.
+ if (!resp.success) {
+ this._log.debug(`Uploading records failed: ${resp.status}`);
+ resp.failureCode =
+ resp.status == 412 ? ENGINE_BATCH_INTERRUPTED : ENGINE_UPLOAD_FAIL;
+ throw resp;
+ }
+
+ // Update server timestamp from the upload.
+ failed = failed.concat(Object.keys(resp.obj.failed));
+ successful = successful.concat(resp.obj.success);
+
+ if (batchOngoing) {
+ // Nothing to do yet
+ return;
+ }
+
+ if (failed.length && this._log.level <= Log.Level.Debug) {
+ this._log.debug(
+ "Records that will be uploaded again because " +
+ "the server couldn't store them: " +
+ failed.join(", ")
+ );
+ }
+
+ counts.failed += failed.length;
+ Object.values(failed).forEach(message => {
+ countTelemetry.addOutgoingFailedReason(message);
+ });
+
+ for (let id of successful) {
+ this._modified.delete(id);
+ }
+
+ await this._onRecordsWritten(
+ successful,
+ failed,
+ postQueue.lastModified
+ );
+
+ // Advance lastSync since we've finished the batch.
+ if (postQueue.lastModified > lastSync) {
+ lastSync = postQueue.lastModified;
+ await this.setLastSync(lastSync);
+ }
+
+ // clear for next batch
+ failed.length = 0;
+ successful.length = 0;
+ };
+
+ let postQueue = up.newPostQueue(this._log, lastSync, handleResponse);
+
+ for (let id of modifiedIDs) {
+ let out;
+ let ok = false;
+ try {
+ out = await this._createRecord(id);
+ if (this._log.level <= Log.Level.Trace) {
+ this._log.trace("Outgoing: " + out);
+ }
+ await out.encrypt(
+ this.service.collectionKeys.keyForCollection(this.name)
+ );
+ ok = true;
+ } catch (ex) {
+ this._log.warn("Error creating record", ex);
+ ++counts.failed;
+ countTelemetry.addOutgoingFailedReason(ex.message);
+ if (Async.isShutdownException(ex) || !this.allowSkippedRecord) {
+ if (!this.allowSkippedRecord) {
+ // Don't bother for shutdown errors
+ Observers.notify("weave:engine:sync:uploaded", counts, this.name);
+ }
+ throw ex;
+ }
+ }
+ if (ok) {
+ let { enqueued, error } = await postQueue.enqueue(out);
+ if (!enqueued) {
+ ++counts.failed;
+ countTelemetry.addOutgoingFailedReason(error.message);
+ if (!this.allowSkippedRecord) {
+ Observers.notify("weave:engine:sync:uploaded", counts, this.name);
+ this._log.warn(
+ `Failed to enqueue record "${id}" (aborting)`,
+ error
+ );
+ throw error;
+ }
+ this._modified.delete(id);
+ this._log.warn(
+ `Failed to enqueue record "${id}" (skipping)`,
+ error
+ );
+ }
+ }
+ await Async.promiseYield();
+ }
+ await postQueue.flush(true);
+ }
+
+ if (counts.sent || counts.failed) {
+ Observers.notify("weave:engine:sync:uploaded", counts, this.name);
+ }
+ },
+
+ async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
+ // Implement this method to take specific actions against successfully
+ // uploaded records and failed records.
+ },
+
+ // Any cleanup necessary.
+ // Save the current snapshot so as to calculate changes at next sync
+ async _syncFinish() {
+ this._log.trace("Finishing up sync");
+
+ let doDelete = async (key, val) => {
+ let coll = new Collection(this.engineURL, this._recordObj, this.service);
+ coll[key] = val;
+ await coll.delete();
+ };
+
+ for (let [key, val] of Object.entries(this._delete)) {
+ // Remove the key for future uses
+ delete this._delete[key];
+
+ this._log.trace("doing post-sync deletions", { key, val });
+ // Send a simple delete for the property
+ if (key != "ids" || val.length <= 100) {
+ await doDelete(key, val);
+ } else {
+ // For many ids, split into chunks of at most 100
+ while (val.length) {
+ await doDelete(key, val.slice(0, 100));
+ val = val.slice(100);
+ }
+ }
+ }
+ this.hasSyncedThisSession = true;
+ await this._tracker.asyncObserver.promiseObserversComplete();
+ },
+
+ async _syncCleanup() {
+ try {
+ // Mark failed WBOs as changed again so they are reuploaded next time.
+ await this.trackRemainingChanges();
+ } finally {
+ this._modified.clear();
+ }
+ },
+
+ async _sync() {
+ try {
+ Async.checkAppReady();
+ await this._syncStartup();
+ Async.checkAppReady();
+ Observers.notify("weave:engine:sync:status", "process-incoming");
+ await this._processIncoming();
+ Async.checkAppReady();
+ Observers.notify("weave:engine:sync:status", "upload-outgoing");
+ try {
+ await this._uploadOutgoing();
+ Async.checkAppReady();
+ await this._syncFinish();
+ } catch (ex) {
+ if (!ex.status || ex.status != 412) {
+ throw ex;
+ }
+ // a 412 posting just means another client raced - but we don't want
+ // to treat that as a sync error - the next sync is almost certain
+ // to work.
+ this._log.warn("412 error during sync - will retry.");
+ }
+ } finally {
+ await this._syncCleanup();
+ }
+ },
+
+ async canDecrypt() {
+ // Report failure even if there's nothing to decrypt
+ let canDecrypt = false;
+
+ // Fetch the most recently uploaded record and try to decrypt it
+ let test = new Collection(this.engineURL, this._recordObj, this.service);
+ test.limit = 1;
+ test.sort = "newest";
+ test.full = true;
+
+ let key = this.service.collectionKeys.keyForCollection(this.name);
+
+ // Any failure fetching/decrypting will just result in false
+ try {
+ this._log.trace("Trying to decrypt a record from the server..");
+ let json = (await test.get()).obj[0];
+ let record = new this._recordObj();
+ record.deserialize(json);
+ await record.decrypt(key);
+ canDecrypt = true;
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Failed test decrypt", ex);
+ }
+
+ return canDecrypt;
+ },
+
+ /**
+ * Deletes the collection for this engine on the server, and removes all local
+ * Sync metadata for this engine. This does *not* remove any existing data on
+ * other clients. This is called when we reset the sync ID.
+ */
+ async wipeServer() {
+ await this._deleteServerCollection();
+ await this._resetClient();
+ },
+
+ /**
+ * Deletes the collection for this engine on the server, without removing
+ * any local Sync metadata or user data. Deleting the collection will not
+ * remove any user data on other clients, but will force other clients to
+ * start over as a first sync.
+ */
+ async _deleteServerCollection() {
+ let response = await this.service.resource(this.engineURL).delete();
+ if (response.status != 200 && response.status != 404) {
+ throw response;
+ }
+ },
+
+ async removeClientData() {
+ // Implement this method in engines that store client specific data
+ // on the server.
+ },
+
+ /*
+ * Decide on (and partially effect) an error-handling strategy.
+ *
+ * Asks the Service to respond to an HMAC error, which might result in keys
+ * being downloaded. That call returns true if an action which might allow a
+ * retry to occur.
+ *
+ * If `mayRetry` is truthy, and the Service suggests a retry,
+ * handleHMACMismatch returns kRecoveryStrategy.retry. Otherwise, it returns
+ * kRecoveryStrategy.error.
+ *
+ * Subclasses of SyncEngine can override this method to allow for different
+ * behavior -- e.g., to delete and ignore erroneous entries.
+ *
+ * All return values will be part of the kRecoveryStrategy enumeration.
+ */
+ async handleHMACMismatch(item, mayRetry) {
+ // By default we either try again, or bail out noisily.
+ return (await this.service.handleHMACEvent()) && mayRetry
+ ? SyncEngine.kRecoveryStrategy.retry
+ : SyncEngine.kRecoveryStrategy.error;
+ },
+
+ /**
+ * Returns a changeset containing all items in the store. The default
+ * implementation returns a changeset with timestamps from long ago, to
+ * ensure we always use the remote version if one exists.
+ *
+ * This function is only called for the first sync. Subsequent syncs call
+ * `pullNewChanges`.
+ *
+ * @return A `Changeset` object.
+ */
+ async pullAllChanges() {
+ let changes = {};
+ let ids = await this._store.getAllIDs();
+ for (let id in ids) {
+ changes[id] = 0;
+ }
+ return changes;
+ },
+
+ /*
+ * Returns a changeset containing entries for all currently tracked items.
+ * The default implementation returns a changeset with timestamps indicating
+ * when the item was added to the tracker.
+ *
+ * @return A `Changeset` object.
+ */
+ async pullNewChanges() {
+ await this._tracker.asyncObserver.promiseObserversComplete();
+ return this.getChangedIDs();
+ },
+
+ /**
+ * Adds all remaining changeset entries back to the tracker, typically for
+ * items that failed to upload. This method is called at the end of each sync.
+ *
+ */
+ async trackRemainingChanges() {
+ for (let [id, change] of this._modified.entries()) {
+ await this._tracker.addChangedID(id, change);
+ }
+ },
+
+ /**
+ * Removes all local Sync metadata for this engine, but keeps all existing
+ * local user data.
+ */
+ async resetClient() {
+ return this._notify("reset-client", this.name, this._resetClient)();
+ },
+
+ async _resetClient() {
+ await this.resetLastSync();
+ this.hasSyncedThisSession = false;
+ this.previousFailed = new SerializableSet();
+ this.toFetch = new SerializableSet();
+ },
+
+ /**
+ * Removes all local Sync metadata and user data for this engine.
+ */
+ async wipeClient() {
+ return this._notify("wipe-client", this.name, this._wipeClient)();
+ },
+
+ async _wipeClient() {
+ await this.resetClient();
+ this._log.debug("Deleting all local data");
+ this._tracker.ignoreAll = true;
+ await this._store.wipe();
+ this._tracker.ignoreAll = false;
+ this._tracker.clearChangedIDs();
+ },
+
+ /**
+ * If one exists, initialize and return a validator for this engine (which
+ * must have a `validate(engine)` method that returns a promise to an object
+ * with a getSummary method). Otherwise return null.
+ */
+ getValidator() {
+ return null;
+ },
+
+ async finalize() {
+ Svc.Prefs.ignore(`engine.${this.prefName}`, this.asyncObserver);
+ await this.asyncObserver.promiseObserversComplete();
+ await this._tracker.finalize();
+ await this._toFetchStorage.finalize();
+ await this._previousFailedStorage.finalize();
+ },
+
+ // Returns a new watchdog. Exposed for tests.
+ _newWatchdog() {
+ return Async.watchdog();
+ },
+};
+
+/**
+ * A changeset is created for each sync in `Engine::get{Changed, All}IDs`,
+ * and stores opaque change data for tracked IDs. The default implementation
+ * only records timestamps, though engines can extend this to store additional
+ * data for each entry.
+ */
+export class Changeset {
+ // Creates an empty changeset.
+ constructor() {
+ this.changes = {};
+ }
+
+ // Returns the last modified time, in seconds, for an entry in the changeset.
+ // `id` is guaranteed to be in the set.
+ getModifiedTimestamp(id) {
+ return this.changes[id];
+ }
+
+ // Adds a change for a tracked ID to the changeset.
+ set(id, change) {
+ this.changes[id] = change;
+ }
+
+ // Adds multiple entries to the changeset, preserving existing entries.
+ insert(changes) {
+ Object.assign(this.changes, changes);
+ }
+
+ // Overwrites the existing set of tracked changes with new entries.
+ replace(changes) {
+ this.changes = changes;
+ }
+
+ // Indicates whether an entry is in the changeset.
+ has(id) {
+ return id in this.changes;
+ }
+
+ // Deletes an entry from the changeset. Used to clean up entries for
+ // reconciled and successfully uploaded records.
+ delete(id) {
+ delete this.changes[id];
+ }
+
+ // Changes the ID of an entry in the changeset. Used when reconciling
+ // duplicates that have local changes.
+ changeID(oldID, newID) {
+ this.changes[newID] = this.changes[oldID];
+ delete this.changes[oldID];
+ }
+
+ // Returns an array of all tracked IDs in this changeset.
+ ids() {
+ return Object.keys(this.changes);
+ }
+
+ // Returns an array of `[id, change]` tuples. Used to repopulate the tracker
+ // with entries for failed uploads at the end of a sync.
+ entries() {
+ return Object.entries(this.changes);
+ }
+
+ // Returns the number of entries in this changeset.
+ count() {
+ return this.ids().length;
+ }
+
+ // Clears the changeset.
+ clear() {
+ this.changes = {};
+ }
+}
diff --git a/services/sync/modules/engines/addons.sys.mjs b/services/sync/modules/engines/addons.sys.mjs
new file mode 100644
index 0000000000..d1b766a957
--- /dev/null
+++ b/services/sync/modules/engines/addons.sys.mjs
@@ -0,0 +1,820 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file defines the add-on sync functionality.
+ *
+ * There are currently a number of known limitations:
+ * - We only sync XPI extensions and themes available from addons.mozilla.org.
+ * We hope to expand support for other add-ons eventually.
+ * - We only attempt syncing of add-ons between applications of the same type.
+ * This means add-ons will not synchronize between Firefox desktop and
+ * Firefox mobile, for example. This is because of significant add-on
+ * incompatibility between application types.
+ *
+ * Add-on records exist for each known {add-on, app-id} pair in the Sync client
+ * set. Each record has a randomly chosen GUID. The records then contain
+ * basic metadata about the add-on.
+ *
+ * We currently synchronize:
+ *
+ * - Installations
+ * - Uninstallations
+ * - User enabling and disabling
+ *
+ * Synchronization is influenced by the following preferences:
+ *
+ * - services.sync.addons.ignoreUserEnabledChanges
+ * - services.sync.addons.trustedSourceHostnames
+ *
+ * and also influenced by whether addons have repository caching enabled and
+ * whether they allow installation of addons from insecure options (both of
+ * which are themselves influenced by the "extensions." pref branch)
+ *
+ * See the documentation in all.js for the behavior of these prefs.
+ */
+
+import { Preferences } from "resource://gre/modules/Preferences.sys.mjs";
+
+import { AddonUtils } from "resource://services-sync/addonutils.sys.mjs";
+import { AddonsReconciler } from "resource://services-sync/addonsreconciler.sys.mjs";
+import {
+ Store,
+ SyncEngine,
+ LegacyTracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs";
+import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
+ AddonRepository: "resource://gre/modules/addons/AddonRepository.sys.mjs",
+});
+
+// 7 days in milliseconds.
+const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
+
+/**
+ * AddonRecord represents the state of an add-on in an application.
+ *
+ * Each add-on has its own record for each application ID it is installed
+ * on.
+ *
+ * The ID of add-on records is a randomly-generated GUID. It is random instead
+ * of deterministic so the URIs of the records cannot be guessed and so
+ * compromised server credentials won't result in disclosure of the specific
+ * add-ons present in a Sync account.
+ *
+ * The record contains the following fields:
+ *
+ * addonID
+ * ID of the add-on. This correlates to the "id" property on an Addon type.
+ *
+ * applicationID
+ * The application ID this record is associated with.
+ *
+ * enabled
+ * Boolean stating whether add-on is enabled or disabled by the user.
+ *
+ * source
+ * String indicating where an add-on is from. Currently, we only support
+ * the value "amo" which indicates that the add-on came from the official
+ * add-ons repository, addons.mozilla.org. In the future, we may support
+ * installing add-ons from other sources. This provides a future-compatible
+ * mechanism for clients to only apply records they know how to handle.
+ */
+function AddonRecord(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+AddonRecord.prototype = {
+ _logName: "Record.Addon",
+};
+Object.setPrototypeOf(AddonRecord.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(AddonRecord, "cleartext", [
+ "addonID",
+ "applicationID",
+ "enabled",
+ "source",
+]);
+
+/**
+ * The AddonsEngine handles synchronization of add-ons between clients.
+ *
+ * The engine maintains an instance of an AddonsReconciler, which is the entity
+ * maintaining state for add-ons. It provides the history and tracking APIs
+ * that AddonManager doesn't.
+ *
+ * The engine instance overrides a handful of functions on the base class. The
+ * rationale for each is documented by that function.
+ */
+export function AddonsEngine(service) {
+ SyncEngine.call(this, "Addons", service);
+
+ this._reconciler = new AddonsReconciler(this._tracker.asyncObserver);
+}
+
+AddonsEngine.prototype = {
+ _storeObj: AddonsStore,
+ _trackerObj: AddonsTracker,
+ _recordObj: AddonRecord,
+ version: 1,
+
+ syncPriority: 5,
+
+ _reconciler: null,
+
+ async initialize() {
+ await SyncEngine.prototype.initialize.call(this);
+ await this._reconciler.ensureStateLoaded();
+ },
+
+ /**
+ * Override parent method to find add-ons by their public ID, not Sync GUID.
+ */
+ async _findDupe(item) {
+ let id = item.addonID;
+
+ // The reconciler should have been updated at the top of the sync, so we
+ // can assume it is up to date when this function is called.
+ let addons = this._reconciler.addons;
+ if (!(id in addons)) {
+ return null;
+ }
+
+ let addon = addons[id];
+ if (addon.guid != item.id) {
+ return addon.guid;
+ }
+
+ return null;
+ },
+
+ /**
+ * Override getChangedIDs to pull in tracker changes plus changes from the
+ * reconciler log.
+ */
+ async getChangedIDs() {
+ let changes = {};
+ const changedIDs = await this._tracker.getChangedIDs();
+ for (let [id, modified] of Object.entries(changedIDs)) {
+ changes[id] = modified;
+ }
+
+ let lastSync = await this.getLastSync();
+ let lastSyncDate = new Date(lastSync * 1000);
+
+ // The reconciler should have been refreshed at the beginning of a sync and
+ // we assume this function is only called from within a sync.
+ let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
+ let addons = this._reconciler.addons;
+ for (let change of reconcilerChanges) {
+ let changeTime = change[0];
+ let id = change[2];
+
+ if (!(id in addons)) {
+ continue;
+ }
+
+ // Keep newest modified time.
+ if (id in changes && changeTime < changes[id]) {
+ continue;
+ }
+
+ if (!(await this.isAddonSyncable(addons[id]))) {
+ continue;
+ }
+
+ this._log.debug("Adding changed add-on from changes log: " + id);
+ let addon = addons[id];
+ changes[addon.guid] = changeTime.getTime() / 1000;
+ }
+
+ return changes;
+ },
+
+ /**
+ * Override start of sync function to refresh reconciler.
+ *
+ * Many functions in this class assume the reconciler is refreshed at the
+ * top of a sync. If this ever changes, those functions should be revisited.
+ *
+ * Technically speaking, we don't need to refresh the reconciler on every
+ * sync since it is installed as an AddonManager listener. However, add-ons
+ * are complicated and we force a full refresh, just in case the listeners
+ * missed something.
+ */
+ async _syncStartup() {
+ // We refresh state before calling parent because syncStartup in the parent
+ // looks for changed IDs, which is dependent on add-on state being up to
+ // date.
+ await this._refreshReconcilerState();
+ return SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ /**
+ * Override end of sync to perform a little housekeeping on the reconciler.
+ *
+ * We prune changes to prevent the reconciler state from growing without
+ * bound. Even if it grows unbounded, there would have to be many add-on
+ * changes (thousands) for it to slow things down significantly. This is
+ * highly unlikely to occur. Still, we exercise defense just in case.
+ */
+ async _syncCleanup() {
+ let lastSync = await this.getLastSync();
+ let ms = 1000 * lastSync - PRUNE_ADDON_CHANGES_THRESHOLD;
+ this._reconciler.pruneChangesBeforeDate(new Date(ms));
+ return SyncEngine.prototype._syncCleanup.call(this);
+ },
+
+ /**
+ * Helper function to ensure reconciler is up to date.
+ *
+ * This will load the reconciler's state from the file
+ * system (if needed) and refresh the state of the reconciler.
+ */
+ async _refreshReconcilerState() {
+ this._log.debug("Refreshing reconciler state");
+ return this._reconciler.refreshGlobalState();
+ },
+
+ // Returns a promise
+ isAddonSyncable(addon, ignoreRepoCheck) {
+ return this._store.isAddonSyncable(addon, ignoreRepoCheck);
+ },
+};
+Object.setPrototypeOf(AddonsEngine.prototype, SyncEngine.prototype);
+
+/**
+ * This is the primary interface between Sync and the Addons Manager.
+ *
+ * In addition to the core store APIs, we provide convenience functions to wrap
+ * Add-on Manager APIs with Sync-specific semantics.
+ */
+function AddonsStore(name, engine) {
+ Store.call(this, name, engine);
+}
+AddonsStore.prototype = {
+ // Define the add-on types (.type) that we support.
+ _syncableTypes: ["extension", "theme"],
+
+ _extensionsPrefs: new Preferences("extensions."),
+
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ /**
+ * Override applyIncoming to filter out records we can't handle.
+ */
+ async applyIncoming(record) {
+ // The fields we look at aren't present when the record is deleted.
+ if (!record.deleted) {
+ // Ignore records not belonging to our application ID because that is the
+ // current policy.
+ if (record.applicationID != Services.appinfo.ID) {
+ this._log.info(
+ "Ignoring incoming record from other App ID: " + record.id
+ );
+ return;
+ }
+
+ // Ignore records that aren't from the official add-on repository, as that
+ // is our current policy.
+ if (record.source != "amo") {
+ this._log.info(
+ "Ignoring unknown add-on source (" +
+ record.source +
+ ")" +
+ " for " +
+ record.id
+ );
+ return;
+ }
+ }
+
+ // Ignore incoming records for which an existing non-syncable addon
+ // exists. Note that we do not insist that the addon manager already have
+ // metadata for this addon - it's possible our reconciler previously saw the
+ // addon but the addon-manager cache no longer has it - which is fine for a
+ // new incoming addon.
+ // (Note that most other cases where the addon-manager cache is invalid
+ // doesn't get this treatment because that cache self-repairs after some
+ // time - but it only re-populates addons which are currently installed.)
+ let existingMeta = this.reconciler.addons[record.addonID];
+ if (
+ existingMeta &&
+ !(await this.isAddonSyncable(existingMeta, /* ignoreRepoCheck */ true))
+ ) {
+ this._log.info(
+ "Ignoring incoming record for an existing but non-syncable addon",
+ record.addonID
+ );
+ return;
+ }
+
+ await Store.prototype.applyIncoming.call(this, record);
+ },
+
+ /**
+ * Provides core Store API to create/install an add-on from a record.
+ */
+ async create(record) {
+ // This will throw if there was an error. This will get caught by the sync
+ // engine and the record will try to be applied later.
+ const results = await AddonUtils.installAddons([
+ {
+ id: record.addonID,
+ syncGUID: record.id,
+ enabled: record.enabled,
+ requireSecureURI: this._extensionsPrefs.get(
+ "install.requireSecureOrigin",
+ true
+ ),
+ },
+ ]);
+
+ if (results.skipped.includes(record.addonID)) {
+ this._log.info("Add-on skipped: " + record.addonID);
+ // Just early-return for skipped addons - we don't want to arrange to
+ // try again next time because the condition that caused up to skip
+ // will remain true for this addon forever.
+ return;
+ }
+
+ let addon;
+ for (let a of results.addons) {
+ if (a.id == record.addonID) {
+ addon = a;
+ break;
+ }
+ }
+
+ // This should never happen, but is present as a fail-safe.
+ if (!addon) {
+ throw new Error("Add-on not found after install: " + record.addonID);
+ }
+
+ this._log.info("Add-on installed: " + record.addonID);
+ },
+
+ /**
+ * Provides core Store API to remove/uninstall an add-on from a record.
+ */
+ async remove(record) {
+ // If this is called, the payload is empty, so we have to find by GUID.
+ let addon = await this.getAddonByGUID(record.id);
+ if (!addon) {
+ // We don't throw because if the add-on could not be found then we assume
+ // it has already been uninstalled and there is nothing for this function
+ // to do.
+ return;
+ }
+
+ this._log.info("Uninstalling add-on: " + addon.id);
+ await AddonUtils.uninstallAddon(addon);
+ },
+
+ /**
+ * Provides core Store API to update an add-on from a record.
+ */
+ async update(record) {
+ let addon = await this.getAddonByID(record.addonID);
+
+ // update() is called if !this.itemExists. And, since itemExists consults
+ // the reconciler only, we need to take care of some corner cases.
+ //
+ // First, the reconciler could know about an add-on that was uninstalled
+ // and no longer present in the add-ons manager.
+ if (!addon) {
+ await this.create(record);
+ return;
+ }
+
+ // It's also possible that the add-on is non-restartless and has pending
+ // install/uninstall activity.
+ //
+ // We wouldn't get here if the incoming record was for a deletion. So,
+ // check for pending uninstall and cancel if necessary.
+ if (addon.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL) {
+ addon.cancelUninstall();
+
+ // We continue with processing because there could be state or ID change.
+ }
+
+ await this.updateUserDisabled(addon, !record.enabled);
+ },
+
+ /**
+ * Provide core Store API to determine if a record exists.
+ */
+ async itemExists(guid) {
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ return !!addon;
+ },
+
+ /**
+ * Create an add-on record from its GUID.
+ *
+ * @param guid
+ * Add-on GUID (from extensions DB)
+ * @param collection
+ * Collection to add record to.
+ *
+ * @return AddonRecord instance
+ */
+ async createRecord(guid, collection) {
+ let record = new AddonRecord(collection, guid);
+ record.applicationID = Services.appinfo.ID;
+
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ // If we don't know about this GUID or if it has been uninstalled, we mark
+ // the record as deleted.
+ if (!addon || !addon.installed) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.modified = addon.modified.getTime() / 1000;
+
+ record.addonID = addon.id;
+ record.enabled = addon.enabled;
+
+ // This needs to be dynamic when add-ons don't come from AddonRepository.
+ record.source = "amo";
+
+ return record;
+ },
+
+ /**
+ * Changes the id of an add-on.
+ *
+ * This implements a core API of the store.
+ */
+ async changeItemID(oldID, newID) {
+ // We always update the GUID in the reconciler because it will be
+ // referenced later in the sync process.
+ let state = this.reconciler.getAddonStateFromSyncGUID(oldID);
+ if (state) {
+ state.guid = newID;
+ await this.reconciler.saveState();
+ }
+
+ let addon = await this.getAddonByGUID(oldID);
+ if (!addon) {
+ this._log.debug(
+ "Cannot change item ID (" +
+ oldID +
+ ") in Add-on " +
+ "Manager because old add-on not present: " +
+ oldID
+ );
+ return;
+ }
+
+ addon.syncGUID = newID;
+ },
+
+ /**
+ * Obtain the set of all syncable add-on Sync GUIDs.
+ *
+ * This implements a core Store API.
+ */
+ async getAllIDs() {
+ let ids = {};
+
+ let addons = this.reconciler.addons;
+ for (let id in addons) {
+ let addon = addons[id];
+ if (await this.isAddonSyncable(addon)) {
+ ids[addon.guid] = true;
+ }
+ }
+
+ return ids;
+ },
+
+ /**
+ * Wipe engine data.
+ *
+ * This uninstalls all syncable addons from the application. In case of
+ * error, it logs the error and keeps trying with other add-ons.
+ */
+ async wipe() {
+ this._log.info("Processing wipe.");
+
+ await this.engine._refreshReconcilerState();
+
+ // We only wipe syncable add-ons. Wipe is a Sync feature not a security
+ // feature.
+ let ids = await this.getAllIDs();
+ for (let guid in ids) {
+ let addon = await this.getAddonByGUID(guid);
+ if (!addon) {
+ this._log.debug(
+ "Ignoring add-on because it couldn't be obtained: " + guid
+ );
+ continue;
+ }
+
+ this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
+ await Utils.catch.call(this, () => addon.uninstall())();
+ }
+ },
+
+ /** *************************************************************************
+ * Functions below are unique to this store and not part of the Store API *
+ ***************************************************************************/
+
+ /**
+ * Obtain an add-on from its public ID.
+ *
+ * @param id
+ * Add-on ID
+ * @return Addon or undefined if not found
+ */
+ async getAddonByID(id) {
+ return lazy.AddonManager.getAddonByID(id);
+ },
+
+ /**
+ * Obtain an add-on from its Sync GUID.
+ *
+ * @param guid
+ * Add-on Sync GUID
+ * @return DBAddonInternal or null
+ */
+ async getAddonByGUID(guid) {
+ return lazy.AddonManager.getAddonBySyncGUID(guid);
+ },
+
+ /**
+ * Determines whether an add-on is suitable for Sync.
+ *
+ * @param addon
+ * Addon instance
+ * @param ignoreRepoCheck
+ * Should we skip checking the Addons repository (primarially useful
+ * for testing and validation).
+ * @return Boolean indicating whether it is appropriate for Sync
+ */
+ async isAddonSyncable(addon, ignoreRepoCheck = false) {
+ // Currently, we limit syncable add-ons to those that are:
+ // 1) In a well-defined set of types
+ // 2) Installed in the current profile
+ // 3) Not installed by a foreign entity (i.e. installed by the app)
+ // since they act like global extensions.
+ // 4) Is not a hotfix.
+ // 5) The addons XPIProvider doesn't veto it (i.e not being installed in
+ // the profile directory, or any other reasons it says the addon can't
+ // be synced)
+ // 6) Are installed from AMO
+
+ // We could represent the test as a complex boolean expression. We go the
+ // verbose route so the failure reason is logged.
+ if (!addon) {
+ this._log.debug("Null object passed to isAddonSyncable.");
+ return false;
+ }
+
+ if (!this._syncableTypes.includes(addon.type)) {
+ this._log.debug(
+ addon.id + " not syncable: type not in allowed list: " + addon.type
+ );
+ return false;
+ }
+
+ if (!(addon.scope & lazy.AddonManager.SCOPE_PROFILE)) {
+ this._log.debug(addon.id + " not syncable: not installed in profile.");
+ return false;
+ }
+
+ // If the addon manager says it's not syncable, we skip it.
+ if (!addon.isSyncable) {
+ this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
+ return false;
+ }
+
+ // This may be too aggressive. If an add-on is downloaded from AMO and
+ // manually placed in the profile directory, foreignInstall will be set.
+ // Arguably, that add-on should be syncable.
+ // TODO Address the edge case and come up with more robust heuristics.
+ if (addon.foreignInstall) {
+ this._log.debug(addon.id + " not syncable: is foreign install.");
+ return false;
+ }
+
+ // If the AddonRepository's cache isn't enabled (which it typically isn't
+ // in tests), getCachedAddonByID always returns null - so skip the check
+ // in that case. We also provide a way to specifically opt-out of the check
+ // even if the cache is enabled, which is used by the validators.
+ if (ignoreRepoCheck || !lazy.AddonRepository.cacheEnabled) {
+ return true;
+ }
+
+ let result = await new Promise(res => {
+ lazy.AddonRepository.getCachedAddonByID(addon.id, res);
+ });
+
+ if (!result) {
+ this._log.debug(
+ addon.id + " not syncable: add-on not found in add-on repository."
+ );
+ return false;
+ }
+
+ return this.isSourceURITrusted(result.sourceURI);
+ },
+
+ /**
+ * Determine whether an add-on's sourceURI field is trusted and the add-on
+ * can be installed.
+ *
+ * This function should only ever be called from isAddonSyncable(). It is
+ * exposed as a separate function to make testing easier.
+ *
+ * @param uri
+ * nsIURI instance to validate
+ * @return bool
+ */
+ isSourceURITrusted: function isSourceURITrusted(uri) {
+ // For security reasons, we currently limit synced add-ons to those
+ // installed from trusted hostname(s). We additionally require TLS with
+ // the add-ons site to help prevent forgeries.
+ let trustedHostnames = Svc.Prefs.get(
+ "addons.trustedSourceHostnames",
+ ""
+ ).split(",");
+
+ if (!uri) {
+ this._log.debug("Undefined argument to isSourceURITrusted().");
+ return false;
+ }
+
+ // Scheme is validated before the hostname because uri.host may not be
+ // populated for certain schemes. It appears to always be populated for
+ // https, so we avoid the potential NS_ERROR_FAILURE on field access.
+ if (uri.scheme != "https") {
+ this._log.debug("Source URI not HTTPS: " + uri.spec);
+ return false;
+ }
+
+ if (!trustedHostnames.includes(uri.host)) {
+ this._log.debug("Source hostname not trusted: " + uri.host);
+ return false;
+ }
+
+ return true;
+ },
+
+ /**
+ * Update the userDisabled flag on an add-on.
+ *
+ * This will enable or disable an add-on. It has no return value and does
+ * not catch or handle exceptions thrown by the addon manager. If no action
+ * is needed it will return immediately.
+ *
+ * @param addon
+ * Addon instance to manipulate.
+ * @param value
+ * Boolean to which to set userDisabled on the passed Addon.
+ */
+ async updateUserDisabled(addon, value) {
+ if (addon.userDisabled == value) {
+ return;
+ }
+
+ // A pref allows changes to the enabled flag to be ignored.
+ if (Svc.Prefs.get("addons.ignoreUserEnabledChanges", false)) {
+ this._log.info(
+ "Ignoring enabled state change due to preference: " + addon.id
+ );
+ return;
+ }
+
+ AddonUtils.updateUserDisabled(addon, value);
+ // updating this flag doesn't send a notification for appDisabled addons,
+ // meaning the reconciler will not update its state and may resync the
+ // addon - so explicitly rectify the state (bug 1366994)
+ if (addon.appDisabled) {
+ await this.reconciler.rectifyStateFromAddon(addon);
+ }
+ },
+};
+
+Object.setPrototypeOf(AddonsStore.prototype, Store.prototype);
+
+/**
+ * The add-ons tracker keeps track of real-time changes to add-ons.
+ *
+ * It hooks up to the reconciler and receives notifications directly from it.
+ */
+function AddonsTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+AddonsTracker.prototype = {
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ get store() {
+ return this.engine._store;
+ },
+
+ /**
+ * This callback is executed whenever the AddonsReconciler sends out a change
+ * notification. See AddonsReconciler.addChangeListener().
+ */
+ async changeListener(date, change, addon) {
+ this._log.debug("changeListener invoked: " + change + " " + addon.id);
+ // Ignore changes that occur during sync.
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (!(await this.store.isAddonSyncable(addon))) {
+ this._log.debug(
+ "Ignoring change because add-on isn't syncable: " + addon.id
+ );
+ return;
+ }
+
+ const added = await this.addChangedID(addon.guid, date.getTime() / 1000);
+ if (added) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ }
+ },
+
+ onStart() {
+ this.reconciler.startListening();
+ this.reconciler.addChangeListener(this);
+ },
+
+ onStop() {
+ this.reconciler.removeChangeListener(this);
+ this.reconciler.stopListening();
+ },
+};
+
+Object.setPrototypeOf(AddonsTracker.prototype, LegacyTracker.prototype);
+
+export class AddonValidator extends CollectionValidator {
+ constructor(engine = null) {
+ super("addons", "id", ["addonID", "enabled", "applicationID", "source"]);
+ this.engine = engine;
+ }
+
+ async getClientItems() {
+ return lazy.AddonManager.getAllAddons();
+ }
+
+ normalizeClientItem(item) {
+ let enabled = !item.userDisabled;
+ if (item.pendingOperations & lazy.AddonManager.PENDING_ENABLE) {
+ enabled = true;
+ } else if (item.pendingOperations & lazy.AddonManager.PENDING_DISABLE) {
+ enabled = false;
+ }
+ return {
+ enabled,
+ id: item.syncGUID,
+ addonID: item.id,
+ applicationID: Services.appinfo.ID,
+ source: "amo", // check item.foreignInstall?
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ let guid = await this.engine._findDupe(item);
+ if (guid) {
+ item.id = guid;
+ }
+ return item;
+ }
+
+ clientUnderstands(item) {
+ return item.applicationID === Services.appinfo.ID;
+ }
+
+ async syncedByClient(item) {
+ return (
+ !item.original.hidden &&
+ !item.original.isSystem &&
+ !(
+ item.original.pendingOperations & lazy.AddonManager.PENDING_UNINSTALL
+ ) &&
+ // No need to await the returned promise explicitely:
+ // |expr1 && expr2| evaluates to expr2 if expr1 is true.
+ this.engine.isAddonSyncable(item.original, true)
+ );
+ }
+}
diff --git a/services/sync/modules/engines/bookmarks.sys.mjs b/services/sync/modules/engines/bookmarks.sys.mjs
new file mode 100644
index 0000000000..8724344084
--- /dev/null
+++ b/services/sync/modules/engines/bookmarks.sys.mjs
@@ -0,0 +1,953 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+
+import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs";
+import {
+ Changeset,
+ Store,
+ SyncEngine,
+ Tracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Observers: "resource://services-common/observers.sys.mjs",
+ PlacesBackups: "resource://gre/modules/PlacesBackups.sys.mjs",
+ PlacesDBUtils: "resource://gre/modules/PlacesDBUtils.sys.mjs",
+ PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+ Resource: "resource://services-sync/resource.sys.mjs",
+ SyncedBookmarksMirror: "resource://gre/modules/SyncedBookmarksMirror.sys.mjs",
+});
+
+const PLACES_MAINTENANCE_INTERVAL_SECONDS = 4 * 60 * 60; // 4 hours.
+
+const FOLDER_SORTINDEX = 1000000;
+
+// Roots that should be deleted from the server, instead of applied locally.
+// This matches `AndroidBrowserBookmarksRepositorySession::forbiddenGUID`,
+// but allows tags because we don't want to reparent tag folders or tag items
+// to "unfiled".
+const FORBIDDEN_INCOMING_IDS = ["pinned", "places", "readinglist"];
+
+// Items with these parents should be deleted from the server. We allow
+// children of the Places root, to avoid orphaning left pane queries and other
+// descendants of custom roots.
+const FORBIDDEN_INCOMING_PARENT_IDS = ["pinned", "readinglist"];
+
+// The tracker ignores changes made by import and restore, to avoid bumping the
+// score and triggering syncs during the process, as well as changes made by
+// Sync.
+XPCOMUtils.defineLazyGetter(lazy, "IGNORED_SOURCES", () => [
+ lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
+ lazy.PlacesUtils.bookmarks.SOURCES.IMPORT,
+ lazy.PlacesUtils.bookmarks.SOURCES.RESTORE,
+ lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP,
+ lazy.PlacesUtils.bookmarks.SOURCES.SYNC_REPARENT_REMOVED_FOLDER_CHILDREN,
+]);
+
+// The validation telemetry version for the engine. Version 1 is collected
+// by `bookmark_validator.js`, and checks value as well as structure
+// differences. Version 2 is collected by the engine as part of building the
+// remote tree, and checks structure differences only.
+const BOOKMARK_VALIDATOR_VERSION = 2;
+
+// The maximum time that the engine should wait before aborting a bookmark
+// merge.
+const BOOKMARK_APPLY_TIMEOUT_MS = 5 * 60 * 60 * 1000; // 5 minutes
+
+// The default frecency value to use when not known.
+const FRECENCY_UNKNOWN = -1;
+
+// Returns the constructor for a bookmark record type.
+function getTypeObject(type) {
+ switch (type) {
+ case "bookmark":
+ return Bookmark;
+ case "query":
+ return BookmarkQuery;
+ case "folder":
+ return BookmarkFolder;
+ case "livemark":
+ return Livemark;
+ case "separator":
+ return BookmarkSeparator;
+ case "item":
+ return PlacesItem;
+ }
+ return null;
+}
+
+export function PlacesItem(collection, id, type) {
+ CryptoWrapper.call(this, collection, id);
+ this.type = type || "item";
+}
+
+PlacesItem.prototype = {
+ async decrypt(keyBundle) {
+ // Do the normal CryptoWrapper decrypt, but change types before returning
+ let clear = await CryptoWrapper.prototype.decrypt.call(this, keyBundle);
+
+ // Convert the abstract places item to the actual object type
+ if (!this.deleted) {
+ Object.setPrototypeOf(this, this.getTypeObject(this.type).prototype);
+ }
+
+ return clear;
+ },
+
+ getTypeObject: function PlacesItem_getTypeObject(type) {
+ let recordObj = getTypeObject(type);
+ if (!recordObj) {
+ throw new Error("Unknown places item object type: " + type);
+ }
+ return recordObj;
+ },
+
+ _logName: "Sync.Record.PlacesItem",
+
+ // Converts the record to a Sync bookmark object that can be passed to
+ // `PlacesSyncUtils.bookmarks.{insert, update}`.
+ toSyncBookmark() {
+ let result = {
+ kind: this.type,
+ recordId: this.id,
+ parentRecordId: this.parentid,
+ };
+ let dateAdded = lazy.PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
+ this.dateAdded,
+ +this.modified * 1000
+ );
+ if (dateAdded > 0) {
+ result.dateAdded = dateAdded;
+ }
+ return result;
+ },
+
+ // Populates the record from a Sync bookmark object returned from
+ // `PlacesSyncUtils.bookmarks.fetch`.
+ fromSyncBookmark(item) {
+ this.parentid = item.parentRecordId;
+ this.parentName = item.parentTitle;
+ if (item.dateAdded) {
+ this.dateAdded = item.dateAdded;
+ }
+ },
+};
+
+Object.setPrototypeOf(PlacesItem.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(PlacesItem, "cleartext", [
+ "hasDupe",
+ "parentid",
+ "parentName",
+ "type",
+ "dateAdded",
+]);
+
+export function Bookmark(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "bookmark");
+}
+
+Bookmark.prototype = {
+ _logName: "Sync.Record.Bookmark",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.title = this.title;
+ info.url = this.bmkUri;
+ info.description = this.description;
+ info.tags = this.tags;
+ info.keyword = this.keyword;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.bmkUri = item.url.href;
+ this.description = item.description;
+ this.tags = item.tags;
+ this.keyword = item.keyword;
+ },
+};
+
+Object.setPrototypeOf(Bookmark.prototype, PlacesItem.prototype);
+
+Utils.deferGetSet(Bookmark, "cleartext", [
+ "title",
+ "bmkUri",
+ "description",
+ "tags",
+ "keyword",
+]);
+
+export function BookmarkQuery(collection, id) {
+ Bookmark.call(this, collection, id, "query");
+}
+
+BookmarkQuery.prototype = {
+ _logName: "Sync.Record.BookmarkQuery",
+
+ toSyncBookmark() {
+ let info = Bookmark.prototype.toSyncBookmark.call(this);
+ info.folder = this.folderName || undefined; // empty string -> undefined
+ info.query = this.queryId;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ Bookmark.prototype.fromSyncBookmark.call(this, item);
+ this.folderName = item.folder || undefined; // empty string -> undefined
+ this.queryId = item.query;
+ },
+};
+
+Object.setPrototypeOf(BookmarkQuery.prototype, Bookmark.prototype);
+
+Utils.deferGetSet(BookmarkQuery, "cleartext", ["folderName", "queryId"]);
+
+export function BookmarkFolder(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "folder");
+}
+
+BookmarkFolder.prototype = {
+ _logName: "Sync.Record.Folder",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.description = this.description;
+ info.title = this.title;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.description = item.description;
+ this.children = item.childRecordIds;
+ },
+};
+
+Object.setPrototypeOf(BookmarkFolder.prototype, PlacesItem.prototype);
+
+Utils.deferGetSet(BookmarkFolder, "cleartext", [
+ "description",
+ "title",
+ "children",
+]);
+
+export function Livemark(collection, id) {
+ BookmarkFolder.call(this, collection, id, "livemark");
+}
+
+Livemark.prototype = {
+ _logName: "Sync.Record.Livemark",
+
+ toSyncBookmark() {
+ let info = BookmarkFolder.prototype.toSyncBookmark.call(this);
+ info.feed = this.feedUri;
+ info.site = this.siteUri;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ BookmarkFolder.prototype.fromSyncBookmark.call(this, item);
+ this.feedUri = item.feed.href;
+ if (item.site) {
+ this.siteUri = item.site.href;
+ }
+ },
+};
+
+Object.setPrototypeOf(Livemark.prototype, BookmarkFolder.prototype);
+
+Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]);
+
+export function BookmarkSeparator(collection, id) {
+ PlacesItem.call(this, collection, id, "separator");
+}
+
+BookmarkSeparator.prototype = {
+ _logName: "Sync.Record.Separator",
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.pos = item.index;
+ },
+};
+
+Object.setPrototypeOf(BookmarkSeparator.prototype, PlacesItem.prototype);
+
+Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos");
+
+/**
+ * The bookmarks engine uses a different store that stages downloaded bookmarks
+ * in a separate database, instead of writing directly to Places. The buffer
+ * handles reconciliation, so we stub out `_reconcile`, and wait to pull changes
+ * until we're ready to upload.
+ */
+export function BookmarksEngine(service) {
+ SyncEngine.call(this, "Bookmarks", service);
+}
+
+BookmarksEngine.prototype = {
+ _recordObj: PlacesItem,
+ _trackerObj: BookmarksTracker,
+ _storeObj: BookmarksStore,
+ version: 2,
+ // Used to override the engine name in telemetry, so that we can distinguish
+ // this engine from the old, now removed non-buffered engine.
+ overrideTelemetryName: "bookmarks-buffered",
+
+ // Needed to ensure we don't miss items when resuming a sync that failed or
+ // aborted early.
+ _defaultSort: "oldest",
+
+ syncPriority: 4,
+ allowSkippedRecord: false,
+
+ async _ensureCurrentSyncID(newSyncID) {
+ await lazy.PlacesSyncUtils.bookmarks.ensureCurrentSyncId(newSyncID);
+ let buf = await this._store.ensureOpenMirror();
+ await buf.ensureCurrentSyncId(newSyncID);
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ let shouldWipeRemote =
+ await lazy.PlacesSyncUtils.bookmarks.shouldWipeRemote();
+ if (!shouldWipeRemote) {
+ this._log.debug(
+ "Checking if server sync ID ${newSyncID} matches existing",
+ { newSyncID }
+ );
+ await this._ensureCurrentSyncID(newSyncID);
+ return newSyncID;
+ }
+ // We didn't take the new sync ID because we need to wipe the server
+ // and other clients after a restore. Send the command, wipe the
+ // server, and reset our sync ID to reupload everything.
+ this._log.debug(
+ "Ignoring server sync ID ${newSyncID} after restore; " +
+ "wiping server and resetting sync ID",
+ { newSyncID }
+ );
+ await this.service.clientsEngine.sendCommand(
+ "wipeEngine",
+ [this.name],
+ null,
+ { reason: "bookmark-restore" }
+ );
+ let assignedSyncID = await this.resetSyncID();
+ return assignedSyncID;
+ },
+
+ async getSyncID() {
+ return lazy.PlacesSyncUtils.bookmarks.getSyncId();
+ },
+
+ async resetSyncID() {
+ await this._deleteServerCollection();
+ return this.resetLocalSyncID();
+ },
+
+ async resetLocalSyncID() {
+ let newSyncID = await lazy.PlacesSyncUtils.bookmarks.resetSyncId();
+ this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID });
+ let buf = await this._store.ensureOpenMirror();
+ await buf.ensureCurrentSyncId(newSyncID);
+ return newSyncID;
+ },
+
+ async getLastSync() {
+ let mirror = await this._store.ensureOpenMirror();
+ return mirror.getCollectionHighWaterMark();
+ },
+
+ async setLastSync(lastSync) {
+ let mirror = await this._store.ensureOpenMirror();
+ await mirror.setCollectionLastModified(lastSync);
+ // Update the last sync time in Places so that reverting to the original
+ // bookmarks engine doesn't download records we've already applied.
+ await lazy.PlacesSyncUtils.bookmarks.setLastSync(lastSync);
+ },
+
+ async _syncStartup() {
+ await super._syncStartup();
+
+ try {
+ // For first syncs, back up the user's bookmarks.
+ let lastSync = await this.getLastSync();
+ if (!lastSync) {
+ this._log.debug("Bookmarks backup starting");
+ await lazy.PlacesBackups.create(null, true);
+ this._log.debug("Bookmarks backup done");
+ }
+ } catch (ex) {
+ // Failure to create a backup is somewhat bad, but probably not bad
+ // enough to prevent syncing of bookmarks - so just log the error and
+ // continue.
+ this._log.warn(
+ "Error while backing up bookmarks, but continuing with sync",
+ ex
+ );
+ }
+ },
+
+ async _sync() {
+ try {
+ await super._sync();
+ if (this._ranMaintenanceOnLastSync) {
+ // If the last sync failed, we ran maintenance, and this sync succeeded,
+ // maintenance likely fixed the issue.
+ this._ranMaintenanceOnLastSync = false;
+ this.service.recordTelemetryEvent("maintenance", "fix", "bookmarks");
+ }
+ } catch (ex) {
+ if (
+ Async.isShutdownException(ex) ||
+ ex.status > 0 ||
+ ex.name == "InterruptedError"
+ ) {
+ // Don't run maintenance on shutdown or HTTP errors, or if we aborted
+ // the sync because the user changed their bookmarks during merging.
+ throw ex;
+ }
+ if (ex.name == "MergeConflictError") {
+ this._log.warn(
+ "Bookmark syncing ran into a merge conflict error...will retry later"
+ );
+ return;
+ }
+ // Run Places maintenance periodically to try to recover from corruption
+ // that might have caused the sync to fail. We cap the interval because
+ // persistent failures likely indicate a problem that won't be fixed by
+ // running maintenance after every failed sync.
+ let elapsedSinceMaintenance =
+ Date.now() / 1000 -
+ Services.prefs.getIntPref("places.database.lastMaintenance", 0);
+ if (elapsedSinceMaintenance >= PLACES_MAINTENANCE_INTERVAL_SECONDS) {
+ this._log.error(
+ "Bookmark sync failed, ${elapsedSinceMaintenance}s " +
+ "elapsed since last run; running Places maintenance",
+ { elapsedSinceMaintenance }
+ );
+ await lazy.PlacesDBUtils.maintenanceOnIdle();
+ this._ranMaintenanceOnLastSync = true;
+ this.service.recordTelemetryEvent("maintenance", "run", "bookmarks");
+ } else {
+ this._ranMaintenanceOnLastSync = false;
+ }
+ throw ex;
+ }
+ },
+
+ async _syncFinish() {
+ await SyncEngine.prototype._syncFinish.call(this);
+ await lazy.PlacesSyncUtils.bookmarks.ensureMobileQuery();
+ },
+
+ async pullAllChanges() {
+ return this.pullNewChanges();
+ },
+
+ async trackRemainingChanges() {
+ let changes = this._modified.changes;
+ await lazy.PlacesSyncUtils.bookmarks.pushChanges(changes);
+ },
+
+ _deleteId(id) {
+ this._noteDeletedId(id);
+ },
+
+ // The bookmarks engine rarely calls this method directly, except in tests or
+ // when handling a `reset{All, Engine}` command from another client. We
+ // usually reset local Sync metadata on a sync ID mismatch, which both engines
+ // override with logic that lives in Places and the mirror.
+ async _resetClient() {
+ await super._resetClient();
+ await lazy.PlacesSyncUtils.bookmarks.reset();
+ let buf = await this._store.ensureOpenMirror();
+ await buf.reset();
+ },
+
+ // Cleans up the Places root, reading list items (ignored in bug 762118,
+ // removed in bug 1155684), and pinned sites.
+ _shouldDeleteRemotely(incomingItem) {
+ return (
+ FORBIDDEN_INCOMING_IDS.includes(incomingItem.id) ||
+ FORBIDDEN_INCOMING_PARENT_IDS.includes(incomingItem.parentid)
+ );
+ },
+
+ emptyChangeset() {
+ return new BookmarksChangeset();
+ },
+
+ async _apply() {
+ let buf = await this._store.ensureOpenMirror();
+ let watchdog = this._newWatchdog();
+ watchdog.start(BOOKMARK_APPLY_TIMEOUT_MS);
+
+ try {
+ let recordsToUpload = await buf.apply({
+ remoteTimeSeconds: lazy.Resource.serverTime,
+ signal: watchdog.signal,
+ });
+ this._modified.replace(recordsToUpload);
+ } finally {
+ watchdog.stop();
+ if (watchdog.abortReason) {
+ this._log.warn(`Aborting bookmark merge: ${watchdog.abortReason}`);
+ }
+ }
+ },
+
+ async _processIncoming(newitems) {
+ await super._processIncoming(newitems);
+ await this._apply();
+ },
+
+ async _reconcile(item) {
+ return true;
+ },
+
+ async _createRecord(id) {
+ let record = await this._doCreateRecord(id);
+ if (!record.deleted) {
+ // Set hasDupe on all (non-deleted) records since we don't use it and we
+ // want to minimize the risk of older clients corrupting records. Note
+ // that the SyncedBookmarksMirror sets it for all records that it created,
+ // but we would like to ensure that weakly uploaded records are marked as
+ // hasDupe as well.
+ record.hasDupe = true;
+ }
+ return record;
+ },
+
+ async _doCreateRecord(id) {
+ let change = this._modified.changes[id];
+ if (!change) {
+ this._log.error(
+ "Creating record for item ${id} not in strong changeset",
+ { id }
+ );
+ throw new TypeError("Can't create record for unchanged item");
+ }
+ let record = this._recordFromCleartext(id, change.cleartext);
+ record.sortindex = await this._store._calculateIndex(record);
+ return record;
+ },
+
+ _recordFromCleartext(id, cleartext) {
+ let recordObj = getTypeObject(cleartext.type);
+ if (!recordObj) {
+ this._log.warn(
+ "Creating record for item ${id} with unknown type ${type}",
+ { id, type: cleartext.type }
+ );
+ recordObj = PlacesItem;
+ }
+ let record = new recordObj(this.name, id);
+ record.cleartext = cleartext;
+ return record;
+ },
+
+ async pullChanges() {
+ return {};
+ },
+
+ /**
+ * Writes successfully uploaded records back to the mirror, so that the
+ * mirror matches the server. We update the mirror before updating Places,
+ * which has implications for interrupted syncs.
+ *
+ * 1. Sync interrupted during upload; server doesn't support atomic uploads.
+ * We'll download and reapply everything that we uploaded before the
+ * interruption. All locally changed items retain their change counters.
+ * 2. Sync interrupted during upload; atomic uploads enabled. The server
+ * discards the batch. All changed local items retain their change
+ * counters, so the next sync resumes cleanly.
+ * 3. Sync interrupted during upload; outgoing records can't fit in a single
+ * batch. We'll download and reapply all records through the most recent
+ * committed batch. This is a variation of (1).
+ * 4. Sync interrupted after we update the mirror, but before cleanup. The
+ * mirror matches the server, but locally changed items retain their change
+ * counters. Reuploading them on the next sync should be idempotent, though
+ * unnecessary. If another client makes a conflicting remote change before
+ * we sync again, we may incorrectly prefer the local state.
+ * 5. Sync completes successfully. We'll update the mirror, and reset the
+ * change counters for all items.
+ */
+ async _onRecordsWritten(succeeded, failed, serverModifiedTime) {
+ let records = [];
+ for (let id of succeeded) {
+ let change = this._modified.changes[id];
+ if (!change) {
+ // TODO (Bug 1433178): Write weakly uploaded records back to the mirror.
+ this._log.info("Uploaded record not in strong changeset", id);
+ continue;
+ }
+ if (!change.synced) {
+ this._log.info("Record in strong changeset not uploaded", id);
+ continue;
+ }
+ let cleartext = change.cleartext;
+ if (!cleartext) {
+ this._log.error(
+ "Missing Sync record cleartext for ${id} in ${change}",
+ { id, change }
+ );
+ throw new TypeError("Missing cleartext for uploaded Sync record");
+ }
+ let record = this._recordFromCleartext(id, cleartext);
+ record.modified = serverModifiedTime;
+ records.push(record);
+ }
+ let buf = await this._store.ensureOpenMirror();
+ await buf.store(records, { needsMerge: false });
+ },
+
+ async finalize() {
+ await super.finalize();
+ await this._store.finalize();
+ },
+};
+
+Object.setPrototypeOf(BookmarksEngine.prototype, SyncEngine.prototype);
+
+/**
+ * The bookmarks store delegates to the mirror for staging and applying
+ * records. Most `Store` methods intentionally remain abstract, so you can't use
+ * this store to create or update bookmarks in Places. All changes must go
+ * through the mirror, which takes care of merging and producing a valid tree.
+ */
+function BookmarksStore(name, engine) {
+ Store.call(this, name, engine);
+}
+
+BookmarksStore.prototype = {
+ _openMirrorPromise: null,
+
+ // For tests.
+ _batchChunkSize: 500,
+
+ // Create a record starting from the weave id (places guid)
+ async createRecord(id, collection) {
+ let item = await lazy.PlacesSyncUtils.bookmarks.fetch(id);
+ if (!item) {
+ // deleted item
+ let record = new PlacesItem(collection, id);
+ record.deleted = true;
+ return record;
+ }
+
+ let recordObj = getTypeObject(item.kind);
+ if (!recordObj) {
+ this._log.warn("Unknown item type, cannot serialize: " + item.kind);
+ recordObj = PlacesItem;
+ }
+ let record = new recordObj(collection, id);
+ record.fromSyncBookmark(item);
+
+ record.sortindex = await this._calculateIndex(record);
+
+ return record;
+ },
+
+ async _calculateIndex(record) {
+ // Ensure folders have a very high sort index so they're not synced last.
+ if (record.type == "folder") {
+ return FOLDER_SORTINDEX;
+ }
+
+ // For anything directly under the toolbar, give it a boost of more than an
+ // unvisited bookmark
+ let index = 0;
+ if (record.parentid == "toolbar") {
+ index += 150;
+ }
+
+ // Add in the bookmark's frecency if we have something.
+ if (record.bmkUri != null) {
+ let frecency = FRECENCY_UNKNOWN;
+ try {
+ frecency = await lazy.PlacesSyncUtils.history.fetchURLFrecency(
+ record.bmkUri
+ );
+ } catch (ex) {
+ this._log.warn(
+ `Failed to fetch frecency for ${record.id}; assuming default`,
+ ex
+ );
+ this._log.trace("Record {id} has invalid URL ${bmkUri}", record);
+ }
+ if (frecency != FRECENCY_UNKNOWN) {
+ index += frecency;
+ }
+ }
+
+ return index;
+ },
+
+ async wipe() {
+ // Save a backup before clearing out all bookmarks.
+ await lazy.PlacesBackups.create(null, true);
+ await lazy.PlacesSyncUtils.bookmarks.wipe();
+ },
+
+ ensureOpenMirror() {
+ if (!this._openMirrorPromise) {
+ this._openMirrorPromise = this._openMirror().catch(err => {
+ // We may have failed to open the mirror temporarily; for example, if
+ // the database is locked. Clear the promise so that subsequent
+ // `ensureOpenMirror` calls can try to open the mirror again.
+ this._openMirrorPromise = null;
+ throw err;
+ });
+ }
+ return this._openMirrorPromise;
+ },
+
+ async _openMirror() {
+ let mirrorPath = PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ "bookmarks.sqlite"
+ );
+ await IOUtils.makeDirectory(PathUtils.parent(mirrorPath), {
+ createAncestors: true,
+ });
+
+ return lazy.SyncedBookmarksMirror.open({
+ path: mirrorPath,
+ recordStepTelemetry: (name, took, counts) => {
+ lazy.Observers.notify(
+ "weave:engine:sync:step",
+ {
+ name,
+ took,
+ counts,
+ },
+ this.name
+ );
+ },
+ recordValidationTelemetry: (took, checked, problems) => {
+ lazy.Observers.notify(
+ "weave:engine:validate:finish",
+ {
+ version: BOOKMARK_VALIDATOR_VERSION,
+ took,
+ checked,
+ problems,
+ },
+ this.name
+ );
+ },
+ });
+ },
+
+ async applyIncomingBatch(records, countTelemetry) {
+ let buf = await this.ensureOpenMirror();
+ for (let chunk of lazy.PlacesUtils.chunkArray(
+ records,
+ this._batchChunkSize
+ )) {
+ await buf.store(chunk);
+ }
+ // Array of failed records.
+ return [];
+ },
+
+ async applyIncoming(record) {
+ let buf = await this.ensureOpenMirror();
+ await buf.store([record]);
+ },
+
+ async finalize() {
+ if (!this._openMirrorPromise) {
+ return;
+ }
+ let buf = await this._openMirrorPromise;
+ await buf.finalize();
+ },
+};
+
+Object.setPrototypeOf(BookmarksStore.prototype, Store.prototype);
+
+// The bookmarks tracker is a special flower. Instead of listening for changes
+// via observer notifications, it queries Places for the set of items that have
+// changed since the last sync. Because it's a "pull-based" tracker, it ignores
+// all concepts of "add a changed ID." However, it still registers an observer
+// to bump the score, so that changed bookmarks are synced immediately.
+function BookmarksTracker(name, engine) {
+ Tracker.call(this, name, engine);
+}
+BookmarksTracker.prototype = {
+ onStart() {
+ this._placesListener = new PlacesWeakCallbackWrapper(
+ this.handlePlacesEvents.bind(this)
+ );
+ lazy.PlacesUtils.observers.addListener(
+ [
+ "bookmark-added",
+ "bookmark-removed",
+ "bookmark-moved",
+ "bookmark-guid-changed",
+ "bookmark-keyword-changed",
+ "bookmark-tags-changed",
+ "bookmark-time-changed",
+ "bookmark-title-changed",
+ "bookmark-url-changed",
+ ],
+ this._placesListener
+ );
+ Svc.Obs.add("bookmarks-restore-begin", this);
+ Svc.Obs.add("bookmarks-restore-success", this);
+ Svc.Obs.add("bookmarks-restore-failed", this);
+ },
+
+ onStop() {
+ lazy.PlacesUtils.observers.removeListener(
+ [
+ "bookmark-added",
+ "bookmark-removed",
+ "bookmark-moved",
+ "bookmark-guid-changed",
+ "bookmark-keyword-changed",
+ "bookmark-tags-changed",
+ "bookmark-time-changed",
+ "bookmark-title-changed",
+ "bookmark-url-changed",
+ ],
+ this._placesListener
+ );
+ Svc.Obs.remove("bookmarks-restore-begin", this);
+ Svc.Obs.remove("bookmarks-restore-success", this);
+ Svc.Obs.remove("bookmarks-restore-failed", this);
+ },
+
+ async getChangedIDs() {
+ return lazy.PlacesSyncUtils.bookmarks.pullChanges();
+ },
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ case "bookmarks-restore-begin":
+ this._log.debug("Ignoring changes from importing bookmarks.");
+ break;
+ case "bookmarks-restore-success":
+ this._log.debug("Tracking all items on successful import.");
+
+ if (data == "json") {
+ this._log.debug(
+ "Restore succeeded: wiping server and other clients."
+ );
+ // Trigger an immediate sync. `ensureCurrentSyncID` will notice we
+ // restored, wipe the server and other clients, reset the sync ID, and
+ // upload the restored tree.
+ this.score += SCORE_INCREMENT_XLARGE;
+ } else {
+ // "html", "html-initial", or "json-append"
+ this._log.debug("Import succeeded.");
+ }
+ break;
+ case "bookmarks-restore-failed":
+ this._log.debug("Tracking all items on failed import.");
+ break;
+ }
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
+
+ /* Every add/remove/change will trigger a sync for MULTI_DEVICE */
+ _upScore: function BMT__upScore() {
+ this.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ handlePlacesEvents(events) {
+ for (let event of events) {
+ switch (event.type) {
+ case "bookmark-added":
+ case "bookmark-removed":
+ case "bookmark-moved":
+ case "bookmark-keyword-changed":
+ case "bookmark-tags-changed":
+ case "bookmark-time-changed":
+ case "bookmark-title-changed":
+ case "bookmark-url-changed":
+ if (lazy.IGNORED_SOURCES.includes(event.source)) {
+ continue;
+ }
+
+ this._log.trace(`'${event.type}': ${event.id}`);
+ this._upScore();
+ break;
+ case "bookmark-guid-changed":
+ if (event.source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC) {
+ this._log.warn(
+ "The source of bookmark-guid-changed event shoud be sync."
+ );
+ continue;
+ }
+
+ this._log.trace(`'${event.type}': ${event.id}`);
+ this._upScore();
+ break;
+ case "purge-caches":
+ this._log.trace("purge-caches");
+ this._upScore();
+ break;
+ }
+ }
+ },
+};
+
+Object.setPrototypeOf(BookmarksTracker.prototype, Tracker.prototype);
+
+/**
+ * A changeset that stores extra metadata in a change record for each ID. The
+ * engine updates this metadata when uploading Sync records, and writes it back
+ * to Places in `BookmarksEngine#trackRemainingChanges`.
+ *
+ * The `synced` property on a change record means its corresponding item has
+ * been uploaded, and we should pretend it doesn't exist in the changeset.
+ */
+class BookmarksChangeset extends Changeset {
+ // Only `_reconcile` calls `getModifiedTimestamp` and `has`, and the engine
+ // does its own reconciliation.
+ getModifiedTimestamp(id) {
+ throw new Error("Don't use timestamps to resolve bookmark conflicts");
+ }
+
+ has(id) {
+ throw new Error("Don't use the changeset to resolve bookmark conflicts");
+ }
+
+ delete(id) {
+ let change = this.changes[id];
+ if (change) {
+ // Mark the change as synced without removing it from the set. We do this
+ // so that we can update Places in `trackRemainingChanges`.
+ change.synced = true;
+ }
+ }
+
+ ids() {
+ let results = new Set();
+ for (let id in this.changes) {
+ if (!this.changes[id].synced) {
+ results.add(id);
+ }
+ }
+ return [...results];
+ }
+}
diff --git a/services/sync/modules/engines/clients.sys.mjs b/services/sync/modules/engines/clients.sys.mjs
new file mode 100644
index 0000000000..f3cebc8a54
--- /dev/null
+++ b/services/sync/modules/engines/clients.sys.mjs
@@ -0,0 +1,1123 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * How does the clients engine work?
+ *
+ * - We use 2 files - commands.json and commands-syncing.json.
+ *
+ * - At sync upload time, we attempt a rename of commands.json to
+ * commands-syncing.json, and ignore errors (helps for crash during sync!).
+ * - We load commands-syncing.json and stash the contents in
+ * _currentlySyncingCommands which lives for the duration of the upload process.
+ * - We use _currentlySyncingCommands to build the outgoing records
+ * - Immediately after successful upload, we delete commands-syncing.json from
+ * disk (and clear _currentlySyncingCommands). We reconcile our local records
+ * with what we just wrote in the server, and add failed IDs commands
+ * back in commands.json
+ * - Any time we need to "save" a command for future syncs, we load
+ * commands.json, update it, and write it back out.
+ */
+
+import { Async } from "resource://services-common/async.sys.mjs";
+
+import {
+ DEVICE_TYPE_DESKTOP,
+ DEVICE_TYPE_MOBILE,
+ SINGLE_USER_THRESHOLD,
+ SYNC_API_VERSION,
+} from "resource://services-sync/constants.sys.mjs";
+
+import {
+ Store,
+ SyncEngine,
+ LegacyTracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Resource } from "resource://services-sync/resource.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+
+const { PREF_ACCOUNT_ROOT } = ChromeUtils.import(
+ "resource://gre/modules/FxAccountsCommon.js"
+);
+
+const CLIENTS_TTL = 15552000; // 180 days
+const CLIENTS_TTL_REFRESH = 604800; // 7 days
+const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
+
+// TTL of the message sent to another device when sending a tab
+const NOTIFY_TAB_SENT_TTL_SECS = 1 * 3600; // 1 hour
+
+// How often we force a refresh of the FxA device list.
+const REFRESH_FXA_DEVICE_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
+
+// Reasons behind sending collection_changed push notifications.
+const COLLECTION_MODIFIED_REASON_SENDTAB = "sendtab";
+const COLLECTION_MODIFIED_REASON_FIRSTSYNC = "firstsync";
+
+const SUPPORTED_PROTOCOL_VERSIONS = [SYNC_API_VERSION];
+const LAST_MODIFIED_ON_PROCESS_COMMAND_PREF =
+ "services.sync.clients.lastModifiedOnProcessCommands";
+
+function hasDupeCommand(commands, action) {
+ if (!commands) {
+ return false;
+ }
+ return commands.some(
+ other =>
+ other.command == action.command &&
+ Utils.deepEquals(other.args, action.args)
+ );
+}
+
+export function ClientsRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+
+ClientsRec.prototype = {
+ _logName: "Sync.Record.Clients",
+ ttl: CLIENTS_TTL,
+};
+Object.setPrototypeOf(ClientsRec.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(ClientsRec, "cleartext", [
+ "name",
+ "type",
+ "commands",
+ "version",
+ "protocols",
+ "formfactor",
+ "os",
+ "appPackage",
+ "application",
+ "device",
+ "fxaDeviceId",
+]);
+
+export function ClientEngine(service) {
+ SyncEngine.call(this, "Clients", service);
+
+ this.fxAccounts = lazy.fxAccounts;
+ this.addClientCommandQueue = Async.asyncQueueCaller(this._log);
+ Utils.defineLazyIDProperty(this, "localID", "services.sync.client.GUID");
+}
+
+ClientEngine.prototype = {
+ _storeObj: ClientStore,
+ _recordObj: ClientsRec,
+ _trackerObj: ClientsTracker,
+ allowSkippedRecord: false,
+ _knownStaleFxADeviceIds: null,
+ _lastDeviceCounts: null,
+ _lastFxaDeviceRefresh: 0,
+
+ async initialize() {
+ // Reset the last sync timestamp on every startup so that we fetch all clients
+ await this.resetLastSync();
+ },
+
+ // These two properties allow us to avoid replaying the same commands
+ // continuously if we cannot manage to upload our own record.
+ _localClientLastModified: 0,
+ get _lastModifiedOnProcessCommands() {
+ return Services.prefs.getIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, -1);
+ },
+
+ set _lastModifiedOnProcessCommands(value) {
+ Services.prefs.setIntPref(LAST_MODIFIED_ON_PROCESS_COMMAND_PREF, value);
+ },
+
+ get isFirstSync() {
+ return !this.lastRecordUpload;
+ },
+
+ // Always sync client data as it controls other sync behavior
+ get enabled() {
+ return true;
+ },
+
+ get lastRecordUpload() {
+ return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
+ },
+ set lastRecordUpload(value) {
+ Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
+ },
+
+ get remoteClients() {
+ // return all non-stale clients for external consumption.
+ return Object.values(this._store._remoteClients).filter(v => !v.stale);
+ },
+
+ remoteClient(id) {
+ let client = this._store._remoteClients[id];
+ return client && !client.stale ? client : null;
+ },
+
+ remoteClientExists(id) {
+ return !!this.remoteClient(id);
+ },
+
+ // Aggregate some stats on the composition of clients on this account
+ get stats() {
+ let stats = {
+ hasMobile: this.localType == DEVICE_TYPE_MOBILE,
+ names: [this.localName],
+ numClients: 1,
+ };
+
+ for (let id in this._store._remoteClients) {
+ let { name, type, stale } = this._store._remoteClients[id];
+ if (!stale) {
+ stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
+ stats.names.push(name);
+ stats.numClients++;
+ }
+ }
+
+ return stats;
+ },
+
+ /**
+ * Obtain information about device types.
+ *
+ * Returns a Map of device types to integer counts. Guaranteed to include
+ * "desktop" (which will have at least 1 - this device) and "mobile" (which
+ * may have zero) counts. It almost certainly will include only these 2.
+ */
+ get deviceTypes() {
+ let counts = new Map();
+
+ counts.set(this.localType, 1); // currently this must be DEVICE_TYPE_DESKTOP
+ counts.set(DEVICE_TYPE_MOBILE, 0);
+
+ for (let id in this._store._remoteClients) {
+ let record = this._store._remoteClients[id];
+ if (record.stale) {
+ continue; // pretend "stale" records don't exist.
+ }
+ let type = record.type;
+ if (!counts.has(type)) {
+ counts.set(type, 0);
+ }
+
+ counts.set(type, counts.get(type) + 1);
+ }
+
+ return counts;
+ },
+
+ get brandName() {
+ let brand = Services.strings.createBundle(
+ "chrome://branding/locale/brand.properties"
+ );
+ return brand.GetStringFromName("brandShortName");
+ },
+
+ get localName() {
+ return this.fxAccounts.device.getLocalName();
+ },
+ set localName(value) {
+ this.fxAccounts.device.setLocalName(value);
+ },
+
+ get localType() {
+ return this.fxAccounts.device.getLocalType();
+ },
+
+ getClientName(id) {
+ if (id == this.localID) {
+ return this.localName;
+ }
+ let client = this._store._remoteClients[id];
+ if (!client) {
+ return "";
+ }
+ // Sometimes the sync clients don't always correctly update the device name
+ // However FxA always does, so try to pull the name from there first
+ let fxaDevice = this.fxAccounts.device.recentDeviceList?.find(
+ device => device.id === client.fxaDeviceId
+ );
+
+ // should be very rare, but could happen if we have yet to fetch devices,
+ // or the client recently disconnected
+ if (!fxaDevice) {
+ this._log.warn(
+ "Couldn't find associated FxA device, falling back to client name"
+ );
+ return client.name;
+ }
+ return fxaDevice.name;
+ },
+
+ getClientFxaDeviceId(id) {
+ if (this._store._remoteClients[id]) {
+ return this._store._remoteClients[id].fxaDeviceId;
+ }
+ return null;
+ },
+
+ getClientByFxaDeviceId(fxaDeviceId) {
+ for (let id in this._store._remoteClients) {
+ let client = this._store._remoteClients[id];
+ if (client.stale) {
+ continue;
+ }
+ if (client.fxaDeviceId == fxaDeviceId) {
+ return client;
+ }
+ }
+ return null;
+ },
+
+ getClientType(id) {
+ const client = this._store._remoteClients[id];
+ if (client.type == DEVICE_TYPE_DESKTOP) {
+ return "desktop";
+ }
+ if (client.formfactor && client.formfactor.includes("tablet")) {
+ return "tablet";
+ }
+ return "phone";
+ },
+
+ async _readCommands() {
+ let commands = await Utils.jsonLoad("commands", this);
+ return commands || {};
+ },
+
+ /**
+ * Low level function, do not use directly (use _addClientCommand instead).
+ */
+ async _saveCommands(commands) {
+ try {
+ await Utils.jsonSave("commands", this, commands);
+ } catch (error) {
+ this._log.error("Failed to save JSON outgoing commands", error);
+ }
+ },
+
+ async _prepareCommandsForUpload() {
+ try {
+ await Utils.jsonMove("commands", "commands-syncing", this);
+ } catch (e) {
+ // Ignore errors
+ }
+ let commands = await Utils.jsonLoad("commands-syncing", this);
+ return commands || {};
+ },
+
+ async _deleteUploadedCommands() {
+ delete this._currentlySyncingCommands;
+ try {
+ await Utils.jsonRemove("commands-syncing", this);
+ } catch (err) {
+ this._log.error("Failed to delete syncing-commands file", err);
+ }
+ },
+
+ // Gets commands for a client we are yet to write to the server. Doesn't
+ // include commands for that client which are already on the server.
+ // We should rename this!
+ async getClientCommands(clientId) {
+ const allCommands = await this._readCommands();
+ return allCommands[clientId] || [];
+ },
+
+ async removeLocalCommand(command) {
+ // the implementation of this engine is such that adding a command to
+ // the local client is how commands are deleted! ¯\_(ツ)_/¯
+ await this._addClientCommand(this.localID, command);
+ },
+
+ async _addClientCommand(clientId, command) {
+ this.addClientCommandQueue.enqueueCall(async () => {
+ try {
+ const localCommands = await this._readCommands();
+ const localClientCommands = localCommands[clientId] || [];
+ const remoteClient = this._store._remoteClients[clientId];
+ let remoteClientCommands = [];
+ if (remoteClient && remoteClient.commands) {
+ remoteClientCommands = remoteClient.commands;
+ }
+ const clientCommands = localClientCommands.concat(remoteClientCommands);
+ if (hasDupeCommand(clientCommands, command)) {
+ return false;
+ }
+ localCommands[clientId] = localClientCommands.concat(command);
+ await this._saveCommands(localCommands);
+ return true;
+ } catch (e) {
+ // Failing to save a command should not "break the queue" of pending operations.
+ this._log.error(e);
+ return false;
+ }
+ });
+
+ return this.addClientCommandQueue.promiseCallsComplete();
+ },
+
+ async _removeClientCommands(clientId) {
+ const allCommands = await this._readCommands();
+ delete allCommands[clientId];
+ await this._saveCommands(allCommands);
+ },
+
+ async updateKnownStaleClients() {
+ this._log.debug("Updating the known stale clients");
+ // _fetchFxADevices side effect updates this._knownStaleFxADeviceIds.
+ await this._fetchFxADevices();
+ let localFxADeviceId = await lazy.fxAccounts.device.getLocalId();
+ // Process newer records first, so that if we hit a record with a device ID
+ // we've seen before, we can mark it stale immediately.
+ let clientList = Object.values(this._store._remoteClients).sort(
+ (a, b) => b.serverLastModified - a.serverLastModified
+ );
+ let seenDeviceIds = new Set([localFxADeviceId]);
+ for (let client of clientList) {
+ // Clients might not have an `fxaDeviceId` if they fail the FxA
+ // registration process.
+ if (!client.fxaDeviceId) {
+ continue;
+ }
+ if (this._knownStaleFxADeviceIds.includes(client.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${client.id} - in known stale clients list`
+ );
+ client.stale = true;
+ } else if (seenDeviceIds.has(client.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${client.id}` +
+ ` - duplicate device id ${client.fxaDeviceId}`
+ );
+ client.stale = true;
+ } else {
+ seenDeviceIds.add(client.fxaDeviceId);
+ }
+ }
+ },
+
+ async _fetchFxADevices() {
+ // We only force a refresh periodically to keep the load on the servers
+ // down, and because we expect FxA to have received a push message in
+ // most cases when the FxA device list would have changed. For this reason
+ // we still go ahead and check the stale list even if we didn't force a
+ // refresh.
+ let now = this.fxAccounts._internal.now(); // tests mock this .now() impl.
+ if (now - REFRESH_FXA_DEVICE_INTERVAL_MS > this._lastFxaDeviceRefresh) {
+ this._lastFxaDeviceRefresh = now;
+ try {
+ await this.fxAccounts.device.refreshDeviceList();
+ } catch (e) {
+ this._log.error("Could not refresh the FxA device list", e);
+ }
+ }
+
+ // We assume that clients not present in the FxA Device Manager list have been
+ // disconnected and so are stale
+ this._log.debug("Refreshing the known stale clients list");
+ let localClients = Object.values(this._store._remoteClients)
+ .filter(client => client.fxaDeviceId) // iOS client records don't have fxaDeviceId
+ .map(client => client.fxaDeviceId);
+ const fxaClients = this.fxAccounts.device.recentDeviceList
+ ? this.fxAccounts.device.recentDeviceList.map(device => device.id)
+ : [];
+ this._knownStaleFxADeviceIds = Utils.arraySub(localClients, fxaClients);
+ },
+
+ async _syncStartup() {
+ // Reupload new client record periodically.
+ if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
+ await this._tracker.addChangedID(this.localID);
+ }
+ return SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ async _processIncoming() {
+ // Fetch all records from the server.
+ await this.resetLastSync();
+ this._incomingClients = {};
+ try {
+ await SyncEngine.prototype._processIncoming.call(this);
+ // Update FxA Device list.
+ await this._fetchFxADevices();
+ // Since clients are synced unconditionally, any records in the local store
+ // that don't exist on the server must be for disconnected clients. Remove
+ // them, so that we don't upload records with commands for clients that will
+ // never see them. We also do this to filter out stale clients from the
+ // tabs collection, since showing their list of tabs is confusing.
+ for (let id in this._store._remoteClients) {
+ if (!this._incomingClients[id]) {
+ this._log.info(`Removing local state for deleted client ${id}`);
+ await this._removeRemoteClient(id);
+ }
+ }
+ let localFxADeviceId = await lazy.fxAccounts.device.getLocalId();
+ // Bug 1264498: Mobile clients don't remove themselves from the clients
+ // collection when the user disconnects Sync, so we mark as stale clients
+ // with the same name that haven't synced in over a week.
+ // (Note we can't simply delete them, or we re-apply them next sync - see
+ // bug 1287687)
+ this._localClientLastModified = Math.round(
+ this._incomingClients[this.localID]
+ );
+ delete this._incomingClients[this.localID];
+ let names = new Set([this.localName]);
+ let seenDeviceIds = new Set([localFxADeviceId]);
+ let idToLastModifiedList = Object.entries(this._incomingClients).sort(
+ (a, b) => b[1] - a[1]
+ );
+ for (let [id, serverLastModified] of idToLastModifiedList) {
+ let record = this._store._remoteClients[id];
+ // stash the server last-modified time on the record.
+ record.serverLastModified = serverLastModified;
+ if (
+ record.fxaDeviceId &&
+ this._knownStaleFxADeviceIds.includes(record.fxaDeviceId)
+ ) {
+ this._log.info(
+ `Hiding stale client ${id} - in known stale clients list`
+ );
+ record.stale = true;
+ }
+ if (!names.has(record.name)) {
+ if (record.fxaDeviceId) {
+ seenDeviceIds.add(record.fxaDeviceId);
+ }
+ names.add(record.name);
+ continue;
+ }
+ let remoteAge = Resource.serverTime - this._incomingClients[id];
+ if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
+ this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
+ record.stale = true;
+ continue;
+ }
+ if (record.fxaDeviceId && seenDeviceIds.has(record.fxaDeviceId)) {
+ this._log.info(
+ `Hiding stale client ${record.id}` +
+ ` - duplicate device id ${record.fxaDeviceId}`
+ );
+ record.stale = true;
+ } else if (record.fxaDeviceId) {
+ seenDeviceIds.add(record.fxaDeviceId);
+ }
+ }
+ } finally {
+ this._incomingClients = null;
+ }
+ },
+
+ async _uploadOutgoing() {
+ this._currentlySyncingCommands = await this._prepareCommandsForUpload();
+ const clientWithPendingCommands = Object.keys(
+ this._currentlySyncingCommands
+ );
+ for (let clientId of clientWithPendingCommands) {
+ if (this._store._remoteClients[clientId] || this.localID == clientId) {
+ this._modified.set(clientId, 0);
+ }
+ }
+ let updatedIDs = this._modified.ids();
+ await SyncEngine.prototype._uploadOutgoing.call(this);
+ // Record the response time as the server time for each item we uploaded.
+ let lastSync = await this.getLastSync();
+ for (let id of updatedIDs) {
+ if (id == this.localID) {
+ this.lastRecordUpload = lastSync;
+ } else {
+ this._store._remoteClients[id].serverLastModified = lastSync;
+ }
+ }
+ },
+
+ async _onRecordsWritten(succeeded, failed) {
+ // Reconcile the status of the local records with what we just wrote on the
+ // server
+ for (let id of succeeded) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (id == this.localID) {
+ if (this.isFirstSync) {
+ this._log.info(
+ "Uploaded our client record for the first time, notifying other clients."
+ );
+ this._notifyClientRecordUploaded();
+ }
+ if (this.localCommands) {
+ this.localCommands = this.localCommands.filter(
+ command => !hasDupeCommand(commandChanges, command)
+ );
+ }
+ } else {
+ const clientRecord = this._store._remoteClients[id];
+ if (!commandChanges || !clientRecord) {
+ // should be impossible, else we wouldn't have been writing it.
+ this._log.warn(
+ "No command/No record changes for a client we uploaded"
+ );
+ continue;
+ }
+ // fixup the client record, so our copy of _remoteClients matches what we uploaded.
+ this._store._remoteClients[id] = await this._store.createRecord(id);
+ // we could do better and pass the reference to the record we just uploaded,
+ // but this will do for now
+ }
+ }
+
+ // Re-add failed commands
+ for (let id of failed) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (!commandChanges) {
+ continue;
+ }
+ await this._addClientCommand(id, commandChanges);
+ }
+
+ await this._deleteUploadedCommands();
+
+ // Notify other devices that their own client collection changed
+ const idsToNotify = succeeded.reduce((acc, id) => {
+ if (id == this.localID) {
+ return acc;
+ }
+ const fxaDeviceId = this.getClientFxaDeviceId(id);
+ return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
+ }, []);
+ if (idsToNotify.length) {
+ this._notifyOtherClientsModified(idsToNotify);
+ }
+ },
+
+ _notifyOtherClientsModified(ids) {
+ // We are not waiting on this promise on purpose.
+ this._notifyCollectionChanged(
+ ids,
+ NOTIFY_TAB_SENT_TTL_SECS,
+ COLLECTION_MODIFIED_REASON_SENDTAB
+ );
+ },
+
+ _notifyClientRecordUploaded() {
+ // We are not waiting on this promise on purpose.
+ this._notifyCollectionChanged(
+ null,
+ 0,
+ COLLECTION_MODIFIED_REASON_FIRSTSYNC
+ );
+ },
+
+ /**
+ * @param {?string[]} ids FxA Client IDs to notify. null means everyone else.
+ * @param {number} ttl TTL of the push notification.
+ * @param {string} reason Reason for sending this push notification.
+ */
+ async _notifyCollectionChanged(ids, ttl, reason) {
+ const message = {
+ version: 1,
+ command: "sync:collection_changed",
+ data: {
+ collections: ["clients"],
+ reason,
+ },
+ };
+ let excludedIds = null;
+ if (!ids) {
+ const localFxADeviceId = await lazy.fxAccounts.device.getLocalId();
+ excludedIds = [localFxADeviceId];
+ }
+ try {
+ await this.fxAccounts.notifyDevices(ids, excludedIds, message, ttl);
+ } catch (e) {
+ this._log.error("Could not notify of changes in the collection", e);
+ }
+ },
+
+ async _syncFinish() {
+ // Record histograms for our device types, and also write them to a pref
+ // so non-histogram telemetry (eg, UITelemetry) and the sync scheduler
+ // has easy access to them, and so they are accurate even before we've
+ // successfully synced the first time after startup.
+ let deviceTypeCounts = this.deviceTypes;
+ for (let [deviceType, count] of deviceTypeCounts) {
+ let hid;
+ let prefName = this.name + ".devices.";
+ switch (deviceType) {
+ case DEVICE_TYPE_DESKTOP:
+ hid = "WEAVE_DEVICE_COUNT_DESKTOP";
+ prefName += "desktop";
+ break;
+ case DEVICE_TYPE_MOBILE:
+ hid = "WEAVE_DEVICE_COUNT_MOBILE";
+ prefName += "mobile";
+ break;
+ default:
+ this._log.warn(
+ `Unexpected deviceType "${deviceType}" recording device telemetry.`
+ );
+ continue;
+ }
+ Services.telemetry.getHistogramById(hid).add(count);
+ // Optimization: only write the pref if it changed since our last sync.
+ if (
+ this._lastDeviceCounts == null ||
+ this._lastDeviceCounts.get(prefName) != count
+ ) {
+ Svc.Prefs.set(prefName, count);
+ }
+ }
+ this._lastDeviceCounts = deviceTypeCounts;
+ return SyncEngine.prototype._syncFinish.call(this);
+ },
+
+ async _reconcile(item) {
+ // Every incoming record is reconciled, so we use this to track the
+ // contents of the collection on the server.
+ this._incomingClients[item.id] = item.modified;
+
+ if (!(await this._store.itemExists(item.id))) {
+ return true;
+ }
+ // Clients are synced unconditionally, so we'll always have new records.
+ // Unfortunately, this will cause the scheduler to use the immediate sync
+ // interval for the multi-device case, instead of the active interval. We
+ // work around this by updating the record during reconciliation, and
+ // returning false to indicate that the record doesn't need to be applied
+ // later.
+ await this._store.update(item);
+ return false;
+ },
+
+ // Treat reset the same as wiping for locally cached clients
+ async _resetClient() {
+ await this._wipeClient();
+ },
+
+ async _wipeClient() {
+ await SyncEngine.prototype._resetClient.call(this);
+ this._knownStaleFxADeviceIds = null;
+ delete this.localCommands;
+ await this._store.wipe();
+ try {
+ await Utils.jsonRemove("commands", this);
+ } catch (err) {
+ this._log.warn("Could not delete commands.json", err);
+ }
+ try {
+ await Utils.jsonRemove("commands-syncing", this);
+ } catch (err) {
+ this._log.warn("Could not delete commands-syncing.json", err);
+ }
+ },
+
+ async removeClientData() {
+ let res = this.service.resource(this.engineURL + "/" + this.localID);
+ await res.delete();
+ },
+
+ // Override the default behavior to delete bad records from the server.
+ async handleHMACMismatch(item, mayRetry) {
+ this._log.debug("Handling HMAC mismatch for " + item.id);
+
+ let base = await SyncEngine.prototype.handleHMACMismatch.call(
+ this,
+ item,
+ mayRetry
+ );
+ if (base != SyncEngine.kRecoveryStrategy.error) {
+ return base;
+ }
+
+ // It's a bad client record. Save it to be deleted at the end of the sync.
+ this._log.debug("Bad client record detected. Scheduling for deletion.");
+ await this._deleteId(item.id);
+
+ // Neither try again nor error; we're going to delete it.
+ return SyncEngine.kRecoveryStrategy.ignore;
+ },
+
+ /**
+ * A hash of valid commands that the client knows about. The key is a command
+ * and the value is a hash containing information about the command such as
+ * number of arguments, description, and importance (lower importance numbers
+ * indicate higher importance.
+ */
+ _commands: {
+ resetAll: {
+ args: 0,
+ importance: 0,
+ desc: "Clear temporary local data for all engines",
+ },
+ resetEngine: {
+ args: 1,
+ importance: 0,
+ desc: "Clear temporary local data for engine",
+ },
+ wipeEngine: {
+ args: 1,
+ importance: 0,
+ desc: "Delete all client data for engine",
+ },
+ logout: { args: 0, importance: 0, desc: "Log out client" },
+ },
+
+ /**
+ * Sends a command+args pair to a specific client.
+ *
+ * @param command Command string
+ * @param args Array of arguments/data for command
+ * @param clientId Client to send command to
+ */
+ async _sendCommandToClient(command, args, clientId, telemetryExtra) {
+ this._log.trace("Sending " + command + " to " + clientId);
+
+ let client = this._store._remoteClients[clientId];
+ if (!client) {
+ throw new Error("Unknown remote client ID: '" + clientId + "'.");
+ }
+ if (client.stale) {
+ throw new Error("Stale remote client ID: '" + clientId + "'.");
+ }
+
+ let action = {
+ command,
+ args,
+ // We send the flowID to the other client so *it* can report it in its
+ // telemetry - we record it in ours below.
+ flowID: telemetryExtra.flowID,
+ };
+
+ if (await this._addClientCommand(clientId, action)) {
+ this._log.trace(`Client ${clientId} got a new action`, [command, args]);
+ await this._tracker.addChangedID(clientId);
+ try {
+ telemetryExtra.deviceID =
+ this.service.identity.hashedDeviceID(clientId);
+ } catch (_) {}
+
+ this.service.recordTelemetryEvent(
+ "sendcommand",
+ command,
+ undefined,
+ telemetryExtra
+ );
+ } else {
+ this._log.trace(`Client ${clientId} got a duplicate action`, [
+ command,
+ args,
+ ]);
+ }
+ },
+
+ /**
+ * Check if the local client has any remote commands and perform them.
+ *
+ * @return false to abort sync
+ */
+ async processIncomingCommands() {
+ return this._notify("clients:process-commands", "", async function () {
+ if (
+ !this.localCommands ||
+ (this._lastModifiedOnProcessCommands == this._localClientLastModified &&
+ !this.ignoreLastModifiedOnProcessCommands)
+ ) {
+ return true;
+ }
+ this._lastModifiedOnProcessCommands = this._localClientLastModified;
+
+ const clearedCommands = await this._readCommands()[this.localID];
+ const commands = this.localCommands.filter(
+ command => !hasDupeCommand(clearedCommands, command)
+ );
+ let didRemoveCommand = false;
+ // Process each command in order.
+ for (let rawCommand of commands) {
+ let shouldRemoveCommand = true; // most commands are auto-removed.
+ let { command, args, flowID } = rawCommand;
+ this._log.debug("Processing command " + command, args);
+
+ this.service.recordTelemetryEvent(
+ "processcommand",
+ command,
+ undefined,
+ { flowID }
+ );
+
+ let engines = [args[0]];
+ switch (command) {
+ case "resetAll":
+ engines = null;
+ // Fallthrough
+ case "resetEngine":
+ await this.service.resetClient(engines);
+ break;
+ case "wipeEngine":
+ await this.service.wipeClient(engines);
+ break;
+ case "logout":
+ this.service.logout();
+ return false;
+ default:
+ this._log.warn("Received an unknown command: " + command);
+ break;
+ }
+ // Add the command to the "cleared" commands list
+ if (shouldRemoveCommand) {
+ await this.removeLocalCommand(rawCommand);
+ didRemoveCommand = true;
+ }
+ }
+ if (didRemoveCommand) {
+ await this._tracker.addChangedID(this.localID);
+ }
+
+ return true;
+ })();
+ },
+
+ /**
+ * Validates and sends a command to a client or all clients.
+ *
+ * Calling this does not actually sync the command data to the server. If the
+ * client already has the command/args pair, it won't receive a duplicate
+ * command.
+ * This method is async since it writes the command to a file.
+ *
+ * @param command
+ * Command to invoke on remote clients
+ * @param args
+ * Array of arguments to give to the command
+ * @param clientId
+ * Client ID to send command to. If undefined, send to all remote
+ * clients.
+ * @param flowID
+ * A unique identifier used to track success for this operation across
+ * devices.
+ */
+ async sendCommand(command, args, clientId = null, telemetryExtra = {}) {
+ let commandData = this._commands[command];
+ // Don't send commands that we don't know about.
+ if (!commandData) {
+ this._log.error("Unknown command to send: " + command);
+ return;
+ } else if (!args || args.length != commandData.args) {
+ // Don't send a command with the wrong number of arguments.
+ this._log.error(
+ "Expected " +
+ commandData.args +
+ " args for '" +
+ command +
+ "', but got " +
+ args
+ );
+ return;
+ }
+
+ // We allocate a "flowID" here, so it is used for each client.
+ telemetryExtra = Object.assign({}, telemetryExtra); // don't clobber the caller's object
+ if (!telemetryExtra.flowID) {
+ telemetryExtra.flowID = Utils.makeGUID();
+ }
+
+ if (clientId) {
+ await this._sendCommandToClient(command, args, clientId, telemetryExtra);
+ } else {
+ for (let [id, record] of Object.entries(this._store._remoteClients)) {
+ if (!record.stale) {
+ await this._sendCommandToClient(command, args, id, telemetryExtra);
+ }
+ }
+ }
+ },
+
+ async _removeRemoteClient(id) {
+ delete this._store._remoteClients[id];
+ await this._tracker.removeChangedID(id);
+ await this._removeClientCommands(id);
+ this._modified.delete(id);
+ },
+};
+Object.setPrototypeOf(ClientEngine.prototype, SyncEngine.prototype);
+
+function ClientStore(name, engine) {
+ Store.call(this, name, engine);
+}
+ClientStore.prototype = {
+ _remoteClients: {},
+
+ async create(record) {
+ await this.update(record);
+ },
+
+ async update(record) {
+ if (record.id == this.engine.localID) {
+ // Only grab commands from the server; local name/type always wins
+ this.engine.localCommands = record.commands;
+ } else {
+ this._remoteClients[record.id] = record.cleartext;
+ }
+ },
+
+ async createRecord(id, collection) {
+ let record = new ClientsRec(collection, id);
+
+ const commandsChanges = this.engine._currentlySyncingCommands
+ ? this.engine._currentlySyncingCommands[id]
+ : [];
+
+ // Package the individual components into a record for the local client
+ if (id == this.engine.localID) {
+ try {
+ record.fxaDeviceId = await this.engine.fxAccounts.device.getLocalId();
+ } catch (error) {
+ this._log.warn("failed to get fxa device id", error);
+ }
+ record.name = this.engine.localName;
+ record.type = this.engine.localType;
+ record.version = Services.appinfo.version;
+ record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
+
+ // Substract the commands we recorded that we've already executed
+ if (
+ commandsChanges &&
+ commandsChanges.length &&
+ this.engine.localCommands &&
+ this.engine.localCommands.length
+ ) {
+ record.commands = this.engine.localCommands.filter(
+ command => !hasDupeCommand(commandsChanges, command)
+ );
+ }
+
+ // Optional fields.
+ record.os = Services.appinfo.OS; // "Darwin"
+ record.appPackage = Services.appinfo.ID;
+ record.application = this.engine.brandName; // "Nightly"
+
+ // We can't compute these yet.
+ // record.device = ""; // Bug 1100723
+ // record.formfactor = ""; // Bug 1100722
+ } else {
+ record.cleartext = Object.assign({}, this._remoteClients[id]);
+ delete record.cleartext.serverLastModified; // serverLastModified is a local only attribute.
+
+ // Add the commands we have to send
+ if (commandsChanges && commandsChanges.length) {
+ const recordCommands = record.cleartext.commands || [];
+ const newCommands = commandsChanges.filter(
+ command => !hasDupeCommand(recordCommands, command)
+ );
+ record.cleartext.commands = recordCommands.concat(newCommands);
+ }
+
+ if (record.cleartext.stale) {
+ // It's almost certainly a logic error for us to upload a record we
+ // consider stale, so make log noise, but still remove the flag.
+ this._log.error(
+ `Preparing to upload record ${id} that we consider stale`
+ );
+ delete record.cleartext.stale;
+ }
+ }
+ if (record.commands) {
+ const maxPayloadSize =
+ this.engine.service.getMemcacheMaxRecordPayloadSize();
+ let origOrder = new Map(record.commands.map((c, i) => [c, i]));
+ // we sort first by priority, and second by age (indicated by order in the
+ // original list)
+ let commands = record.commands.slice().sort((a, b) => {
+ let infoA = this.engine._commands[a.command];
+ let infoB = this.engine._commands[b.command];
+ // Treat unknown command types as highest priority, to allow us to add
+ // high priority commands in the future without worrying about clients
+ // removing them on each-other unnecessarially.
+ let importA = infoA ? infoA.importance : 0;
+ let importB = infoB ? infoB.importance : 0;
+ // Higher importantance numbers indicate that we care less, so they
+ // go to the end of the list where they'll be popped off.
+ let importDelta = importA - importB;
+ if (importDelta != 0) {
+ return importDelta;
+ }
+ let origIdxA = origOrder.get(a);
+ let origIdxB = origOrder.get(b);
+ // Within equivalent priorities, we put older entries near the end
+ // of the list, so that they are removed first.
+ return origIdxB - origIdxA;
+ });
+ let truncatedCommands = Utils.tryFitItems(commands, maxPayloadSize);
+ if (truncatedCommands.length != record.commands.length) {
+ this._log.warn(
+ `Removing commands from client ${id} (from ${record.commands.length} to ${truncatedCommands.length})`
+ );
+ // Restore original order.
+ record.commands = truncatedCommands.sort(
+ (a, b) => origOrder.get(a) - origOrder.get(b)
+ );
+ }
+ }
+ return record;
+ },
+
+ async itemExists(id) {
+ return id in (await this.getAllIDs());
+ },
+
+ async getAllIDs() {
+ let ids = {};
+ ids[this.engine.localID] = true;
+ for (let id in this._remoteClients) {
+ ids[id] = true;
+ }
+ return ids;
+ },
+
+ async wipe() {
+ this._remoteClients = {};
+ },
+};
+Object.setPrototypeOf(ClientStore.prototype, Store.prototype);
+
+function ClientsTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+ClientsTracker.prototype = {
+ _enabled: false,
+
+ onStart() {
+ Svc.Obs.add("fxaccounts:new_device_id", this.asyncObserver);
+ Services.prefs.addObserver(
+ PREF_ACCOUNT_ROOT + "device.name",
+ this.asyncObserver
+ );
+ },
+ onStop() {
+ Services.prefs.removeObserver(
+ PREF_ACCOUNT_ROOT + "device.name",
+ this.asyncObserver
+ );
+ Svc.Obs.remove("fxaccounts:new_device_id", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "nsPref:changed":
+ this._log.debug("client.name preference changed");
+ // Fallthrough intended.
+ case "fxaccounts:new_device_id":
+ await this.addChangedID(this.engine.localID);
+ this.score += SINGLE_USER_THRESHOLD + 1; // ALWAYS SYNC NOW.
+ break;
+ }
+ },
+};
+Object.setPrototypeOf(ClientsTracker.prototype, LegacyTracker.prototype);
diff --git a/services/sync/modules/engines/extension-storage.sys.mjs b/services/sync/modules/engines/extension-storage.sys.mjs
new file mode 100644
index 0000000000..f0ec21811a
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.sys.mjs
@@ -0,0 +1,303 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import {
+ BridgedEngine,
+ BridgeWrapperXPCOM,
+ LogAdapter,
+} from "resource://services-sync/bridged_engine.sys.mjs";
+import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ MULTI_DEVICE_THRESHOLD: "resource://services-sync/constants.sys.mjs",
+ Observers: "resource://services-common/observers.sys.mjs",
+ SCORE_INCREMENT_MEDIUM: "resource://services-sync/constants.sys.mjs",
+ Svc: "resource://services-sync/util.sys.mjs",
+ extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs",
+
+ extensionStorageSyncKinto:
+ "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs",
+});
+
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "StorageSyncService",
+ "@mozilla.org/extensions/storage/sync;1",
+ "nsIInterfaceRequestor"
+);
+
+const PREF_FORCE_ENABLE = "engine.extension-storage.force";
+
+// A helper to indicate whether extension-storage is enabled - it's based on
+// the "addons" pref. The same logic is shared between both engine impls.
+function getEngineEnabled() {
+ // By default, we sync extension storage if we sync addons. This
+ // lets us simplify the UX since users probably don't consider
+ // "extension preferences" a separate category of syncing.
+ // However, we also respect engine.extension-storage.force, which
+ // can be set to true or false, if a power user wants to customize
+ // the behavior despite the lack of UI.
+ const forced = lazy.Svc.Prefs.get(PREF_FORCE_ENABLE, undefined);
+ if (forced !== undefined) {
+ return forced;
+ }
+ return lazy.Svc.Prefs.get("engine.addons", false);
+}
+
+function setEngineEnabled(enabled) {
+ // This will be called by the engine manager when declined on another device.
+ // Things will go a bit pear-shaped if the engine manager tries to end up
+ // with 'addons' and 'extension-storage' in different states - however, this
+ // *can* happen given we support the `engine.extension-storage.force`
+ // preference. So if that pref exists, we set it to this value. If that pref
+ // doesn't exist, we just ignore it and hope that the 'addons' engine is also
+ // going to be set to the same state.
+ if (lazy.Svc.Prefs.has(PREF_FORCE_ENABLE)) {
+ lazy.Svc.Prefs.set(PREF_FORCE_ENABLE, enabled);
+ }
+}
+
+// A "bridged engine" to our webext-storage component.
+export function ExtensionStorageEngineBridge(service) {
+ this.component = lazy.StorageSyncService.getInterface(
+ Ci.mozIBridgedSyncEngine
+ );
+ BridgedEngine.call(this, "Extension-Storage", service);
+ this._bridge = new BridgeWrapperXPCOM(this.component);
+
+ let app_services_logger = Cc["@mozilla.org/appservices/logger;1"].getService(
+ Ci.mozIAppServicesLogger
+ );
+ let logger_target = "app-services:webext_storage:sync";
+ app_services_logger.register(logger_target, new LogAdapter(this._log));
+}
+
+ExtensionStorageEngineBridge.prototype = {
+ syncPriority: 10,
+
+ // Used to override the engine name in telemetry, so that we can distinguish .
+ overrideTelemetryName: "rust-webext-storage",
+
+ _notifyPendingChanges() {
+ return new Promise(resolve => {
+ this.component
+ .QueryInterface(Ci.mozISyncedExtensionStorageArea)
+ .fetchPendingSyncChanges({
+ QueryInterface: ChromeUtils.generateQI([
+ "mozIExtensionStorageListener",
+ "mozIExtensionStorageCallback",
+ ]),
+ onChanged: (extId, json) => {
+ try {
+ lazy.extensionStorageSync.notifyListeners(
+ extId,
+ JSON.parse(json)
+ );
+ } catch (ex) {
+ this._log.warn(
+ `Error notifying change listeners for ${extId}`,
+ ex
+ );
+ }
+ },
+ handleSuccess: resolve,
+ handleError: (code, message) => {
+ this._log.warn(
+ "Error fetching pending synced changes",
+ message,
+ code
+ );
+ resolve();
+ },
+ });
+ });
+ },
+
+ _takeMigrationInfo() {
+ return new Promise((resolve, reject) => {
+ this.component
+ .QueryInterface(Ci.mozIExtensionStorageArea)
+ .takeMigrationInfo({
+ QueryInterface: ChromeUtils.generateQI([
+ "mozIExtensionStorageCallback",
+ ]),
+ handleSuccess: result => {
+ resolve(result ? JSON.parse(result) : null);
+ },
+ handleError: (code, message) => {
+ this._log.warn("Error fetching migration info", message, code);
+ // `takeMigrationInfo` doesn't actually perform the migration,
+ // just reads (and clears) any data stored in the DB from the
+ // previous migration.
+ //
+ // Any errors here are very likely occurring a good while
+ // after the migration ran, so we just warn and pretend
+ // nothing was there.
+ resolve(null);
+ },
+ });
+ });
+ },
+
+ async _syncStartup() {
+ let result = await super._syncStartup();
+ let info = await this._takeMigrationInfo();
+ if (info) {
+ lazy.Observers.notify(
+ "weave:telemetry:migration",
+ info,
+ "webext-storage"
+ );
+ }
+ return result;
+ },
+
+ async _processIncoming() {
+ await super._processIncoming();
+ try {
+ await this._notifyPendingChanges();
+ } catch (ex) {
+ // Failing to notify `storage.onChanged` observers is bad, but shouldn't
+ // interrupt syncing.
+ this._log.warn("Error notifying about synced changes", ex);
+ }
+ },
+
+ get enabled() {
+ return getEngineEnabled();
+ },
+ set enabled(enabled) {
+ setEngineEnabled(enabled);
+ },
+};
+Object.setPrototypeOf(
+ ExtensionStorageEngineBridge.prototype,
+ BridgedEngine.prototype
+);
+
+/**
+ *****************************************************************************
+ *
+ * Deprecated support for Kinto
+ *
+ *****************************************************************************
+ */
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+export function ExtensionStorageEngineKinto(service) {
+ SyncEngine.call(this, "Extension-Storage", service);
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_skipPercentageChance",
+ "services.sync.extension-storage.skipPercentageChance",
+ 0
+ );
+}
+
+ExtensionStorageEngineKinto.prototype = {
+ _trackerObj: ExtensionStorageTracker,
+ // we don't need these since we implement our own sync logic
+ _storeObj: undefined,
+ _recordObj: undefined,
+
+ syncPriority: 10,
+ allowSkippedRecord: false,
+
+ async _sync() {
+ return lazy.extensionStorageSyncKinto.syncAll();
+ },
+
+ get enabled() {
+ return getEngineEnabled();
+ },
+ // We only need the enabled setter for the edge-case where info/collections
+ // has `extension-storage` - which could happen if the pref to flip the new
+ // engine on was once set but no longer is.
+ set enabled(enabled) {
+ setEngineEnabled(enabled);
+ },
+
+ _wipeClient() {
+ return lazy.extensionStorageSyncKinto.clearAll();
+ },
+
+ shouldSkipSync(syncReason) {
+ if (syncReason == "user" || syncReason == "startup") {
+ this._log.info(
+ `Not skipping extension storage sync: reason == ${syncReason}`
+ );
+ // Always sync if a user clicks the button, or if we're starting up.
+ return false;
+ }
+ // Ensure this wouldn't cause a resync...
+ if (this._tracker.score >= lazy.MULTI_DEVICE_THRESHOLD) {
+ this._log.info(
+ "Not skipping extension storage sync: Would trigger resync anyway"
+ );
+ return false;
+ }
+
+ let probability = this._skipPercentageChance / 100.0;
+ // Math.random() returns a value in the interval [0, 1), so `>` is correct:
+ // if `probability` is 1 skip every time, and if it's 0, never skip.
+ let shouldSkip = probability > Math.random();
+
+ this._log.info(
+ `Skipping extension-storage sync with a chance of ${probability}: ${shouldSkip}`
+ );
+ return shouldSkip;
+ },
+};
+Object.setPrototypeOf(
+ ExtensionStorageEngineKinto.prototype,
+ SyncEngine.prototype
+);
+
+function ExtensionStorageTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ this._ignoreAll = false;
+}
+ExtensionStorageTracker.prototype = {
+ get ignoreAll() {
+ return this._ignoreAll;
+ },
+
+ set ignoreAll(value) {
+ this._ignoreAll = value;
+ },
+
+ onStart() {
+ lazy.Svc.Obs.add("ext.storage.sync-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ lazy.Svc.Obs.remove("ext.storage.sync-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (topic !== "ext.storage.sync-changed") {
+ return;
+ }
+
+ // Single adds, removes and changes are not so important on their
+ // own, so let's just increment score a bit.
+ this.score += lazy.SCORE_INCREMENT_MEDIUM;
+ },
+};
+Object.setPrototypeOf(ExtensionStorageTracker.prototype, Tracker.prototype);
diff --git a/services/sync/modules/engines/forms.sys.mjs b/services/sync/modules/engines/forms.sys.mjs
new file mode 100644
index 0000000000..3516327659
--- /dev/null
+++ b/services/sync/modules/engines/forms.sys.mjs
@@ -0,0 +1,298 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ Store,
+ SyncEngine,
+ LegacyTracker,
+} from "resource://services-sync/engines.sys.mjs";
+
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { SCORE_INCREMENT_MEDIUM } from "resource://services-sync/constants.sys.mjs";
+import {
+ CollectionProblemData,
+ CollectionValidator,
+} from "resource://services-sync/collection_validator.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+const lazy = {};
+ChromeUtils.defineESModuleGetters(lazy, {
+ FormHistory: "resource://gre/modules/FormHistory.sys.mjs",
+});
+
+const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds.
+
+export function FormRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+
+FormRec.prototype = {
+ _logName: "Sync.Record.Form",
+ ttl: FORMS_TTL,
+};
+Object.setPrototypeOf(FormRec.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
+
+var FormWrapper = {
+ _log: Log.repository.getLogger("Sync.Engine.Forms"),
+
+ _getEntryCols: ["fieldname", "value"],
+ _guidCols: ["guid"],
+
+ _search(terms, searchData) {
+ return lazy.FormHistory.search(terms, searchData);
+ },
+
+ async _update(changes) {
+ if (!lazy.FormHistory.enabled) {
+ return; // update isn't going to do anything.
+ }
+ await lazy.FormHistory.update(changes).catch(console.error);
+ },
+
+ async getEntry(guid) {
+ let results = await this._search(this._getEntryCols, { guid });
+ if (!results.length) {
+ return null;
+ }
+ return { name: results[0].fieldname, value: results[0].value };
+ },
+
+ async getGUID(name, value) {
+ // Query for the provided entry.
+ let query = { fieldname: name, value };
+ let results = await this._search(this._guidCols, query);
+ return results.length ? results[0].guid : null;
+ },
+
+ async hasGUID(guid) {
+ // We could probably use a count function here, but search exists...
+ let results = await this._search(this._guidCols, { guid });
+ return !!results.length;
+ },
+
+ async replaceGUID(oldGUID, newGUID) {
+ let changes = {
+ op: "update",
+ guid: oldGUID,
+ newGuid: newGUID,
+ };
+ await this._update(changes);
+ },
+};
+
+export function FormEngine(service) {
+ SyncEngine.call(this, "Forms", service);
+}
+
+FormEngine.prototype = {
+ _storeObj: FormStore,
+ _trackerObj: FormTracker,
+ _recordObj: FormRec,
+
+ syncPriority: 6,
+
+ get prefName() {
+ return "history";
+ },
+
+ async _findDupe(item) {
+ return FormWrapper.getGUID(item.name, item.value);
+ },
+};
+Object.setPrototypeOf(FormEngine.prototype, SyncEngine.prototype);
+
+function FormStore(name, engine) {
+ Store.call(this, name, engine);
+}
+FormStore.prototype = {
+ async _processChange(change) {
+ // If this._changes is defined, then we are applying a batch, so we
+ // can defer it.
+ if (this._changes) {
+ this._changes.push(change);
+ return;
+ }
+
+ // Otherwise we must handle the change right now.
+ await FormWrapper._update(change);
+ },
+
+ async applyIncomingBatch(records, countTelemetry) {
+ Async.checkAppReady();
+ // We collect all the changes to be made then apply them all at once.
+ this._changes = [];
+ let failures = await Store.prototype.applyIncomingBatch.call(
+ this,
+ records,
+ countTelemetry
+ );
+ if (this._changes.length) {
+ await FormWrapper._update(this._changes);
+ }
+ delete this._changes;
+ return failures;
+ },
+
+ async getAllIDs() {
+ let results = await FormWrapper._search(["guid"], []);
+ let guids = {};
+ for (let result of results) {
+ guids[result.guid] = true;
+ }
+ return guids;
+ },
+
+ async changeItemID(oldID, newID) {
+ await FormWrapper.replaceGUID(oldID, newID);
+ },
+
+ async itemExists(id) {
+ return FormWrapper.hasGUID(id);
+ },
+
+ async createRecord(id, collection) {
+ let record = new FormRec(collection, id);
+ let entry = await FormWrapper.getEntry(id);
+ if (entry != null) {
+ record.name = entry.name;
+ record.value = entry.value;
+ } else {
+ record.deleted = true;
+ }
+ return record;
+ },
+
+ async create(record) {
+ this._log.trace("Adding form record for " + record.name);
+ let change = {
+ op: "add",
+ guid: record.id,
+ fieldname: record.name,
+ value: record.value,
+ };
+ await this._processChange(change);
+ },
+
+ async remove(record) {
+ this._log.trace("Removing form record: " + record.id);
+ let change = {
+ op: "remove",
+ guid: record.id,
+ };
+ await this._processChange(change);
+ },
+
+ async update(record) {
+ this._log.trace("Ignoring form record update request!");
+ },
+
+ async wipe() {
+ let change = {
+ op: "remove",
+ };
+ await FormWrapper._update(change);
+ },
+};
+Object.setPrototypeOf(FormStore.prototype, Store.prototype);
+
+function FormTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+FormTracker.prototype = {
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ onStart() {
+ Svc.Obs.add("satchel-storage-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ Svc.Obs.remove("satchel-storage-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+ switch (topic) {
+ case "satchel-storage-changed":
+ if (data == "formhistory-add" || data == "formhistory-remove") {
+ let guid = subject.QueryInterface(Ci.nsISupportsString).toString();
+ await this.trackEntry(guid);
+ }
+ break;
+ }
+ },
+
+ async trackEntry(guid) {
+ const added = await this.addChangedID(guid);
+ if (added) {
+ this.score += SCORE_INCREMENT_MEDIUM;
+ }
+ },
+};
+Object.setPrototypeOf(FormTracker.prototype, LegacyTracker.prototype);
+
+class FormsProblemData extends CollectionProblemData {
+ getSummary() {
+ // We don't support syncing deleted form data, so "clientMissing" isn't a problem
+ return super.getSummary().filter(entry => entry.name !== "clientMissing");
+ }
+}
+
+export class FormValidator extends CollectionValidator {
+ constructor() {
+ super("forms", "id", ["name", "value"]);
+ this.ignoresMissingClients = true;
+ }
+
+ emptyProblemData() {
+ return new FormsProblemData();
+ }
+
+ async getClientItems() {
+ return FormWrapper._search(["guid", "fieldname", "value"], {});
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ name: item.fieldname,
+ fieldname: item.fieldname,
+ value: item.value,
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ let res = Object.assign(
+ {
+ guid: item.id,
+ fieldname: item.name,
+ original: item,
+ },
+ item
+ );
+ // Missing `name` or `value` causes the getGUID call to throw
+ if (item.name !== undefined && item.value !== undefined) {
+ let guid = await FormWrapper.getGUID(item.name, item.value);
+ if (guid) {
+ res.guid = guid;
+ res.id = guid;
+ res.duped = true;
+ }
+ }
+
+ return res;
+ }
+}
diff --git a/services/sync/modules/engines/history.sys.mjs b/services/sync/modules/engines/history.sys.mjs
new file mode 100644
index 0000000000..491e7c8a89
--- /dev/null
+++ b/services/sync/modules/engines/history.sys.mjs
@@ -0,0 +1,585 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const HISTORY_TTL = 5184000; // 60 days in milliseconds
+const THIRTY_DAYS_IN_MS = 2592000000; // 30 days in milliseconds
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+
+import {
+ MAX_HISTORY_DOWNLOAD,
+ MAX_HISTORY_UPLOAD,
+ SCORE_INCREMENT_SMALL,
+ SCORE_INCREMENT_XLARGE,
+} from "resource://services-sync/constants.sys.mjs";
+
+import {
+ Store,
+ SyncEngine,
+ LegacyTracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Utils } from "resource://services-sync/util.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs",
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+});
+
+export function HistoryRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+
+HistoryRec.prototype = {
+ _logName: "Sync.Record.History",
+ ttl: HISTORY_TTL,
+};
+Object.setPrototypeOf(HistoryRec.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]);
+
+export function HistoryEngine(service) {
+ SyncEngine.call(this, "History", service);
+}
+
+HistoryEngine.prototype = {
+ _recordObj: HistoryRec,
+ _storeObj: HistoryStore,
+ _trackerObj: HistoryTracker,
+ downloadLimit: MAX_HISTORY_DOWNLOAD,
+
+ syncPriority: 7,
+
+ async getSyncID() {
+ return lazy.PlacesSyncUtils.history.getSyncId();
+ },
+
+ async ensureCurrentSyncID(newSyncID) {
+ this._log.debug(
+ "Checking if server sync ID ${newSyncID} matches existing",
+ { newSyncID }
+ );
+ await lazy.PlacesSyncUtils.history.ensureCurrentSyncId(newSyncID);
+ return newSyncID;
+ },
+
+ async resetSyncID() {
+ // First, delete the collection on the server. It's fine if we're
+ // interrupted here: on the next sync, we'll detect that our old sync ID is
+ // now stale, and start over as a first sync.
+ await this._deleteServerCollection();
+ // Then, reset our local sync ID.
+ return this.resetLocalSyncID();
+ },
+
+ async resetLocalSyncID() {
+ let newSyncID = await lazy.PlacesSyncUtils.history.resetSyncId();
+ this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID });
+ return newSyncID;
+ },
+
+ async getLastSync() {
+ let lastSync = await lazy.PlacesSyncUtils.history.getLastSync();
+ return lastSync;
+ },
+
+ async setLastSync(lastSync) {
+ await lazy.PlacesSyncUtils.history.setLastSync(lastSync);
+ },
+
+ shouldSyncURL(url) {
+ return !url.startsWith("file:");
+ },
+
+ async pullNewChanges() {
+ const changedIDs = await this._tracker.getChangedIDs();
+ let modifiedGUIDs = Object.keys(changedIDs);
+ if (!modifiedGUIDs.length) {
+ return {};
+ }
+
+ let guidsToRemove =
+ await lazy.PlacesSyncUtils.history.determineNonSyncableGuids(
+ modifiedGUIDs
+ );
+ await this._tracker.removeChangedID(...guidsToRemove);
+ return changedIDs;
+ },
+
+ async _resetClient() {
+ await super._resetClient();
+ await lazy.PlacesSyncUtils.history.reset();
+ },
+};
+Object.setPrototypeOf(HistoryEngine.prototype, SyncEngine.prototype);
+
+function HistoryStore(name, engine) {
+ Store.call(this, name, engine);
+}
+
+HistoryStore.prototype = {
+ // We try and only update this many visits at one time.
+ MAX_VISITS_PER_INSERT: 500,
+
+ // Some helper functions to handle GUIDs
+ async setGUID(uri, guid) {
+ if (!guid) {
+ guid = Utils.makeGUID();
+ }
+
+ try {
+ await lazy.PlacesSyncUtils.history.changeGuid(uri, guid);
+ } catch (e) {
+ this._log.error("Error setting GUID ${guid} for URI ${uri}", guid, uri);
+ }
+
+ return guid;
+ },
+
+ async GUIDForUri(uri, create) {
+ // Use the existing GUID if it exists
+ let guid;
+ try {
+ guid = await lazy.PlacesSyncUtils.history.fetchGuidForURL(uri);
+ } catch (e) {
+ this._log.error("Error fetching GUID for URL ${uri}", uri);
+ }
+
+ // If the URI has an existing GUID, return it.
+ if (guid) {
+ return guid;
+ }
+
+ // If the URI doesn't have a GUID and we were indicated to create one.
+ if (create) {
+ return this.setGUID(uri);
+ }
+
+ // If the URI doesn't have a GUID and we didn't create one for it.
+ return null;
+ },
+
+ async changeItemID(oldID, newID) {
+ let info = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(oldID);
+ if (!info) {
+ throw new Error(`Can't change ID for nonexistent history entry ${oldID}`);
+ }
+ this.setGUID(info.url, newID);
+ },
+
+ async getAllIDs() {
+ let urls = await lazy.PlacesSyncUtils.history.getAllURLs({
+ since: new Date(Date.now() - THIRTY_DAYS_IN_MS),
+ limit: MAX_HISTORY_UPLOAD,
+ });
+
+ let urlsByGUID = {};
+ for (let url of urls) {
+ if (!this.engine.shouldSyncURL(url)) {
+ continue;
+ }
+ let guid = await this.GUIDForUri(url, true);
+ urlsByGUID[guid] = url;
+ }
+ return urlsByGUID;
+ },
+
+ async applyIncomingBatch(records, countTelemetry) {
+ // Convert incoming records to mozIPlaceInfo objects which are applied as
+ // either history additions or removals.
+ let failed = [];
+ let toAdd = [];
+ let toRemove = [];
+ await Async.yieldingForEach(records, async record => {
+ if (record.deleted) {
+ toRemove.push(record);
+ } else {
+ try {
+ let pageInfo = await this._recordToPlaceInfo(record);
+ if (pageInfo) {
+ toAdd.push(pageInfo);
+ }
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.error("Failed to create a place info", ex);
+ this._log.trace("The record that failed", record);
+ failed.push(record.id);
+ countTelemetry.addIncomingFailedReason(ex.message);
+ }
+ }
+ });
+ if (toAdd.length || toRemove.length) {
+ if (toRemove.length) {
+ // PlacesUtils.history.remove takes an array of visits to remove,
+ // but the error semantics are tricky - a single "bad" entry will cause
+ // an exception before anything is removed. So we do remove them one at
+ // a time.
+ await Async.yieldingForEach(toRemove, async record => {
+ try {
+ await this.remove(record);
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.error("Failed to delete a place info", ex);
+ this._log.trace("The record that failed", record);
+ failed.push(record.id);
+ countTelemetry.addIncomingFailedReason(ex.message);
+ }
+ });
+ }
+ for (let chunk of this._generateChunks(toAdd)) {
+ // Per bug 1415560, we ignore any exceptions returned by insertMany
+ // as they are likely to be spurious. We do supply an onError handler
+ // and log the exceptions seen there as they are likely to be
+ // informative, but we still never abort the sync based on them.
+ try {
+ await lazy.PlacesUtils.history.insertMany(
+ chunk,
+ null,
+ failedVisit => {
+ this._log.info(
+ "Failed to insert a history record",
+ failedVisit.guid
+ );
+ this._log.trace("The record that failed", failedVisit);
+ failed.push(failedVisit.guid);
+ }
+ );
+ } catch (ex) {
+ this._log.info("Failed to insert history records", ex);
+ countTelemetry.addIncomingFailedReason(ex.message);
+ }
+ }
+ }
+
+ return failed;
+ },
+
+ /**
+ * Returns a generator that splits records into sanely sized chunks suitable
+ * for passing to places to prevent places doing bad things at shutdown.
+ */
+ *_generateChunks(records) {
+ // We chunk based on the number of *visits* inside each record. However,
+ // we do not split a single record into multiple records, because at some
+ // time in the future, we intend to ensure these records are ordered by
+ // lastModified, and advance the engine's timestamp as we process them,
+ // meaning we can resume exactly where we left off next sync - although
+ // currently that's not done, so we will retry the entire batch next sync
+ // if interrupted.
+ // ie, this means that if a single record has more than MAX_VISITS_PER_INSERT
+ // visits, we will call insertMany() with exactly 1 record, but with
+ // more than MAX_VISITS_PER_INSERT visits.
+ let curIndex = 0;
+ this._log.debug(`adding ${records.length} records to history`);
+ while (curIndex < records.length) {
+ Async.checkAppReady(); // may throw if we are shutting down.
+ let toAdd = []; // what we are going to insert.
+ let count = 0; // a counter which tells us when toAdd is full.
+ do {
+ let record = records[curIndex];
+ curIndex += 1;
+ toAdd.push(record);
+ count += record.visits.length;
+ } while (
+ curIndex < records.length &&
+ count + records[curIndex].visits.length <= this.MAX_VISITS_PER_INSERT
+ );
+ this._log.trace(`adding ${toAdd.length} items in this chunk`);
+ yield toAdd;
+ }
+ },
+
+ /* An internal helper to determine if we can add an entry to places.
+ Exists primarily so tests can override it.
+ */
+ _canAddURI(uri) {
+ return lazy.PlacesUtils.history.canAddURI(uri);
+ },
+
+ /**
+ * Converts a Sync history record to a mozIPlaceInfo.
+ *
+ * Throws if an invalid record is encountered (invalid URI, etc.),
+ * returns a new PageInfo object if the record is to be applied, null
+ * otherwise (no visits to add, etc.),
+ */
+ async _recordToPlaceInfo(record) {
+ // Sort out invalid URIs and ones Places just simply doesn't want.
+ record.url = lazy.PlacesUtils.normalizeToURLOrGUID(record.histUri);
+ record.uri = CommonUtils.makeURI(record.histUri);
+
+ if (!Utils.checkGUID(record.id)) {
+ this._log.warn("Encountered record with invalid GUID: " + record.id);
+ return null;
+ }
+ record.guid = record.id;
+
+ if (
+ !this._canAddURI(record.uri) ||
+ !this.engine.shouldSyncURL(record.uri.spec)
+ ) {
+ this._log.trace(
+ "Ignoring record " +
+ record.id +
+ " with URI " +
+ record.uri.spec +
+ ": can't add this URI."
+ );
+ return null;
+ }
+
+ // We dupe visits by date and type. So an incoming visit that has
+ // the same timestamp and type as a local one won't get applied.
+ // To avoid creating new objects, we rewrite the query result so we
+ // can simply check for containment below.
+ let curVisitsAsArray = [];
+ let curVisits = new Set();
+ try {
+ curVisitsAsArray = await lazy.PlacesSyncUtils.history.fetchVisitsForURL(
+ record.histUri
+ );
+ } catch (e) {
+ this._log.error(
+ "Error while fetching visits for URL ${record.histUri}",
+ record.histUri
+ );
+ }
+ let oldestAllowed =
+ lazy.PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP;
+ if (curVisitsAsArray.length == 20) {
+ let oldestVisit = curVisitsAsArray[curVisitsAsArray.length - 1];
+ oldestAllowed = lazy.PlacesSyncUtils.history.clampVisitDate(
+ lazy.PlacesUtils.toDate(oldestVisit.date).getTime()
+ );
+ }
+
+ let i, k;
+ for (i = 0; i < curVisitsAsArray.length; i++) {
+ // Same logic as used in the loop below to generate visitKey.
+ let { date, type } = curVisitsAsArray[i];
+ let dateObj = lazy.PlacesUtils.toDate(date);
+ let millis = lazy.PlacesSyncUtils.history
+ .clampVisitDate(dateObj)
+ .getTime();
+ curVisits.add(`${millis},${type}`);
+ }
+
+ // Walk through the visits, make sure we have sound data, and eliminate
+ // dupes. The latter is done by rewriting the array in-place.
+ for (i = 0, k = 0; i < record.visits.length; i++) {
+ let visit = (record.visits[k] = record.visits[i]);
+
+ if (
+ !visit.date ||
+ typeof visit.date != "number" ||
+ !Number.isInteger(visit.date)
+ ) {
+ this._log.warn(
+ "Encountered record with invalid visit date: " + visit.date
+ );
+ continue;
+ }
+
+ if (
+ !visit.type ||
+ !Object.values(lazy.PlacesUtils.history.TRANSITIONS).includes(
+ visit.type
+ )
+ ) {
+ this._log.warn(
+ "Encountered record with invalid visit type: " +
+ visit.type +
+ "; ignoring."
+ );
+ continue;
+ }
+
+ // Dates need to be integers. Future and far past dates are clamped to the
+ // current date and earliest sensible date, respectively.
+ let originalVisitDate = lazy.PlacesUtils.toDate(Math.round(visit.date));
+ visit.date =
+ lazy.PlacesSyncUtils.history.clampVisitDate(originalVisitDate);
+
+ if (visit.date.getTime() < oldestAllowed) {
+ // Visit is older than the oldest visit we have, and we have so many
+ // visits for this uri that we hit our limit when inserting.
+ continue;
+ }
+ let visitKey = `${visit.date.getTime()},${visit.type}`;
+ if (curVisits.has(visitKey)) {
+ // Visit is a dupe, don't increment 'k' so the element will be
+ // overwritten.
+ continue;
+ }
+
+ // Note the visit key, so that we don't add duplicate visits with
+ // clamped timestamps.
+ curVisits.add(visitKey);
+
+ visit.transition = visit.type;
+ k += 1;
+ }
+ record.visits.length = k; // truncate array
+
+ // No update if there aren't any visits to apply.
+ // History wants at least one visit.
+ // In any case, the only thing we could change would be the title
+ // and that shouldn't change without a visit.
+ if (!record.visits.length) {
+ this._log.trace(
+ "Ignoring record " +
+ record.id +
+ " with URI " +
+ record.uri.spec +
+ ": no visits to add."
+ );
+ return null;
+ }
+
+ // PageInfo is validated using validateItemProperties which does a shallow
+ // copy of the properties. Since record uses getters some of the properties
+ // are not copied over. Thus we create and return a new object.
+ let pageInfo = {
+ title: record.title,
+ url: record.url,
+ guid: record.guid,
+ visits: record.visits,
+ };
+
+ return pageInfo;
+ },
+
+ async remove(record) {
+ this._log.trace("Removing page: " + record.id);
+ let removed = await lazy.PlacesUtils.history.remove(record.id);
+ if (removed) {
+ this._log.trace("Removed page: " + record.id);
+ } else {
+ this._log.debug("Page already removed: " + record.id);
+ }
+ },
+
+ async itemExists(id) {
+ return !!(await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id));
+ },
+
+ async createRecord(id, collection) {
+ let foo = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id);
+ let record = new HistoryRec(collection, id);
+ if (foo) {
+ record.histUri = foo.url;
+ record.title = foo.title;
+ record.sortindex = foo.frecency;
+ try {
+ record.visits = await lazy.PlacesSyncUtils.history.fetchVisitsForURL(
+ record.histUri
+ );
+ } catch (e) {
+ this._log.error(
+ "Error while fetching visits for URL ${record.histUri}",
+ record.histUri
+ );
+ record.visits = [];
+ }
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ async wipe() {
+ return lazy.PlacesSyncUtils.history.wipe();
+ },
+};
+Object.setPrototypeOf(HistoryStore.prototype, Store.prototype);
+
+function HistoryTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+HistoryTracker.prototype = {
+ onStart() {
+ this._log.info("Adding Places observer.");
+ this._placesObserver = new PlacesWeakCallbackWrapper(
+ this.handlePlacesEvents.bind(this)
+ );
+ PlacesObservers.addListener(
+ ["page-visited", "history-cleared", "page-removed"],
+ this._placesObserver
+ );
+ },
+
+ onStop() {
+ this._log.info("Removing Places observer.");
+ if (this._placesObserver) {
+ PlacesObservers.removeListener(
+ ["page-visited", "history-cleared", "page-removed"],
+ this._placesObserver
+ );
+ }
+ },
+
+ QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]),
+
+ handlePlacesEvents(aEvents) {
+ this.asyncObserver.enqueueCall(() => this._handlePlacesEvents(aEvents));
+ },
+
+ async _handlePlacesEvents(aEvents) {
+ if (this.ignoreAll) {
+ this._log.trace(
+ "ignoreAll: ignoring visits [" +
+ aEvents.map(v => v.guid).join(",") +
+ "]"
+ );
+ return;
+ }
+ for (let event of aEvents) {
+ switch (event.type) {
+ case "page-visited": {
+ this._log.trace("'page-visited': " + event.url);
+ if (
+ this.engine.shouldSyncURL(event.url) &&
+ (await this.addChangedID(event.pageGuid))
+ ) {
+ this.score += SCORE_INCREMENT_SMALL;
+ }
+ break;
+ }
+ case "history-cleared": {
+ this._log.trace("history-cleared");
+ // Note that we're going to trigger a sync, but none of the cleared
+ // pages are tracked, so the deletions will not be propagated.
+ // See Bug 578694.
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ case "page-removed": {
+ if (event.reason === PlacesVisitRemoved.REASON_EXPIRED) {
+ return;
+ }
+
+ this._log.trace(
+ "page-removed: " + event.url + ", reason " + event.reason
+ );
+ const added = await this.addChangedID(event.pageGuid);
+ if (added) {
+ this.score += event.isRemovedFromStore
+ ? SCORE_INCREMENT_XLARGE
+ : SCORE_INCREMENT_SMALL;
+ }
+ break;
+ }
+ }
+ }
+ },
+};
+Object.setPrototypeOf(HistoryTracker.prototype, LegacyTracker.prototype);
diff --git a/services/sync/modules/engines/passwords.sys.mjs b/services/sync/modules/engines/passwords.sys.mjs
new file mode 100644
index 0000000000..8f24d7333a
--- /dev/null
+++ b/services/sync/modules/engines/passwords.sys.mjs
@@ -0,0 +1,565 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ Collection,
+ CryptoWrapper,
+} from "resource://services-sync/record.sys.mjs";
+
+import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs";
+import { CollectionValidator } from "resource://services-sync/collection_validator.sys.mjs";
+import {
+ Store,
+ SyncEngine,
+ LegacyTracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+
+// These are valid fields the server could have for a logins record
+// we mainly use this to detect if there are any unknownFields and
+// store (but don't process) those fields to roundtrip them back
+const VALID_LOGIN_FIELDS = [
+ "id",
+ "displayOrigin",
+ "formSubmitURL",
+ "formActionOrigin",
+ "httpRealm",
+ "hostname",
+ "origin",
+ "password",
+ "passwordField",
+ "timeCreated",
+ "timeLastUsed",
+ "timePasswordChanged",
+ "timesUsed",
+ "username",
+ "usernameField",
+ "unknownFields",
+];
+
+const SYNCABLE_LOGIN_FIELDS = [
+ // `nsILoginInfo` fields.
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "username",
+ "password",
+ "usernameField",
+ "passwordField",
+
+ // `nsILoginMetaInfo` fields.
+ "timeCreated",
+ "timePasswordChanged",
+];
+
+// Compares two logins to determine if their syncable fields changed. The login
+// manager fires `modifyLogin` for changes to all fields, including ones we
+// don't sync. In particular, `timeLastUsed` changes shouldn't mark the login
+// for upload; otherwise, we might overwrite changed passwords before they're
+// downloaded (bug 973166).
+function isSyncableChange(oldLogin, newLogin) {
+ oldLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+ newLogin.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+ for (let property of SYNCABLE_LOGIN_FIELDS) {
+ if (oldLogin[property] != newLogin[property]) {
+ return true;
+ }
+ }
+ return false;
+}
+
+export function LoginRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+
+LoginRec.prototype = {
+ _logName: "Sync.Record.Login",
+
+ cleartextToString() {
+ let o = Object.assign({}, this.cleartext);
+ if (o.password) {
+ o.password = "X".repeat(o.password.length);
+ }
+ return JSON.stringify(o);
+ },
+};
+Object.setPrototypeOf(LoginRec.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(LoginRec, "cleartext", [
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "username",
+ "password",
+ "usernameField",
+ "passwordField",
+ "timeCreated",
+ "timePasswordChanged",
+]);
+
+export function PasswordEngine(service) {
+ SyncEngine.call(this, "Passwords", service);
+}
+
+PasswordEngine.prototype = {
+ _storeObj: PasswordStore,
+ _trackerObj: PasswordTracker,
+ _recordObj: LoginRec,
+
+ syncPriority: 2,
+
+ // Metadata for syncing is stored in the login manager
+ async ensureCurrentSyncID(newSyncID) {
+ return Services.logins.ensureCurrentSyncID(newSyncID);
+ },
+
+ async getLastSync() {
+ let legacyValue = await super.getLastSync();
+ if (legacyValue) {
+ await this.setLastSync(legacyValue);
+ Svc.Prefs.reset(this.name + ".lastSync");
+ this._log.debug(
+ `migrated timestamp of ${legacyValue} to the logins store`
+ );
+ return legacyValue;
+ }
+ return Services.logins.getLastSync();
+ },
+
+ async setLastSync(timestamp) {
+ await Services.logins.setLastSync(timestamp);
+ },
+
+ async _syncFinish() {
+ await SyncEngine.prototype._syncFinish.call(this);
+
+ // Delete the Weave credentials from the server once.
+ if (!Svc.Prefs.get("deletePwdFxA", false)) {
+ try {
+ let ids = [];
+ for (let host of Utils.getSyncCredentialsHosts()) {
+ for (let info of Services.logins.findLogins(host, "", "")) {
+ ids.push(info.QueryInterface(Ci.nsILoginMetaInfo).guid);
+ }
+ }
+ if (ids.length) {
+ let coll = new Collection(this.engineURL, null, this.service);
+ coll.ids = ids;
+ let ret = await coll.delete();
+ this._log.debug("Delete result: " + ret);
+ if (!ret.success && ret.status != 400) {
+ // A non-400 failure means try again next time.
+ return;
+ }
+ } else {
+ this._log.debug("Didn't find any passwords to delete");
+ }
+ // If there were no ids to delete, or we succeeded, or got a 400,
+ // record success.
+ Svc.Prefs.set("deletePwdFxA", true);
+ Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Password deletes failed", ex);
+ }
+ }
+ },
+
+ async _findDupe(item) {
+ let login = this._store._nsLoginInfoFromRecord(item);
+ if (!login) {
+ return null;
+ }
+
+ let logins = Services.logins.findLogins(
+ login.origin,
+ login.formActionOrigin,
+ login.httpRealm
+ );
+
+ await Async.promiseYield(); // Yield back to main thread after synchronous operation.
+
+ // Look for existing logins that match the origin, but ignore the password.
+ for (let local of logins) {
+ if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
+ return local.guid;
+ }
+ }
+
+ return null;
+ },
+
+ async pullAllChanges() {
+ let changes = {};
+ let ids = await this._store.getAllIDs();
+ for (let [id, info] of Object.entries(ids)) {
+ changes[id] = info.timePasswordChanged / 1000;
+ }
+ return changes;
+ },
+
+ getValidator() {
+ return new PasswordValidator();
+ },
+};
+Object.setPrototypeOf(PasswordEngine.prototype, SyncEngine.prototype);
+
+function PasswordStore(name, engine) {
+ Store.call(this, name, engine);
+ this._nsLoginInfo = new Components.Constructor(
+ "@mozilla.org/login-manager/loginInfo;1",
+ Ci.nsILoginInfo,
+ "init"
+ );
+}
+PasswordStore.prototype = {
+ _newPropertyBag() {
+ return Cc["@mozilla.org/hash-property-bag;1"].createInstance(
+ Ci.nsIWritablePropertyBag2
+ );
+ },
+
+ // Returns an stringified object of any fields not "known" by this client
+ // mainly used to to prevent data loss for other clients by roundtripping
+ // these fields without processing them
+ _processUnknownFields(record) {
+ let unknownFields = {};
+ let keys = Object.keys(record);
+ keys
+ .filter(key => !VALID_LOGIN_FIELDS.includes(key))
+ .forEach(key => {
+ unknownFields[key] = record[key];
+ });
+ // If we found some unknown fields, we stringify it to be able
+ // to properly encrypt it for roundtripping since we can't know if
+ // it contained sensitive fields or not
+ if (Object.keys(unknownFields).length) {
+ return JSON.stringify(unknownFields);
+ }
+ return null;
+ },
+
+ /**
+ * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
+ */
+ _nsLoginInfoFromRecord(record) {
+ function nullUndefined(x) {
+ return x == undefined ? null : x;
+ }
+
+ function stringifyNullUndefined(x) {
+ return x == undefined || x == null ? "" : x;
+ }
+
+ if (record.formSubmitURL && record.httpRealm) {
+ this._log.warn(
+ "Record " +
+ record.id +
+ " has both formSubmitURL and httpRealm. Skipping."
+ );
+ return null;
+ }
+
+ // Passing in "undefined" results in an empty string, which later
+ // counts as a value. Explicitly `|| null` these fields according to JS
+ // truthiness. Records with empty strings or null will be unmolested.
+ let info = new this._nsLoginInfo(
+ record.hostname,
+ nullUndefined(record.formSubmitURL),
+ nullUndefined(record.httpRealm),
+ stringifyNullUndefined(record.username),
+ record.password,
+ record.usernameField,
+ record.passwordField
+ );
+
+ info.QueryInterface(Ci.nsILoginMetaInfo);
+ info.guid = record.id;
+ if (record.timeCreated && !isNaN(new Date(record.timeCreated).getTime())) {
+ info.timeCreated = record.timeCreated;
+ }
+ if (
+ record.timePasswordChanged &&
+ !isNaN(new Date(record.timePasswordChanged).getTime())
+ ) {
+ info.timePasswordChanged = record.timePasswordChanged;
+ }
+
+ // Check the record if there are any unknown fields from other clients
+ // that we want to roundtrip during sync to prevent data loss
+ let unknownFields = this._processUnknownFields(record.cleartext);
+ if (unknownFields) {
+ info.unknownFields = unknownFields;
+ }
+ return info;
+ },
+
+ async _getLoginFromGUID(id) {
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", id);
+
+ let logins = Services.logins.searchLogins(prop);
+ await Async.promiseYield(); // Yield back to main thread after synchronous operation.
+
+ if (logins.length) {
+ this._log.trace(logins.length + " items matching " + id + " found.");
+ return logins[0];
+ }
+
+ this._log.trace("No items matching " + id + " found. Ignoring");
+ return null;
+ },
+
+ async getAllIDs() {
+ let items = {};
+ let logins = Services.logins.getAllLogins();
+
+ for (let i = 0; i < logins.length; i++) {
+ // Skip over Weave password/passphrase entries.
+ let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo);
+ if (Utils.getSyncCredentialsHosts().has(metaInfo.origin)) {
+ continue;
+ }
+
+ items[metaInfo.guid] = metaInfo;
+ }
+
+ return items;
+ },
+
+ async changeItemID(oldID, newID) {
+ this._log.trace("Changing item ID: " + oldID + " to " + newID);
+
+ let oldLogin = await this._getLoginFromGUID(oldID);
+ if (!oldLogin) {
+ this._log.trace("Can't change item ID: item doesn't exist");
+ return;
+ }
+ if (await this._getLoginFromGUID(newID)) {
+ this._log.trace("Can't change item ID: new ID already in use");
+ return;
+ }
+
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", newID);
+
+ Services.logins.modifyLogin(oldLogin, prop);
+ },
+
+ async itemExists(id) {
+ return !!(await this._getLoginFromGUID(id));
+ },
+
+ async createRecord(id, collection) {
+ let record = new LoginRec(collection, id);
+ let login = await this._getLoginFromGUID(id);
+
+ if (!login) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.hostname = login.origin;
+ record.formSubmitURL = login.formActionOrigin;
+ record.httpRealm = login.httpRealm;
+ record.username = login.username;
+ record.password = login.password;
+ record.usernameField = login.usernameField;
+ record.passwordField = login.passwordField;
+
+ // Optional fields.
+ login.QueryInterface(Ci.nsILoginMetaInfo);
+ record.timeCreated = login.timeCreated;
+ record.timePasswordChanged = login.timePasswordChanged;
+
+ // put the unknown fields back to the top-level record
+ // during upload
+ if (login.unknownFields) {
+ let unknownFields = JSON.parse(login.unknownFields);
+ if (unknownFields) {
+ Object.keys(unknownFields).forEach(key => {
+ // We have to manually add it to the cleartext since that's
+ // what gets processed during upload
+ record.cleartext[key] = unknownFields[key];
+ });
+ }
+ }
+
+ return record;
+ },
+
+ async create(record) {
+ let login = this._nsLoginInfoFromRecord(record);
+ if (!login) {
+ return;
+ }
+
+ this._log.trace("Adding login for " + record.hostname);
+ this._log.trace(
+ "httpRealm: " +
+ JSON.stringify(login.httpRealm) +
+ "; " +
+ "formSubmitURL: " +
+ JSON.stringify(login.formActionOrigin)
+ );
+ await Services.logins.addLoginAsync(login);
+ },
+
+ async remove(record) {
+ this._log.trace("Removing login " + record.id);
+
+ let loginItem = await this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.trace("Asked to remove record that doesn't exist, ignoring");
+ return;
+ }
+
+ Services.logins.removeLogin(loginItem);
+ },
+
+ async update(record) {
+ let loginItem = await this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.trace("Skipping update for unknown item: " + record.hostname);
+ return;
+ }
+
+ this._log.trace("Updating " + record.hostname);
+ let newinfo = this._nsLoginInfoFromRecord(record);
+ if (!newinfo) {
+ return;
+ }
+
+ Services.logins.modifyLogin(loginItem, newinfo);
+ },
+
+ async wipe() {
+ Services.logins.removeAllUserFacingLogins();
+ },
+};
+Object.setPrototypeOf(PasswordStore.prototype, Store.prototype);
+
+function PasswordTracker(name, engine) {
+ LegacyTracker.call(this, name, engine);
+}
+PasswordTracker.prototype = {
+ onStart() {
+ Svc.Obs.add("passwordmgr-storage-changed", this.asyncObserver);
+ },
+
+ onStop() {
+ Svc.Obs.remove("passwordmgr-storage-changed", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ if (this.ignoreAll) {
+ return;
+ }
+
+ // A single add, remove or change or removing all items
+ // will trigger a sync for MULTI_DEVICE.
+ switch (data) {
+ case "modifyLogin": {
+ subject.QueryInterface(Ci.nsIArrayExtensions);
+ let oldLogin = subject.GetElementAt(0);
+ let newLogin = subject.GetElementAt(1);
+ if (!isSyncableChange(oldLogin, newLogin)) {
+ this._log.trace(`${data}: Ignoring change for ${newLogin.guid}`);
+ break;
+ }
+ const tracked = await this._trackLogin(newLogin);
+ if (tracked) {
+ this._log.trace(`${data}: Tracking change for ${newLogin.guid}`);
+ }
+ break;
+ }
+
+ case "addLogin":
+ case "removeLogin":
+ subject
+ .QueryInterface(Ci.nsILoginMetaInfo)
+ .QueryInterface(Ci.nsILoginInfo);
+ const tracked = await this._trackLogin(subject);
+ if (tracked) {
+ this._log.trace(data + ": " + subject.guid);
+ }
+ break;
+
+ // Bug 1613620: We iterate through the removed logins and track them to ensure
+ // the logins are deleted across synced devices/accounts
+ case "removeAllLogins":
+ subject.QueryInterface(Ci.nsIArrayExtensions);
+ let count = subject.Count();
+ for (let i = 0; i < count; i++) {
+ let currentSubject = subject.GetElementAt(i);
+ let tracked = await this._trackLogin(currentSubject);
+ if (tracked) {
+ this._log.trace(data + ": " + currentSubject.guid);
+ }
+ }
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ },
+
+ async _trackLogin(login) {
+ if (Utils.getSyncCredentialsHosts().has(login.origin)) {
+ // Skip over Weave password/passphrase changes.
+ return false;
+ }
+ const added = await this.addChangedID(login.guid);
+ if (!added) {
+ return false;
+ }
+ this.score += SCORE_INCREMENT_XLARGE;
+ return true;
+ },
+};
+Object.setPrototypeOf(PasswordTracker.prototype, LegacyTracker.prototype);
+
+export class PasswordValidator extends CollectionValidator {
+ constructor() {
+ super("passwords", "id", [
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "password",
+ "passwordField",
+ "username",
+ "usernameField",
+ ]);
+ }
+
+ getClientItems() {
+ let logins = Services.logins.getAllLogins();
+ let syncHosts = Utils.getSyncCredentialsHosts();
+ let result = logins
+ .map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
+ .filter(l => !syncHosts.has(l.origin));
+ return Promise.resolve(result);
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ hostname: item.hostname,
+ formSubmitURL: item.formSubmitURL,
+ httpRealm: item.httpRealm,
+ password: item.password,
+ passwordField: item.passwordField,
+ username: item.username,
+ usernameField: item.usernameField,
+ original: item,
+ };
+ }
+
+ async normalizeServerItem(item) {
+ return Object.assign({ guid: item.id }, item);
+ }
+}
diff --git a/services/sync/modules/engines/prefs.sys.mjs b/services/sync/modules/engines/prefs.sys.mjs
new file mode 100644
index 0000000000..13d8a63070
--- /dev/null
+++ b/services/sync/modules/engines/prefs.sys.mjs
@@ -0,0 +1,467 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Prefs which start with this prefix are our "control" prefs - they indicate
+// which preferences should be synced.
+const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
+
+// Prefs which have a default value are usually not synced - however, if the
+// preference exists under this prefix and the value is:
+// * `true`, then we do sync default values.
+// * `false`, then as soon as we ever sync a non-default value out, or sync
+// any value in, then we toggle the value to `true`.
+//
+// We never explicitly set this pref back to false, so it's one-shot.
+// Some preferences which are known to have a different default value on
+// different platforms have this preference with a default value of `false`,
+// so they don't sync until one device changes to the non-default value, then
+// that value forever syncs, even if it gets reset back to the default.
+// Note that preferences handled this way *must also* have the "normal"
+// control pref set.
+// A possible future enhancement would be to sync these prefs so that
+// other distributions can flag them if they change the default, but that
+// doesn't seem worthwhile until we can be confident they'd actually create
+// this special control pref at the same time they flip the default.
+const PREF_SYNC_SEEN_PREFIX = "services.sync.prefs.sync-seen.";
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+import { Preferences } from "resource://gre/modules/Preferences.sys.mjs";
+
+import {
+ Store,
+ SyncEngine,
+ Tracker,
+} from "resource://services-sync/engines.sys.mjs";
+import { CryptoWrapper } from "resource://services-sync/record.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+import { SCORE_INCREMENT_XLARGE } from "resource://services-sync/constants.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+
+const lazy = {};
+
+XPCOMUtils.defineLazyGetter(lazy, "PREFS_GUID", () =>
+ CommonUtils.encodeBase64URL(Services.appinfo.ID)
+);
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
+});
+
+// In bug 1538015, we decided that it isn't always safe to allow all "incoming"
+// preferences to be applied locally. So we have introduced another preference,
+// which if false (the default) will ignore all incoming preferences which don't
+// already have the "control" preference locally set. If this new
+// preference is set to true, then we continue our old behavior of allowing all
+// preferences to be updated, even those which don't already have a local
+// "control" pref.
+const PREF_SYNC_PREFS_ARBITRARY =
+ "services.sync.prefs.dangerously_allow_arbitrary";
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "ALLOW_ARBITRARY",
+ PREF_SYNC_PREFS_ARBITRARY
+);
+
+// The SUMO supplied URL we log with more information about how custom prefs can
+// continue to be synced. SUMO have told us that this URL will remain "stable".
+const PREFS_DOC_URL_TEMPLATE =
+ "https://support.mozilla.org/1/firefox/%VERSION%/%OS%/%LOCALE%/sync-custom-preferences";
+XPCOMUtils.defineLazyGetter(lazy, "PREFS_DOC_URL", () =>
+ Services.urlFormatter.formatURL(PREFS_DOC_URL_TEMPLATE)
+);
+
+// Check for a local control pref or PREF_SYNC_PREFS_ARBITRARY
+function isAllowedPrefName(prefName) {
+ if (prefName == PREF_SYNC_PREFS_ARBITRARY) {
+ return false; // never allow this.
+ }
+ if (lazy.ALLOW_ARBITRARY) {
+ // user has set the "dangerous" pref, so everything is allowed.
+ return true;
+ }
+ // The pref must already have a control pref set, although it doesn't matter
+ // here whether that value is true or false. We can't use prefHasUserValue
+ // here because we also want to check prefs still with default values.
+ try {
+ Services.prefs.getBoolPref(PREF_SYNC_PREFS_PREFIX + prefName);
+ // pref exists!
+ return true;
+ } catch (_) {
+ return false;
+ }
+}
+
+export function PrefRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+
+PrefRec.prototype = {
+ _logName: "Sync.Record.Pref",
+};
+Object.setPrototypeOf(PrefRec.prototype, CryptoWrapper.prototype);
+
+Utils.deferGetSet(PrefRec, "cleartext", ["value"]);
+
+export function PrefsEngine(service) {
+ SyncEngine.call(this, "Prefs", service);
+}
+
+PrefsEngine.prototype = {
+ _storeObj: PrefStore,
+ _trackerObj: PrefTracker,
+ _recordObj: PrefRec,
+ version: 2,
+
+ syncPriority: 1,
+ allowSkippedRecord: false,
+
+ async getChangedIDs() {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified) {
+ changedIDs[lazy.PREFS_GUID] = 0;
+ }
+ return changedIDs;
+ },
+
+ async _wipeClient() {
+ await SyncEngine.prototype._wipeClient.call(this);
+ this.justWiped = true;
+ },
+
+ async _reconcile(item) {
+ // Apply the incoming item if we don't care about the local data
+ if (this.justWiped) {
+ this.justWiped = false;
+ return true;
+ }
+ return SyncEngine.prototype._reconcile.call(this, item);
+ },
+
+ async trackRemainingChanges() {
+ if (this._modified.count() > 0) {
+ this._tracker.modified = true;
+ }
+ },
+};
+Object.setPrototypeOf(PrefsEngine.prototype, SyncEngine.prototype);
+
+// We don't use services.sync.engine.tabs.filteredSchemes since it includes
+// about: pages and the like, which we want to be syncable in preferences.
+// Blob, moz-extension, data and file uris are never safe to sync,
+// so we limit our check to those.
+const UNSYNCABLE_URL_REGEXP = /^(moz-extension|blob|data|file):/i;
+function isUnsyncableURLPref(prefName) {
+ if (Services.prefs.getPrefType(prefName) != Ci.nsIPrefBranch.PREF_STRING) {
+ return false;
+ }
+ const prefValue = Services.prefs.getStringPref(prefName, "");
+ return UNSYNCABLE_URL_REGEXP.test(prefValue);
+}
+
+function PrefStore(name, engine) {
+ Store.call(this, name, engine);
+ Svc.Obs.add(
+ "profile-before-change",
+ function () {
+ this.__prefs = null;
+ },
+ this
+ );
+}
+PrefStore.prototype = {
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ _getSyncPrefs() {
+ let syncPrefs = Services.prefs
+ .getBranch(PREF_SYNC_PREFS_PREFIX)
+ .getChildList("")
+ .filter(pref => isAllowedPrefName(pref) && !isUnsyncableURLPref(pref));
+ // Also sync preferences that determine which prefs get synced.
+ let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
+ return controlPrefs.concat(syncPrefs);
+ },
+
+ _isSynced(pref) {
+ if (pref.startsWith(PREF_SYNC_PREFS_PREFIX)) {
+ // this is an incoming control pref, which is ignored if there's not already
+ // a local control pref for the preference.
+ let controlledPref = pref.slice(PREF_SYNC_PREFS_PREFIX.length);
+ return isAllowedPrefName(controlledPref);
+ }
+
+ // This is the pref itself - it must be both allowed, and have a control
+ // pref which is true.
+ if (!this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false)) {
+ return false;
+ }
+ return isAllowedPrefName(pref);
+ },
+
+ _getAllPrefs() {
+ let values = {};
+ for (let pref of this._getSyncPrefs()) {
+ // Note: _isSynced doesn't call isUnsyncableURLPref since it would cause
+ // us not to apply (syncable) changes to preferences that are set locally
+ // which have unsyncable urls.
+ if (this._isSynced(pref) && !isUnsyncableURLPref(pref)) {
+ let isSet = this._prefs.isSet(pref);
+ // Missing and default prefs get the null value, unless that `seen`
+ // pref is set, in which case it always gets the value.
+ let forceValue = this._prefs.get(PREF_SYNC_SEEN_PREFIX + pref, false);
+ values[pref] = isSet || forceValue ? this._prefs.get(pref, null) : null;
+ // If this is a special "sync-seen" pref, and it's not the default value,
+ // set the seen pref to true.
+ if (
+ isSet &&
+ this._prefs.get(PREF_SYNC_SEEN_PREFIX + pref, false) === false
+ ) {
+ this._log.trace(`toggling sync-seen pref for '${pref}' to true`);
+ this._prefs.set(PREF_SYNC_SEEN_PREFIX + pref, true);
+ }
+ }
+ }
+ return values;
+ },
+
+ _setAllPrefs(values) {
+ const selectedThemeIDPref = "extensions.activeThemeID";
+ let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
+ let selectedThemeIDAfter = selectedThemeIDBefore;
+
+ // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
+ // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
+ let prefs = Object.keys(values).sort(
+ a => -a.indexOf(PREF_SYNC_PREFS_PREFIX)
+ );
+ for (let pref of prefs) {
+ let value = values[pref];
+ if (!this._isSynced(pref)) {
+ // An extra complication just so we can warn when we decline to sync a
+ // preference due to no local control pref.
+ if (!pref.startsWith(PREF_SYNC_PREFS_PREFIX)) {
+ // this is an incoming pref - if the incoming value is not null and
+ // there's no local control pref, then it means we would have previously
+ // applied a value, but now will decline to.
+ // We need to check this here rather than in _isSynced because the
+ // default list of prefs we sync has changed, so we don't want to report
+ // this message when we wouldn't have actually applied a value.
+ // We should probably remove all of this in ~ Firefox 80.
+ if (value !== null) {
+ // null means "use the default value"
+ let controlPref = PREF_SYNC_PREFS_PREFIX + pref;
+ let controlPrefExists;
+ try {
+ Services.prefs.getBoolPref(controlPref);
+ controlPrefExists = true;
+ } catch (ex) {
+ controlPrefExists = false;
+ }
+ if (!controlPrefExists) {
+ // This is a long message and written to both the sync log and the
+ // console, but note that users who have not customized the control
+ // prefs will never see this.
+ let msg =
+ `Not syncing the preference '${pref}' because it has no local ` +
+ `control preference (${PREF_SYNC_PREFS_PREFIX}${pref}) and ` +
+ `the preference ${PREF_SYNC_PREFS_ARBITRARY} isn't true. ` +
+ `See ${lazy.PREFS_DOC_URL} for more information`;
+ console.warn(msg);
+ this._log.warn(msg);
+ }
+ }
+ }
+ continue;
+ }
+
+ if (typeof value == "string" && UNSYNCABLE_URL_REGEXP.test(value)) {
+ this._log.trace(`Skipping incoming unsyncable url for pref: ${pref}`);
+ continue;
+ }
+
+ switch (pref) {
+ // Some special prefs we don't want to set directly.
+ case selectedThemeIDPref:
+ selectedThemeIDAfter = value;
+ break;
+
+ // default is to just set the pref
+ default:
+ if (value == null) {
+ // Pref has gone missing. The best we can do is reset it.
+ this._prefs.reset(pref);
+ } else {
+ try {
+ this._prefs.set(pref, value);
+ } catch (ex) {
+ this._log.trace(`Failed to set pref: ${pref}`, ex);
+ }
+ }
+ // If there's a "sync-seen" pref for this it gets toggled to true
+ // regardless of the value.
+ let seenPref = PREF_SYNC_SEEN_PREFIX + pref;
+ if (this._prefs.get(seenPref, undefined) === false) {
+ this._prefs.set(PREF_SYNC_SEEN_PREFIX + pref, true);
+ }
+ }
+ }
+ // Themes are a little messy. Themes which have been installed are handled
+ // by the addons engine - but default themes aren't seen by that engine.
+ // So if there's a new default theme ID and that ID corresponds to a
+ // system addon, then we arrange to enable that addon here.
+ if (selectedThemeIDBefore != selectedThemeIDAfter) {
+ this._maybeEnableBuiltinTheme(selectedThemeIDAfter).catch(e => {
+ this._log.error("Failed to maybe update the default theme", e);
+ });
+ }
+ },
+
+ async _maybeEnableBuiltinTheme(themeId) {
+ let addon = null;
+ try {
+ addon = await lazy.AddonManager.getAddonByID(themeId);
+ } catch (ex) {
+ this._log.trace(
+ `There's no addon with ID '${themeId} - it can't be a builtin theme`
+ );
+ return;
+ }
+ if (addon && addon.isBuiltin && addon.type == "theme") {
+ this._log.trace(`Enabling builtin theme '${themeId}'`);
+ await addon.enable();
+ } else {
+ this._log.trace(
+ `Have incoming theme ID of '${themeId}' but it's not a builtin theme`
+ );
+ }
+ },
+
+ async getAllIDs() {
+ /* We store all prefs in just one WBO, with just one GUID */
+ let allprefs = {};
+ allprefs[lazy.PREFS_GUID] = true;
+ return allprefs;
+ },
+
+ async changeItemID(oldID, newID) {
+ this._log.trace("PrefStore GUID is constant!");
+ },
+
+ async itemExists(id) {
+ return id === lazy.PREFS_GUID;
+ },
+
+ async createRecord(id, collection) {
+ let record = new PrefRec(collection, id);
+
+ if (id == lazy.PREFS_GUID) {
+ record.value = this._getAllPrefs();
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ async create(record) {
+ this._log.trace("Ignoring create request");
+ },
+
+ async remove(record) {
+ this._log.trace("Ignoring remove request");
+ },
+
+ async update(record) {
+ // Silently ignore pref updates that are for other apps.
+ if (record.id != lazy.PREFS_GUID) {
+ return;
+ }
+
+ this._log.trace("Received pref updates, applying...");
+ this._setAllPrefs(record.value);
+ },
+
+ async wipe() {
+ this._log.trace("Ignoring wipe request");
+ },
+};
+Object.setPrototypeOf(PrefStore.prototype, Store.prototype);
+
+function PrefTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ this._ignoreAll = false;
+ Svc.Obs.add("profile-before-change", this.asyncObserver);
+}
+PrefTracker.prototype = {
+ get ignoreAll() {
+ return this._ignoreAll;
+ },
+
+ set ignoreAll(value) {
+ this._ignoreAll = value;
+ },
+
+ get modified() {
+ return Svc.Prefs.get("engine.prefs.modified", false);
+ },
+ set modified(value) {
+ Svc.Prefs.set("engine.prefs.modified", value);
+ },
+
+ clearChangedIDs: function clearChangedIDs() {
+ this.modified = false;
+ },
+
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ onStart() {
+ Services.prefs.addObserver("", this.asyncObserver);
+ },
+
+ onStop() {
+ this.__prefs = null;
+ Services.prefs.removeObserver("", this.asyncObserver);
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "profile-before-change":
+ await this.stop();
+ break;
+ case "nsPref:changed":
+ if (this.ignoreAll) {
+ break;
+ }
+ // Trigger a sync for MULTI-DEVICE for a change that determines
+ // which prefs are synced or a regular pref change.
+ if (
+ data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
+ this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)
+ ) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ this.modified = true;
+ this._log.trace("Preference " + data + " changed");
+ }
+ break;
+ }
+ },
+};
+Object.setPrototypeOf(PrefTracker.prototype, Tracker.prototype);
+
+export function getPrefsGUIDForTest() {
+ return lazy.PREFS_GUID;
+}
diff --git a/services/sync/modules/engines/tabs.sys.mjs b/services/sync/modules/engines/tabs.sys.mjs
new file mode 100644
index 0000000000..45242b71a7
--- /dev/null
+++ b/services/sync/modules/engines/tabs.sys.mjs
@@ -0,0 +1,624 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const STORAGE_VERSION = 1; // This needs to be kept in-sync with the rust storage version
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+import { SyncEngine, Tracker } from "resource://services-sync/engines.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+import {
+ SCORE_INCREMENT_SMALL,
+ STATUS_OK,
+ URI_LENGTH_MAX,
+} from "resource://services-sync/constants.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { Async } from "resource://services-common/async.sys.mjs";
+import {
+ SyncRecord,
+ SyncTelemetry,
+} from "resource://services-sync/telemetry.sys.mjs";
+import { BridgedEngine } from "resource://services-sync/bridged_engine.sys.mjs";
+
+const FAR_FUTURE = 4102405200000; // 2100/01/01
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
+ PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.sys.mjs",
+ ReaderMode: "resource://gre/modules/ReaderMode.sys.mjs",
+ TabsStore: "resource://gre/modules/RustTabs.sys.mjs",
+});
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "TABS_FILTERED_SCHEMES",
+ "services.sync.engine.tabs.filteredSchemes",
+ "",
+ null,
+ val => {
+ return new Set(val.split("|"));
+ }
+);
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "SYNC_AFTER_DELAY_MS",
+ "services.sync.syncedTabs.syncDelayAfterTabChange",
+ 0
+);
+
+// A "bridged engine" to our tabs component.
+export function TabEngine(service) {
+ BridgedEngine.call(this, "Tabs", service);
+}
+
+TabEngine.prototype = {
+ _trackerObj: TabTracker,
+ syncPriority: 3,
+
+ async prepareTheBridge(isQuickWrite) {
+ let clientsEngine = this.service.clientsEngine;
+ // Tell the bridged engine about clients.
+ // This is the same shape as ClientData in app-services.
+ // schema: https://github.com/mozilla/application-services/blob/a1168751231ed4e88c44d85f6dccc09c3b412bd2/components/sync15/src/client_types.rs#L14
+ let clientData = {
+ local_client_id: clientsEngine.localID,
+ recent_clients: {},
+ };
+
+ // We shouldn't upload tabs past what the server will accept
+ let tabs = await this.getTabsWithinPayloadSize();
+ await this._rustStore.setLocalTabs(
+ tabs.map(tab => {
+ // rust wants lastUsed in MS but the provider gives it in seconds
+ tab.lastUsed = tab.lastUsed * 1000;
+ return tab;
+ })
+ );
+
+ for (let remoteClient of clientsEngine.remoteClients) {
+ let id = remoteClient.id;
+ if (!id) {
+ throw new Error("Remote client somehow did not have an id");
+ }
+ let client = {
+ fxa_device_id: remoteClient.fxaDeviceId,
+ // device_name and device_type are soft-deprecated - every client
+ // prefers what's in the FxA record. But fill them correctly anyway.
+ device_name: clientsEngine.getClientName(id) ?? "",
+ device_type: clientsEngine.getClientType(id),
+ };
+ clientData.recent_clients[id] = client;
+ }
+
+ // put ourself in there too so we record the correct device info in our sync record.
+ clientData.recent_clients[clientsEngine.localID] = {
+ fxa_device_id: await clientsEngine.fxAccounts.device.getLocalId(),
+ device_name: clientsEngine.localName,
+ device_type: clientsEngine.localType,
+ };
+
+ // Quick write needs to adjust the lastSync so we can POST to the server
+ // see quickWrite() for details
+ if (isQuickWrite) {
+ await this.setLastSync(FAR_FUTURE);
+ await this._bridge.prepareForSync(JSON.stringify(clientData));
+ return;
+ }
+
+ // Just incase we crashed while the lastSync timestamp was FAR_FUTURE, we
+ // reset it to zero
+ if ((await this.getLastSync()) === FAR_FUTURE) {
+ await this._bridge.setLastSync(0);
+ }
+ await this._bridge.prepareForSync(JSON.stringify(clientData));
+ },
+
+ async _syncStartup() {
+ await super._syncStartup();
+ await this.prepareTheBridge();
+ },
+
+ async initialize() {
+ await SyncEngine.prototype.initialize.call(this);
+
+ let path = PathUtils.join(PathUtils.profileDir, "synced-tabs.db");
+ this._rustStore = await lazy.TabsStore.init(path);
+ this._bridge = await this._rustStore.bridgedEngine();
+
+ // Uniffi doesn't currently only support async methods, so we'll need to hardcode
+ // these values for now (which is fine for now as these hardly ever change)
+ this._bridge.storageVersion = STORAGE_VERSION;
+ this._bridge.allowSkippedRecord = true;
+
+ this._log.info("Got a bridged engine!");
+ this._tracker.modified = true;
+ },
+
+ async getChangedIDs() {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified) {
+ changedIDs[this.service.clientsEngine.localID] = 0;
+ }
+ return changedIDs;
+ },
+
+ // API for use by Sync UI code to give user choices of tabs to open.
+ async getAllClients() {
+ let remoteTabs = await this._rustStore.getAll();
+ let remoteClientTabs = [];
+ for (let remoteClient of this.service.clientsEngine.remoteClients) {
+ // We get the some client info from the rust tabs engine and some from
+ // the clients engine.
+ let rustClient = remoteTabs.find(
+ x => x.clientId === remoteClient.fxaDeviceId
+ );
+ if (!rustClient) {
+ continue;
+ }
+ let client = {
+ // rust gives us ms but js uses seconds, so fix them up.
+ tabs: rustClient.remoteTabs.map(tab => {
+ tab.lastUsed = tab.lastUsed / 1000;
+ return tab;
+ }),
+ lastModified: rustClient.lastModified / 1000,
+ ...remoteClient,
+ };
+ remoteClientTabs.push(client);
+ }
+ return remoteClientTabs;
+ },
+
+ async removeClientData() {
+ let url = this.engineURL + "/" + this.service.clientsEngine.localID;
+ await this.service.resource(url).delete();
+ },
+
+ async trackRemainingChanges() {
+ if (this._modified.count() > 0) {
+ this._tracker.modified = true;
+ }
+ },
+
+ async getTabsWithinPayloadSize() {
+ const maxPayloadSize = this.service.getMaxRecordPayloadSize();
+ // See bug 535326 comment 8 for an explanation of the estimation
+ const maxSerializedSize = (maxPayloadSize / 4) * 3 - 1500;
+ return TabProvider.getAllTabsWithEstimatedMax(true, maxSerializedSize);
+ },
+
+ // Support for "quick writes"
+ _engineLock: Utils.lock,
+ _engineLocked: false,
+
+ // Tabs has a special lock to help support its "quick write"
+ get locked() {
+ return this._engineLocked;
+ },
+ lock() {
+ if (this._engineLocked) {
+ return false;
+ }
+ this._engineLocked = true;
+ return true;
+ },
+ unlock() {
+ this._engineLocked = false;
+ },
+
+ // Quickly do a POST of our current tabs if possible.
+ // This does things that would be dangerous for other engines - eg, posting
+ // without checking what's on the server could cause data-loss for other
+ // engines, but because each device exclusively owns exactly 1 tabs record
+ // with a known ID, it's safe here.
+ // Returns true if we successfully synced, false otherwise (either on error
+ // or because we declined to sync for any reason.) The return value is
+ // primarily for tests.
+ async quickWrite() {
+ if (!this.enabled) {
+ // this should be very rare, and only if tabs are disabled after the
+ // timer is created.
+ this._log.info("Can't do a quick-sync as tabs is disabled");
+ return false;
+ }
+ // This quick-sync doesn't drive the login state correctly, so just
+ // decline to sync if out status is bad
+ if (this.service.status.checkSetup() != STATUS_OK) {
+ this._log.info(
+ "Can't do a quick-sync due to the service status",
+ this.service.status.toString()
+ );
+ return false;
+ }
+ if (!this.service.serverConfiguration) {
+ this._log.info("Can't do a quick sync before the first full sync");
+ return false;
+ }
+ try {
+ return await this._engineLock("tabs.js: quickWrite", async () => {
+ // We want to restore the lastSync timestamp when complete so next sync
+ // takes tabs written by other devices since our last real sync.
+ // And for this POST we don't want the protections offered by
+ // X-If-Unmodified-Since - we want the POST to work even if the remote
+ // has moved on and we will catch back up next full sync.
+ const origLastSync = await this.getLastSync();
+ try {
+ return this._doQuickWrite();
+ } finally {
+ // set the lastSync to it's original value for regular sync
+ await this.setLastSync(origLastSync);
+ }
+ })();
+ } catch (ex) {
+ if (!Utils.isLockException(ex)) {
+ throw ex;
+ }
+ this._log.info(
+ "Can't do a quick-write as another tab sync is in progress"
+ );
+ return false;
+ }
+ },
+
+ // The guts of the quick-write sync, after we've taken the lock, checked
+ // the service status etc.
+ async _doQuickWrite() {
+ // We need to track telemetry for these syncs too!
+ const name = "tabs";
+ let telemetryRecord = new SyncRecord(
+ SyncTelemetry.allowedEngines,
+ "quick-write"
+ );
+ telemetryRecord.onEngineStart(name);
+ try {
+ Async.checkAppReady();
+ // We need to prep the bridge before we try to POST since it grabs
+ // the most recent local client id and properly sets a lastSync
+ // which is needed for a proper POST request
+ await this.prepareTheBridge(true);
+ this._tracker.clearChangedIDs();
+ this._tracker.resetScore();
+
+ Async.checkAppReady();
+ // now just the "upload" part of a sync,
+ // which for a rust engine is not obvious.
+ // We need to do is ask the rust engine for the changes. Although
+ // this is kinda abusing the bridged-engine interface, we know the tabs
+ // implementation of it works ok
+ let outgoing = await this._bridge.apply();
+ // We know we always have exactly 1 record.
+ let mine = outgoing[0];
+ this._log.trace("outgoing bso", mine);
+ // `this._recordObj` is a `BridgedRecord`, which isn't exported.
+ let record = this._recordObj.fromOutgoingBso(this.name, JSON.parse(mine));
+ let changeset = {};
+ changeset[record.id] = { synced: false, record };
+ this._modified.replace(changeset);
+
+ Async.checkAppReady();
+ await this._uploadOutgoing();
+ telemetryRecord.onEngineStop(name, null);
+ return true;
+ } catch (ex) {
+ this._log.warn("quicksync sync failed", ex);
+ telemetryRecord.onEngineStop(name, ex);
+ return false;
+ } finally {
+ // The top-level sync is never considered to fail here, just the engine
+ telemetryRecord.finished(null);
+ SyncTelemetry.takeTelemetryRecord(telemetryRecord);
+ }
+ },
+
+ async _sync() {
+ try {
+ await this._engineLock("tabs.js: fullSync", async () => {
+ await super._sync();
+ })();
+ } catch (ex) {
+ if (!Utils.isLockException(ex)) {
+ throw ex;
+ }
+ this._log.info(
+ "Can't do full tabs sync as a quick-write is currently running"
+ );
+ }
+ },
+};
+Object.setPrototypeOf(TabEngine.prototype, BridgedEngine.prototype);
+
+export const TabProvider = {
+ getWindowEnumerator() {
+ return Services.wm.getEnumerator("navigator:browser");
+ },
+
+ shouldSkipWindow(win) {
+ return win.closed || lazy.PrivateBrowsingUtils.isWindowPrivate(win);
+ },
+
+ getAllBrowserTabs() {
+ let tabs = [];
+ for (let win of this.getWindowEnumerator()) {
+ if (this.shouldSkipWindow(win)) {
+ continue;
+ }
+ // Get all the tabs from the browser
+ for (let tab of win.gBrowser.tabs) {
+ tabs.push(tab);
+ }
+ }
+
+ return tabs.sort(function (a, b) {
+ return b.lastAccessed - a.lastAccessed;
+ });
+ },
+
+ // This function creates tabs records up to a specified amount of bytes
+ // It is an "estimation" since we don't accurately calculate how much the
+ // favicon and JSON overhead is and give a rough estimate (for optimization purposes)
+ async getAllTabsWithEstimatedMax(filter, bytesMax) {
+ let log = Log.repository.getLogger(`Sync.Engine.Tabs.Provider`);
+ let tabRecords = [];
+ let iconPromises = [];
+ let runningByteLength = 0;
+ let encoder = new TextEncoder();
+
+ // Fetch all the tabs the user has open
+ let winTabs = this.getAllBrowserTabs();
+
+ for (let tab of winTabs) {
+ // We don't want to process any more tabs than we can sync
+ if (runningByteLength >= bytesMax) {
+ log.warn(
+ `Can't fit all tabs in sync payload: have ${winTabs.length},
+ but can only fit ${tabRecords.length}.`
+ );
+ break;
+ }
+
+ // Note that we used to sync "tab history" (ie, the "back button") state,
+ // but in practice this hasn't been used - only the current URI is of
+ // interest to clients.
+ // We stopped recording this in bug 1783991.
+ if (!tab?.linkedBrowser) {
+ continue;
+ }
+ let acceptable = !filter
+ ? url => url
+ : url =>
+ url &&
+ !lazy.TABS_FILTERED_SCHEMES.has(Services.io.extractScheme(url));
+
+ let url = tab.linkedBrowser.currentURI?.spec;
+ // Special case for reader mode.
+ if (url && url.startsWith("about:reader?")) {
+ url = lazy.ReaderMode.getOriginalUrl(url);
+ }
+ // We ignore the tab completely if the current entry url is
+ // not acceptable (we need something accurate to open).
+ if (!acceptable(url)) {
+ continue;
+ }
+
+ if (url.length > URI_LENGTH_MAX) {
+ log.trace("Skipping over-long URL.");
+ continue;
+ }
+
+ let thisTab = {
+ title: tab.linkedBrowser.contentTitle || "",
+ urlHistory: [url],
+ icon: "",
+ lastUsed: Math.floor((tab.lastAccessed || 0) / 1000),
+ };
+ tabRecords.push(thisTab);
+
+ // we don't want to wait for each favicon to resolve to get the bytes
+ // so we estimate a conservative 100 chars for the favicon and json overhead
+ // Rust will further optimize and trim if we happened to be wildly off
+ runningByteLength +=
+ encoder.encode(thisTab.title + thisTab.lastUsed + url).byteLength + 100;
+
+ // Use the favicon service for the icon url - we can wait for the promises at the end.
+ let iconPromise = lazy.PlacesUtils.promiseFaviconData(url)
+ .then(iconData => {
+ thisTab.icon = iconData.uri.spec;
+ })
+ .catch(ex => {
+ log.trace(
+ `Failed to fetch favicon for ${url}`,
+ thisTab.urlHistory[0]
+ );
+ });
+ iconPromises.push(iconPromise);
+ }
+
+ await Promise.allSettled(iconPromises);
+ return tabRecords;
+ },
+};
+
+function TabTracker(name, engine) {
+ Tracker.call(this, name, engine);
+
+ // Make sure "this" pointer is always set correctly for event listeners.
+ this.onTab = Utils.bind2(this, this.onTab);
+ this._unregisterListeners = Utils.bind2(this, this._unregisterListeners);
+}
+TabTracker.prototype = {
+ QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
+
+ clearChangedIDs() {
+ this.modified = false;
+ },
+
+ // We do not track TabSelect because that almost always triggers
+ // the web progress listeners (onLocationChange), which we already track
+ _topics: ["TabOpen", "TabClose"],
+
+ _registerListenersForWindow(window) {
+ this._log.trace("Registering tab listeners in window");
+ for (let topic of this._topics) {
+ window.addEventListener(topic, this.onTab);
+ }
+ window.addEventListener("unload", this._unregisterListeners);
+ // If it's got a tab browser we can listen for things like navigation.
+ if (window.gBrowser) {
+ window.gBrowser.addProgressListener(this);
+ }
+ },
+
+ _unregisterListeners(event) {
+ this._unregisterListenersForWindow(event.target);
+ },
+
+ _unregisterListenersForWindow(window) {
+ this._log.trace("Removing tab listeners in window");
+ window.removeEventListener("unload", this._unregisterListeners);
+ for (let topic of this._topics) {
+ window.removeEventListener(topic, this.onTab);
+ }
+ if (window.gBrowser) {
+ window.gBrowser.removeProgressListener(this);
+ }
+ },
+
+ onStart() {
+ Svc.Obs.add("domwindowopened", this.asyncObserver);
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ this._registerListenersForWindow(win);
+ }
+ },
+
+ onStop() {
+ Svc.Obs.remove("domwindowopened", this.asyncObserver);
+ for (let win of Services.wm.getEnumerator("navigator:browser")) {
+ this._unregisterListenersForWindow(win);
+ }
+ },
+
+ async observe(subject, topic, data) {
+ switch (topic) {
+ case "domwindowopened":
+ let onLoad = () => {
+ subject.removeEventListener("load", onLoad);
+ // Only register after the window is done loading to avoid unloads.
+ this._registerListenersForWindow(subject);
+ };
+
+ // Add tab listeners now that a window has opened.
+ subject.addEventListener("load", onLoad);
+ break;
+ }
+ },
+
+ onTab(event) {
+ if (event.originalTarget.linkedBrowser) {
+ let browser = event.originalTarget.linkedBrowser;
+ if (
+ lazy.PrivateBrowsingUtils.isBrowserPrivate(browser) &&
+ !lazy.PrivateBrowsingUtils.permanentPrivateBrowsing
+ ) {
+ this._log.trace("Ignoring tab event from private browsing.");
+ return;
+ }
+ }
+ this._log.trace("onTab event: " + event.type);
+
+ switch (event.type) {
+ case "TabOpen":
+ /* We do not have a reliable way of checking the URI on the TabOpen
+ * so we will rely on the other methods (onLocationChange, getAllTabsWithEstimatedMax)
+ * to filter these when going through sync
+ */
+ this.callScheduleSync(SCORE_INCREMENT_SMALL);
+ break;
+ case "TabClose":
+ // If event target has `linkedBrowser`, the event target can be assumed <tab> element.
+ // Else, event target is assumed <browser> element, use the target as it is.
+ const tab = event.target.linkedBrowser || event.target;
+
+ // TabClose means the tab has already loaded and we can check the URI
+ // and ignore if it's a scheme we don't care about
+ if (lazy.TABS_FILTERED_SCHEMES.has(tab.currentURI.scheme)) {
+ return;
+ }
+ this.callScheduleSync(SCORE_INCREMENT_SMALL);
+ break;
+ }
+ },
+
+ // web progress listeners.
+ onLocationChange(webProgress, request, locationURI, flags) {
+ // We only care about top-level location changes. We do want location changes in the
+ // same document because if a page uses the `pushState()` API, they *appear* as though
+ // they are in the same document even if the URL changes. It also doesn't hurt to accurately
+ // reflect the fragment changing - so we allow LOCATION_CHANGE_SAME_DOCUMENT
+ if (
+ flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_RELOAD ||
+ !webProgress.isTopLevel ||
+ !locationURI
+ ) {
+ return;
+ }
+
+ // We can't filter out tabs that we don't sync here, because we might be
+ // navigating from a tab that we *did* sync to one we do not, and that
+ // tab we *did* sync should no longer be synced.
+ this.callScheduleSync();
+ },
+
+ callScheduleSync(scoreIncrement) {
+ this.modified = true;
+ let { scheduler } = this.engine.service;
+ let delayInMs = lazy.SYNC_AFTER_DELAY_MS;
+
+ // Schedule a sync once we detect a tab change
+ // to ensure the server always has the most up to date tabs
+ if (
+ delayInMs > 0 &&
+ scheduler.numClients > 1 // Only schedule quick syncs for multi client users
+ ) {
+ if (this.tabsQuickWriteTimer) {
+ this._log.debug(
+ "Detected a tab change, but a quick-write is already scheduled"
+ );
+ return;
+ }
+ this._log.debug(
+ "Detected a tab change: scheduling a quick-write in " + delayInMs + "ms"
+ );
+ CommonUtils.namedTimer(
+ () => {
+ this._log.trace("tab quick-sync timer fired.");
+ this.engine
+ .quickWrite()
+ .then(() => {
+ this._log.trace("tab quick-sync done.");
+ })
+ .catch(ex => {
+ this._log.error("tab quick-sync failed.", ex);
+ });
+ },
+ delayInMs,
+ this,
+ "tabsQuickWriteTimer"
+ );
+ } else if (scoreIncrement) {
+ this._log.debug(
+ "Detected a tab change, but conditions aren't met for a quick write - bumping score"
+ );
+ this.score += scoreIncrement;
+ } else {
+ this._log.debug(
+ "Detected a tab change, but conditions aren't met for a quick write or a score bump"
+ );
+ }
+ },
+};
+Object.setPrototypeOf(TabTracker.prototype, Tracker.prototype);
diff --git a/services/sync/modules/keys.sys.mjs b/services/sync/modules/keys.sys.mjs
new file mode 100644
index 0000000000..b6a1dce19a
--- /dev/null
+++ b/services/sync/modules/keys.sys.mjs
@@ -0,0 +1,166 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Weave } from "resource://services-sync/main.sys.mjs";
+
+/**
+ * Represents a pair of keys.
+ *
+ * Each key stored in a key bundle is 256 bits. One key is used for symmetric
+ * encryption. The other is used for HMAC.
+ *
+ * A KeyBundle by itself is just an anonymous pair of keys. Other types
+ * deriving from this one add semantics, such as associated collections or
+ * generating a key bundle via HKDF from another key.
+ */
+function KeyBundle() {
+ this._encrypt = null;
+ this._encryptB64 = null;
+ this._hmac = null;
+ this._hmacB64 = null;
+}
+KeyBundle.prototype = {
+ _encrypt: null,
+ _encryptB64: null,
+ _hmac: null,
+ _hmacB64: null,
+
+ equals: function equals(bundle) {
+ return (
+ bundle &&
+ bundle.hmacKey == this.hmacKey &&
+ bundle.encryptionKey == this.encryptionKey
+ );
+ },
+
+ /*
+ * Accessors for the two keys.
+ */
+ get encryptionKey() {
+ return this._encrypt;
+ },
+
+ set encryptionKey(value) {
+ if (!value || typeof value != "string") {
+ throw new Error("Encryption key can only be set to string values.");
+ }
+
+ if (value.length < 16) {
+ throw new Error("Encryption key must be at least 128 bits long.");
+ }
+
+ this._encrypt = value;
+ this._encryptB64 = btoa(value);
+ },
+
+ get encryptionKeyB64() {
+ return this._encryptB64;
+ },
+
+ get hmacKey() {
+ return this._hmac;
+ },
+
+ set hmacKey(value) {
+ if (!value || typeof value != "string") {
+ throw new Error("HMAC key can only be set to string values.");
+ }
+
+ if (value.length < 16) {
+ throw new Error("HMAC key must be at least 128 bits long.");
+ }
+
+ this._hmac = value;
+ this._hmacB64 = btoa(value);
+ },
+
+ get hmacKeyB64() {
+ return this._hmacB64;
+ },
+
+ /**
+ * Populate this key pair with 2 new, randomly generated keys.
+ */
+ async generateRandom() {
+ // Compute both at that same time
+ let [generatedHMAC, generatedEncr] = await Promise.all([
+ Weave.Crypto.generateRandomKey(),
+ Weave.Crypto.generateRandomKey(),
+ ]);
+ this.keyPairB64 = [generatedEncr, generatedHMAC];
+ },
+};
+
+/**
+ * Represents a KeyBundle associated with a collection.
+ *
+ * This is just a KeyBundle with a collection attached.
+ */
+export function BulkKeyBundle(collection) {
+ let log = Log.repository.getLogger("Sync.BulkKeyBundle");
+ log.info("BulkKeyBundle being created for " + collection);
+ KeyBundle.call(this);
+
+ this._collection = collection;
+}
+
+BulkKeyBundle.fromHexKey = function (hexKey) {
+ let key = CommonUtils.hexToBytes(hexKey);
+ let bundle = new BulkKeyBundle();
+ // [encryptionKey, hmacKey]
+ bundle.keyPair = [key.slice(0, 32), key.slice(32, 64)];
+ return bundle;
+};
+
+BulkKeyBundle.fromJWK = function (jwk) {
+ if (!jwk || !jwk.k || jwk.kty !== "oct") {
+ throw new Error("Invalid JWK provided to BulkKeyBundle.fromJWK");
+ }
+ return BulkKeyBundle.fromHexKey(CommonUtils.base64urlToHex(jwk.k));
+};
+
+BulkKeyBundle.prototype = {
+ get collection() {
+ return this._collection;
+ },
+
+ /**
+ * Obtain the key pair in this key bundle.
+ *
+ * The returned keys are represented as raw byte strings.
+ */
+ get keyPair() {
+ return [this.encryptionKey, this.hmacKey];
+ },
+
+ set keyPair(value) {
+ if (!Array.isArray(value) || value.length != 2) {
+ throw new Error("BulkKeyBundle.keyPair value must be array of 2 keys.");
+ }
+
+ this.encryptionKey = value[0];
+ this.hmacKey = value[1];
+ },
+
+ get keyPairB64() {
+ return [this.encryptionKeyB64, this.hmacKeyB64];
+ },
+
+ set keyPairB64(value) {
+ if (!Array.isArray(value) || value.length != 2) {
+ throw new Error(
+ "BulkKeyBundle.keyPairB64 value must be an array of 2 keys."
+ );
+ }
+
+ this.encryptionKey = CommonUtils.safeAtoB(value[0]);
+ this.hmacKey = CommonUtils.safeAtoB(value[1]);
+ },
+};
+
+Object.setPrototypeOf(BulkKeyBundle.prototype, KeyBundle.prototype);
diff --git a/services/sync/modules/main.sys.mjs b/services/sync/modules/main.sys.mjs
new file mode 100644
index 0000000000..283d4c7f09
--- /dev/null
+++ b/services/sync/modules/main.sys.mjs
@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export { lazy as Weave };
+
+const lazy = {};
+
+// We want these to be lazily loaded, which helps performance and also tests
+// to not have these loaded before they are ready.
+// eslint-disable-next-line mozilla/valid-lazy
+ChromeUtils.defineESModuleGetters(lazy, {
+ Service: "resource://services-sync/service.sys.mjs",
+ Status: "resource://services-sync/status.sys.mjs",
+ Svc: "resource://services-sync/util.sys.mjs",
+ Utils: "resource://services-sync/util.sys.mjs",
+});
+
+// eslint-disable-next-line mozilla/valid-lazy
+ChromeUtils.defineLazyGetter(lazy, "Crypto", () => {
+ let { WeaveCrypto } = ChromeUtils.importESModule(
+ "resource://services-crypto/WeaveCrypto.sys.mjs"
+ );
+ return new WeaveCrypto();
+});
diff --git a/services/sync/modules/policies.sys.mjs b/services/sync/modules/policies.sys.mjs
new file mode 100644
index 0000000000..68a6645ef4
--- /dev/null
+++ b/services/sync/modules/policies.sys.mjs
@@ -0,0 +1,1057 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import {
+ CREDENTIALS_CHANGED,
+ ENGINE_APPLY_FAIL,
+ ENGINE_UNKNOWN_FAIL,
+ IDLE_OBSERVER_BACK_DELAY,
+ LOGIN_FAILED_INVALID_PASSPHRASE,
+ LOGIN_FAILED_LOGIN_REJECTED,
+ LOGIN_FAILED_NETWORK_ERROR,
+ LOGIN_FAILED_NO_PASSPHRASE,
+ LOGIN_SUCCEEDED,
+ MASTER_PASSWORD_LOCKED,
+ MASTER_PASSWORD_LOCKED_RETRY_INTERVAL,
+ MAX_ERROR_COUNT_BEFORE_BACKOFF,
+ MINIMUM_BACKOFF_INTERVAL,
+ MULTI_DEVICE_THRESHOLD,
+ NO_SYNC_NODE_FOUND,
+ NO_SYNC_NODE_INTERVAL,
+ OVER_QUOTA,
+ RESPONSE_OVER_QUOTA,
+ SCORE_UPDATE_DELAY,
+ SERVER_MAINTENANCE,
+ SINGLE_USER_THRESHOLD,
+ STATUS_OK,
+ SYNC_FAILED_PARTIAL,
+ SYNC_SUCCEEDED,
+ kSyncBackoffNotMet,
+ kSyncMasterPasswordLocked,
+} from "resource://services-sync/constants.sys.mjs";
+
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+const { logManager } = ChromeUtils.import(
+ "resource://gre/modules/FxAccountsCommon.js"
+);
+import { Async } from "resource://services-common/async.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
+ Status: "resource://services-sync/status.sys.mjs",
+});
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "IdleService",
+ "@mozilla.org/widget/useridleservice;1",
+ "nsIUserIdleService"
+);
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "CaptivePortalService",
+ "@mozilla.org/network/captive-portal-service;1",
+ "nsICaptivePortalService"
+);
+
+// Get the value for an interval that's stored in preferences. To save users
+// from themselves (and us from them!) the minimum time they can specify
+// is 60s.
+function getThrottledIntervalPreference(prefName) {
+ return Math.max(Svc.Prefs.get(prefName), 60) * 1000;
+}
+
+export function SyncScheduler(service) {
+ this.service = service;
+ this.init();
+}
+
+SyncScheduler.prototype = {
+ _log: Log.repository.getLogger("Sync.SyncScheduler"),
+
+ _fatalLoginStatus: [
+ LOGIN_FAILED_NO_PASSPHRASE,
+ LOGIN_FAILED_INVALID_PASSPHRASE,
+ LOGIN_FAILED_LOGIN_REJECTED,
+ ],
+
+ /**
+ * The nsITimer object that schedules the next sync. See scheduleNextSync().
+ */
+ syncTimer: null,
+
+ setDefaults: function setDefaults() {
+ this._log.trace("Setting SyncScheduler policy values to defaults.");
+
+ this.singleDeviceInterval = getThrottledIntervalPreference(
+ "scheduler.fxa.singleDeviceInterval"
+ );
+ this.idleInterval = getThrottledIntervalPreference(
+ "scheduler.idleInterval"
+ );
+ this.activeInterval = getThrottledIntervalPreference(
+ "scheduler.activeInterval"
+ );
+ this.immediateInterval = getThrottledIntervalPreference(
+ "scheduler.immediateInterval"
+ );
+
+ // A user is non-idle on startup by default.
+ this.idle = false;
+
+ this.hasIncomingItems = false;
+ // This is the last number of clients we saw when previously updating the
+ // client mode. If this != currentNumClients (obtained from prefs written
+ // by the clients engine) then we need to transition to and from
+ // single and multi-device mode.
+ this.numClientsLastSync = 0;
+
+ this._resyncs = 0;
+
+ this.clearSyncTriggers();
+ },
+
+ // nextSync is in milliseconds, but prefs can't hold that much
+ get nextSync() {
+ return Svc.Prefs.get("nextSync", 0) * 1000;
+ },
+ set nextSync(value) {
+ Svc.Prefs.set("nextSync", Math.floor(value / 1000));
+ },
+
+ get missedFxACommandsFetchInterval() {
+ return Services.prefs.getIntPref(
+ "identity.fxaccounts.commands.missed.fetch_interval"
+ );
+ },
+
+ get missedFxACommandsLastFetch() {
+ return Services.prefs.getIntPref(
+ "identity.fxaccounts.commands.missed.last_fetch",
+ 0
+ );
+ },
+
+ set missedFxACommandsLastFetch(val) {
+ Services.prefs.setIntPref(
+ "identity.fxaccounts.commands.missed.last_fetch",
+ val
+ );
+ },
+
+ get syncInterval() {
+ return this._syncInterval;
+ },
+ set syncInterval(value) {
+ if (value != this._syncInterval) {
+ Services.prefs.setIntPref("services.sync.syncInterval", value);
+ }
+ },
+
+ get syncThreshold() {
+ return this._syncThreshold;
+ },
+ set syncThreshold(value) {
+ if (value != this._syncThreshold) {
+ Services.prefs.setIntPref("services.sync.syncThreshold", value);
+ }
+ },
+
+ get globalScore() {
+ return this._globalScore;
+ },
+ set globalScore(value) {
+ if (this._globalScore != value) {
+ Services.prefs.setIntPref("services.sync.globalScore", value);
+ }
+ },
+
+ // Managed by the clients engine (by way of prefs)
+ get numClients() {
+ return this.numDesktopClients + this.numMobileClients;
+ },
+ set numClients(value) {
+ throw new Error("Don't set numClients - the clients engine manages it.");
+ },
+
+ get offline() {
+ // Services.io.offline has slowly become fairly useless over the years - it
+ // no longer attempts to track the actual network state by default, but one
+ // thing stays true: if it says we're offline then we are definitely not online.
+ //
+ // We also ask the captive portal service if we are behind a locked captive
+ // portal.
+ //
+ // We don't check on the NetworkLinkService however, because it gave us
+ // false positives in the past in a vm environment.
+ try {
+ if (
+ Services.io.offline ||
+ lazy.CaptivePortalService.state ==
+ lazy.CaptivePortalService.LOCKED_PORTAL
+ ) {
+ return true;
+ }
+ } catch (ex) {
+ this._log.warn("Could not determine network status.", ex);
+ }
+ return false;
+ },
+
+ _initPrefGetters() {
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "idleTime",
+ "services.sync.scheduler.idleTime"
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "maxResyncs",
+ "services.sync.maxResyncs",
+ 0
+ );
+
+ // The number of clients we have is maintained in preferences via the
+ // clients engine, and only updated after a successsful sync.
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "numDesktopClients",
+ "services.sync.clients.devices.desktop",
+ 0
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "numMobileClients",
+ "services.sync.clients.devices.mobile",
+ 0
+ );
+
+ // Scheduler state that seems to be read more often than it's written.
+ // We also check if the value has changed before writing in the setters.
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_syncThreshold",
+ "services.sync.syncThreshold",
+ SINGLE_USER_THRESHOLD
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_syncInterval",
+ "services.sync.syncInterval",
+ this.singleDeviceInterval
+ );
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_globalScore",
+ "services.sync.globalScore",
+ 0
+ );
+ },
+
+ init: function init() {
+ this._log.manageLevelFromPref("services.sync.log.logger.service.main");
+ this.setDefaults();
+ this._initPrefGetters();
+ Svc.Obs.add("weave:engine:score:updated", this);
+ Svc.Obs.add("network:offline-status-changed", this);
+ Svc.Obs.add("network:link-status-changed", this);
+ Svc.Obs.add("captive-portal-detected", this);
+ Svc.Obs.add("weave:service:sync:start", this);
+ Svc.Obs.add("weave:service:sync:finish", this);
+ Svc.Obs.add("weave:engine:sync:finish", this);
+ Svc.Obs.add("weave:engine:sync:error", this);
+ Svc.Obs.add("weave:service:login:error", this);
+ Svc.Obs.add("weave:service:logout:finish", this);
+ Svc.Obs.add("weave:service:sync:error", this);
+ Svc.Obs.add("weave:service:backoff:interval", this);
+ Svc.Obs.add("weave:engine:sync:applied", this);
+ Svc.Obs.add("weave:service:setup-complete", this);
+ Svc.Obs.add("weave:service:start-over", this);
+ Svc.Obs.add("FxA:hawk:backoff:interval", this);
+
+ if (lazy.Status.checkSetup() == STATUS_OK) {
+ Svc.Obs.add("wake_notification", this);
+ Svc.Obs.add("captive-portal-login-success", this);
+ Svc.Obs.add("sleep_notification", this);
+ lazy.IdleService.addIdleObserver(this, this.idleTime);
+ }
+ },
+
+ // eslint-disable-next-line complexity
+ observe: function observe(subject, topic, data) {
+ this._log.trace("Handling " + topic);
+ switch (topic) {
+ case "weave:engine:score:updated":
+ if (lazy.Status.login == LOGIN_SUCCEEDED) {
+ CommonUtils.namedTimer(
+ this.calculateScore,
+ SCORE_UPDATE_DELAY,
+ this,
+ "_scoreTimer"
+ );
+ }
+ break;
+ case "network:link-status-changed":
+ // Note: NetworkLinkService is unreliable, we get false negatives for it
+ // in cases such as VMs (bug 1420802), so we don't want to use it in
+ // `get offline`, but we assume that it's probably reliable if we're
+ // getting status changed events. (We might be wrong about this, but
+ // if that's true, then the only downside is that we won't sync as
+ // promptly).
+ let isOffline = this.offline;
+ this._log.debug(
+ `Network link status changed to "${data}". Offline?`,
+ isOffline
+ );
+ // Data may be one of `up`, `down`, `change`, or `unknown`. We only want
+ // to sync if it's "up".
+ if (data == "up" && !isOffline) {
+ this._log.debug("Network link looks up. Syncing.");
+ this.scheduleNextSync(0, { why: topic });
+ } else if (data == "down") {
+ // Unschedule pending syncs if we know we're going down. We don't do
+ // this via `checkSyncStatus`, since link status isn't reflected in
+ // `this.offline`.
+ this.clearSyncTriggers();
+ }
+ break;
+ case "network:offline-status-changed":
+ case "captive-portal-detected":
+ // Whether online or offline, we'll reschedule syncs
+ this._log.trace("Network offline status change: " + data);
+ this.checkSyncStatus();
+ break;
+ case "weave:service:sync:start":
+ // Clear out any potentially pending syncs now that we're syncing
+ this.clearSyncTriggers();
+
+ // reset backoff info, if the server tells us to continue backing off,
+ // we'll handle that later
+ lazy.Status.resetBackoff();
+
+ this.globalScore = 0;
+ break;
+ case "weave:service:sync:finish":
+ this.nextSync = 0;
+ this.adjustSyncInterval();
+
+ if (
+ lazy.Status.service == SYNC_FAILED_PARTIAL &&
+ this.requiresBackoff
+ ) {
+ this.requiresBackoff = false;
+ this.handleSyncError();
+ return;
+ }
+
+ let sync_interval;
+ let nextSyncReason = "schedule";
+ this.updateGlobalScore();
+ if (
+ this.globalScore > this.syncThreshold &&
+ lazy.Status.service == STATUS_OK
+ ) {
+ // The global score should be 0 after a sync. If it's not, either
+ // items were changed during the last sync (and we should schedule an
+ // immediate follow-up sync), or an engine skipped
+ this._resyncs++;
+ if (this._resyncs <= this.maxResyncs) {
+ sync_interval = 0;
+ nextSyncReason = "resync";
+ } else {
+ this._log.warn(
+ `Resync attempt ${this._resyncs} exceeded ` +
+ `maximum ${this.maxResyncs}`
+ );
+ Svc.Obs.notify("weave:service:resyncs-finished");
+ }
+ } else {
+ this._resyncs = 0;
+ Svc.Obs.notify("weave:service:resyncs-finished");
+ }
+
+ this._syncErrors = 0;
+ if (lazy.Status.sync == NO_SYNC_NODE_FOUND) {
+ // If we don't have a Sync node, override the interval, even if we've
+ // scheduled a follow-up sync.
+ this._log.trace("Scheduling a sync at interval NO_SYNC_NODE_FOUND.");
+ sync_interval = NO_SYNC_NODE_INTERVAL;
+ }
+ this.scheduleNextSync(sync_interval, { why: nextSyncReason });
+ break;
+ case "weave:engine:sync:finish":
+ if (data == "clients") {
+ // Update the client mode because it might change what we sync.
+ this.updateClientMode();
+ }
+ break;
+ case "weave:engine:sync:error":
+ // `subject` is the exception thrown by an engine's sync() method.
+ let exception = subject;
+ if (exception.status >= 500 && exception.status <= 504) {
+ this.requiresBackoff = true;
+ }
+ break;
+ case "weave:service:login:error":
+ this.clearSyncTriggers();
+
+ if (lazy.Status.login == MASTER_PASSWORD_LOCKED) {
+ // Try again later, just as if we threw an error... only without the
+ // error count.
+ this._log.debug("Couldn't log in: master password is locked.");
+ this._log.trace(
+ "Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL"
+ );
+ this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL);
+ } else if (!this._fatalLoginStatus.includes(lazy.Status.login)) {
+ // Not a fatal login error, just an intermittent network or server
+ // issue. Keep on syncin'.
+ this.checkSyncStatus();
+ }
+ break;
+ case "weave:service:logout:finish":
+ // Start or cancel the sync timer depending on if
+ // logged in or logged out
+ this.checkSyncStatus();
+ break;
+ case "weave:service:sync:error":
+ // There may be multiple clients but if the sync fails, client mode
+ // should still be updated so that the next sync has a correct interval.
+ this.updateClientMode();
+ this.adjustSyncInterval();
+ this.nextSync = 0;
+ this.handleSyncError();
+ break;
+ case "FxA:hawk:backoff:interval":
+ case "weave:service:backoff:interval":
+ let requested_interval = subject * 1000;
+ this._log.debug(
+ "Got backoff notification: " + requested_interval + "ms"
+ );
+ // Leave up to 25% more time for the back off.
+ let interval = requested_interval * (1 + Math.random() * 0.25);
+ lazy.Status.backoffInterval = interval;
+ lazy.Status.minimumNextSync = Date.now() + requested_interval;
+ this._log.debug(
+ "Fuzzed minimum next sync: " + lazy.Status.minimumNextSync
+ );
+ break;
+ case "weave:engine:sync:applied":
+ let numItems = subject.succeeded;
+ this._log.trace(
+ "Engine " + data + " successfully applied " + numItems + " items."
+ );
+ // Bug 1800186 - the tabs engine always reports incoming items, so we don't
+ // want special scheduling in this scenario.
+ // (However, even when we fix the underlying cause of that, we probably still can
+ // ignore tabs here - new incoming tabs don't need to trigger the extra syncs we do
+ // based on this flag.)
+ if (data != "tabs" && numItems) {
+ this.hasIncomingItems = true;
+ }
+ if (subject.newFailed) {
+ this._log.error(
+ `Engine ${data} found ${subject.newFailed} new records that failed to apply`
+ );
+ }
+ break;
+ case "weave:service:setup-complete":
+ Services.prefs.savePrefFile(null);
+ lazy.IdleService.addIdleObserver(this, this.idleTime);
+ Svc.Obs.add("wake_notification", this);
+ Svc.Obs.add("captive-portal-login-success", this);
+ Svc.Obs.add("sleep_notification", this);
+ break;
+ case "weave:service:start-over":
+ this.setDefaults();
+ try {
+ lazy.IdleService.removeIdleObserver(this, this.idleTime);
+ } catch (ex) {
+ if (ex.result != Cr.NS_ERROR_FAILURE) {
+ throw ex;
+ }
+ // In all likelihood we didn't have an idle observer registered yet.
+ // It's all good.
+ }
+ break;
+ case "idle":
+ this._log.trace("We're idle.");
+ this.idle = true;
+ // Adjust the interval for future syncs. This won't actually have any
+ // effect until the next pending sync (which will happen soon since we
+ // were just active.)
+ this.adjustSyncInterval();
+ break;
+ case "active":
+ this._log.trace("Received notification that we're back from idle.");
+ this.idle = false;
+ CommonUtils.namedTimer(
+ function onBack() {
+ if (this.idle) {
+ this._log.trace(
+ "... and we're idle again. " +
+ "Ignoring spurious back notification."
+ );
+ return;
+ }
+
+ this._log.trace("Genuine return from idle. Syncing.");
+ // Trigger a sync if we have multiple clients.
+ if (this.numClients > 1) {
+ this.scheduleNextSync(0, { why: topic });
+ }
+ },
+ IDLE_OBSERVER_BACK_DELAY,
+ this,
+ "idleDebouncerTimer"
+ );
+ break;
+ case "wake_notification":
+ this._log.debug("Woke from sleep.");
+ CommonUtils.nextTick(() => {
+ // Trigger a sync if we have multiple clients. We give it 2 seconds
+ // so the browser can recover from the wake and do more important
+ // operations first (timers etc).
+ if (this.numClients > 1) {
+ if (!this.offline) {
+ this._log.debug("Online, will sync in 2s.");
+ this.scheduleNextSync(2000, { why: topic });
+ }
+ }
+ });
+ break;
+ case "captive-portal-login-success":
+ this._log.debug("Captive portal login success. Scheduling a sync.");
+ CommonUtils.nextTick(() => {
+ this.scheduleNextSync(3000, { why: topic });
+ });
+ break;
+ case "sleep_notification":
+ if (this.service.engineManager.get("tabs")._tracker.modified) {
+ this._log.debug("Going to sleep, doing a quick sync.");
+ this.scheduleNextSync(0, { engines: ["tabs"], why: "sleep" });
+ }
+ break;
+ }
+ },
+
+ adjustSyncInterval: function adjustSyncInterval() {
+ if (this.numClients <= 1) {
+ this._log.trace("Adjusting syncInterval to singleDeviceInterval.");
+ this.syncInterval = this.singleDeviceInterval;
+ return;
+ }
+
+ // Only MULTI_DEVICE clients will enter this if statement
+ // since SINGLE_USER clients will be handled above.
+ if (this.idle) {
+ this._log.trace("Adjusting syncInterval to idleInterval.");
+ this.syncInterval = this.idleInterval;
+ return;
+ }
+
+ if (this.hasIncomingItems) {
+ this._log.trace("Adjusting syncInterval to immediateInterval.");
+ this.hasIncomingItems = false;
+ this.syncInterval = this.immediateInterval;
+ } else {
+ this._log.trace("Adjusting syncInterval to activeInterval.");
+ this.syncInterval = this.activeInterval;
+ }
+ },
+
+ updateGlobalScore() {
+ let engines = [this.service.clientsEngine].concat(
+ this.service.engineManager.getEnabled()
+ );
+ let globalScore = this.globalScore;
+ for (let i = 0; i < engines.length; i++) {
+ this._log.trace(engines[i].name + ": score: " + engines[i].score);
+ globalScore += engines[i].score;
+ engines[i]._tracker.resetScore();
+ }
+ this.globalScore = globalScore;
+ this._log.trace("Global score updated: " + globalScore);
+ },
+
+ calculateScore() {
+ this.updateGlobalScore();
+ this.checkSyncStatus();
+ },
+
+ /**
+ * Query the number of known clients to figure out what mode to be in
+ */
+ updateClientMode: function updateClientMode() {
+ // Nothing to do if it's the same amount
+ let numClients = this.numClients;
+ if (numClients == this.numClientsLastSync) {
+ return;
+ }
+
+ this._log.debug(
+ `Client count: ${this.numClientsLastSync} -> ${numClients}`
+ );
+ this.numClientsLastSync = numClients;
+
+ if (numClients <= 1) {
+ this._log.trace("Adjusting syncThreshold to SINGLE_USER_THRESHOLD");
+ this.syncThreshold = SINGLE_USER_THRESHOLD;
+ } else {
+ this._log.trace("Adjusting syncThreshold to MULTI_DEVICE_THRESHOLD");
+ this.syncThreshold = MULTI_DEVICE_THRESHOLD;
+ }
+ this.adjustSyncInterval();
+ },
+
+ /**
+ * Check if we should be syncing and schedule the next sync, if it's not scheduled
+ */
+ checkSyncStatus: function checkSyncStatus() {
+ // Should we be syncing now, if not, cancel any sync timers and return
+ // if we're in backoff, we'll schedule the next sync.
+ let ignore = [kSyncBackoffNotMet, kSyncMasterPasswordLocked];
+ let skip = this.service._checkSync(ignore);
+ this._log.trace('_checkSync returned "' + skip + '".');
+ if (skip) {
+ this.clearSyncTriggers();
+ return;
+ }
+
+ let why = "schedule";
+ // Only set the wait time to 0 if we need to sync right away
+ let wait;
+ if (this.globalScore > this.syncThreshold) {
+ this._log.debug("Global Score threshold hit, triggering sync.");
+ wait = 0;
+ why = "score";
+ }
+ this.scheduleNextSync(wait, { why });
+ },
+
+ /**
+ * Call sync() if Master Password is not locked.
+ *
+ * Otherwise, reschedule a sync for later.
+ */
+ syncIfMPUnlocked(engines, why) {
+ // No point if we got kicked out by the master password dialog.
+ if (lazy.Status.login == MASTER_PASSWORD_LOCKED && Utils.mpLocked()) {
+ this._log.debug(
+ "Not initiating sync: Login status is " + lazy.Status.login
+ );
+
+ // If we're not syncing now, we need to schedule the next one.
+ this._log.trace(
+ "Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL"
+ );
+ this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL);
+ return;
+ }
+
+ if (!Async.isAppReady()) {
+ this._log.debug("Not initiating sync: app is shutting down");
+ return;
+ }
+ Services.tm.dispatchToMainThread(() => {
+ this.service.sync({ engines, why });
+ const now = Math.round(new Date().getTime() / 1000);
+ // Only fetch missed messages in a "scheduled" sync so we don't race against
+ // the Push service reconnecting on a network link change for example.
+ if (
+ why == "schedule" &&
+ now >=
+ this.missedFxACommandsLastFetch + this.missedFxACommandsFetchInterval
+ ) {
+ lazy.fxAccounts.commands
+ .pollDeviceCommands()
+ .then(() => {
+ this.missedFxACommandsLastFetch = now;
+ })
+ .catch(e => {
+ this._log.error("Fetching missed remote commands failed.", e);
+ });
+ }
+ });
+ },
+
+ /**
+ * Set a timer for the next sync
+ */
+ scheduleNextSync(interval, { engines = null, why = null } = {}) {
+ // If no interval was specified, use the current sync interval.
+ if (interval == null) {
+ interval = this.syncInterval;
+ }
+
+ // Ensure the interval is set to no less than the backoff.
+ if (lazy.Status.backoffInterval && interval < lazy.Status.backoffInterval) {
+ this._log.trace(
+ "Requested interval " +
+ interval +
+ " ms is smaller than the backoff interval. " +
+ "Using backoff interval " +
+ lazy.Status.backoffInterval +
+ " ms instead."
+ );
+ interval = lazy.Status.backoffInterval;
+ }
+ let nextSync = this.nextSync;
+ if (nextSync != 0) {
+ // There's already a sync scheduled. Don't reschedule if there's already
+ // a timer scheduled for sooner than requested.
+ let currentInterval = nextSync - Date.now();
+ this._log.trace(
+ "There's already a sync scheduled in " + currentInterval + " ms."
+ );
+ if (currentInterval < interval && this.syncTimer) {
+ this._log.trace(
+ "Ignoring scheduling request for next sync in " + interval + " ms."
+ );
+ return;
+ }
+ }
+
+ // Start the sync right away if we're already late.
+ if (interval <= 0) {
+ this._log.trace(`Requested sync should happen right away. (why=${why})`);
+ this.syncIfMPUnlocked(engines, why);
+ return;
+ }
+
+ this._log.debug(`Next sync in ${interval} ms. (why=${why})`);
+ CommonUtils.namedTimer(
+ () => {
+ this.syncIfMPUnlocked(engines, why);
+ },
+ interval,
+ this,
+ "syncTimer"
+ );
+
+ // Save the next sync time in-case sync is disabled (logout/offline/etc.)
+ this.nextSync = Date.now() + interval;
+ },
+
+ /**
+ * Incorporates the backoff/retry logic used in error handling and elective
+ * non-syncing.
+ */
+ scheduleAtInterval: function scheduleAtInterval(minimumInterval) {
+ let interval = Utils.calculateBackoff(
+ this._syncErrors,
+ MINIMUM_BACKOFF_INTERVAL,
+ lazy.Status.backoffInterval
+ );
+ if (minimumInterval) {
+ interval = Math.max(minimumInterval, interval);
+ }
+
+ this._log.debug(
+ "Starting client-initiated backoff. Next sync in " + interval + " ms."
+ );
+ this.scheduleNextSync(interval, { why: "client-backoff-schedule" });
+ },
+
+ autoConnect: function autoConnect() {
+ if (this.service._checkSetup() == STATUS_OK && !this.service._checkSync()) {
+ // Schedule a sync based on when a previous sync was scheduled.
+ // scheduleNextSync() will do the right thing if that time lies in
+ // the past.
+ this.scheduleNextSync(this.nextSync - Date.now(), { why: "startup" });
+ }
+ },
+
+ _syncErrors: 0,
+ /**
+ * Deal with sync errors appropriately
+ */
+ handleSyncError: function handleSyncError() {
+ this._log.trace("In handleSyncError. Error count: " + this._syncErrors);
+ this._syncErrors++;
+
+ // Do nothing on the first couple of failures, if we're not in
+ // backoff due to 5xx errors.
+ if (!lazy.Status.enforceBackoff) {
+ if (this._syncErrors < MAX_ERROR_COUNT_BEFORE_BACKOFF) {
+ this.scheduleNextSync(null, { why: "reschedule" });
+ return;
+ }
+ this._log.debug(
+ "Sync error count has exceeded " +
+ MAX_ERROR_COUNT_BEFORE_BACKOFF +
+ "; enforcing backoff."
+ );
+ lazy.Status.enforceBackoff = true;
+ }
+
+ this.scheduleAtInterval();
+ },
+
+ /**
+ * Remove any timers/observers that might trigger a sync
+ */
+ clearSyncTriggers: function clearSyncTriggers() {
+ this._log.debug("Clearing sync triggers and the global score.");
+ this.globalScore = this.nextSync = 0;
+
+ // Clear out any scheduled syncs
+ if (this.syncTimer) {
+ this.syncTimer.clear();
+ }
+ },
+};
+
+export function ErrorHandler(service) {
+ this.service = service;
+ this.init();
+}
+
+ErrorHandler.prototype = {
+ init() {
+ Svc.Obs.add("weave:engine:sync:applied", this);
+ Svc.Obs.add("weave:engine:sync:error", this);
+ Svc.Obs.add("weave:service:login:error", this);
+ Svc.Obs.add("weave:service:sync:error", this);
+ Svc.Obs.add("weave:service:sync:finish", this);
+ Svc.Obs.add("weave:service:start-over:finish", this);
+
+ this.initLogs();
+ },
+
+ initLogs: function initLogs() {
+ // Set the root Sync logger level based on a pref. All other logs will
+ // inherit this level unless they specifically override it.
+ Log.repository
+ .getLogger("Sync")
+ .manageLevelFromPref(`services.sync.log.logger`);
+ // And allow our specific log to have a custom level via a pref.
+ this._log = Log.repository.getLogger("Sync.ErrorHandler");
+ this._log.manageLevelFromPref("services.sync.log.logger.service.main");
+ },
+
+ observe(subject, topic, data) {
+ this._log.trace("Handling " + topic);
+ switch (topic) {
+ case "weave:engine:sync:applied":
+ if (subject.newFailed) {
+ // An engine isn't able to apply one or more incoming records.
+ // We don't fail hard on this, but it usually indicates a bug,
+ // so for now treat it as sync error (c.f. Service._syncEngine())
+ lazy.Status.engines = [data, ENGINE_APPLY_FAIL];
+ this._log.debug(data + " failed to apply some records.");
+ }
+ break;
+ case "weave:engine:sync:error": {
+ let exception = subject; // exception thrown by engine's sync() method
+ let engine_name = data; // engine name that threw the exception
+
+ this.checkServerError(exception);
+
+ lazy.Status.engines = [
+ engine_name,
+ exception.failureCode || ENGINE_UNKNOWN_FAIL,
+ ];
+ if (Async.isShutdownException(exception)) {
+ this._log.debug(
+ engine_name +
+ " was interrupted due to the application shutting down"
+ );
+ } else {
+ this._log.debug(engine_name + " failed", exception);
+ }
+ break;
+ }
+ case "weave:service:login:error":
+ this._log.error("Sync encountered a login error");
+ this.resetFileLog();
+ break;
+ case "weave:service:sync:error": {
+ if (lazy.Status.sync == CREDENTIALS_CHANGED) {
+ this.service.logout();
+ }
+
+ let exception = subject;
+ if (Async.isShutdownException(exception)) {
+ // If we are shutting down we just log the fact, attempt to flush
+ // the log file and get out of here!
+ this._log.error(
+ "Sync was interrupted due to the application shutting down"
+ );
+ this.resetFileLog();
+ break;
+ }
+
+ // Not a shutdown related exception...
+ this._log.error("Sync encountered an error", exception);
+ this.resetFileLog();
+ break;
+ }
+ case "weave:service:sync:finish":
+ this._log.trace("Status.service is " + lazy.Status.service);
+
+ // Check both of these status codes: in the event of a failure in one
+ // engine, Status.service will be SYNC_FAILED_PARTIAL despite
+ // Status.sync being SYNC_SUCCEEDED.
+ // *facepalm*
+ if (
+ lazy.Status.sync == SYNC_SUCCEEDED &&
+ lazy.Status.service == STATUS_OK
+ ) {
+ // Great. Let's clear our mid-sync 401 note.
+ this._log.trace("Clearing lastSyncReassigned.");
+ Svc.Prefs.reset("lastSyncReassigned");
+ }
+
+ if (lazy.Status.service == SYNC_FAILED_PARTIAL) {
+ this._log.error("Some engines did not sync correctly.");
+ }
+ this.resetFileLog();
+ break;
+ case "weave:service:start-over:finish":
+ // ensure we capture any logs between the last sync and the reset completing.
+ this.resetFileLog()
+ .then(() => {
+ // although for privacy reasons we also delete all logs (but we allow
+ // a preference to avoid this to help with debugging.)
+ if (!Svc.Prefs.get("log.keepLogsOnReset", false)) {
+ return logManager.removeAllLogs().then(() => {
+ Svc.Obs.notify("weave:service:remove-file-log");
+ });
+ }
+ return null;
+ })
+ .catch(err => {
+ // So we failed to delete the logs - take the ironic option of
+ // writing this error to the logs we failed to delete!
+ this._log.error("Failed to delete logs on reset", err);
+ });
+ break;
+ }
+ },
+
+ async _dumpAddons() {
+ // Just dump the items that sync may be concerned with. Specifically,
+ // active extensions that are not hidden.
+ let addons = [];
+ try {
+ addons = await lazy.AddonManager.getAddonsByTypes(["extension"]);
+ } catch (e) {
+ this._log.warn("Failed to dump addons", e);
+ }
+
+ let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
+ this._log.trace("Addons installed", relevantAddons.length);
+ for (let addon of relevantAddons) {
+ this._log.trace(" - ${name}, version ${version}, id ${id}", addon);
+ }
+ },
+
+ /**
+ * Generate a log file for the sync that just completed
+ * and refresh the input & output streams.
+ */
+ async resetFileLog() {
+ // If we're writing an error log, dump extensions that may be causing problems.
+ if (logManager.sawError) {
+ await this._dumpAddons();
+ }
+ const logType = await logManager.resetFileLog();
+ if (logType == logManager.ERROR_LOG_WRITTEN) {
+ console.error(
+ "Sync encountered an error - see about:sync-log for the log file."
+ );
+ }
+ Svc.Obs.notify("weave:service:reset-file-log");
+ },
+
+ /**
+ * Handle HTTP response results or exceptions and set the appropriate
+ * Status.* bits.
+ *
+ * This method also looks for "side-channel" warnings.
+ */
+ checkServerError(resp) {
+ // In this case we were passed a resolved value of Resource#_doRequest.
+ switch (resp.status) {
+ case 400:
+ if (resp == RESPONSE_OVER_QUOTA) {
+ lazy.Status.sync = OVER_QUOTA;
+ }
+ break;
+
+ case 401:
+ this.service.logout();
+ this._log.info("Got 401 response; resetting clusterURL.");
+ this.service.clusterURL = null;
+
+ let delay = 0;
+ if (Svc.Prefs.get("lastSyncReassigned")) {
+ // We got a 401 in the middle of the previous sync, and we just got
+ // another. Login must have succeeded in order for us to get here, so
+ // the password should be correct.
+ // This is likely to be an intermittent server issue, so back off and
+ // give it time to recover.
+ this._log.warn("Last sync also failed for 401. Delaying next sync.");
+ delay = MINIMUM_BACKOFF_INTERVAL;
+ } else {
+ this._log.debug("New mid-sync 401 failure. Making a note.");
+ Svc.Prefs.set("lastSyncReassigned", true);
+ }
+ this._log.info("Attempting to schedule another sync.");
+ this.service.scheduler.scheduleNextSync(delay, { why: "reschedule" });
+ break;
+
+ case 500:
+ case 502:
+ case 503:
+ case 504:
+ lazy.Status.enforceBackoff = true;
+ if (resp.status == 503 && resp.headers["retry-after"]) {
+ let retryAfter = resp.headers["retry-after"];
+ this._log.debug("Got Retry-After: " + retryAfter);
+ if (this.service.isLoggedIn) {
+ lazy.Status.sync = SERVER_MAINTENANCE;
+ } else {
+ lazy.Status.login = SERVER_MAINTENANCE;
+ }
+ Svc.Obs.notify(
+ "weave:service:backoff:interval",
+ parseInt(retryAfter, 10)
+ );
+ }
+ break;
+ }
+
+ // In this other case we were passed a rejection value.
+ switch (resp.result) {
+ case Cr.NS_ERROR_UNKNOWN_HOST:
+ case Cr.NS_ERROR_CONNECTION_REFUSED:
+ case Cr.NS_ERROR_NET_TIMEOUT:
+ case Cr.NS_ERROR_NET_RESET:
+ case Cr.NS_ERROR_NET_INTERRUPT:
+ case Cr.NS_ERROR_PROXY_CONNECTION_REFUSED:
+ // The constant says it's about login, but in fact it just
+ // indicates general network error.
+ if (this.service.isLoggedIn) {
+ lazy.Status.sync = LOGIN_FAILED_NETWORK_ERROR;
+ } else {
+ lazy.Status.login = LOGIN_FAILED_NETWORK_ERROR;
+ }
+ break;
+ }
+ },
+};
diff --git a/services/sync/modules/record.sys.mjs b/services/sync/modules/record.sys.mjs
new file mode 100644
index 0000000000..7d5918a8ca
--- /dev/null
+++ b/services/sync/modules/record.sys.mjs
@@ -0,0 +1,1335 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const CRYPTO_COLLECTION = "crypto";
+const KEYS_WBO = "keys";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import {
+ DEFAULT_DOWNLOAD_BATCH_SIZE,
+ DEFAULT_KEYBUNDLE_NAME,
+} from "resource://services-sync/constants.sys.mjs";
+import { BulkKeyBundle } from "resource://services-sync/keys.sys.mjs";
+import { Weave } from "resource://services-sync/main.sys.mjs";
+import { Resource } from "resource://services-sync/resource.sys.mjs";
+import { Utils } from "resource://services-sync/util.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs";
+
+/**
+ * The base class for all Sync basic storage objects (BSOs). This is the format
+ * used to store all records on the Sync server. In an earlier version of the
+ * Sync protocol, BSOs used to be called WBOs, or Weave Basic Objects. This
+ * class retains the old name.
+ *
+ * @class
+ * @param {String} collection The collection name for this BSO.
+ * @param {String} id The ID of this BSO.
+ */
+export function WBORecord(collection, id) {
+ this.data = {};
+ this.payload = {};
+ this.collection = collection; // Optional.
+ this.id = id; // Optional.
+}
+
+WBORecord.prototype = {
+ _logName: "Sync.Record.WBO",
+
+ get sortindex() {
+ if (this.data.sortindex) {
+ return this.data.sortindex;
+ }
+ return 0;
+ },
+
+ // Get thyself from your URI, then deserialize.
+ // Set thine 'response' field.
+ async fetch(resource) {
+ if (!(resource instanceof Resource)) {
+ throw new Error("First argument must be a Resource instance.");
+ }
+
+ let r = await resource.get();
+ if (r.success) {
+ this.deserialize(r.obj); // Warning! Muffles exceptions!
+ }
+ this.response = r;
+ return this;
+ },
+
+ upload(resource) {
+ if (!(resource instanceof Resource)) {
+ throw new Error("First argument must be a Resource instance.");
+ }
+
+ return resource.put(this);
+ },
+
+ // Take a base URI string, with trailing slash, and return the URI of this
+ // WBO based on collection and ID.
+ uri(base) {
+ if (this.collection && this.id) {
+ let url = CommonUtils.makeURI(base + this.collection + "/" + this.id);
+ url.QueryInterface(Ci.nsIURL);
+ return url;
+ }
+ return null;
+ },
+
+ deserialize: function deserialize(json) {
+ if (!json || typeof json !== "object") {
+ throw new TypeError("Can't deserialize record from: " + json);
+ }
+ this.data = json;
+ try {
+ // The payload is likely to be JSON, but if not, keep it as a string
+ this.payload = JSON.parse(this.payload);
+ } catch (ex) {}
+ },
+
+ toJSON: function toJSON() {
+ // Copy fields from data to be stringified, making sure payload is a string
+ let obj = {};
+ for (let [key, val] of Object.entries(this.data)) {
+ obj[key] = key == "payload" ? JSON.stringify(val) : val;
+ }
+ if (this.ttl) {
+ obj.ttl = this.ttl;
+ }
+ return obj;
+ },
+
+ toString: function toString() {
+ return (
+ "{ " +
+ "id: " +
+ this.id +
+ " " +
+ "index: " +
+ this.sortindex +
+ " " +
+ "modified: " +
+ this.modified +
+ " " +
+ "ttl: " +
+ this.ttl +
+ " " +
+ "payload: " +
+ JSON.stringify(this.payload) +
+ " }"
+ );
+ },
+};
+
+Utils.deferGetSet(WBORecord, "data", [
+ "id",
+ "modified",
+ "sortindex",
+ "payload",
+]);
+
+/**
+ * An encrypted BSO record. This subclass handles encrypting and decrypting the
+ * BSO payload, but doesn't parse or interpret the cleartext string. Subclasses
+ * must override `transformBeforeEncrypt` and `transformAfterDecrypt` to process
+ * the cleartext.
+ *
+ * This class is only exposed for bridged engines, which handle serialization
+ * and deserialization in Rust. Sync engines implemented in JS should subclass
+ * `CryptoWrapper` instead, which takes care of transforming the cleartext into
+ * an object, and ensuring its contents are valid.
+ *
+ * @class
+ * @template Cleartext
+ * @param {String} collection The collection name for this BSO.
+ * @param {String} id The ID of this BSO.
+ */
+export function RawCryptoWrapper(collection, id) {
+ // Setting properties before calling the superclass constructor isn't allowed
+ // in new-style classes (`class MyRecord extends RawCryptoWrapper`), but
+ // allowed with plain functions. This is also why `defaultCleartext` is a
+ // method, and not simply set in the subclass constructor.
+ this.cleartext = this.defaultCleartext();
+ WBORecord.call(this, collection, id);
+ this.ciphertext = null;
+}
+
+RawCryptoWrapper.prototype = {
+ _logName: "Sync.Record.RawCryptoWrapper",
+
+ /**
+ * Returns the default empty cleartext for this record type. This is exposed
+ * as a method so that subclasses can override it, and access the default
+ * cleartext in their constructors. `CryptoWrapper`, for example, overrides
+ * this to return an empty object, so that initializing the `id` in its
+ * constructor calls its overridden `id` setter.
+ *
+ * @returns {Cleartext} An empty cleartext.
+ */
+ defaultCleartext() {
+ return null;
+ },
+
+ /**
+ * Transforms the cleartext into a string that can be encrypted and wrapped
+ * in a BSO payload. This is called before uploading the record to the server.
+ *
+ * @param {Cleartext} outgoingCleartext The cleartext to upload.
+ * @returns {String} The serialized cleartext.
+ */
+ transformBeforeEncrypt(outgoingCleartext) {
+ throw new TypeError("Override to stringify outgoing records");
+ },
+
+ /**
+ * Transforms an incoming cleartext string into an instance of the
+ * `Cleartext` type. This is called when fetching the record from the
+ * server.
+ *
+ * @param {String} incomingCleartext The decrypted cleartext string.
+ * @returns {Cleartext} The parsed cleartext.
+ */
+ transformAfterDecrypt(incomingCleartext) {
+ throw new TypeError("Override to parse incoming records");
+ },
+
+ ciphertextHMAC: async function ciphertextHMAC(keyBundle) {
+ let hmacKeyByteString = keyBundle.hmacKey;
+ if (!hmacKeyByteString) {
+ throw new Error("Cannot compute HMAC without an HMAC key.");
+ }
+ let hmacKey = CommonUtils.byteStringToArrayBuffer(hmacKeyByteString);
+ // NB: this.ciphertext is a base64-encoded string. For some reason this
+ // implementation computes the HMAC on the encoded value.
+ let data = CommonUtils.byteStringToArrayBuffer(this.ciphertext);
+ let hmac = await CryptoUtils.hmac("SHA-256", hmacKey, data);
+ return CommonUtils.bytesAsHex(CommonUtils.arrayBufferToByteString(hmac));
+ },
+
+ /*
+ * Don't directly use the sync key. Instead, grab a key for this
+ * collection, which is decrypted with the sync key.
+ *
+ * Cache those keys; invalidate the cache if the time on the keys collection
+ * changes, or other auth events occur.
+ *
+ * Optional key bundle overrides the collection key lookup.
+ */
+ async encrypt(keyBundle) {
+ if (!keyBundle) {
+ throw new Error("A key bundle must be supplied to encrypt.");
+ }
+
+ this.IV = Weave.Crypto.generateRandomIV();
+ this.ciphertext = await Weave.Crypto.encrypt(
+ this.transformBeforeEncrypt(this.cleartext),
+ keyBundle.encryptionKeyB64,
+ this.IV
+ );
+ this.hmac = await this.ciphertextHMAC(keyBundle);
+ this.cleartext = null;
+ },
+
+ // Optional key bundle.
+ async decrypt(keyBundle) {
+ if (!this.ciphertext) {
+ throw new Error("No ciphertext: nothing to decrypt?");
+ }
+
+ if (!keyBundle) {
+ throw new Error("A key bundle must be supplied to decrypt.");
+ }
+
+ // Authenticate the encrypted blob with the expected HMAC
+ let computedHMAC = await this.ciphertextHMAC(keyBundle);
+
+ if (computedHMAC != this.hmac) {
+ Utils.throwHMACMismatch(this.hmac, computedHMAC);
+ }
+
+ let cleartext = await Weave.Crypto.decrypt(
+ this.ciphertext,
+ keyBundle.encryptionKeyB64,
+ this.IV
+ );
+ this.cleartext = this.transformAfterDecrypt(cleartext);
+ this.ciphertext = null;
+
+ return this.cleartext;
+ },
+};
+
+Object.setPrototypeOf(RawCryptoWrapper.prototype, WBORecord.prototype);
+
+Utils.deferGetSet(RawCryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]);
+
+/**
+ * An encrypted BSO record with a JSON payload. All engines implemented in JS
+ * should subclass this class to describe their own record types.
+ *
+ * @class
+ * @param {String} collection The collection name for this BSO.
+ * @param {String} id The ID of this BSO.
+ */
+export function CryptoWrapper(collection, id) {
+ RawCryptoWrapper.call(this, collection, id);
+}
+
+CryptoWrapper.prototype = {
+ _logName: "Sync.Record.CryptoWrapper",
+
+ defaultCleartext() {
+ return {};
+ },
+
+ transformBeforeEncrypt(cleartext) {
+ return JSON.stringify(cleartext);
+ },
+
+ transformAfterDecrypt(cleartext) {
+ // Handle invalid data here. Elsewhere we assume that cleartext is an object.
+ let json_result = JSON.parse(cleartext);
+
+ if (!(json_result && json_result instanceof Object)) {
+ throw new Error(
+ `Decryption failed: result is <${json_result}>, not an object.`
+ );
+ }
+
+ // If the payload has an encrypted id ensure it matches the requested record's id.
+ if (json_result.id && json_result.id != this.id) {
+ throw new Error(`Record id mismatch: ${json_result.id} != ${this.id}`);
+ }
+
+ return json_result;
+ },
+
+ cleartextToString() {
+ return JSON.stringify(this.cleartext);
+ },
+
+ toString: function toString() {
+ let payload = this.deleted ? "DELETED" : this.cleartextToString();
+
+ return (
+ "{ " +
+ "id: " +
+ this.id +
+ " " +
+ "index: " +
+ this.sortindex +
+ " " +
+ "modified: " +
+ this.modified +
+ " " +
+ "ttl: " +
+ this.ttl +
+ " " +
+ "payload: " +
+ payload +
+ " " +
+ "collection: " +
+ (this.collection || "undefined") +
+ " }"
+ );
+ },
+
+ // The custom setter below masks the parent's getter, so explicitly call it :(
+ get id() {
+ return super.id;
+ },
+
+ // Keep both plaintext and encrypted versions of the id to verify integrity
+ set id(val) {
+ super.id = val;
+ this.cleartext.id = val;
+ },
+};
+
+Object.setPrototypeOf(CryptoWrapper.prototype, RawCryptoWrapper.prototype);
+
+Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted");
+
+/**
+ * An interface and caching layer for records.
+ */
+export function RecordManager(service) {
+ this.service = service;
+
+ this._log = Log.repository.getLogger(this._logName);
+ this._records = {};
+}
+
+RecordManager.prototype = {
+ _recordType: CryptoWrapper,
+ _logName: "Sync.RecordManager",
+
+ async import(url) {
+ this._log.trace("Importing record: " + (url.spec ? url.spec : url));
+ try {
+ // Clear out the last response with empty object if GET fails
+ this.response = {};
+ this.response = await this.service.resource(url).get();
+
+ // Don't parse and save the record on failure
+ if (!this.response.success) {
+ return null;
+ }
+
+ let record = new this._recordType(url);
+ record.deserialize(this.response.obj);
+
+ return this.set(url, record);
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Failed to import record", ex);
+ return null;
+ }
+ },
+
+ get(url) {
+ // Use a url string as the key to the hash
+ let spec = url.spec ? url.spec : url;
+ if (spec in this._records) {
+ return Promise.resolve(this._records[spec]);
+ }
+ return this.import(url);
+ },
+
+ set: function RecordMgr_set(url, record) {
+ let spec = url.spec ? url.spec : url;
+ return (this._records[spec] = record);
+ },
+
+ contains: function RecordMgr_contains(url) {
+ if ((url.spec || url) in this._records) {
+ return true;
+ }
+ return false;
+ },
+
+ clearCache: function recordMgr_clearCache() {
+ this._records = {};
+ },
+
+ del: function RecordMgr_del(url) {
+ delete this._records[url];
+ },
+};
+
+/**
+ * Keeps track of mappings between collection names ('tabs') and KeyBundles.
+ *
+ * You can update this thing simply by giving it /info/collections. It'll
+ * use the last modified time to bring itself up to date.
+ */
+export function CollectionKeyManager(lastModified, default_, collections) {
+ this.lastModified = lastModified || 0;
+ this._default = default_ || null;
+ this._collections = collections || {};
+
+ this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
+}
+
+// TODO: persist this locally as an Identity. Bug 610913.
+// Note that the last modified time needs to be preserved.
+CollectionKeyManager.prototype = {
+ /**
+ * Generate a new CollectionKeyManager that has the same attributes
+ * as this one.
+ */
+ clone() {
+ const newCollections = {};
+ for (let c in this._collections) {
+ newCollections[c] = this._collections[c];
+ }
+
+ return new CollectionKeyManager(
+ this.lastModified,
+ this._default,
+ newCollections
+ );
+ },
+
+ // Return information about old vs new keys:
+ // * same: true if two collections are equal
+ // * changed: an array of collection names that changed.
+ _compareKeyBundleCollections: function _compareKeyBundleCollections(m1, m2) {
+ let changed = [];
+
+ function process(m1, m2) {
+ for (let k1 in m1) {
+ let v1 = m1[k1];
+ let v2 = m2[k1];
+ if (!(v1 && v2 && v1.equals(v2))) {
+ changed.push(k1);
+ }
+ }
+ }
+
+ // Diffs both ways.
+ process(m1, m2);
+ process(m2, m1);
+
+ // Return a sorted, unique array.
+ changed.sort();
+ let last;
+ changed = changed.filter(x => x != last && (last = x));
+ return { same: !changed.length, changed };
+ },
+
+ get isClear() {
+ return !this._default;
+ },
+
+ clear: function clear() {
+ this._log.info("Clearing collection keys...");
+ this.lastModified = 0;
+ this._collections = {};
+ this._default = null;
+ },
+
+ keyForCollection(collection) {
+ if (collection && this._collections[collection]) {
+ return this._collections[collection];
+ }
+
+ return this._default;
+ },
+
+ /**
+ * If `collections` (an array of strings) is provided, iterate
+ * over it and generate random keys for each collection.
+ * Create a WBO for the given data.
+ */
+ _makeWBO(collections, defaultBundle) {
+ let wbo = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
+ let c = {};
+ for (let k in collections) {
+ c[k] = collections[k].keyPairB64;
+ }
+ wbo.cleartext = {
+ default: defaultBundle ? defaultBundle.keyPairB64 : null,
+ collections: c,
+ collection: CRYPTO_COLLECTION,
+ id: KEYS_WBO,
+ };
+ return wbo;
+ },
+
+ /**
+ * Create a WBO for the current keys.
+ */
+ asWBO(collection, id) {
+ return this._makeWBO(this._collections, this._default);
+ },
+
+ /**
+ * Compute a new default key, and new keys for any specified collections.
+ */
+ async newKeys(collections) {
+ let newDefaultKeyBundle = await this.newDefaultKeyBundle();
+
+ let newColls = {};
+ if (collections) {
+ for (let c of collections) {
+ let b = new BulkKeyBundle(c);
+ await b.generateRandom();
+ newColls[c] = b;
+ }
+ }
+ return [newDefaultKeyBundle, newColls];
+ },
+
+ /**
+ * Generates new keys, but does not replace our local copy. Use this to
+ * verify an upload before storing.
+ */
+ async generateNewKeysWBO(collections) {
+ let newDefaultKey, newColls;
+ [newDefaultKey, newColls] = await this.newKeys(collections);
+
+ return this._makeWBO(newColls, newDefaultKey);
+ },
+
+ /**
+ * Create a new default key.
+ *
+ * @returns {BulkKeyBundle}
+ */
+ async newDefaultKeyBundle() {
+ const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
+ await key.generateRandom();
+ return key;
+ },
+
+ /**
+ * Create a new default key and store it as this._default, since without one you cannot use setContents.
+ */
+ async generateDefaultKey() {
+ this._default = await this.newDefaultKeyBundle();
+ },
+
+ /**
+ * Return true if keys are already present for each of the given
+ * collections.
+ */
+ hasKeysFor(collections) {
+ // We can't use filter() here because sometimes collections is an iterator.
+ for (let collection of collections) {
+ if (!this._collections[collection]) {
+ return false;
+ }
+ }
+ return true;
+ },
+
+ /**
+ * Return a new CollectionKeyManager that has keys for each of the
+ * given collections (creating new ones for collections where we
+ * don't already have keys).
+ */
+ async ensureKeysFor(collections) {
+ const newKeys = Object.assign({}, this._collections);
+ for (let c of collections) {
+ if (newKeys[c]) {
+ continue; // don't replace existing keys
+ }
+
+ const b = new BulkKeyBundle(c);
+ await b.generateRandom();
+ newKeys[c] = b;
+ }
+ return new CollectionKeyManager(this.lastModified, this._default, newKeys);
+ },
+
+ // Take the fetched info/collections WBO, checking the change
+ // time of the crypto collection.
+ updateNeeded(info_collections) {
+ this._log.info(
+ "Testing for updateNeeded. Last modified: " + this.lastModified
+ );
+
+ // No local record of modification time? Need an update.
+ if (!this.lastModified) {
+ return true;
+ }
+
+ // No keys on the server? We need an update, though our
+ // update handling will be a little more drastic...
+ if (!(CRYPTO_COLLECTION in info_collections)) {
+ return true;
+ }
+
+ // Otherwise, we need an update if our modification time is stale.
+ return info_collections[CRYPTO_COLLECTION] > this.lastModified;
+ },
+
+ //
+ // Set our keys and modified time to the values fetched from the server.
+ // Returns one of three values:
+ //
+ // * If the default key was modified, return true.
+ // * If the default key was not modified, but per-collection keys were,
+ // return an array of such.
+ // * Otherwise, return false -- we were up-to-date.
+ //
+ setContents: function setContents(payload, modified) {
+ let self = this;
+
+ this._log.info(
+ "Setting collection keys contents. Our last modified: " +
+ this.lastModified +
+ ", input modified: " +
+ modified +
+ "."
+ );
+
+ if (!payload) {
+ throw new Error("No payload in CollectionKeyManager.setContents().");
+ }
+
+ if (!payload.default) {
+ this._log.warn("No downloaded default key: this should not occur.");
+ this._log.warn("Not clearing local keys.");
+ throw new Error(
+ "No default key in CollectionKeyManager.setContents(). Cannot proceed."
+ );
+ }
+
+ // Process the incoming default key.
+ let b = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
+ b.keyPairB64 = payload.default;
+ let newDefault = b;
+
+ // Process the incoming collections.
+ let newCollections = {};
+ if ("collections" in payload) {
+ this._log.info("Processing downloaded per-collection keys.");
+ let colls = payload.collections;
+ for (let k in colls) {
+ let v = colls[k];
+ if (v) {
+ let keyObj = new BulkKeyBundle(k);
+ keyObj.keyPairB64 = v;
+ newCollections[k] = keyObj;
+ }
+ }
+ }
+
+ // Check to see if these are already our keys.
+ let sameDefault = this._default && this._default.equals(newDefault);
+ let collComparison = this._compareKeyBundleCollections(
+ newCollections,
+ this._collections
+ );
+ let sameColls = collComparison.same;
+
+ if (sameDefault && sameColls) {
+ self._log.info("New keys are the same as our old keys!");
+ if (modified) {
+ self._log.info("Bumped local modified time.");
+ self.lastModified = modified;
+ }
+ return false;
+ }
+
+ // Make sure things are nice and tidy before we set.
+ this.clear();
+
+ this._log.info("Saving downloaded keys.");
+ this._default = newDefault;
+ this._collections = newCollections;
+
+ // Always trust the server.
+ if (modified) {
+ self._log.info("Bumping last modified to " + modified);
+ self.lastModified = modified;
+ }
+
+ return sameDefault ? collComparison.changed : true;
+ },
+
+ async updateContents(syncKeyBundle, storage_keys) {
+ let log = this._log;
+ log.info("Updating collection keys...");
+
+ // storage_keys is a WBO, fetched from storage/crypto/keys.
+ // Its payload is the default key, and a map of collections to keys.
+ // We lazily compute the key objects from the strings we're given.
+
+ let payload;
+ try {
+ payload = await storage_keys.decrypt(syncKeyBundle);
+ } catch (ex) {
+ log.warn("Got exception decrypting storage keys with sync key.", ex);
+ log.info("Aborting updateContents. Rethrowing.");
+ throw ex;
+ }
+
+ let r = this.setContents(payload, storage_keys.modified);
+ log.info("Collection keys updated.");
+ return r;
+ },
+};
+
+export function Collection(uri, recordObj, service) {
+ if (!service) {
+ throw new Error("Collection constructor requires a service.");
+ }
+
+ Resource.call(this, uri);
+
+ // This is a bit hacky, but gets the job done.
+ let res = service.resource(uri);
+ this.authenticator = res.authenticator;
+
+ this._recordObj = recordObj;
+ this._service = service;
+
+ this._full = false;
+ this._ids = null;
+ this._limit = 0;
+ this._older = 0;
+ this._newer = 0;
+ this._data = [];
+ // optional members used by batch upload operations.
+ this._batch = null;
+ this._commit = false;
+ // Used for batch download operations -- note that this is explicitly an
+ // opaque value and not (necessarily) a number.
+ this._offset = null;
+}
+
+Collection.prototype = {
+ _logName: "Sync.Collection",
+
+ _rebuildURL: function Coll__rebuildURL() {
+ // XXX should consider what happens if it's not a URL...
+ this.uri.QueryInterface(Ci.nsIURL);
+
+ let args = [];
+ if (this.older) {
+ args.push("older=" + this.older);
+ }
+ if (this.newer) {
+ args.push("newer=" + this.newer);
+ }
+ if (this.full) {
+ args.push("full=1");
+ }
+ if (this.sort) {
+ args.push("sort=" + this.sort);
+ }
+ if (this.ids != null) {
+ args.push("ids=" + this.ids);
+ }
+ if (this.limit > 0 && this.limit != Infinity) {
+ args.push("limit=" + this.limit);
+ }
+ if (this._batch) {
+ args.push("batch=" + encodeURIComponent(this._batch));
+ }
+ if (this._commit) {
+ args.push("commit=true");
+ }
+ if (this._offset) {
+ args.push("offset=" + encodeURIComponent(this._offset));
+ }
+
+ this.uri = this.uri
+ .mutate()
+ .setQuery(args.length ? "?" + args.join("&") : "")
+ .finalize();
+ },
+
+ // get full items
+ get full() {
+ return this._full;
+ },
+ set full(value) {
+ this._full = value;
+ this._rebuildURL();
+ },
+
+ // Apply the action to a certain set of ids
+ get ids() {
+ return this._ids;
+ },
+ set ids(value) {
+ this._ids = value;
+ this._rebuildURL();
+ },
+
+ // Limit how many records to get
+ get limit() {
+ return this._limit;
+ },
+ set limit(value) {
+ this._limit = value;
+ this._rebuildURL();
+ },
+
+ // get only items modified before some date
+ get older() {
+ return this._older;
+ },
+ set older(value) {
+ this._older = value;
+ this._rebuildURL();
+ },
+
+ // get only items modified since some date
+ get newer() {
+ return this._newer;
+ },
+ set newer(value) {
+ this._newer = value;
+ this._rebuildURL();
+ },
+
+ // get items sorted by some criteria. valid values:
+ // oldest (oldest first)
+ // newest (newest first)
+ // index
+ get sort() {
+ return this._sort;
+ },
+ set sort(value) {
+ if (value && value != "oldest" && value != "newest" && value != "index") {
+ throw new TypeError(
+ `Illegal value for sort: "${value}" (should be "oldest", "newest", or "index").`
+ );
+ }
+ this._sort = value;
+ this._rebuildURL();
+ },
+
+ get offset() {
+ return this._offset;
+ },
+ set offset(value) {
+ this._offset = value;
+ this._rebuildURL();
+ },
+
+ // Set information about the batch for this request.
+ get batch() {
+ return this._batch;
+ },
+ set batch(value) {
+ this._batch = value;
+ this._rebuildURL();
+ },
+
+ get commit() {
+ return this._commit;
+ },
+ set commit(value) {
+ this._commit = value && true;
+ this._rebuildURL();
+ },
+
+ // Similar to get(), but will page through the items `batchSize` at a time,
+ // deferring calling the record handler until we've gotten them all.
+ //
+ // Returns the last response processed, and doesn't run the record handler
+ // on any items if a non-success status is received while downloading the
+ // records (or if a network error occurs).
+ async getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) {
+ let totalLimit = Number(this.limit) || Infinity;
+ if (batchSize <= 0 || batchSize >= totalLimit) {
+ throw new Error("Invalid batch size");
+ }
+
+ if (!this.full) {
+ throw new Error("getBatched is unimplemented for guid-only GETs");
+ }
+
+ // _onComplete and _onProgress are reset after each `get` by Resource.
+ let { _onComplete, _onProgress } = this;
+ let recordBuffer = [];
+ let resp;
+ try {
+ let lastModifiedTime;
+ this.limit = batchSize;
+
+ do {
+ this._onProgress = _onProgress;
+ this._onComplete = _onComplete;
+ if (batchSize + recordBuffer.length > totalLimit) {
+ this.limit = totalLimit - recordBuffer.length;
+ }
+ this._log.trace("Performing batched GET", {
+ limit: this.limit,
+ offset: this.offset,
+ });
+ // Actually perform the request
+ resp = await this.get();
+ if (!resp.success) {
+ recordBuffer = [];
+ break;
+ }
+ for (let json of resp.obj) {
+ let record = new this._recordObj();
+ record.deserialize(json);
+ recordBuffer.push(record);
+ }
+
+ // Initialize last modified, or check that something broken isn't happening.
+ let lastModified = resp.headers["x-last-modified"];
+ if (!lastModifiedTime) {
+ lastModifiedTime = lastModified;
+ this.setHeader("X-If-Unmodified-Since", lastModified);
+ } else if (lastModified != lastModifiedTime) {
+ // Should be impossible -- We'd get a 412 in this case.
+ throw new Error(
+ "X-Last-Modified changed in the middle of a download batch! " +
+ `${lastModified} => ${lastModifiedTime}`
+ );
+ }
+
+ // If this is missing, we're finished.
+ this.offset = resp.headers["x-weave-next-offset"];
+ } while (this.offset && totalLimit > recordBuffer.length);
+ } finally {
+ // Ensure we undo any temporary state so that subsequent calls to get()
+ // or getBatched() work properly. We do this before calling the record
+ // handler so that we can more convincingly pretend to be a normal get()
+ // call. Note: we're resetting these to the values they had before this
+ // function was called.
+ this._limit = totalLimit;
+ this._offset = null;
+ delete this._headers["x-if-unmodified-since"];
+ this._rebuildURL();
+ }
+ return { response: resp, records: recordBuffer };
+ },
+
+ // This object only supports posting via the postQueue object.
+ post() {
+ throw new Error(
+ "Don't directly post to a collection - use newPostQueue instead"
+ );
+ },
+
+ newPostQueue(log, timestamp, postCallback) {
+ let poster = (data, headers, batch, commit) => {
+ this.batch = batch;
+ this.commit = commit;
+ for (let [header, value] of headers) {
+ this.setHeader(header, value);
+ }
+ return Resource.prototype.post.call(this, data);
+ };
+ return new PostQueue(
+ poster,
+ timestamp,
+ this._service.serverConfiguration || {},
+ log,
+ postCallback
+ );
+ },
+};
+
+Object.setPrototypeOf(Collection.prototype, Resource.prototype);
+
+// These are limits for requests provided by the server at the
+// info/configuration endpoint -- server documentation is available here:
+// http://moz-services-docs.readthedocs.io/en/latest/storage/apis-1.5.html#api-instructions
+//
+// All are optional, however we synthesize (non-infinite) default values for the
+// "max_request_bytes" and "max_record_payload_bytes" options. For the others,
+// we ignore them (we treat the limit is infinite) if they're missing.
+//
+// These are also the only ones that all servers (even batching-disabled
+// servers) should support, at least once this sync-serverstorage patch is
+// everywhere https://github.com/mozilla-services/server-syncstorage/pull/74
+//
+// Batching enabled servers also limit the amount of payload data and number
+// of and records we can send in a single post as well as in the whole batch.
+// Note that the byte limits for these there are just with respect to the
+// *payload* data, e.g. the data appearing in the payload property (a
+// string) of the object.
+//
+// Note that in practice, these limits should be sensible, but the code makes
+// no assumptions about this. If we hit any of the limits, we perform the
+// corresponding action (e.g. submit a request, possibly committing the
+// current batch).
+const DefaultPostQueueConfig = Object.freeze({
+ // Number of total bytes allowed in a request
+ max_request_bytes: 260 * 1024,
+
+ // Maximum number of bytes allowed in the "payload" property of a record.
+ max_record_payload_bytes: 256 * 1024,
+
+ // The limit for how many bytes worth of data appearing in "payload"
+ // properties are allowed in a single post.
+ max_post_bytes: Infinity,
+
+ // The limit for the number of records allowed in a single post.
+ max_post_records: Infinity,
+
+ // The limit for how many bytes worth of data appearing in "payload"
+ // properties are allowed in a batch. (Same as max_post_bytes, but for
+ // batches).
+ max_total_bytes: Infinity,
+
+ // The limit for the number of records allowed in a single post. (Same
+ // as max_post_records, but for batches).
+ max_total_records: Infinity,
+});
+
+// Manages a pair of (byte, count) limits for a PostQueue, such as
+// (max_post_bytes, max_post_records) or (max_total_bytes, max_total_records).
+class LimitTracker {
+ constructor(maxBytes, maxRecords) {
+ this.maxBytes = maxBytes;
+ this.maxRecords = maxRecords;
+ this.curBytes = 0;
+ this.curRecords = 0;
+ }
+
+ clear() {
+ this.curBytes = 0;
+ this.curRecords = 0;
+ }
+
+ canAddRecord(payloadSize) {
+ // The record counts are inclusive, but depending on the version of the
+ // server, the byte counts may or may not be inclusive (See
+ // https://github.com/mozilla-services/server-syncstorage/issues/73).
+ return (
+ this.curRecords + 1 <= this.maxRecords &&
+ this.curBytes + payloadSize < this.maxBytes
+ );
+ }
+
+ canNeverAdd(recordSize) {
+ return recordSize >= this.maxBytes;
+ }
+
+ didAddRecord(recordSize) {
+ if (!this.canAddRecord(recordSize)) {
+ // This is a bug, caller is expected to call canAddRecord first.
+ throw new Error(
+ "LimitTracker.canAddRecord must be checked before adding record"
+ );
+ }
+ this.curRecords += 1;
+ this.curBytes += recordSize;
+ }
+}
+
+/* A helper to manage the posting of records while respecting the various
+ size limits.
+
+ This supports the concept of a server-side "batch". The general idea is:
+ * We queue as many records as allowed in memory, then make a single POST.
+ * This first POST (optionally) gives us a batch ID, which we use for
+ all subsequent posts, until...
+ * At some point we hit a batch-maximum, and jump through a few hoops to
+ commit the current batch (ie, all previous POSTs) and start a new one.
+ * Eventually commit the final batch.
+
+ In most cases we expect there to be exactly 1 batch consisting of possibly
+ multiple POSTs.
+*/
+export function PostQueue(poster, timestamp, serverConfig, log, postCallback) {
+ // The "post" function we should use when it comes time to do the post.
+ this.poster = poster;
+ this.log = log;
+
+ let config = Object.assign({}, DefaultPostQueueConfig, serverConfig);
+
+ if (!serverConfig.max_request_bytes && serverConfig.max_post_bytes) {
+ // Use max_post_bytes for max_request_bytes if it's missing. Only needed
+ // until server-syncstorage/pull/74 is everywhere, and even then it's
+ // unnecessary if the server limits are configured sanely (there's no
+ // guarantee of -- at least before that is fully deployed)
+ config.max_request_bytes = serverConfig.max_post_bytes;
+ }
+
+ this.log.trace("new PostQueue config (after defaults): ", config);
+
+ // The callback we make with the response when we do get around to making the
+ // post (which could be during any of the enqueue() calls or the final flush())
+ // This callback may be called multiple times and must not add new items to
+ // the queue.
+ // The second argument passed to this callback is a boolean value that is true
+ // if we're in the middle of a batch, and false if either the batch is
+ // complete, or it's a post to a server that does not understand batching.
+ this.postCallback = postCallback;
+
+ // Tracks the count and combined payload size for the records we've queued
+ // so far but are yet to POST.
+ this.postLimits = new LimitTracker(
+ config.max_post_bytes,
+ config.max_post_records
+ );
+
+ // As above, but for the batch size.
+ this.batchLimits = new LimitTracker(
+ config.max_total_bytes,
+ config.max_total_records
+ );
+
+ // Limit for the size of `this.queued` before we do a post.
+ this.maxRequestBytes = config.max_request_bytes;
+
+ // Limit for the size of incoming record payloads.
+ this.maxPayloadBytes = config.max_record_payload_bytes;
+
+ // The string where we are capturing the stringified version of the records
+ // queued so far. It will always be invalid JSON as it is always missing the
+ // closing bracket. It's also used to track whether or not we've gone past
+ // maxRequestBytes.
+ this.queued = "";
+
+ // The ID of our current batch. Can be undefined (meaning we are yet to make
+ // the first post of a patch, so don't know if we have a batch), null (meaning
+ // we've made the first post but the server response indicated no batching
+ // semantics), otherwise we have made the first post and it holds the batch ID
+ // returned from the server.
+ this.batchID = undefined;
+
+ // Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET.
+ this.lastModified = timestamp;
+}
+
+PostQueue.prototype = {
+ async enqueue(record) {
+ // We want to ensure the record has a .toJSON() method defined - even
+ // though JSON.stringify() would implicitly call it, the stringify might
+ // still work even if it isn't defined, which isn't what we want.
+ let jsonRepr = record.toJSON();
+ if (!jsonRepr) {
+ throw new Error(
+ "You must only call this with objects that explicitly support JSON"
+ );
+ }
+
+ let bytes = JSON.stringify(jsonRepr);
+
+ // We use the payload size for the LimitTrackers, since that's what the
+ // byte limits other than max_request_bytes refer to.
+ let payloadLength = jsonRepr.payload.length;
+
+ // The `+ 2` is to account for the 2-byte (maximum) overhead (one byte for
+ // the leading comma or "[", which all records will have, and the other for
+ // the final trailing "]", only present for the last record).
+ let encodedLength = bytes.length + 2;
+
+ // Check first if there's some limit that indicates we cannot ever enqueue
+ // this record.
+ let isTooBig =
+ this.postLimits.canNeverAdd(payloadLength) ||
+ this.batchLimits.canNeverAdd(payloadLength) ||
+ encodedLength >= this.maxRequestBytes ||
+ payloadLength >= this.maxPayloadBytes;
+
+ if (isTooBig) {
+ return {
+ enqueued: false,
+ error: new Error("Single record too large to submit to server"),
+ };
+ }
+
+ let canPostRecord = this.postLimits.canAddRecord(payloadLength);
+ let canBatchRecord = this.batchLimits.canAddRecord(payloadLength);
+ let canSendRecord =
+ this.queued.length + encodedLength < this.maxRequestBytes;
+
+ if (!canPostRecord || !canBatchRecord || !canSendRecord) {
+ this.log.trace("PostQueue flushing: ", {
+ canPostRecord,
+ canSendRecord,
+ canBatchRecord,
+ });
+ // We need to write the queue out before handling this one, but we only
+ // commit the batch (and thus start a new one) if the record couldn't fit
+ // inside the batch.
+ await this.flush(!canBatchRecord);
+ }
+
+ this.postLimits.didAddRecord(payloadLength);
+ this.batchLimits.didAddRecord(payloadLength);
+
+ // Either a ',' or a '[' depending on whether this is the first record.
+ this.queued += this.queued.length ? "," : "[";
+ this.queued += bytes;
+ return { enqueued: true };
+ },
+
+ async flush(finalBatchPost) {
+ if (!this.queued) {
+ // nothing queued - we can't be in a batch, and something has gone very
+ // bad if we think we are.
+ if (this.batchID) {
+ throw new Error(
+ `Flush called when no queued records but we are in a batch ${this.batchID}`
+ );
+ }
+ return;
+ }
+ // the batch query-param and headers we'll send.
+ let batch;
+ let headers = [];
+ if (this.batchID === undefined) {
+ // First commit in a (possible) batch.
+ batch = "true";
+ } else if (this.batchID) {
+ // We have an existing batch.
+ batch = this.batchID;
+ } else {
+ // Not the first post and we know we have no batch semantics.
+ batch = null;
+ }
+
+ headers.push(["x-if-unmodified-since", this.lastModified]);
+
+ let numQueued = this.postLimits.curRecords;
+ this.log.info(
+ `Posting ${numQueued} records of ${
+ this.queued.length + 1
+ } bytes with batch=${batch}`
+ );
+ let queued = this.queued + "]";
+ if (finalBatchPost) {
+ this.batchLimits.clear();
+ }
+ this.postLimits.clear();
+ this.queued = "";
+ let response = await this.poster(
+ queued,
+ headers,
+ batch,
+ !!(finalBatchPost && this.batchID !== null)
+ );
+
+ if (!response.success) {
+ this.log.trace("Server error response during a batch", response);
+ // not clear what we should do here - we expect the consumer of this to
+ // abort by throwing in the postCallback below.
+ await this.postCallback(this, response, !finalBatchPost);
+ return;
+ }
+
+ if (finalBatchPost) {
+ this.log.trace("Committed batch", this.batchID);
+ this.batchID = undefined; // we are now in "first post for the batch" state.
+ this.lastModified = response.headers["x-last-modified"];
+ await this.postCallback(this, response, false);
+ return;
+ }
+
+ if (response.status != 202) {
+ if (this.batchID) {
+ throw new Error(
+ "Server responded non-202 success code while a batch was in progress"
+ );
+ }
+ this.batchID = null; // no batch semantics are in place.
+ this.lastModified = response.headers["x-last-modified"];
+ await this.postCallback(this, response, false);
+ return;
+ }
+
+ // this response is saying the server has batch semantics - we should
+ // always have a batch ID in the response.
+ let responseBatchID = response.obj.batch;
+ this.log.trace("Server responsed 202 with batch", responseBatchID);
+ if (!responseBatchID) {
+ this.log.error(
+ "Invalid server response: 202 without a batch ID",
+ response
+ );
+ throw new Error("Invalid server response: 202 without a batch ID");
+ }
+
+ if (this.batchID === undefined) {
+ this.batchID = responseBatchID;
+ if (!this.lastModified) {
+ this.lastModified = response.headers["x-last-modified"];
+ if (!this.lastModified) {
+ throw new Error("Batch response without x-last-modified");
+ }
+ }
+ }
+
+ if (this.batchID != responseBatchID) {
+ throw new Error(
+ `Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`
+ );
+ }
+
+ await this.postCallback(this, response, true);
+ },
+};
diff --git a/services/sync/modules/resource.sys.mjs b/services/sync/modules/resource.sys.mjs
new file mode 100644
index 0000000000..be7815d534
--- /dev/null
+++ b/services/sync/modules/resource.sys.mjs
@@ -0,0 +1,292 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Observers } from "resource://services-common/observers.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { Utils } from "resource://services-sync/util.sys.mjs";
+import { setTimeout, clearTimeout } from "resource://gre/modules/Timer.sys.mjs";
+
+/* global AbortController */
+
+/*
+ * Resource represents a remote network resource, identified by a URI.
+ * Create an instance like so:
+ *
+ * let resource = new Resource("http://foobar.com/path/to/resource");
+ *
+ * The 'resource' object has the following methods to issue HTTP requests
+ * of the corresponding HTTP methods:
+ *
+ * get(callback)
+ * put(data, callback)
+ * post(data, callback)
+ * delete(callback)
+ */
+export function Resource(uri) {
+ this._log = Log.repository.getLogger(this._logName);
+ this._log.manageLevelFromPref("services.sync.log.logger.network.resources");
+ this.uri = uri;
+ this._headers = {};
+}
+
+// (static) Caches the latest server timestamp (X-Weave-Timestamp header).
+Resource.serverTime = null;
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ Resource,
+ "SEND_VERSION_INFO",
+ "services.sync.sendVersionInfo",
+ true
+);
+Resource.prototype = {
+ _logName: "Sync.Resource",
+
+ /**
+ * Callback to be invoked at request time to add authentication details.
+ * If the callback returns a promise, it will be awaited upon.
+ *
+ * By default, a global authenticator is provided. If this is set, it will
+ * be used instead of the global one.
+ */
+ authenticator: null,
+
+ // Wait 5 minutes before killing a request.
+ ABORT_TIMEOUT: 300000,
+
+ // Headers to be included when making a request for the resource.
+ // Note: Header names should be all lower case, there's no explicit
+ // check for duplicates due to case!
+ get headers() {
+ return this._headers;
+ },
+ set headers(_) {
+ throw new Error("headers can't be mutated directly. Please use setHeader.");
+ },
+ setHeader(header, value) {
+ this._headers[header.toLowerCase()] = value;
+ },
+
+ // URI representing this resource.
+ get uri() {
+ return this._uri;
+ },
+ set uri(value) {
+ if (typeof value == "string") {
+ this._uri = CommonUtils.makeURI(value);
+ } else {
+ this._uri = value;
+ }
+ },
+
+ // Get the string representation of the URI.
+ get spec() {
+ if (this._uri) {
+ return this._uri.spec;
+ }
+ return null;
+ },
+
+ /**
+ * @param {string} method HTTP method
+ * @returns {Headers}
+ */
+ async _buildHeaders(method) {
+ const headers = new Headers(this._headers);
+
+ if (Resource.SEND_VERSION_INFO) {
+ headers.append("user-agent", Utils.userAgent);
+ }
+
+ if (this.authenticator) {
+ const result = await this.authenticator(this, method);
+ if (result && result.headers) {
+ for (const [k, v] of Object.entries(result.headers)) {
+ headers.append(k.toLowerCase(), v);
+ }
+ }
+ } else {
+ this._log.debug("No authenticator found.");
+ }
+
+ // PUT and POST are treated differently because they have payload data.
+ if (("PUT" == method || "POST" == method) && !headers.has("content-type")) {
+ headers.append("content-type", "text/plain");
+ }
+
+ if (this._log.level <= Log.Level.Trace) {
+ for (const [k, v] of headers) {
+ if (k == "authorization" || k == "x-client-state") {
+ this._log.trace(`HTTP Header ${k}: ***** (suppressed)`);
+ } else {
+ this._log.trace(`HTTP Header ${k}: ${v}`);
+ }
+ }
+ }
+
+ if (!headers.has("accept")) {
+ headers.append("accept", "application/json;q=0.9,*/*;q=0.2");
+ }
+
+ return headers;
+ },
+
+ /**
+ * @param {string} method HTTP method
+ * @param {string} data HTTP body
+ * @param {object} signal AbortSignal instance
+ * @returns {Request}
+ */
+ async _createRequest(method, data, signal) {
+ const headers = await this._buildHeaders(method);
+ const init = {
+ cache: "no-store", // No cache.
+ headers,
+ method,
+ signal,
+ mozErrors: true, // Return nsresult error codes instead of a generic
+ // NetworkError when fetch rejects.
+ };
+
+ if (data) {
+ if (!(typeof data == "string" || data instanceof String)) {
+ data = JSON.stringify(data);
+ }
+ this._log.debug(`${method} Length: ${data.length}`);
+ this._log.trace(`${method} Body: ${data}`);
+ init.body = data;
+ }
+ return new Request(this.uri.spec, init);
+ },
+
+ /**
+ * @param {string} method HTTP method
+ * @param {string} [data] HTTP body
+ * @returns {Response}
+ */
+ async _doRequest(method, data = null) {
+ const controller = new AbortController();
+ const request = await this._createRequest(method, data, controller.signal);
+ const responsePromise = fetch(request); // Rejects on network failure.
+ let didTimeout = false;
+ const timeoutId = setTimeout(() => {
+ didTimeout = true;
+ this._log.error(
+ `Request timed out after ${this.ABORT_TIMEOUT}ms. Aborting.`
+ );
+ controller.abort();
+ }, this.ABORT_TIMEOUT);
+ let response;
+ try {
+ response = await responsePromise;
+ } catch (e) {
+ this._log.warn(`${method} request to ${this.uri.spec} failed`, e);
+ if (!didTimeout) {
+ throw e;
+ }
+ throw Components.Exception(
+ "Request aborted (timeout)",
+ Cr.NS_ERROR_NET_TIMEOUT
+ );
+ } finally {
+ clearTimeout(timeoutId);
+ }
+ return this._processResponse(response, method);
+ },
+
+ async _processResponse(response, method) {
+ const data = await response.text();
+ this._logResponse(response, method, data);
+ this._processResponseHeaders(response);
+
+ const ret = {
+ data,
+ url: response.url,
+ status: response.status,
+ success: response.ok,
+ headers: {},
+ };
+ for (const [k, v] of response.headers) {
+ ret.headers[k] = v;
+ }
+
+ // Make a lazy getter to convert the json response into an object.
+ // Note that this can cause a parse error to be thrown far away from the
+ // actual fetch, so be warned!
+ XPCOMUtils.defineLazyGetter(ret, "obj", () => {
+ try {
+ return JSON.parse(ret.data);
+ } catch (ex) {
+ this._log.warn("Got exception parsing response body", ex);
+ // Stringify to avoid possibly printing non-printable characters.
+ this._log.debug(
+ "Parse fail: Response body starts",
+ (ret.data + "").slice(0, 100)
+ );
+ throw ex;
+ }
+ });
+
+ return ret;
+ },
+
+ _logResponse(response, method, data) {
+ const { status, ok: success, url } = response;
+
+ // Log the status of the request.
+ this._log.debug(
+ `${method} ${success ? "success" : "fail"} ${status} ${url}`
+ );
+
+ // Additionally give the full response body when Trace logging.
+ if (this._log.level <= Log.Level.Trace) {
+ this._log.trace(`${method} body`, data);
+ }
+
+ if (!success) {
+ this._log.warn(
+ `${method} request to ${url} failed with status ${status}`
+ );
+ }
+ },
+
+ _processResponseHeaders({ headers, ok: success }) {
+ if (headers.has("x-weave-timestamp")) {
+ Resource.serverTime = parseFloat(headers.get("x-weave-timestamp"));
+ }
+ // This is a server-side safety valve to allow slowing down
+ // clients without hurting performance.
+ if (headers.has("x-weave-backoff")) {
+ let backoff = headers.get("x-weave-backoff");
+ this._log.debug(`Got X-Weave-Backoff: ${backoff}`);
+ Observers.notify("weave:service:backoff:interval", parseInt(backoff, 10));
+ }
+
+ if (success && headers.has("x-weave-quota-remaining")) {
+ Observers.notify(
+ "weave:service:quota:remaining",
+ parseInt(headers.get("x-weave-quota-remaining"), 10)
+ );
+ }
+ },
+
+ get() {
+ return this._doRequest("GET");
+ },
+
+ put(data) {
+ return this._doRequest("PUT", data);
+ },
+
+ post(data) {
+ return this._doRequest("POST", data);
+ },
+
+ delete() {
+ return this._doRequest("DELETE");
+ },
+};
diff --git a/services/sync/modules/service.sys.mjs b/services/sync/modules/service.sys.mjs
new file mode 100644
index 0000000000..7204c18434
--- /dev/null
+++ b/services/sync/modules/service.sys.mjs
@@ -0,0 +1,1630 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const CRYPTO_COLLECTION = "crypto";
+const KEYS_WBO = "keys";
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+
+import {
+ CLIENT_NOT_CONFIGURED,
+ CREDENTIALS_CHANGED,
+ HMAC_EVENT_INTERVAL,
+ LOGIN_FAILED,
+ LOGIN_FAILED_INVALID_PASSPHRASE,
+ LOGIN_FAILED_NETWORK_ERROR,
+ LOGIN_FAILED_NO_PASSPHRASE,
+ LOGIN_FAILED_NO_USERNAME,
+ LOGIN_FAILED_SERVER_ERROR,
+ LOGIN_SUCCEEDED,
+ MASTER_PASSWORD_LOCKED,
+ METARECORD_DOWNLOAD_FAIL,
+ NO_SYNC_NODE_FOUND,
+ PREFS_BRANCH,
+ STATUS_DISABLED,
+ STATUS_OK,
+ STORAGE_VERSION,
+ VERSION_OUT_OF_DATE,
+ WEAVE_VERSION,
+ kFirefoxShuttingDown,
+ kFirstSyncChoiceNotMade,
+ kSyncBackoffNotMet,
+ kSyncMasterPasswordLocked,
+ kSyncNetworkOffline,
+ kSyncNotConfigured,
+ kSyncWeaveDisabled,
+} from "resource://services-sync/constants.sys.mjs";
+
+import { EngineManager } from "resource://services-sync/engines.sys.mjs";
+import { ClientEngine } from "resource://services-sync/engines/clients.sys.mjs";
+import { Weave } from "resource://services-sync/main.sys.mjs";
+import {
+ ErrorHandler,
+ SyncScheduler,
+} from "resource://services-sync/policies.sys.mjs";
+import {
+ CollectionKeyManager,
+ CryptoWrapper,
+ RecordManager,
+ WBORecord,
+} from "resource://services-sync/record.sys.mjs";
+import { Resource } from "resource://services-sync/resource.sys.mjs";
+import { EngineSynchronizer } from "resource://services-sync/stages/enginesync.sys.mjs";
+import { DeclinedEngines } from "resource://services-sync/stages/declined.sys.mjs";
+import { Status } from "resource://services-sync/status.sys.mjs";
+
+ChromeUtils.importESModule("resource://services-sync/telemetry.sys.mjs");
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs";
+
+const fxAccounts = getFxAccountsSingleton();
+
+function getEngineModules() {
+ let result = {
+ Addons: { module: "addons.js", symbol: "AddonsEngine" },
+ Password: { module: "passwords.js", symbol: "PasswordEngine" },
+ Prefs: { module: "prefs.js", symbol: "PrefsEngine" },
+ };
+ if (AppConstants.MOZ_APP_NAME != "thunderbird") {
+ result.Bookmarks = { module: "bookmarks.js", symbol: "BookmarksEngine" };
+ result.Form = { module: "forms.js", symbol: "FormEngine" };
+ result.History = { module: "history.js", symbol: "HistoryEngine" };
+ result.Tab = { module: "tabs.js", symbol: "TabEngine" };
+ }
+ if (Svc.Prefs.get("engine.addresses.available", false)) {
+ result.Addresses = {
+ module: "resource://autofill/FormAutofillSync.jsm",
+ symbol: "AddressesEngine",
+ };
+ }
+ if (Svc.Prefs.get("engine.creditcards.available", false)) {
+ result.CreditCards = {
+ module: "resource://autofill/FormAutofillSync.jsm",
+ symbol: "CreditCardsEngine",
+ };
+ }
+ result["Extension-Storage"] = {
+ module: "extension-storage.js",
+ controllingPref: "webextensions.storage.sync.kinto",
+ whenTrue: "ExtensionStorageEngineKinto",
+ whenFalse: "ExtensionStorageEngineBridge",
+ };
+ return result;
+}
+
+const lazy = {};
+
+// A unique identifier for this browser session. Used for logging so
+// we can easily see whether 2 logs are in the same browser session or
+// after the browser restarted.
+XPCOMUtils.defineLazyGetter(lazy, "browserSessionID", Utils.makeGUID);
+
+function Sync11Service() {
+ this._notify = Utils.notify("weave:service:");
+ Utils.defineLazyIDProperty(this, "syncID", "services.sync.client.syncID");
+}
+Sync11Service.prototype = {
+ _lock: Utils.lock,
+ _locked: false,
+ _loggedIn: false,
+
+ infoURL: null,
+ storageURL: null,
+ metaURL: null,
+ cryptoKeyURL: null,
+ // The cluster URL comes via the identity object, which in the FxA
+ // world is ebbedded in the token returned from the token server.
+ _clusterURL: null,
+
+ get clusterURL() {
+ return this._clusterURL || "";
+ },
+ set clusterURL(value) {
+ if (value != null && typeof value != "string") {
+ throw new Error("cluster must be a string, got " + typeof value);
+ }
+ this._clusterURL = value;
+ this._updateCachedURLs();
+ },
+
+ get isLoggedIn() {
+ return this._loggedIn;
+ },
+
+ get locked() {
+ return this._locked;
+ },
+ lock: function lock() {
+ if (this._locked) {
+ return false;
+ }
+ this._locked = true;
+ return true;
+ },
+ unlock: function unlock() {
+ this._locked = false;
+ },
+
+ // A specialized variant of Utils.catch.
+ // This provides a more informative error message when we're already syncing:
+ // see Bug 616568.
+ _catch(func) {
+ function lockExceptions(ex) {
+ if (Utils.isLockException(ex)) {
+ // This only happens if we're syncing already.
+ this._log.info("Cannot start sync: already syncing?");
+ }
+ }
+
+ return Utils.catch.call(this, func, lockExceptions);
+ },
+
+ get userBaseURL() {
+ // The user URL is the cluster URL.
+ return this.clusterURL;
+ },
+
+ _updateCachedURLs: function _updateCachedURLs() {
+ // Nothing to cache yet if we don't have the building blocks
+ if (!this.clusterURL) {
+ // Also reset all other URLs used by Sync to ensure we aren't accidentally
+ // using one cached earlier - if there's no cluster URL any cached ones
+ // are invalid.
+ this.infoURL = undefined;
+ this.storageURL = undefined;
+ this.metaURL = undefined;
+ this.cryptoKeysURL = undefined;
+ return;
+ }
+
+ this._log.debug(
+ "Caching URLs under storage user base: " + this.userBaseURL
+ );
+
+ // Generate and cache various URLs under the storage API for this user
+ this.infoURL = this.userBaseURL + "info/collections";
+ this.storageURL = this.userBaseURL + "storage/";
+ this.metaURL = this.storageURL + "meta/global";
+ this.cryptoKeysURL = this.storageURL + CRYPTO_COLLECTION + "/" + KEYS_WBO;
+ },
+
+ _checkCrypto: function _checkCrypto() {
+ let ok = false;
+
+ try {
+ let iv = Weave.Crypto.generateRandomIV();
+ if (iv.length == 24) {
+ ok = true;
+ }
+ } catch (e) {
+ this._log.debug("Crypto check failed: " + e);
+ }
+
+ return ok;
+ },
+
+ /**
+ * Here is a disgusting yet reasonable way of handling HMAC errors deep in
+ * the guts of Sync. The astute reader will note that this is a hacky way of
+ * implementing something like continuable conditions.
+ *
+ * A handler function is glued to each engine. If the engine discovers an
+ * HMAC failure, we fetch keys from the server and update our keys, just as
+ * we would on startup.
+ *
+ * If our key collection changed, we signal to the engine (via our return
+ * value) that it should retry decryption.
+ *
+ * If our key collection did not change, it means that we already had the
+ * correct keys... and thus a different client has the wrong ones. Reupload
+ * the bundle that we fetched, which will bump the modified time on the
+ * server and (we hope) prompt a broken client to fix itself.
+ *
+ * We keep track of the time at which we last applied this reasoning, because
+ * thrashing doesn't solve anything. We keep a reasonable interval between
+ * these remedial actions.
+ */
+ lastHMACEvent: 0,
+
+ /*
+ * Returns whether to try again.
+ */
+ async handleHMACEvent() {
+ let now = Date.now();
+
+ // Leave a sizable delay between HMAC recovery attempts. This gives us
+ // time for another client to fix themselves if we touch the record.
+ if (now - this.lastHMACEvent < HMAC_EVENT_INTERVAL) {
+ return false;
+ }
+
+ this._log.info(
+ "Bad HMAC event detected. Attempting recovery " +
+ "or signaling to other clients."
+ );
+
+ // Set the last handled time so that we don't act again.
+ this.lastHMACEvent = now;
+
+ // Fetch keys.
+ let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
+ try {
+ let cryptoResp = (
+ await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))
+ ).response;
+
+ // Save out the ciphertext for when we reupload. If there's a bug in
+ // CollectionKeyManager, this will prevent us from uploading junk.
+ let cipherText = cryptoKeys.ciphertext;
+
+ if (!cryptoResp.success) {
+ this._log.warn("Failed to download keys.");
+ return false;
+ }
+
+ let keysChanged = await this.handleFetchedKeys(
+ this.identity.syncKeyBundle,
+ cryptoKeys,
+ true
+ );
+ if (keysChanged) {
+ // Did they change? If so, carry on.
+ this._log.info("Suggesting retry.");
+ return true; // Try again.
+ }
+
+ // If not, reupload them and continue the current sync.
+ cryptoKeys.ciphertext = cipherText;
+ cryptoKeys.cleartext = null;
+
+ let uploadResp = await this._uploadCryptoKeys(
+ cryptoKeys,
+ cryptoResp.obj.modified
+ );
+ if (uploadResp.success) {
+ this._log.info("Successfully re-uploaded keys. Continuing sync.");
+ } else {
+ this._log.warn(
+ "Got error response re-uploading keys. " +
+ "Continuing sync; let's try again later."
+ );
+ }
+
+ return false; // Don't try again: same keys.
+ } catch (ex) {
+ this._log.warn(
+ "Got exception fetching and handling crypto keys. " +
+ "Will try again later.",
+ ex
+ );
+ return false;
+ }
+ },
+
+ async handleFetchedKeys(syncKey, cryptoKeys, skipReset) {
+ // Don't want to wipe if we're just starting up!
+ let wasBlank = this.collectionKeys.isClear;
+ let keysChanged = await this.collectionKeys.updateContents(
+ syncKey,
+ cryptoKeys
+ );
+
+ if (keysChanged && !wasBlank) {
+ this._log.debug("Keys changed: " + JSON.stringify(keysChanged));
+
+ if (!skipReset) {
+ this._log.info("Resetting client to reflect key change.");
+
+ if (keysChanged.length) {
+ // Collection keys only. Reset individual engines.
+ await this.resetClient(keysChanged);
+ } else {
+ // Default key changed: wipe it all.
+ await this.resetClient();
+ }
+
+ this._log.info("Downloaded new keys, client reset. Proceeding.");
+ }
+ return true;
+ }
+ return false;
+ },
+
+ /**
+ * Prepare to initialize the rest of Weave after waiting a little bit
+ */
+ async onStartup() {
+ this.status = Status;
+ this.identity = Status._authManager;
+ this.collectionKeys = new CollectionKeyManager();
+
+ this.scheduler = new SyncScheduler(this);
+ this.errorHandler = new ErrorHandler(this);
+
+ this._log = Log.repository.getLogger("Sync.Service");
+ this._log.manageLevelFromPref("services.sync.log.logger.service.main");
+
+ this._log.info("Loading Weave " + WEAVE_VERSION);
+
+ this.recordManager = new RecordManager(this);
+
+ this.enabled = true;
+
+ await this._registerEngines();
+
+ let ua = Cc["@mozilla.org/network/protocol;1?name=http"].getService(
+ Ci.nsIHttpProtocolHandler
+ ).userAgent;
+ this._log.info(ua);
+
+ if (!this._checkCrypto()) {
+ this.enabled = false;
+ this._log.info(
+ "Could not load the Weave crypto component. Disabling " +
+ "Weave, since it will not work correctly."
+ );
+ }
+
+ Svc.Obs.add("weave:service:setup-complete", this);
+ Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon
+ Svc.Obs.add("fxaccounts:device_disconnected", this);
+ Services.prefs.addObserver(PREFS_BRANCH + "engine.", this);
+
+ if (!this.enabled) {
+ this._log.info("Firefox Sync disabled.");
+ }
+
+ this._updateCachedURLs();
+
+ let status = this._checkSetup();
+ if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) {
+ this._startTracking();
+ }
+
+ // Send an event now that Weave service is ready. We don't do this
+ // synchronously so that observers can import this module before
+ // registering an observer.
+ CommonUtils.nextTick(() => {
+ this.status.ready = true;
+
+ // UI code uses the flag on the XPCOM service so it doesn't have
+ // to load a bunch of modules.
+ let xps = Cc["@mozilla.org/weave/service;1"].getService(
+ Ci.nsISupports
+ ).wrappedJSObject;
+ xps.ready = true;
+
+ Svc.Obs.notify("weave:service:ready");
+ });
+ },
+
+ _checkSetup: function _checkSetup() {
+ if (!this.enabled) {
+ return (this.status.service = STATUS_DISABLED);
+ }
+ return this.status.checkSetup();
+ },
+
+ /**
+ * Register the built-in engines for certain applications
+ */
+ async _registerEngines() {
+ this.engineManager = new EngineManager(this);
+
+ let engineModules = getEngineModules();
+
+ let engines = [];
+ // We allow a pref, which has no default value, to limit the engines
+ // which are registered. We expect only tests will use this.
+ if (Svc.Prefs.has("registerEngines")) {
+ engines = Svc.Prefs.get("registerEngines").split(",");
+ this._log.info("Registering custom set of engines", engines);
+ } else {
+ // default is all engines.
+ engines = Object.keys(engineModules);
+ }
+
+ let declined = [];
+ let pref = Svc.Prefs.get("declinedEngines");
+ if (pref) {
+ declined = pref.split(",");
+ }
+
+ let clientsEngine = new ClientEngine(this);
+ // Ideally clientsEngine should not exist
+ // (or be a promise that calls initialize() before returning the engine)
+ await clientsEngine.initialize();
+ this.clientsEngine = clientsEngine;
+
+ for (let name of engines) {
+ if (!(name in engineModules)) {
+ this._log.info("Do not know about engine: " + name);
+ continue;
+ }
+ let modInfo = engineModules[name];
+ if (!modInfo.module.includes(":")) {
+ modInfo.module = "resource://services-sync/engines/" + modInfo.module;
+ }
+ try {
+ let ns = ChromeUtils.import(modInfo.module);
+ if (modInfo.symbol) {
+ let symbol = modInfo.symbol;
+ if (!(symbol in ns)) {
+ this._log.warn(
+ "Could not find exported engine instance: " + symbol
+ );
+ continue;
+ }
+ await this.engineManager.register(ns[symbol]);
+ } else {
+ let { whenTrue, whenFalse, controllingPref } = modInfo;
+ if (!(whenTrue in ns) || !(whenFalse in ns)) {
+ this._log.warn("Could not find all exported engine instances", {
+ whenTrue,
+ whenFalse,
+ });
+ continue;
+ }
+ await this.engineManager.registerAlternatives(
+ name.toLowerCase(),
+ controllingPref,
+ ns[whenTrue],
+ ns[whenFalse]
+ );
+ }
+ } catch (ex) {
+ this._log.warn("Could not register engine " + name, ex);
+ }
+ }
+
+ this.engineManager.setDeclined(declined);
+ },
+
+ /**
+ * This method updates the local engines state from an existing meta/global
+ * when Sync is disabled.
+ * Running this code if sync is enabled would end up in very weird results
+ * (but we're nice and we check before doing anything!).
+ */
+ async updateLocalEnginesState() {
+ await this.promiseInitialized;
+
+ // Sanity check, this method is not meant to be run if Sync is enabled!
+ if (Svc.Prefs.get("username", "")) {
+ throw new Error("Sync is enabled!");
+ }
+
+ // For historical reasons the behaviour of setCluster() is bizarre,
+ // so just check what we care about - the meta URL.
+ if (!this.metaURL) {
+ await this.identity.setCluster();
+ if (!this.metaURL) {
+ this._log.warn("Could not find a cluster.");
+ return;
+ }
+ }
+ // Clear the cache so we always fetch the latest meta/global.
+ this.recordManager.clearCache();
+ let meta = await this.recordManager.get(this.metaURL);
+ if (!meta) {
+ this._log.info("Meta record is null, aborting engine state update.");
+ return;
+ }
+ const declinedEngines = meta.payload.declined;
+ const allEngines = this.engineManager.getAll().map(e => e.name);
+ // We don't want our observer of the enabled prefs to treat the change as
+ // a user-change, otherwise we will do the wrong thing with declined etc.
+ this._ignorePrefObserver = true;
+ try {
+ for (const engine of allEngines) {
+ Svc.Prefs.set(`engine.${engine}`, !declinedEngines.includes(engine));
+ }
+ } finally {
+ this._ignorePrefObserver = false;
+ }
+ },
+
+ QueryInterface: ChromeUtils.generateQI([
+ "nsIObserver",
+ "nsISupportsWeakReference",
+ ]),
+
+ observe(subject, topic, data) {
+ switch (topic) {
+ // Ideally this observer should be in the SyncScheduler, but it would require
+ // some work to know about the sync specific engines. We should move this there once it does.
+ case "sync:collection_changed":
+ // We check if we're running TPS here to avoid TPS failing because it
+ // couldn't get to get the sync lock, due to us currently syncing the
+ // clients engine.
+ if (data.includes("clients") && !Svc.Prefs.get("testing.tps", false)) {
+ // Sync in the background (it's fine not to wait on the returned promise
+ // because sync() has a lock).
+ // [] = clients collection only
+ this.sync({ why: "collection_changed", engines: [] }).catch(e => {
+ this._log.error(e);
+ });
+ }
+ break;
+ case "fxaccounts:device_disconnected":
+ data = JSON.parse(data);
+ if (!data.isLocalDevice) {
+ // Refresh the known stale clients list in the background.
+ this.clientsEngine.updateKnownStaleClients().catch(e => {
+ this._log.error(e);
+ });
+ }
+ break;
+ case "weave:service:setup-complete":
+ let status = this._checkSetup();
+ if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) {
+ this._startTracking();
+ }
+ break;
+ case "nsPref:changed":
+ if (this._ignorePrefObserver) {
+ return;
+ }
+ const engine = data.slice((PREFS_BRANCH + "engine.").length);
+ if (engine.includes(".")) {
+ // A sub-preference of the engine was changed. For example
+ // `services.sync.engine.bookmarks.validation.percentageChance`.
+ return;
+ }
+ this._handleEngineStatusChanged(engine);
+ break;
+ }
+ },
+
+ _handleEngineStatusChanged(engine) {
+ this._log.trace("Status for " + engine + " engine changed.");
+ if (Svc.Prefs.get("engineStatusChanged." + engine, false)) {
+ // The enabled status being changed back to what it was before.
+ Svc.Prefs.reset("engineStatusChanged." + engine);
+ } else {
+ // Remember that the engine status changed locally until the next sync.
+ Svc.Prefs.set("engineStatusChanged." + engine, true);
+ }
+ },
+
+ _startTracking() {
+ const engines = [this.clientsEngine, ...this.engineManager.getAll()];
+ for (let engine of engines) {
+ try {
+ engine.startTracking();
+ } catch (e) {
+ this._log.error(`Could not start ${engine.name} engine tracker`, e);
+ }
+ }
+ // This is for TPS. We should try to do better.
+ Svc.Obs.notify("weave:service:tracking-started");
+ },
+
+ async _stopTracking() {
+ const engines = [this.clientsEngine, ...this.engineManager.getAll()];
+ for (let engine of engines) {
+ try {
+ await engine.stopTracking();
+ } catch (e) {
+ this._log.error(`Could not stop ${engine.name} engine tracker`, e);
+ }
+ }
+ Svc.Obs.notify("weave:service:tracking-stopped");
+ },
+
+ /**
+ * Obtain a Resource instance with authentication credentials.
+ */
+ resource: function resource(url) {
+ let res = new Resource(url);
+ res.authenticator = this.identity.getResourceAuthenticator();
+
+ return res;
+ },
+
+ /**
+ * Perform the info fetch as part of a login or key fetch, or
+ * inside engine sync.
+ */
+ async _fetchInfo(url) {
+ let infoURL = url || this.infoURL;
+
+ this._log.trace("In _fetchInfo: " + infoURL);
+ let info;
+ try {
+ info = await this.resource(infoURL).get();
+ } catch (ex) {
+ this.errorHandler.checkServerError(ex);
+ throw ex;
+ }
+
+ // Always check for errors.
+ this.errorHandler.checkServerError(info);
+ if (!info.success) {
+ this._log.error("Aborting sync: failed to get collections.");
+ throw info;
+ }
+ return info;
+ },
+
+ async verifyAndFetchSymmetricKeys(infoResponse) {
+ this._log.debug(
+ "Fetching and verifying -- or generating -- symmetric keys."
+ );
+
+ let syncKeyBundle = this.identity.syncKeyBundle;
+ if (!syncKeyBundle) {
+ this.status.login = LOGIN_FAILED_NO_PASSPHRASE;
+ this.status.sync = CREDENTIALS_CHANGED;
+ return false;
+ }
+
+ try {
+ if (!infoResponse) {
+ infoResponse = await this._fetchInfo(); // Will throw an exception on failure.
+ }
+
+ // This only applies when the server is already at version 4.
+ if (infoResponse.status != 200) {
+ this._log.warn(
+ "info/collections returned non-200 response. Failing key fetch."
+ );
+ this.status.login = LOGIN_FAILED_SERVER_ERROR;
+ this.errorHandler.checkServerError(infoResponse);
+ return false;
+ }
+
+ let infoCollections = infoResponse.obj;
+
+ this._log.info(
+ "Testing info/collections: " + JSON.stringify(infoCollections)
+ );
+
+ if (this.collectionKeys.updateNeeded(infoCollections)) {
+ this._log.info("collection keys reports that a key update is needed.");
+
+ // Don't always set to CREDENTIALS_CHANGED -- we will probably take care of this.
+
+ // Fetch storage/crypto/keys.
+ let cryptoKeys;
+
+ if (infoCollections && CRYPTO_COLLECTION in infoCollections) {
+ try {
+ cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
+ let cryptoResp = (
+ await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))
+ ).response;
+
+ if (cryptoResp.success) {
+ await this.handleFetchedKeys(syncKeyBundle, cryptoKeys);
+ return true;
+ } else if (cryptoResp.status == 404) {
+ // On failure, ask to generate new keys and upload them.
+ // Fall through to the behavior below.
+ this._log.warn(
+ "Got 404 for crypto/keys, but 'crypto' in info/collections. Regenerating."
+ );
+ cryptoKeys = null;
+ } else {
+ // Some other problem.
+ this.status.login = LOGIN_FAILED_SERVER_ERROR;
+ this.errorHandler.checkServerError(cryptoResp);
+ this._log.warn(
+ "Got status " + cryptoResp.status + " fetching crypto keys."
+ );
+ return false;
+ }
+ } catch (ex) {
+ this._log.warn("Got exception fetching cryptoKeys.", ex);
+ // TODO: Um, what exceptions might we get here? Should we re-throw any?
+
+ // One kind of exception: HMAC failure.
+ if (Utils.isHMACMismatch(ex)) {
+ this.status.login = LOGIN_FAILED_INVALID_PASSPHRASE;
+ this.status.sync = CREDENTIALS_CHANGED;
+ } else {
+ // In the absence of further disambiguation or more precise
+ // failure constants, just report failure.
+ this.status.login = LOGIN_FAILED;
+ }
+ return false;
+ }
+ } else {
+ this._log.info(
+ "... 'crypto' is not a reported collection. Generating new keys."
+ );
+ }
+
+ if (!cryptoKeys) {
+ this._log.info("No keys! Generating new ones.");
+
+ // Better make some and upload them, and wipe the server to ensure
+ // consistency. This is all achieved via _freshStart.
+ // If _freshStart fails to clear the server or upload keys, it will
+ // throw.
+ await this._freshStart();
+ return true;
+ }
+
+ // Last-ditch case.
+ return false;
+ }
+ // No update needed: we're good!
+ return true;
+ } catch (ex) {
+ // This means no keys are present, or there's a network error.
+ this._log.debug("Failed to fetch and verify keys", ex);
+ this.errorHandler.checkServerError(ex);
+ return false;
+ }
+ },
+
+ getMaxRecordPayloadSize() {
+ let config = this.serverConfiguration;
+ if (!config || !config.max_record_payload_bytes) {
+ this._log.warn(
+ "No config or incomplete config in getMaxRecordPayloadSize." +
+ " Are we running tests?"
+ );
+ return 256 * 1024;
+ }
+ let payloadMax = config.max_record_payload_bytes;
+ if (config.max_post_bytes && payloadMax <= config.max_post_bytes) {
+ return config.max_post_bytes - 4096;
+ }
+ return payloadMax;
+ },
+
+ getMemcacheMaxRecordPayloadSize() {
+ // Collections stored in memcached ("tabs", "clients" or "meta") have a
+ // different max size than ones stored in the normal storage server db.
+ // In practice, the real limit here is 1M (bug 1300451 comment 40), but
+ // there's overhead involved that is hard to calculate on the client, so we
+ // use 512k to be safe (at the recommendation of the server team). Note
+ // that if the server reports a lower limit (via info/configuration), we
+ // respect that limit instead. See also bug 1403052.
+ return Math.min(512 * 1024, this.getMaxRecordPayloadSize());
+ },
+
+ async verifyLogin(allow40XRecovery = true) {
+ // Attaching auth credentials to a request requires access to
+ // passwords, which means that Resource.get can throw MP-related
+ // exceptions!
+ // So we ask the identity to verify the login state after unlocking the
+ // master password (ie, this call is expected to prompt for MP unlock
+ // if necessary) while we still have control.
+ this.status.login = await this.identity.unlockAndVerifyAuthState();
+ this._log.debug(
+ "Fetching unlocked auth state returned " + this.status.login
+ );
+ if (this.status.login != STATUS_OK) {
+ return false;
+ }
+
+ try {
+ // Make sure we have a cluster to verify against.
+ // This is a little weird, if we don't get a node we pretend
+ // to succeed, since that probably means we just don't have storage.
+ if (this.clusterURL == "" && !(await this.identity.setCluster())) {
+ this.status.sync = NO_SYNC_NODE_FOUND;
+ return true;
+ }
+
+ // Fetch collection info on every startup.
+ let test = await this.resource(this.infoURL).get();
+
+ switch (test.status) {
+ case 200:
+ // The user is authenticated.
+
+ // We have no way of verifying the passphrase right now,
+ // so wait until remoteSetup to do so.
+ // Just make the most trivial checks.
+ if (!this.identity.syncKeyBundle) {
+ this._log.warn("No passphrase in verifyLogin.");
+ this.status.login = LOGIN_FAILED_NO_PASSPHRASE;
+ return false;
+ }
+
+ // Go ahead and do remote setup, so that we can determine
+ // conclusively that our passphrase is correct.
+ if (await this._remoteSetup(test)) {
+ // Username/password verified.
+ this.status.login = LOGIN_SUCCEEDED;
+ return true;
+ }
+
+ this._log.warn("Remote setup failed.");
+ // Remote setup must have failed.
+ return false;
+
+ case 401:
+ this._log.warn("401: login failed.");
+ // Fall through to the 404 case.
+
+ case 404:
+ // Check that we're verifying with the correct cluster
+ if (allow40XRecovery && (await this.identity.setCluster())) {
+ return await this.verifyLogin(false);
+ }
+
+ // We must have the right cluster, but the server doesn't expect us.
+ // For FxA this almost certainly means "transient error fetching token".
+ this.status.login = LOGIN_FAILED_NETWORK_ERROR;
+ return false;
+
+ default:
+ // Server didn't respond with something that we expected
+ this.status.login = LOGIN_FAILED_SERVER_ERROR;
+ this.errorHandler.checkServerError(test);
+ return false;
+ }
+ } catch (ex) {
+ // Must have failed on some network issue
+ this._log.debug("verifyLogin failed", ex);
+ this.status.login = LOGIN_FAILED_NETWORK_ERROR;
+ this.errorHandler.checkServerError(ex);
+ return false;
+ }
+ },
+
+ async generateNewSymmetricKeys() {
+ this._log.info("Generating new keys WBO...");
+ let wbo = await this.collectionKeys.generateNewKeysWBO();
+ this._log.info("Encrypting new key bundle.");
+ await wbo.encrypt(this.identity.syncKeyBundle);
+
+ let uploadRes = await this._uploadCryptoKeys(wbo, 0);
+ if (uploadRes.status != 200) {
+ this._log.warn(
+ "Got status " +
+ uploadRes.status +
+ " uploading new keys. What to do? Throw!"
+ );
+ this.errorHandler.checkServerError(uploadRes);
+ throw new Error("Unable to upload symmetric keys.");
+ }
+ this._log.info("Got status " + uploadRes.status + " uploading keys.");
+ let serverModified = uploadRes.obj; // Modified timestamp according to server.
+ this._log.debug("Server reports crypto modified: " + serverModified);
+
+ // Now verify that info/collections shows them!
+ this._log.debug("Verifying server collection records.");
+ let info = await this._fetchInfo();
+ this._log.debug("info/collections is: " + info.data);
+
+ if (info.status != 200) {
+ this._log.warn("Non-200 info/collections response. Aborting.");
+ throw new Error("Unable to upload symmetric keys.");
+ }
+
+ info = info.obj;
+ if (!(CRYPTO_COLLECTION in info)) {
+ this._log.error(
+ "Consistency failure: info/collections excludes " +
+ "crypto after successful upload."
+ );
+ throw new Error("Symmetric key upload failed.");
+ }
+
+ // Can't check against local modified: clock drift.
+ if (info[CRYPTO_COLLECTION] < serverModified) {
+ this._log.error(
+ "Consistency failure: info/collections crypto entry " +
+ "is stale after successful upload."
+ );
+ throw new Error("Symmetric key upload failed.");
+ }
+
+ // Doesn't matter if the timestamp is ahead.
+
+ // Download and install them.
+ let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
+ let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL)))
+ .response;
+ if (cryptoResp.status != 200) {
+ this._log.warn("Failed to download keys.");
+ throw new Error("Symmetric key download failed.");
+ }
+ let keysChanged = await this.handleFetchedKeys(
+ this.identity.syncKeyBundle,
+ cryptoKeys,
+ true
+ );
+ if (keysChanged) {
+ this._log.info("Downloaded keys differed, as expected.");
+ }
+ },
+
+ // configures/enabled/turns-on sync. There must be an FxA user signed in.
+ async configure() {
+ // We don't, and must not, throw if sync is already configured, because we
+ // might end up being called as part of a "reconnect" flow. We also want to
+ // avoid checking the FxA user is the same as the pref because the email
+ // address for the FxA account can change - we'd need to use the uid.
+ let user = await fxAccounts.getSignedInUser();
+ if (!user) {
+ throw new Error("No FxA user is signed in");
+ }
+ this._log.info("Configuring sync with current FxA user");
+ Svc.Prefs.set("username", user.email);
+ Svc.Obs.notify("weave:connected");
+ },
+
+ // resets/turns-off sync.
+ async startOver() {
+ this._log.trace("Invoking Service.startOver.");
+ await this._stopTracking();
+ this.status.resetSync();
+
+ // Deletion doesn't make sense if we aren't set up yet!
+ if (this.clusterURL != "") {
+ // Clear client-specific data from the server, including disabled engines.
+ const engines = [this.clientsEngine, ...this.engineManager.getAll()];
+ for (let engine of engines) {
+ try {
+ await engine.removeClientData();
+ } catch (ex) {
+ this._log.warn(`Deleting client data for ${engine.name} failed`, ex);
+ }
+ }
+ this._log.debug("Finished deleting client data.");
+ } else {
+ this._log.debug("Skipping client data removal: no cluster URL.");
+ }
+
+ this.identity.resetCredentials();
+ this.status.login = LOGIN_FAILED_NO_USERNAME;
+ this.logout();
+ Svc.Obs.notify("weave:service:start-over");
+
+ // Reset all engines and clear keys.
+ await this.resetClient();
+ this.collectionKeys.clear();
+ this.status.resetBackoff();
+
+ // Reset Weave prefs.
+ this._ignorePrefObserver = true;
+ Svc.Prefs.resetBranch("");
+ this._ignorePrefObserver = false;
+ this.clusterURL = null;
+
+ Svc.Prefs.set("lastversion", WEAVE_VERSION);
+
+ try {
+ this.identity.finalize();
+ this.status.__authManager = null;
+ this.identity = Status._authManager;
+ Svc.Obs.notify("weave:service:start-over:finish");
+ } catch (err) {
+ this._log.error(
+ "startOver failed to re-initialize the identity manager",
+ err
+ );
+ // Still send the observer notification so the current state is
+ // reflected in the UI.
+ Svc.Obs.notify("weave:service:start-over:finish");
+ }
+ },
+
+ async login() {
+ async function onNotify() {
+ this._loggedIn = false;
+ if (this.scheduler.offline) {
+ this.status.login = LOGIN_FAILED_NETWORK_ERROR;
+ throw new Error("Application is offline, login should not be called");
+ }
+
+ this._log.info("User logged in successfully - verifying login.");
+ if (!(await this.verifyLogin())) {
+ // verifyLogin sets the failure states here.
+ throw new Error(`Login failed: ${this.status.login}`);
+ }
+
+ this._updateCachedURLs();
+
+ this._loggedIn = true;
+
+ return true;
+ }
+
+ let notifier = this._notify("login", "", onNotify.bind(this));
+ return this._catch(this._lock("service.js: login", notifier))();
+ },
+
+ logout: function logout() {
+ // If we failed during login, we aren't going to have this._loggedIn set,
+ // but we still want to ask the identity to logout, so it doesn't try and
+ // reuse any old credentials next time we sync.
+ this._log.info("Logging out");
+ this.identity.logout();
+ this._loggedIn = false;
+
+ Svc.Obs.notify("weave:service:logout:finish");
+ },
+
+ // Note: returns false if we failed for a reason other than the server not yet
+ // supporting the api.
+ async _fetchServerConfiguration() {
+ // This is similar to _fetchInfo, but with different error handling.
+
+ let infoURL = this.userBaseURL + "info/configuration";
+ this._log.debug("Fetching server configuration", infoURL);
+ let configResponse;
+ try {
+ configResponse = await this.resource(infoURL).get();
+ } catch (ex) {
+ // This is probably a network or similar error.
+ this._log.warn("Failed to fetch info/configuration", ex);
+ this.errorHandler.checkServerError(ex);
+ return false;
+ }
+
+ if (configResponse.status == 404) {
+ // This server doesn't support the URL yet - that's OK.
+ this._log.debug(
+ "info/configuration returned 404 - using default upload semantics"
+ );
+ } else if (configResponse.status != 200) {
+ this._log.warn(
+ `info/configuration returned ${configResponse.status} - using default configuration`
+ );
+ this.errorHandler.checkServerError(configResponse);
+ return false;
+ } else {
+ this.serverConfiguration = configResponse.obj;
+ }
+ this._log.trace(
+ "info/configuration for this server",
+ this.serverConfiguration
+ );
+ return true;
+ },
+
+ // Stuff we need to do after login, before we can really do
+ // anything (e.g. key setup).
+ async _remoteSetup(infoResponse, fetchConfig = true) {
+ if (fetchConfig && !(await this._fetchServerConfiguration())) {
+ return false;
+ }
+
+ this._log.debug("Fetching global metadata record");
+ let meta = await this.recordManager.get(this.metaURL);
+
+ // Checking modified time of the meta record.
+ if (
+ infoResponse &&
+ infoResponse.obj.meta != this.metaModified &&
+ (!meta || !meta.isNew)
+ ) {
+ // Delete the cached meta record...
+ this._log.debug(
+ "Clearing cached meta record. metaModified is " +
+ JSON.stringify(this.metaModified) +
+ ", setting to " +
+ JSON.stringify(infoResponse.obj.meta)
+ );
+
+ this.recordManager.del(this.metaURL);
+
+ // ... fetch the current record from the server, and COPY THE FLAGS.
+ let newMeta = await this.recordManager.get(this.metaURL);
+
+ // If we got a 401, we do not want to create a new meta/global - we
+ // should be able to get the existing meta after we get a new node.
+ if (this.recordManager.response.status == 401) {
+ this._log.debug(
+ "Fetching meta/global record on the server returned 401."
+ );
+ this.errorHandler.checkServerError(this.recordManager.response);
+ return false;
+ }
+
+ if (this.recordManager.response.status == 404) {
+ this._log.debug("No meta/global record on the server. Creating one.");
+ try {
+ await this._uploadNewMetaGlobal();
+ } catch (uploadRes) {
+ this._log.warn(
+ "Unable to upload new meta/global. Failing remote setup."
+ );
+ this.errorHandler.checkServerError(uploadRes);
+ return false;
+ }
+ } else if (!newMeta) {
+ this._log.warn("Unable to get meta/global. Failing remote setup.");
+ this.errorHandler.checkServerError(this.recordManager.response);
+ return false;
+ } else {
+ // If newMeta, then it stands to reason that meta != null.
+ newMeta.isNew = meta.isNew;
+ newMeta.changed = meta.changed;
+ }
+
+ // Switch in the new meta object and record the new time.
+ meta = newMeta;
+ this.metaModified = infoResponse.obj.meta;
+ }
+
+ let remoteVersion =
+ meta && meta.payload.storageVersion ? meta.payload.storageVersion : "";
+
+ this._log.debug(
+ [
+ "Weave Version:",
+ WEAVE_VERSION,
+ "Local Storage:",
+ STORAGE_VERSION,
+ "Remote Storage:",
+ remoteVersion,
+ ].join(" ")
+ );
+
+ // Check for cases that require a fresh start. When comparing remoteVersion,
+ // we need to convert it to a number as older clients used it as a string.
+ if (
+ !meta ||
+ !meta.payload.storageVersion ||
+ !meta.payload.syncID ||
+ STORAGE_VERSION > parseFloat(remoteVersion)
+ ) {
+ this._log.info(
+ "One of: no meta, no meta storageVersion, or no meta syncID. Fresh start needed."
+ );
+
+ // abort the server wipe if the GET status was anything other than 404 or 200
+ let status = this.recordManager.response.status;
+ if (status != 200 && status != 404) {
+ this.status.sync = METARECORD_DOWNLOAD_FAIL;
+ this.errorHandler.checkServerError(this.recordManager.response);
+ this._log.warn(
+ "Unknown error while downloading metadata record. Aborting sync."
+ );
+ return false;
+ }
+
+ if (!meta) {
+ this._log.info("No metadata record, server wipe needed");
+ }
+ if (meta && !meta.payload.syncID) {
+ this._log.warn("No sync id, server wipe needed");
+ }
+
+ this._log.info("Wiping server data");
+ await this._freshStart();
+
+ if (status == 404) {
+ this._log.info(
+ "Metadata record not found, server was wiped to ensure " +
+ "consistency."
+ );
+ } else {
+ // 200
+ this._log.info("Wiped server; incompatible metadata: " + remoteVersion);
+ }
+ return true;
+ } else if (remoteVersion > STORAGE_VERSION) {
+ this.status.sync = VERSION_OUT_OF_DATE;
+ this._log.warn("Upgrade required to access newer storage version.");
+ return false;
+ } else if (meta.payload.syncID != this.syncID) {
+ this._log.info(
+ "Sync IDs differ. Local is " +
+ this.syncID +
+ ", remote is " +
+ meta.payload.syncID
+ );
+ await this.resetClient();
+ this.collectionKeys.clear();
+ this.syncID = meta.payload.syncID;
+ this._log.debug("Clear cached values and take syncId: " + this.syncID);
+
+ if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) {
+ this._log.warn("Failed to fetch symmetric keys. Failing remote setup.");
+ return false;
+ }
+
+ // bug 545725 - re-verify creds and fail sanely
+ if (!(await this.verifyLogin())) {
+ this.status.sync = CREDENTIALS_CHANGED;
+ this._log.info(
+ "Credentials have changed, aborting sync and forcing re-login."
+ );
+ return false;
+ }
+
+ return true;
+ }
+ if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) {
+ this._log.warn("Failed to fetch symmetric keys. Failing remote setup.");
+ return false;
+ }
+
+ return true;
+ },
+
+ /**
+ * Return whether we should attempt login at the start of a sync.
+ *
+ * Note that this function has strong ties to _checkSync: callers
+ * of this function should typically use _checkSync to verify that
+ * any necessary login took place.
+ */
+ _shouldLogin: function _shouldLogin() {
+ return (
+ this.enabled &&
+ !this.scheduler.offline &&
+ !this.isLoggedIn &&
+ Async.isAppReady()
+ );
+ },
+
+ /**
+ * Determine if a sync should run.
+ *
+ * @param ignore [optional]
+ * array of reasons to ignore when checking
+ *
+ * @return Reason for not syncing; not-truthy if sync should run
+ */
+ _checkSync: function _checkSync(ignore) {
+ let reason = "";
+ // Ideally we'd call _checkSetup() here but that has too many side-effects.
+ if (Status.service == CLIENT_NOT_CONFIGURED) {
+ reason = kSyncNotConfigured;
+ } else if (Status.service == STATUS_DISABLED || !this.enabled) {
+ reason = kSyncWeaveDisabled;
+ } else if (this.scheduler.offline) {
+ reason = kSyncNetworkOffline;
+ } else if (this.status.minimumNextSync > Date.now()) {
+ reason = kSyncBackoffNotMet;
+ } else if (
+ this.status.login == MASTER_PASSWORD_LOCKED &&
+ Utils.mpLocked()
+ ) {
+ reason = kSyncMasterPasswordLocked;
+ } else if (Svc.Prefs.get("firstSync") == "notReady") {
+ reason = kFirstSyncChoiceNotMade;
+ } else if (!Async.isAppReady()) {
+ reason = kFirefoxShuttingDown;
+ }
+
+ if (ignore && ignore.includes(reason)) {
+ return "";
+ }
+
+ return reason;
+ },
+
+ async sync({ engines, why } = {}) {
+ let dateStr = Utils.formatTimestamp(new Date());
+ this._log.debug("User-Agent: " + Utils.userAgent);
+ await this.promiseInitialized;
+ this._log.info(
+ `Starting sync at ${dateStr} in browser session ${lazy.browserSessionID}`
+ );
+ return this._catch(async function () {
+ // Make sure we're logged in.
+ if (this._shouldLogin()) {
+ this._log.debug("In sync: should login.");
+ if (!(await this.login())) {
+ this._log.debug("Not syncing: login returned false.");
+ return;
+ }
+ } else {
+ this._log.trace("In sync: no need to login.");
+ }
+ await this._lockedSync(engines, why);
+ })();
+ },
+
+ /**
+ * Sync up engines with the server.
+ */
+ async _lockedSync(engineNamesToSync, why) {
+ return this._lock(
+ "service.js: sync",
+ this._notify("sync", JSON.stringify({ why }), async function onNotify() {
+ let histogram =
+ Services.telemetry.getHistogramById("WEAVE_START_COUNT");
+ histogram.add(1);
+
+ let synchronizer = new EngineSynchronizer(this);
+ await synchronizer.sync(engineNamesToSync, why); // Might throw!
+
+ histogram = Services.telemetry.getHistogramById(
+ "WEAVE_COMPLETE_SUCCESS_COUNT"
+ );
+ histogram.add(1);
+
+ // We successfully synchronized.
+ // Check if the identity wants to pre-fetch a migration sentinel from
+ // the server.
+ // If we have no clusterURL, we are probably doing a node reassignment
+ // so don't attempt to get it in that case.
+ if (this.clusterURL) {
+ this.identity.prefetchMigrationSentinel(this);
+ }
+
+ // Now let's update our declined engines
+ await this._maybeUpdateDeclined();
+ })
+ )();
+ },
+
+ /**
+ * Update the "declined" information in meta/global if necessary.
+ */
+ async _maybeUpdateDeclined() {
+ // if Sync failed due to no node we will not have a meta URL, so can't
+ // update anything.
+ if (!this.metaURL) {
+ return;
+ }
+ let meta = await this.recordManager.get(this.metaURL);
+ if (!meta) {
+ this._log.warn("No meta/global; can't update declined state.");
+ return;
+ }
+
+ let declinedEngines = new DeclinedEngines(this);
+ let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
+ if (!didChange) {
+ this._log.info(
+ "No change to declined engines. Not reuploading meta/global."
+ );
+ return;
+ }
+
+ await this.uploadMetaGlobal(meta);
+ },
+
+ /**
+ * Upload a fresh meta/global record
+ * @throws the response object if the upload request was not a success
+ */
+ async _uploadNewMetaGlobal() {
+ let meta = new WBORecord("meta", "global");
+ meta.payload.syncID = this.syncID;
+ meta.payload.storageVersion = STORAGE_VERSION;
+ meta.payload.declined = this.engineManager.getDeclined();
+ meta.modified = 0;
+ meta.isNew = true;
+
+ await this.uploadMetaGlobal(meta);
+ },
+
+ /**
+ * Upload meta/global, throwing the response on failure
+ * @param {WBORecord} meta meta/global record
+ * @throws the response object if the request was not a success
+ */
+ async uploadMetaGlobal(meta) {
+ this._log.debug("Uploading meta/global", meta);
+ let res = this.resource(this.metaURL);
+ res.setHeader("X-If-Unmodified-Since", meta.modified);
+ let response = await res.put(meta);
+ if (!response.success) {
+ throw response;
+ }
+ // From https://docs.services.mozilla.com/storage/apis-1.5.html:
+ // "Successful responses will return the new last-modified time for the collection."
+ meta.modified = response.obj;
+ this.recordManager.set(this.metaURL, meta);
+ },
+
+ /**
+ * Upload crypto/keys
+ * @param {WBORecord} cryptoKeys crypto/keys record
+ * @param {Number} lastModified known last modified timestamp (in decimal seconds),
+ * will be used to set the X-If-Unmodified-Since header
+ */
+ async _uploadCryptoKeys(cryptoKeys, lastModified) {
+ this._log.debug(`Uploading crypto/keys (lastModified: ${lastModified})`);
+ let res = this.resource(this.cryptoKeysURL);
+ res.setHeader("X-If-Unmodified-Since", lastModified);
+ return res.put(cryptoKeys);
+ },
+
+ async _freshStart() {
+ this._log.info("Fresh start. Resetting client.");
+ await this.resetClient();
+ this.collectionKeys.clear();
+
+ // Wipe the server.
+ await this.wipeServer();
+
+ // Upload a new meta/global record.
+ // _uploadNewMetaGlobal throws on failure -- including race conditions.
+ // If we got into a race condition, we'll abort the sync this way, too.
+ // That's fine. We'll just wait till the next sync. The client that we're
+ // racing is probably busy uploading stuff right now anyway.
+ await this._uploadNewMetaGlobal();
+
+ // Wipe everything we know about except meta because we just uploaded it
+ // TODO: there's a bug here. We should be calling resetClient, no?
+
+ // Generate, upload, and download new keys. Do this last so we don't wipe
+ // them...
+ await this.generateNewSymmetricKeys();
+ },
+
+ /**
+ * Wipe user data from the server.
+ *
+ * @param collections [optional]
+ * Array of collections to wipe. If not given, all collections are
+ * wiped by issuing a DELETE request for `storageURL`.
+ *
+ * @return the server's timestamp of the (last) DELETE.
+ */
+ async wipeServer(collections) {
+ let response;
+ let histogram = Services.telemetry.getHistogramById(
+ "WEAVE_WIPE_SERVER_SUCCEEDED"
+ );
+ if (!collections) {
+ // Strip the trailing slash.
+ let res = this.resource(this.storageURL.slice(0, -1));
+ res.setHeader("X-Confirm-Delete", "1");
+ try {
+ response = await res.delete();
+ } catch (ex) {
+ this._log.debug("Failed to wipe server", ex);
+ histogram.add(false);
+ throw ex;
+ }
+ if (response.status != 200 && response.status != 404) {
+ this._log.debug(
+ "Aborting wipeServer. Server responded with " +
+ response.status +
+ " response for " +
+ this.storageURL
+ );
+ histogram.add(false);
+ throw response;
+ }
+ histogram.add(true);
+ return response.headers["x-weave-timestamp"];
+ }
+
+ let timestamp;
+ for (let name of collections) {
+ let url = this.storageURL + name;
+ try {
+ response = await this.resource(url).delete();
+ } catch (ex) {
+ this._log.debug("Failed to wipe '" + name + "' collection", ex);
+ histogram.add(false);
+ throw ex;
+ }
+
+ if (response.status != 200 && response.status != 404) {
+ this._log.debug(
+ "Aborting wipeServer. Server responded with " +
+ response.status +
+ " response for " +
+ url
+ );
+ histogram.add(false);
+ throw response;
+ }
+
+ if ("x-weave-timestamp" in response.headers) {
+ timestamp = response.headers["x-weave-timestamp"];
+ }
+ }
+ histogram.add(true);
+ return timestamp;
+ },
+
+ /**
+ * Wipe all local user data.
+ *
+ * @param engines [optional]
+ * Array of engine names to wipe. If not given, all engines are used.
+ */
+ async wipeClient(engines) {
+ // If we don't have any engines, reset the service and wipe all engines
+ if (!engines) {
+ // Clear out any service data
+ await this.resetService();
+
+ engines = [this.clientsEngine, ...this.engineManager.getAll()];
+ } else {
+ // Convert the array of names into engines
+ engines = this.engineManager.get(engines);
+ }
+
+ // Fully wipe each engine if it's able to decrypt data
+ for (let engine of engines) {
+ if (await engine.canDecrypt()) {
+ await engine.wipeClient();
+ }
+ }
+ },
+
+ /**
+ * Wipe all remote user data by wiping the server then telling each remote
+ * client to wipe itself.
+ *
+ * @param engines
+ * Array of engine names to wipe.
+ */
+ async wipeRemote(engines) {
+ try {
+ // Make sure stuff gets uploaded.
+ await this.resetClient(engines);
+
+ // Clear out any server data.
+ await this.wipeServer(engines);
+
+ // Only wipe the engines provided.
+ let extra = { reason: "wipe-remote" };
+ for (const e of engines) {
+ await this.clientsEngine.sendCommand("wipeEngine", [e], null, extra);
+ }
+
+ // Make sure the changed clients get updated.
+ await this.clientsEngine.sync();
+ } catch (ex) {
+ this.errorHandler.checkServerError(ex);
+ throw ex;
+ }
+ },
+
+ /**
+ * Reset local service information like logs, sync times, caches.
+ */
+ async resetService() {
+ return this._catch(async function reset() {
+ this._log.info("Service reset.");
+
+ // Pretend we've never synced to the server and drop cached data
+ this.syncID = "";
+ this.recordManager.clearCache();
+ })();
+ },
+
+ /**
+ * Reset the client by getting rid of any local server data and client data.
+ *
+ * @param engines [optional]
+ * Array of engine names to reset. If not given, all engines are used.
+ */
+ async resetClient(engines) {
+ return this._catch(async function doResetClient() {
+ // If we don't have any engines, reset everything including the service
+ if (!engines) {
+ // Clear out any service data
+ await this.resetService();
+
+ engines = [this.clientsEngine, ...this.engineManager.getAll()];
+ } else {
+ // Convert the array of names into engines
+ engines = this.engineManager.get(engines);
+ }
+
+ // Have each engine drop any temporary meta data
+ for (let engine of engines) {
+ await engine.resetClient();
+ }
+ })();
+ },
+
+ recordTelemetryEvent(object, method, value, extra = undefined) {
+ Svc.Obs.notify("weave:telemetry:event", { object, method, value, extra });
+ },
+};
+
+export var Service = new Sync11Service();
+Service.promiseInitialized = new Promise(resolve => {
+ Service.onStartup().then(resolve);
+});
diff --git a/services/sync/modules/stages/declined.sys.mjs b/services/sync/modules/stages/declined.sys.mjs
new file mode 100644
index 0000000000..2c74aab117
--- /dev/null
+++ b/services/sync/modules/stages/declined.sys.mjs
@@ -0,0 +1,78 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains code for maintaining the set of declined engines,
+ * in conjunction with EngineManager.
+ */
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { Observers } from "resource://services-common/observers.sys.mjs";
+
+export var DeclinedEngines = function (service) {
+ this._log = Log.repository.getLogger("Sync.Declined");
+ this._log.manageLevelFromPref("services.sync.log.logger.declined");
+
+ this.service = service;
+};
+
+DeclinedEngines.prototype = {
+ updateDeclined(meta, engineManager = this.service.engineManager) {
+ let enabled = new Set(engineManager.getEnabled().map(e => e.name));
+ let known = new Set(engineManager.getAll().map(e => e.name));
+ let remoteDeclined = new Set(meta.payload.declined || []);
+ let localDeclined = new Set(engineManager.getDeclined());
+
+ this._log.debug(
+ "Handling remote declined: " + JSON.stringify([...remoteDeclined])
+ );
+ this._log.debug(
+ "Handling local declined: " + JSON.stringify([...localDeclined])
+ );
+
+ // Any engines that are locally enabled should be removed from the remote
+ // declined list.
+ //
+ // Any engines that are locally declined should be added to the remote
+ // declined list.
+ let newDeclined = CommonUtils.union(
+ localDeclined,
+ CommonUtils.difference(remoteDeclined, enabled)
+ );
+
+ // If our declined set has changed, put it into the meta object and mark
+ // it as changed.
+ let declinedChanged = !CommonUtils.setEqual(newDeclined, remoteDeclined);
+ this._log.debug("Declined changed? " + declinedChanged);
+ if (declinedChanged) {
+ meta.changed = true;
+ meta.payload.declined = [...newDeclined];
+ }
+
+ // Update the engine manager regardless.
+ engineManager.setDeclined(newDeclined);
+
+ // Any engines that are locally known, locally disabled, and not remotely
+ // or locally declined, are candidates for enablement.
+ let undecided = CommonUtils.difference(
+ CommonUtils.difference(known, enabled),
+ newDeclined
+ );
+ if (undecided.size) {
+ let subject = {
+ declined: newDeclined,
+ enabled,
+ known,
+ undecided,
+ };
+ CommonUtils.nextTick(() => {
+ Observers.notify("weave:engines:notdeclined", subject);
+ });
+ }
+
+ return declinedChanged;
+ },
+};
diff --git a/services/sync/modules/stages/enginesync.sys.mjs b/services/sync/modules/stages/enginesync.sys.mjs
new file mode 100644
index 0000000000..6078a3af0e
--- /dev/null
+++ b/services/sync/modules/stages/enginesync.sys.mjs
@@ -0,0 +1,400 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains code for synchronizing engines.
+ */
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import {
+ ABORT_SYNC_COMMAND,
+ LOGIN_FAILED_NETWORK_ERROR,
+ NO_SYNC_NODE_FOUND,
+ STATUS_OK,
+ SYNC_FAILED_PARTIAL,
+ SYNC_SUCCEEDED,
+ WEAVE_VERSION,
+ kSyncNetworkOffline,
+} from "resource://services-sync/constants.sys.mjs";
+
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+
+const lazy = {};
+ChromeUtils.defineESModuleGetters(lazy, {
+ Doctor: "resource://services-sync/doctor.sys.mjs",
+});
+
+/**
+ * Perform synchronization of engines.
+ *
+ * This was originally split out of service.js. The API needs lots of love.
+ */
+export function EngineSynchronizer(service) {
+ this._log = Log.repository.getLogger("Sync.Synchronizer");
+ this._log.manageLevelFromPref("services.sync.log.logger.synchronizer");
+
+ this.service = service;
+}
+
+EngineSynchronizer.prototype = {
+ async sync(engineNamesToSync, why) {
+ let fastSync = why && why == "sleep";
+ let startTime = Date.now();
+
+ this.service.status.resetSync();
+
+ // Make sure we should sync or record why we shouldn't.
+ let reason = this.service._checkSync();
+ if (reason) {
+ if (reason == kSyncNetworkOffline) {
+ this.service.status.sync = LOGIN_FAILED_NETWORK_ERROR;
+ }
+
+ // this is a purposeful abort rather than a failure, so don't set
+ // any status bits
+ reason = "Can't sync: " + reason;
+ throw new Error(reason);
+ }
+
+ // If we don't have a node, get one. If that fails, retry in 10 minutes.
+ if (
+ !this.service.clusterURL &&
+ !(await this.service.identity.setCluster())
+ ) {
+ this.service.status.sync = NO_SYNC_NODE_FOUND;
+ this._log.info("No cluster URL found. Cannot sync.");
+ return;
+ }
+
+ // Ping the server with a special info request once a day.
+ let infoURL = this.service.infoURL;
+ let now = Math.floor(Date.now() / 1000);
+ let lastPing = Svc.Prefs.get("lastPing", 0);
+ if (now - lastPing > 86400) {
+ // 60 * 60 * 24
+ infoURL += "?v=" + WEAVE_VERSION;
+ Svc.Prefs.set("lastPing", now);
+ }
+
+ let engineManager = this.service.engineManager;
+
+ // Figure out what the last modified time is for each collection
+ let info = await this.service._fetchInfo(infoURL);
+
+ // Convert the response to an object and read out the modified times
+ for (let engine of [this.service.clientsEngine].concat(
+ engineManager.getAll()
+ )) {
+ engine.lastModified = info.obj[engine.name] || 0;
+ }
+
+ if (!(await this.service._remoteSetup(info, !fastSync))) {
+ throw new Error("Aborting sync, remote setup failed");
+ }
+
+ if (!fastSync) {
+ // Make sure we have an up-to-date list of clients before sending commands
+ this._log.debug("Refreshing client list.");
+ if (!(await this._syncEngine(this.service.clientsEngine))) {
+ // Clients is an engine like any other; it can fail with a 401,
+ // and we can elect to abort the sync.
+ this._log.warn("Client engine sync failed. Aborting.");
+ return;
+ }
+ }
+
+ // We only honor the "hint" of what engines to Sync if this isn't
+ // a first sync.
+ let allowEnginesHint = false;
+ // Wipe data in the desired direction if necessary
+ switch (Svc.Prefs.get("firstSync")) {
+ case "resetClient":
+ await this.service.resetClient(engineManager.enabledEngineNames);
+ break;
+ case "wipeClient":
+ await this.service.wipeClient(engineManager.enabledEngineNames);
+ break;
+ case "wipeRemote":
+ await this.service.wipeRemote(engineManager.enabledEngineNames);
+ break;
+ default:
+ allowEnginesHint = true;
+ break;
+ }
+
+ if (!fastSync && this.service.clientsEngine.localCommands) {
+ try {
+ if (!(await this.service.clientsEngine.processIncomingCommands())) {
+ this.service.status.sync = ABORT_SYNC_COMMAND;
+ throw new Error("Processed command aborted sync.");
+ }
+
+ // Repeat remoteSetup in-case the commands forced us to reset
+ if (!(await this.service._remoteSetup(info))) {
+ throw new Error("Remote setup failed after processing commands.");
+ }
+ } finally {
+ // Always immediately attempt to push back the local client (now
+ // without commands).
+ // Note that we don't abort here; if there's a 401 because we've
+ // been reassigned, we'll handle it around another engine.
+ await this._syncEngine(this.service.clientsEngine);
+ }
+ }
+
+ // Update engines because it might change what we sync.
+ try {
+ await this._updateEnabledEngines();
+ } catch (ex) {
+ this._log.debug("Updating enabled engines failed", ex);
+ this.service.errorHandler.checkServerError(ex);
+ throw ex;
+ }
+
+ await this.service.engineManager.switchAlternatives();
+
+ // If the engines to sync has been specified, we sync in the order specified.
+ let enginesToSync;
+ if (allowEnginesHint && engineNamesToSync) {
+ this._log.info("Syncing specified engines", engineNamesToSync);
+ enginesToSync = engineManager
+ .get(engineNamesToSync)
+ .filter(e => e.enabled);
+ } else {
+ this._log.info("Syncing all enabled engines.");
+ enginesToSync = engineManager.getEnabled();
+ }
+ try {
+ // We don't bother validating engines that failed to sync.
+ let enginesToValidate = [];
+ for (let engine of enginesToSync) {
+ if (engine.shouldSkipSync(why)) {
+ this._log.info(`Engine ${engine.name} asked to be skipped`);
+ continue;
+ }
+ // If there's any problems with syncing the engine, report the failure
+ if (
+ !(await this._syncEngine(engine)) ||
+ this.service.status.enforceBackoff
+ ) {
+ this._log.info("Aborting sync for failure in " + engine.name);
+ break;
+ }
+ enginesToValidate.push(engine);
+ }
+
+ // If _syncEngine fails for a 401, we might not have a cluster URL here.
+ // If that's the case, break out of this immediately, rather than
+ // throwing an exception when trying to fetch metaURL.
+ if (!this.service.clusterURL) {
+ this._log.debug(
+ "Aborting sync, no cluster URL: not uploading new meta/global."
+ );
+ return;
+ }
+
+ // Upload meta/global if any engines changed anything.
+ let meta = await this.service.recordManager.get(this.service.metaURL);
+ if (meta.isNew || meta.changed) {
+ this._log.info("meta/global changed locally: reuploading.");
+ try {
+ await this.service.uploadMetaGlobal(meta);
+ delete meta.isNew;
+ delete meta.changed;
+ } catch (error) {
+ this._log.error(
+ "Unable to upload meta/global. Leaving marked as new."
+ );
+ }
+ }
+
+ if (!fastSync) {
+ await lazy.Doctor.consult(enginesToValidate);
+ }
+
+ // If there were no sync engine failures
+ if (this.service.status.service != SYNC_FAILED_PARTIAL) {
+ this.service.status.sync = SYNC_SUCCEEDED;
+ }
+
+ // Even if there were engine failures, bump lastSync even on partial since
+ // it's reflected in the UI (bug 1439777).
+ if (
+ this.service.status.service == SYNC_FAILED_PARTIAL ||
+ this.service.status.service == STATUS_OK
+ ) {
+ Svc.Prefs.set("lastSync", new Date().toString());
+ }
+ } finally {
+ Svc.Prefs.reset("firstSync");
+
+ let syncTime = ((Date.now() - startTime) / 1000).toFixed(2);
+ let dateStr = Utils.formatTimestamp(new Date());
+ this._log.info(
+ "Sync completed at " + dateStr + " after " + syncTime + " secs."
+ );
+ }
+ },
+
+ // Returns true if sync should proceed.
+ // false / no return value means sync should be aborted.
+ async _syncEngine(engine) {
+ try {
+ await engine.sync();
+ } catch (e) {
+ if (e.status == 401) {
+ // Maybe a 401, cluster update perhaps needed?
+ // We rely on ErrorHandler observing the sync failure notification to
+ // schedule another sync and clear node assignment values.
+ // Here we simply want to muffle the exception and return an
+ // appropriate value.
+ return false;
+ }
+ // Note that policies.js has already logged info about the exception...
+ if (Async.isShutdownException(e)) {
+ // Failure due to a shutdown exception should prevent other engines
+ // trying to start and immediately failing.
+ this._log.info(
+ `${engine.name} was interrupted by shutdown; no other engines will sync`
+ );
+ return false;
+ }
+ }
+
+ return true;
+ },
+
+ async _updateEnabledFromMeta(
+ meta,
+ numClients,
+ engineManager = this.service.engineManager
+ ) {
+ this._log.info("Updating enabled engines: " + numClients + " clients.");
+
+ if (meta.isNew || !meta.payload.engines) {
+ this._log.debug(
+ "meta/global isn't new, or is missing engines. Not updating enabled state."
+ );
+ return;
+ }
+
+ // If we're the only client, and no engines are marked as enabled,
+ // thumb our noses at the server data: it can't be right.
+ // Belt-and-suspenders approach to Bug 615926.
+ let hasEnabledEngines = false;
+ for (let e in meta.payload.engines) {
+ if (e != "clients") {
+ hasEnabledEngines = true;
+ break;
+ }
+ }
+
+ if (numClients <= 1 && !hasEnabledEngines) {
+ this._log.info(
+ "One client and no enabled engines: not touching local engine status."
+ );
+ return;
+ }
+
+ this.service._ignorePrefObserver = true;
+
+ let enabled = engineManager.enabledEngineNames;
+
+ let toDecline = new Set();
+ let toUndecline = new Set();
+
+ for (let engineName in meta.payload.engines) {
+ if (engineName == "clients") {
+ // Clients is special.
+ continue;
+ }
+ let index = enabled.indexOf(engineName);
+ if (index != -1) {
+ // The engine is enabled locally. Nothing to do.
+ enabled.splice(index, 1);
+ continue;
+ }
+ let engine = engineManager.get(engineName);
+ if (!engine) {
+ // The engine doesn't exist locally. Nothing to do.
+ continue;
+ }
+
+ let attemptedEnable = false;
+ // If the engine was enabled remotely, enable it locally.
+ if (!Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
+ this._log.trace(
+ "Engine " + engineName + " was enabled. Marking as non-declined."
+ );
+ toUndecline.add(engineName);
+ this._log.trace(engineName + " engine was enabled remotely.");
+ engine.enabled = true;
+ // Note that setting engine.enabled to true might not have worked for
+ // the password engine if a master-password is enabled. However, it's
+ // still OK that we added it to undeclined - the user *tried* to enable
+ // it remotely - so it still winds up as not being flagged as declined
+ // even though it's disabled remotely.
+ attemptedEnable = true;
+ }
+
+ // If either the engine was disabled locally or enabling the engine
+ // failed (see above re master-password) then wipe server data and
+ // disable it everywhere.
+ if (!engine.enabled) {
+ this._log.trace("Wiping data for " + engineName + " engine.");
+ await engine.wipeServer();
+ delete meta.payload.engines[engineName];
+ meta.changed = true; // the new enabled state must propagate
+ // We also here mark the engine as declined, because the pref
+ // was explicitly changed to false - unless we tried, and failed,
+ // to enable it - in which case we leave the declined state alone.
+ if (!attemptedEnable) {
+ // This will be reflected in meta/global in the next stage.
+ this._log.trace(
+ "Engine " +
+ engineName +
+ " was disabled locally. Marking as declined."
+ );
+ toDecline.add(engineName);
+ }
+ }
+ }
+
+ // Any remaining engines were either enabled locally or disabled remotely.
+ for (let engineName of enabled) {
+ let engine = engineManager.get(engineName);
+ if (Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
+ this._log.trace("The " + engineName + " engine was enabled locally.");
+ toUndecline.add(engineName);
+ } else {
+ this._log.trace("The " + engineName + " engine was disabled remotely.");
+
+ // Don't automatically mark it as declined!
+ try {
+ engine.enabled = false;
+ } catch (e) {
+ this._log.trace("Failed to disable engine " + engineName);
+ }
+ }
+ }
+
+ engineManager.decline(toDecline);
+ engineManager.undecline(toUndecline);
+
+ Svc.Prefs.resetBranch("engineStatusChanged.");
+ this.service._ignorePrefObserver = false;
+ },
+
+ async _updateEnabledEngines() {
+ let meta = await this.service.recordManager.get(this.service.metaURL);
+ let numClients = this.service.scheduler.numClients;
+ let engineManager = this.service.engineManager;
+
+ await this._updateEnabledFromMeta(meta, numClients, engineManager);
+ },
+};
+Object.freeze(EngineSynchronizer.prototype);
diff --git a/services/sync/modules/status.sys.mjs b/services/sync/modules/status.sys.mjs
new file mode 100644
index 0000000000..429dbda7b6
--- /dev/null
+++ b/services/sync/modules/status.sys.mjs
@@ -0,0 +1,135 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import {
+ CLIENT_NOT_CONFIGURED,
+ ENGINE_SUCCEEDED,
+ LOGIN_FAILED,
+ LOGIN_FAILED_NO_PASSPHRASE,
+ LOGIN_FAILED_NO_USERNAME,
+ LOGIN_SUCCEEDED,
+ STATUS_OK,
+ SYNC_FAILED,
+ SYNC_FAILED_PARTIAL,
+ SYNC_SUCCEEDED,
+} from "resource://services-sync/constants.sys.mjs";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { SyncAuthManager } from "resource://services-sync/sync_auth.sys.mjs";
+
+export var Status = {
+ _log: Log.repository.getLogger("Sync.Status"),
+ __authManager: null,
+ ready: false,
+
+ get _authManager() {
+ if (this.__authManager) {
+ return this.__authManager;
+ }
+ this.__authManager = new SyncAuthManager();
+ return this.__authManager;
+ },
+
+ get service() {
+ return this._service;
+ },
+
+ set service(code) {
+ this._log.debug(
+ "Status.service: " + (this._service || undefined) + " => " + code
+ );
+ this._service = code;
+ },
+
+ get login() {
+ return this._login;
+ },
+
+ set login(code) {
+ this._log.debug("Status.login: " + this._login + " => " + code);
+ this._login = code;
+
+ if (
+ code == LOGIN_FAILED_NO_USERNAME ||
+ code == LOGIN_FAILED_NO_PASSPHRASE
+ ) {
+ this.service = CLIENT_NOT_CONFIGURED;
+ } else if (code != LOGIN_SUCCEEDED) {
+ this.service = LOGIN_FAILED;
+ } else {
+ this.service = STATUS_OK;
+ }
+ },
+
+ get sync() {
+ return this._sync;
+ },
+
+ set sync(code) {
+ this._log.debug("Status.sync: " + this._sync + " => " + code);
+ this._sync = code;
+ this.service = code == SYNC_SUCCEEDED ? STATUS_OK : SYNC_FAILED;
+ },
+
+ get engines() {
+ return this._engines;
+ },
+
+ set engines([name, code]) {
+ this._log.debug("Status for engine " + name + ": " + code);
+ this._engines[name] = code;
+
+ if (code != ENGINE_SUCCEEDED) {
+ this.service = SYNC_FAILED_PARTIAL;
+ }
+ },
+
+ // Implement toString because adding a logger introduces a cyclic object
+ // value, so we can't trivially debug-print Status as JSON.
+ toString: function toString() {
+ return (
+ "<Status" +
+ ": login: " +
+ Status.login +
+ ", service: " +
+ Status.service +
+ ", sync: " +
+ Status.sync +
+ ">"
+ );
+ },
+
+ checkSetup: function checkSetup() {
+ if (!this._authManager.username) {
+ Status.login = LOGIN_FAILED_NO_USERNAME;
+ Status.service = CLIENT_NOT_CONFIGURED;
+ } else if (Status.login == STATUS_OK) {
+ Status.service = STATUS_OK;
+ }
+ return Status.service;
+ },
+
+ resetBackoff: function resetBackoff() {
+ this.enforceBackoff = false;
+ this.backoffInterval = 0;
+ this.minimumNextSync = 0;
+ },
+
+ resetSync: function resetSync() {
+ // Logger setup.
+ this._log.manageLevelFromPref("services.sync.log.logger.status");
+
+ this._log.info("Resetting Status.");
+ this.service = STATUS_OK;
+ this._login = LOGIN_SUCCEEDED;
+ this._sync = SYNC_SUCCEEDED;
+ this._engines = {};
+ this.partial = false;
+ },
+};
+
+// Initialize various status values.
+Status.resetBackoff();
+Status.resetSync();
diff --git a/services/sync/modules/sync_auth.sys.mjs b/services/sync/modules/sync_auth.sys.mjs
new file mode 100644
index 0000000000..2f2bd2af77
--- /dev/null
+++ b/services/sync/modules/sync_auth.sys.mjs
@@ -0,0 +1,656 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+import { Async } from "resource://services-common/async.sys.mjs";
+import { TokenServerClient } from "resource://services-common/tokenserverclient.sys.mjs";
+import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs";
+import { Svc, Utils } from "resource://services-sync/util.sys.mjs";
+
+import {
+ LOGIN_FAILED_LOGIN_REJECTED,
+ LOGIN_FAILED_NETWORK_ERROR,
+ LOGIN_FAILED_NO_USERNAME,
+ LOGIN_SUCCEEDED,
+ MASTER_PASSWORD_LOCKED,
+ STATUS_OK,
+} from "resource://services-sync/constants.sys.mjs";
+
+const lazy = {};
+
+// Lazy imports to prevent unnecessary load on startup.
+ChromeUtils.defineESModuleGetters(lazy, {
+ BulkKeyBundle: "resource://services-sync/keys.sys.mjs",
+ Weave: "resource://services-sync/main.sys.mjs",
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "log", function () {
+ let log = Log.repository.getLogger("Sync.SyncAuthManager");
+ log.manageLevelFromPref("services.sync.log.logger.identity");
+ return log;
+});
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "IGNORE_CACHED_AUTH_CREDENTIALS",
+ "services.sync.debug.ignoreCachedAuthCredentials"
+);
+
+// FxAccountsCommon.js doesn't use a "namespace", so create one here.
+var fxAccountsCommon = ChromeUtils.import(
+ "resource://gre/modules/FxAccountsCommon.js"
+);
+
+const SCOPE_OLD_SYNC = fxAccountsCommon.SCOPE_OLD_SYNC;
+
+const OBSERVER_TOPICS = [
+ fxAccountsCommon.ONLOGIN_NOTIFICATION,
+ fxAccountsCommon.ONVERIFIED_NOTIFICATION,
+ fxAccountsCommon.ONLOGOUT_NOTIFICATION,
+ fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION,
+ "weave:connected",
+];
+
+/*
+ General authentication error for abstracting authentication
+ errors from multiple sources (e.g., from FxAccounts, TokenServer).
+ details is additional details about the error - it might be a string, or
+ some other error object (which should do the right thing when toString() is
+ called on it)
+*/
+export function AuthenticationError(details, source) {
+ this.details = details;
+ this.source = source;
+}
+
+AuthenticationError.prototype = {
+ toString() {
+ return "AuthenticationError(" + this.details + ")";
+ },
+};
+
+// The `SyncAuthManager` coordinates access authorization to the Sync server.
+// Its job is essentially to get us from having a signed-in Firefox Accounts user,
+// to knowing the user's sync storage node and having the necessary short-lived
+// credentials in order to access it.
+//
+
+export function SyncAuthManager() {
+ // NOTE: _fxaService and _tokenServerClient are replaced with mocks by
+ // the test suite.
+ this._fxaService = lazy.fxAccounts;
+ this._tokenServerClient = new TokenServerClient();
+ this._tokenServerClient.observerPrefix = "weave:service";
+ this._log = lazy.log;
+ XPCOMUtils.defineLazyPreferenceGetter(
+ this,
+ "_username",
+ "services.sync.username"
+ );
+
+ this.asyncObserver = Async.asyncObserver(this, lazy.log);
+ for (let topic of OBSERVER_TOPICS) {
+ Services.obs.addObserver(this.asyncObserver, topic);
+ }
+}
+
+SyncAuthManager.prototype = {
+ _fxaService: null,
+ _tokenServerClient: null,
+ // https://docs.services.mozilla.com/token/apis.html
+ _token: null,
+ // protection against the user changing underneath us - the uid
+ // of the current user.
+ _userUid: null,
+
+ hashedUID() {
+ const id = this._fxaService.telemetry.getSanitizedUID();
+ if (!id) {
+ throw new Error("hashedUID: Don't seem to have previously seen a token");
+ }
+ return id;
+ },
+
+ // Return a hashed version of a deviceID, suitable for telemetry.
+ hashedDeviceID(deviceID) {
+ const id = this._fxaService.telemetry.sanitizeDeviceId(deviceID);
+ if (!id) {
+ throw new Error("hashedUID: Don't seem to have previously seen a token");
+ }
+ return id;
+ },
+
+ // The "node type" reported to telemetry or null if not specified.
+ get telemetryNodeType() {
+ return this._token && this._token.node_type ? this._token.node_type : null;
+ },
+
+ finalize() {
+ // After this is called, we can expect Service.identity != this.
+ for (let topic of OBSERVER_TOPICS) {
+ Services.obs.removeObserver(this.asyncObserver, topic);
+ }
+ this.resetCredentials();
+ this._userUid = null;
+ },
+
+ async getSignedInUser() {
+ let data = await this._fxaService.getSignedInUser();
+ if (!data) {
+ this._userUid = null;
+ return null;
+ }
+ if (this._userUid == null) {
+ this._userUid = data.uid;
+ } else if (this._userUid != data.uid) {
+ throw new Error("The signed in user has changed");
+ }
+ return data;
+ },
+
+ logout() {
+ // This will be called when sync fails (or when the account is being
+ // unlinked etc). It may have failed because we got a 401 from a sync
+ // server, so we nuke the token. Next time sync runs and wants an
+ // authentication header, we will notice the lack of the token and fetch a
+ // new one.
+ this._token = null;
+ },
+
+ async observe(subject, topic, data) {
+ this._log.debug("observed " + topic);
+ if (!this.username) {
+ this._log.info("Sync is not configured, so ignoring the notification");
+ return;
+ }
+ switch (topic) {
+ case "weave:connected":
+ case fxAccountsCommon.ONLOGIN_NOTIFICATION: {
+ this._log.info("Sync has been connected to a logged in user");
+ this.resetCredentials();
+ let accountData = await this.getSignedInUser();
+
+ if (!accountData.verified) {
+ // wait for a verified notification before we kick sync off.
+ this._log.info("The user is not verified");
+ break;
+ }
+ }
+ // We've been configured with an already verified user, so fall-through.
+ // intentional fall-through - the user is verified.
+ case fxAccountsCommon.ONVERIFIED_NOTIFICATION: {
+ this._log.info("The user became verified");
+ lazy.Weave.Status.login = LOGIN_SUCCEEDED;
+
+ // And actually sync. If we've never synced before, we force a full sync.
+ // If we have, then we are probably just reauthenticating so it's a normal sync.
+ // We can use any pref that must be set if we've synced before, and check
+ // the sync lock state because we might already be doing that first sync.
+ let isFirstSync =
+ !lazy.Weave.Service.locked && !Svc.Prefs.get("client.syncID", null);
+ if (isFirstSync) {
+ this._log.info("Doing initial sync actions");
+ Svc.Prefs.set("firstSync", "resetClient");
+ Services.obs.notifyObservers(null, "weave:service:setup-complete");
+ }
+ // There's no need to wait for sync to complete and it would deadlock
+ // our AsyncObserver.
+ if (!Svc.Prefs.get("testing.tps", false)) {
+ lazy.Weave.Service.sync({ why: "login" });
+ }
+ break;
+ }
+
+ case fxAccountsCommon.ONLOGOUT_NOTIFICATION:
+ lazy.Weave.Service.startOver()
+ .then(() => {
+ this._log.trace("startOver completed");
+ })
+ .catch(err => {
+ this._log.warn("Failed to reset sync", err);
+ });
+ // startOver will cause this instance to be thrown away, so there's
+ // nothing else to do.
+ break;
+
+ case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION:
+ // throw away token forcing us to fetch a new one later.
+ this.resetCredentials();
+ break;
+ }
+ },
+
+ /**
+ * Provide override point for testing token expiration.
+ */
+ _now() {
+ return this._fxaService._internal.now();
+ },
+
+ get _localtimeOffsetMsec() {
+ return this._fxaService._internal.localtimeOffsetMsec;
+ },
+
+ get syncKeyBundle() {
+ return this._syncKeyBundle;
+ },
+
+ get username() {
+ return this._username;
+ },
+
+ /**
+ * Set the username value.
+ *
+ * Changing the username has the side-effect of wiping credentials.
+ */
+ set username(value) {
+ // setting .username is an old throwback, but it should no longer happen.
+ throw new Error("don't set the username");
+ },
+
+ /**
+ * Resets all calculated credentials we hold for the current user. This will
+ * *not* force the user to reauthenticate, but instead will force us to
+ * calculate a new key bundle, fetch a new token, etc.
+ */
+ resetCredentials() {
+ this._syncKeyBundle = null;
+ this._token = null;
+ // The cluster URL comes from the token, so resetting it to empty will
+ // force Sync to not accidentally use a value from an earlier token.
+ lazy.Weave.Service.clusterURL = null;
+ },
+
+ /**
+ * Pre-fetches any information that might help with migration away from this
+ * identity. Called after every sync and is really just an optimization that
+ * allows us to avoid a network request for when we actually need the
+ * migration info.
+ */
+ prefetchMigrationSentinel(service) {
+ // nothing to do here until we decide to migrate away from FxA.
+ },
+
+ /**
+ * Verify the current auth state, unlocking the master-password if necessary.
+ *
+ * Returns a promise that resolves with the current auth state after
+ * attempting to unlock.
+ */
+ async unlockAndVerifyAuthState() {
+ let data = await this.getSignedInUser();
+ const fxa = this._fxaService;
+ if (!data) {
+ lazy.log.debug("unlockAndVerifyAuthState has no FxA user");
+ return LOGIN_FAILED_NO_USERNAME;
+ }
+ if (!this.username) {
+ lazy.log.debug(
+ "unlockAndVerifyAuthState finds that sync isn't configured"
+ );
+ return LOGIN_FAILED_NO_USERNAME;
+ }
+ if (!data.verified) {
+ // Treat not verified as if the user needs to re-auth, so the browser
+ // UI reflects the state.
+ lazy.log.debug("unlockAndVerifyAuthState has an unverified user");
+ return LOGIN_FAILED_LOGIN_REJECTED;
+ }
+ if (await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC)) {
+ lazy.log.debug(
+ "unlockAndVerifyAuthState already has (or can fetch) sync keys"
+ );
+ return STATUS_OK;
+ }
+ // so no keys - ensure MP unlocked.
+ if (!Utils.ensureMPUnlocked()) {
+ // user declined to unlock, so we don't know if they are stored there.
+ lazy.log.debug(
+ "unlockAndVerifyAuthState: user declined to unlock master-password"
+ );
+ return MASTER_PASSWORD_LOCKED;
+ }
+ // If we still can't get keys it probably means the user authenticated
+ // without unlocking the MP or cleared the saved logins, so we've now
+ // lost them - the user will need to reauth before continuing.
+ let result;
+ if (await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC)) {
+ result = STATUS_OK;
+ } else {
+ result = LOGIN_FAILED_LOGIN_REJECTED;
+ }
+ lazy.log.debug(
+ "unlockAndVerifyAuthState re-fetched credentials and is returning",
+ result
+ );
+ return result;
+ },
+
+ /**
+ * Do we have a non-null, not yet expired token for the user currently
+ * signed in?
+ */
+ _hasValidToken() {
+ // If pref is set to ignore cached authentication credentials for debugging,
+ // then return false to force the fetching of a new token.
+ if (lazy.IGNORE_CACHED_AUTH_CREDENTIALS) {
+ return false;
+ }
+ if (!this._token) {
+ return false;
+ }
+ if (this._token.expiration < this._now()) {
+ return false;
+ }
+ return true;
+ },
+
+ // Get our tokenServerURL - a private helper. Returns a string.
+ get _tokenServerUrl() {
+ // We used to support services.sync.tokenServerURI but this was a
+ // pain-point for people using non-default servers as Sync may auto-reset
+ // all services.sync prefs. So if that still exists, it wins.
+ let url = Svc.Prefs.get("tokenServerURI"); // Svc.Prefs "root" is services.sync
+ if (!url) {
+ url = Services.prefs.getCharPref("identity.sync.tokenserver.uri");
+ }
+ while (url.endsWith("/")) {
+ // trailing slashes cause problems...
+ url = url.slice(0, -1);
+ }
+ return url;
+ },
+
+ // Refresh the sync token for our user. Returns a promise that resolves
+ // with a token, or rejects with an error.
+ async _fetchTokenForUser() {
+ const fxa = this._fxaService;
+ // We need keys for things to work. If we don't have them, just
+ // return null for the token - sync calling unlockAndVerifyAuthState()
+ // before actually syncing will setup the error states if necessary.
+ if (!(await fxa.keys.canGetKeyForScope(SCOPE_OLD_SYNC))) {
+ this._log.info(
+ "Unable to fetch keys (master-password locked?), so aborting token fetch"
+ );
+ throw new Error("Can't fetch a token as we can't get keys");
+ }
+
+ // Do the token dance, with a retry in case of transient auth failure.
+ // We need to prove that we know the sync key in order to get a token
+ // from the tokenserver.
+ let getToken = async key => {
+ this._log.info("Getting a sync token from", this._tokenServerUrl);
+ let token = await this._fetchTokenUsingOAuth(key);
+ this._log.trace("Successfully got a token");
+ return token;
+ };
+
+ try {
+ let token, key;
+ try {
+ this._log.info("Getting sync key");
+ key = await fxa.keys.getKeyForScope(SCOPE_OLD_SYNC);
+ if (!key) {
+ throw new Error("browser does not have the sync key, cannot sync");
+ }
+ token = await getToken(key);
+ } catch (err) {
+ // If we get a 401 fetching the token it may be that our auth tokens needed
+ // to be regenerated; retry exactly once.
+ if (!err.response || err.response.status !== 401) {
+ throw err;
+ }
+ this._log.warn(
+ "Token server returned 401, retrying token fetch with fresh credentials"
+ );
+ key = await fxa.keys.getKeyForScope(SCOPE_OLD_SYNC);
+ token = await getToken(key);
+ }
+ // TODO: Make it be only 80% of the duration, so refresh the token
+ // before it actually expires. This is to avoid sync storage errors
+ // otherwise, we may briefly enter a "needs reauthentication" state.
+ // (XXX - the above may no longer be true - someone should check ;)
+ token.expiration = this._now() + token.duration * 1000 * 0.8;
+ if (!this._syncKeyBundle) {
+ this._syncKeyBundle = lazy.BulkKeyBundle.fromJWK(key);
+ }
+ lazy.Weave.Status.login = LOGIN_SUCCEEDED;
+ this._token = token;
+ return token;
+ } catch (caughtErr) {
+ let err = caughtErr; // The error we will rethrow.
+
+ // TODO: unify these errors - we need to handle errors thrown by
+ // both tokenserverclient and hawkclient.
+ // A tokenserver error thrown based on a bad response.
+ if (err.response && err.response.status === 401) {
+ err = new AuthenticationError(err, "tokenserver");
+ // A hawkclient error.
+ } else if (err.code && err.code === 401) {
+ err = new AuthenticationError(err, "hawkclient");
+ // An FxAccounts.jsm error.
+ } else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) {
+ err = new AuthenticationError(err, "fxaccounts");
+ }
+
+ // TODO: write tests to make sure that different auth error cases are handled here
+ // properly: auth error getting oauth token, auth error getting sync token (invalid
+ // generation or client-state error)
+ if (err instanceof AuthenticationError) {
+ this._log.error("Authentication error in _fetchTokenForUser", err);
+ // set it to the "fatal" LOGIN_FAILED_LOGIN_REJECTED reason.
+ lazy.Weave.Status.login = LOGIN_FAILED_LOGIN_REJECTED;
+ } else {
+ this._log.error("Non-authentication error in _fetchTokenForUser", err);
+ // for now assume it is just a transient network related problem
+ // (although sadly, it might also be a regular unhandled exception)
+ lazy.Weave.Status.login = LOGIN_FAILED_NETWORK_ERROR;
+ }
+ throw err;
+ }
+ },
+
+ /**
+ * Generates an OAuth access_token using the OLD_SYNC scope and exchanges it
+ * for a TokenServer token.
+ *
+ * @returns {Promise}
+ * @private
+ */
+ async _fetchTokenUsingOAuth(key) {
+ this._log.debug("Getting a token using OAuth");
+ const fxa = this._fxaService;
+ const ttl = fxAccountsCommon.OAUTH_TOKEN_FOR_SYNC_LIFETIME_SECONDS;
+ const accessToken = await fxa.getOAuthToken({ scope: SCOPE_OLD_SYNC, ttl });
+ const headers = {
+ "X-KeyId": key.kid,
+ };
+
+ return this._tokenServerClient
+ .getTokenUsingOAuth(this._tokenServerUrl, accessToken, headers)
+ .catch(async err => {
+ if (err.response && err.response.status === 401) {
+ // remove the cached token if we cannot authorize with it.
+ // we have to do this here because we know which `token` to remove
+ // from cache.
+ await fxa.removeCachedOAuthToken({ token: accessToken });
+ }
+
+ // continue the error chain, so other handlers can deal with the error.
+ throw err;
+ });
+ },
+
+ // Returns a promise that is resolved with a valid token for the current
+ // user, or rejects if one can't be obtained.
+ // NOTE: This does all the authentication for Sync - it both sets the
+ // key bundle (ie, decryption keys) and does the token fetch. These 2
+ // concepts could be decoupled, but there doesn't seem any value in that
+ // currently.
+ async _ensureValidToken(forceNewToken = false) {
+ let signedInUser = await this.getSignedInUser();
+ if (!signedInUser) {
+ throw new Error("no user is logged in");
+ }
+ if (!signedInUser.verified) {
+ throw new Error("user is not verified");
+ }
+
+ await this.asyncObserver.promiseObserversComplete();
+
+ if (!forceNewToken && this._hasValidToken()) {
+ this._log.trace("_ensureValidToken already has one");
+ return this._token;
+ }
+
+ // We are going to grab a new token - re-use the same promise if we are
+ // already fetching one.
+ if (!this._ensureValidTokenPromise) {
+ this._ensureValidTokenPromise = this.__ensureValidToken().finally(() => {
+ this._ensureValidTokenPromise = null;
+ });
+ }
+ return this._ensureValidTokenPromise;
+ },
+
+ async __ensureValidToken() {
+ // reset this._token as a safety net to reduce the possibility of us
+ // repeatedly attempting to use an invalid token if _fetchTokenForUser throws.
+ this._token = null;
+ try {
+ let token = await this._fetchTokenForUser();
+ this._token = token;
+ // This is a little bit of a hack. The tokenserver tells us a HMACed version
+ // of the FxA uid which we can use for metrics purposes without revealing the
+ // user's true uid. It conceptually belongs to FxA but we get it from tokenserver
+ // for legacy reasons. Hand it back to the FxA client code to deal with.
+ this._fxaService.telemetry._setHashedUID(token.hashed_fxa_uid);
+ return token;
+ } finally {
+ Services.obs.notifyObservers(null, "weave:service:login:got-hashed-id");
+ }
+ },
+
+ getResourceAuthenticator() {
+ return this._getAuthenticationHeader.bind(this);
+ },
+
+ /**
+ * @return a Hawk HTTP Authorization Header, lightly wrapped, for the .uri
+ * of a RESTRequest or AsyncResponse object.
+ */
+ async _getAuthenticationHeader(httpObject, method) {
+ // Note that in failure states we return null, causing the request to be
+ // made without authorization headers, thereby presumably causing a 401,
+ // which causes Sync to log out. If we throw, this may not happen as
+ // expected.
+ try {
+ await this._ensureValidToken();
+ } catch (ex) {
+ this._log.error("Failed to fetch a token for authentication", ex);
+ return null;
+ }
+ if (!this._token) {
+ return null;
+ }
+ let credentials = { id: this._token.id, key: this._token.key };
+ method = method || httpObject.method;
+
+ // Get the local clock offset from the Firefox Accounts server. This should
+ // be close to the offset from the storage server.
+ let options = {
+ now: this._now(),
+ localtimeOffsetMsec: this._localtimeOffsetMsec,
+ credentials,
+ };
+
+ let headerValue = await CryptoUtils.computeHAWK(
+ httpObject.uri,
+ method,
+ options
+ );
+ return { headers: { authorization: headerValue.field } };
+ },
+
+ /**
+ * Determine the cluster for the current user and update state.
+ * Returns true if a new cluster URL was found and it is different from
+ * the existing cluster URL, false otherwise.
+ */
+ async setCluster() {
+ // Make sure we didn't get some unexpected response for the cluster.
+ let cluster = await this._findCluster();
+ this._log.debug("Cluster value = " + cluster);
+ if (cluster == null) {
+ return false;
+ }
+
+ // Convert from the funky "String object with additional properties" that
+ // resource.js returns to a plain-old string.
+ cluster = cluster.toString();
+ // Don't update stuff if we already have the right cluster
+ if (cluster == lazy.Weave.Service.clusterURL) {
+ return false;
+ }
+
+ this._log.debug("Setting cluster to " + cluster);
+ lazy.Weave.Service.clusterURL = cluster;
+
+ return true;
+ },
+
+ async _findCluster() {
+ try {
+ // Ensure we are ready to authenticate and have a valid token.
+ // We need to handle node reassignment here. If we are being asked
+ // for a clusterURL while the service already has a clusterURL, then
+ // it's likely a 401 was received using the existing token - in which
+ // case we just discard the existing token and fetch a new one.
+ let forceNewToken = false;
+ if (lazy.Weave.Service.clusterURL) {
+ this._log.debug(
+ "_findCluster has a pre-existing clusterURL, so fetching a new token token"
+ );
+ forceNewToken = true;
+ }
+ let token = await this._ensureValidToken(forceNewToken);
+ let endpoint = token.endpoint;
+ // For Sync 1.5 storage endpoints, we use the base endpoint verbatim.
+ // However, it should end in "/" because we will extend it with
+ // well known path components. So we add a "/" if it's missing.
+ if (!endpoint.endsWith("/")) {
+ endpoint += "/";
+ }
+ this._log.debug("_findCluster returning " + endpoint);
+ return endpoint;
+ } catch (err) {
+ this._log.info("Failed to fetch the cluster URL", err);
+ // service.js's verifyLogin() method will attempt to fetch a cluster
+ // URL when it sees a 401. If it gets null, it treats it as a "real"
+ // auth error and sets Status.login to LOGIN_FAILED_LOGIN_REJECTED, which
+ // in turn causes a notification bar to appear informing the user they
+ // need to re-authenticate.
+ // On the other hand, if fetching the cluster URL fails with an exception,
+ // verifyLogin() assumes it is a transient error, and thus doesn't show
+ // the notification bar under the assumption the issue will resolve
+ // itself.
+ // Thus:
+ // * On a real 401, we must return null.
+ // * On any other problem we must let an exception bubble up.
+ if (err instanceof AuthenticationError) {
+ return null;
+ }
+ throw err;
+ }
+ },
+};
diff --git a/services/sync/modules/telemetry.sys.mjs b/services/sync/modules/telemetry.sys.mjs
new file mode 100644
index 0000000000..9a12be9714
--- /dev/null
+++ b/services/sync/modules/telemetry.sys.mjs
@@ -0,0 +1,1313 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Support for Sync-and-FxA-related telemetry, which is submitted in a special-purpose
+// telemetry ping called the "sync ping", documented here:
+//
+// ../../../toolkit/components/telemetry/docs/data/sync-ping.rst
+//
+// The sync ping contains identifiers that are linked to the user's Firefox Account
+// and are separate from the main telemetry client_id, so this file is also responsible
+// for ensuring that we can delete those pings upon user request, by plumbing its
+// identifiers into the "deletion-request" ping.
+
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+import { Log } from "resource://gre/modules/Log.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+ Async: "resource://services-common/async.sys.mjs",
+ AuthenticationError: "resource://services-sync/sync_auth.sys.mjs",
+ FxAccounts: "resource://gre/modules/FxAccounts.sys.mjs",
+ Observers: "resource://services-common/observers.sys.mjs",
+ Resource: "resource://services-sync/resource.sys.mjs",
+ Status: "resource://services-sync/status.sys.mjs",
+ Svc: "resource://services-sync/util.sys.mjs",
+ TelemetryController: "resource://gre/modules/TelemetryController.sys.mjs",
+ TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.sys.mjs",
+ TelemetryUtils: "resource://gre/modules/TelemetryUtils.sys.mjs",
+ Weave: "resource://services-sync/main.sys.mjs",
+});
+
+XPCOMUtils.defineLazyModuleGetters(lazy, {
+ ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(lazy, "fxAccounts", () => {
+ return ChromeUtils.importESModule(
+ "resource://gre/modules/FxAccounts.sys.mjs"
+ ).getFxAccountsSingleton();
+});
+
+import * as constants from "resource://services-sync/constants.sys.mjs";
+
+XPCOMUtils.defineLazyGetter(
+ lazy,
+ "WeaveService",
+ () => Cc["@mozilla.org/weave/service;1"].getService().wrappedJSObject
+);
+const log = Log.repository.getLogger("Sync.Telemetry");
+
+const TOPICS = [
+ // For tracking change to account/device identifiers.
+ "fxaccounts:new_device_id",
+ "fxaccounts:onlogout",
+ "weave:service:ready",
+ "weave:service:login:got-hashed-id",
+
+ // For whole-of-sync metrics.
+ "weave:service:sync:start",
+ "weave:service:sync:finish",
+ "weave:service:sync:error",
+
+ // For individual engine metrics.
+ "weave:engine:sync:start",
+ "weave:engine:sync:finish",
+ "weave:engine:sync:error",
+ "weave:engine:sync:applied",
+ "weave:engine:sync:step",
+ "weave:engine:sync:uploaded",
+ "weave:engine:validate:finish",
+ "weave:engine:validate:error",
+
+ // For ad-hoc telemetry events.
+ "weave:telemetry:event",
+ "weave:telemetry:histogram",
+ "fxa:telemetry:event",
+
+ "weave:telemetry:migration",
+];
+
+const PING_FORMAT_VERSION = 1;
+
+const EMPTY_UID = "0".repeat(32);
+
+// The set of engines we record telemetry for - any other engines are ignored.
+const ENGINES = new Set([
+ "addons",
+ "bookmarks",
+ "clients",
+ "forms",
+ "history",
+ "passwords",
+ "prefs",
+ "tabs",
+ "extension-storage",
+ "addresses",
+ "creditcards",
+]);
+
+function tryGetMonotonicTimestamp() {
+ try {
+ return Services.telemetry.msSinceProcessStart();
+ } catch (e) {
+ log.warn("Unable to get a monotonic timestamp!");
+ return -1;
+ }
+}
+
+function timeDeltaFrom(monotonicStartTime) {
+ let now = tryGetMonotonicTimestamp();
+ if (monotonicStartTime !== -1 && now !== -1) {
+ return Math.round(now - monotonicStartTime);
+ }
+ return -1;
+}
+
+const NS_ERROR_MODULE_BASE_OFFSET = 0x45;
+const NS_ERROR_MODULE_NETWORK = 6;
+
+// A reimplementation of NS_ERROR_GET_MODULE, which surprisingly doesn't seem
+// to exist anywhere in .js code in a way that can be reused.
+// This is taken from DownloadCore.sys.mjs.
+function NS_ERROR_GET_MODULE(code) {
+ return ((code & 0x7fff0000) >> 16) - NS_ERROR_MODULE_BASE_OFFSET;
+}
+
+// Converts extra integer fields to strings, rounds floats to three
+// decimal places (nanosecond precision for timings), and removes profile
+// directory paths and URLs from potential error messages.
+function normalizeExtraTelemetryFields(extra) {
+ let result = {};
+ for (let key in extra) {
+ let value = extra[key];
+ let type = typeof value;
+ if (type == "string") {
+ result[key] = ErrorSanitizer.cleanErrorMessage(value);
+ } else if (type == "number") {
+ result[key] = Number.isInteger(value)
+ ? value.toString(10)
+ : value.toFixed(3);
+ } else if (type != "undefined") {
+ throw new TypeError(
+ `Invalid type ${type} for extra telemetry field ${key}`
+ );
+ }
+ }
+ return lazy.ObjectUtils.isEmpty(result) ? undefined : result;
+}
+
+// Keps track of the counts of individual records fate during a sync cycle
+// The main reason this is a class is to keep track of reasons individual records
+// failure reasons without huge memory overhead.
+export class SyncedRecordsTelemetry {
+ // applied => number of items that should be applied.
+ // failed => number of items that failed in this sync.
+ // newFailed => number of items that failed for the first time in this sync.
+ // reconciled => number of items that were reconciled.
+ // failedReasons => {name, count} of reasons a record failed
+ incomingCounts = {
+ applied: 0,
+ failed: 0,
+ newFailed: 0,
+ reconciled: 0,
+ failedReasons: null,
+ };
+ outgoingCounts = { failed: 0, sent: 0, failedReasons: null };
+
+ addIncomingFailedReason(reason) {
+ if (!this.incomingCounts.failedReasons) {
+ this.incomingCounts.failedReasons = [];
+ }
+ let transformedReason = SyncTelemetry.transformError(reason);
+ // Some errors like http/nss errors don't have an error object
+ // those will be caught by the higher level telemetry
+ if (!transformedReason.error) {
+ return;
+ }
+
+ let index = this.incomingCounts.failedReasons.findIndex(
+ reasons => reasons.name === transformedReason.error
+ );
+
+ if (index >= 0) {
+ this.incomingCounts.failedReasons[index].count += 1;
+ } else {
+ this.incomingCounts.failedReasons.push({
+ name: transformedReason.error,
+ count: 1,
+ });
+ }
+ }
+
+ addOutgoingFailedReason(reason) {
+ if (!this.outgoingCounts.failedReasons) {
+ this.outgoingCounts.failedReasons = [];
+ }
+ let transformedReason = SyncTelemetry.transformError(reason);
+ // Some errors like http/nss errors don't have an error object
+ // those will be caught by the higher level telemetry
+ if (!transformedReason.error) {
+ return;
+ }
+ let index = this.outgoingCounts.failedReasons.findIndex(
+ reasons => reasons.name === transformedReason.error
+ );
+ if (index >= 0) {
+ this.outgoingCounts.failedReasons[index].count += 1;
+ } else {
+ this.outgoingCounts.failedReasons.push({
+ name: transformedReason.error,
+ count: 1,
+ });
+ }
+ }
+}
+
+// The `ErrorSanitizer` has 2 main jobs:
+// * Remove PII from errors, such as the profile dir or URLs the user might
+// have visited.
+// * Normalize errors so different locales or operating systems etc don't
+// generate different messages for the same underlying error.
+// * [TODO] Normalize errors so environmental factors don't influence message.
+// For example, timestamps or GUIDs should be replaced with something static.
+export class ErrorSanitizer {
+ // Things we normalize - this isn't exhaustive, but covers the common error messages we see.
+ // Eg:
+ // > Win error 112 during operation write on file [profileDir]\weave\addonsreconciler.json (Espacio en disco insuficiente. )
+ // > Win error 112 during operation write on file [profileDir]\weave\addonsreconciler.json (Diskte yeterli yer yok. )
+ // > <snip many other translations of the error>
+ // > Unix error 28 during operation write on file [profileDir]/weave/addonsreconciler.json (No space left on device)
+ // These tend to crowd out other errors we might care about (eg, top 5 errors for some engines are
+ // variations of the "no space left on device")
+
+ // Note that only errors that have same-but-different errors on Windows and Unix are here - we
+ // still sanitize ones that aren't in these maps to remove the translations etc - eg,
+ // `ERROR_SHARING_VIOLATION` doesn't really have a unix equivalent, so doesn't appear here, but
+ // we still strip the translations to avoid the variants.
+ static E_NO_SPACE_ON_DEVICE = "OS error [No space left on device]";
+ static E_PERMISSION_DENIED = "OS error [Permission denied]";
+ static E_NO_FILE_OR_DIR = "OS error [File/Path not found]";
+ static E_NO_MEM = "OS error [No memory]";
+
+ static WindowsErrorSubstitutions = {
+ 2: this.E_NO_FILE_OR_DIR, // ERROR_FILE_NOT_FOUND
+ 3: this.E_NO_FILE_OR_DIR, // ERROR_PATH_NOT_FOUND
+ 5: this.E_PERMISSION_DENIED, // ERROR_ACCESS_DENIED
+ 8: this.E_NO_MEM, // ERROR_NOT_ENOUGH_MEMORY
+ 112: this.E_NO_SPACE_ON_DEVICE, // ERROR_DISK_FULL
+ };
+
+ static UnixErrorSubstitutions = {
+ 2: this.E_NO_FILE_OR_DIR, // ENOENT
+ 12: this.E_NO_MEM, // ENOMEM
+ 13: this.E_PERMISSION_DENIED, // EACCESS
+ 28: this.E_NO_SPACE_ON_DEVICE, // ENOSPC
+ };
+
+ static DOMErrorSubstitutions = {
+ NotFoundError: this.E_NO_FILE_OR_DIR,
+ NotAllowedError: this.E_PERMISSION_DENIED,
+ };
+
+ static reWinError =
+ /^(?<head>Win error (?<errno>\d+))(?<detail>.*) \(.*\r?\n?\)$/m;
+ static reUnixError =
+ /^(?<head>Unix error (?<errno>\d+))(?<detail>.*) \(.*\)$/;
+
+ static #cleanOSErrorMessage(message, error = undefined) {
+ if (DOMException.isInstance(error)) {
+ const sub = this.DOMErrorSubstitutions[error.name];
+ message = message.replaceAll("\\", "/");
+ if (sub) {
+ return `${sub} ${message}`;
+ }
+ }
+
+ let match = this.reWinError.exec(message);
+ if (match) {
+ let head =
+ this.WindowsErrorSubstitutions[match.groups.errno] || match.groups.head;
+ return head + match.groups.detail.replaceAll("\\", "/");
+ }
+ match = this.reUnixError.exec(message);
+ if (match) {
+ let head =
+ this.UnixErrorSubstitutions[match.groups.errno] || match.groups.head;
+ return head + match.groups.detail;
+ }
+ return message;
+ }
+
+ // A regex we can use to replace the profile dir in error messages. We use a
+ // regexp so we can simply replace all case-insensitive occurences.
+ // This escaping function is from:
+ // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
+ static reProfileDir = new RegExp(
+ PathUtils.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
+ "gi"
+ );
+
+ /**
+ * Clean an error message, removing PII and normalizing OS-specific messages.
+ *
+ * @param {string} message The error message
+ * @param {Error?} error The error class instance, if any.
+ */
+ static cleanErrorMessage(message, error = undefined) {
+ // There's a chance the profiledir is in the error string which is PII we
+ // want to avoid including in the ping.
+ message = message.replace(this.reProfileDir, "[profileDir]");
+ // MSG_INVALID_URL from /dom/bindings/Errors.msg -- no way to access this
+ // directly from JS.
+ if (message.endsWith("is not a valid URL.")) {
+ message = "<URL> is not a valid URL.";
+ }
+ // Try to filter things that look somewhat like a URL (in that they contain a
+ // colon in the middle of non-whitespace), in case anything else is including
+ // these in error messages. Note that JSON.stringified stuff comes through
+ // here, so we explicitly ignore double-quotes as well.
+ message = message.replace(/[^\s"]+:[^\s"]+/g, "<URL>");
+
+ // Anywhere that's normalized the guid in errors we can easily filter
+ // to make it easier to aggregate these types of errors
+ message = message.replace(/<guid: ([^>]+)>/g, "<GUID>");
+
+ return this.#cleanOSErrorMessage(message, error);
+ }
+}
+
+// This function validates the payload of a telemetry "event" - this can be
+// removed once there are APIs available for the telemetry modules to collect
+// these events (bug 1329530) - but for now we simulate that planned API as
+// best we can.
+function validateTelemetryEvent(eventDetails) {
+ let { object, method, value, extra } = eventDetails;
+ // Do do basic validation of the params - everything except "extra" must
+ // be a string. method and object are required.
+ if (
+ typeof method != "string" ||
+ typeof object != "string" ||
+ (value && typeof value != "string") ||
+ (extra && typeof extra != "object")
+ ) {
+ log.warn("Invalid event parameters - wrong types", eventDetails);
+ return false;
+ }
+ // length checks.
+ if (
+ method.length > 20 ||
+ object.length > 20 ||
+ (value && value.length > 80)
+ ) {
+ log.warn("Invalid event parameters - wrong lengths", eventDetails);
+ return false;
+ }
+
+ // extra can be falsey, or an object with string names and values.
+ if (extra) {
+ if (Object.keys(extra).length > 10) {
+ log.warn("Invalid event parameters - too many extra keys", eventDetails);
+ return false;
+ }
+ for (let [ename, evalue] of Object.entries(extra)) {
+ if (
+ typeof ename != "string" ||
+ ename.length > 15 ||
+ typeof evalue != "string" ||
+ evalue.length > 85
+ ) {
+ log.warn(
+ `Invalid event parameters: extra item "${ename} is invalid`,
+ eventDetails
+ );
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+class EngineRecord {
+ constructor(name) {
+ // startTime is in ms from process start, but is monotonic (unlike Date.now())
+ // so we need to keep both it and when.
+ this.startTime = tryGetMonotonicTimestamp();
+ this.name = name;
+
+ // This allows cases like bookmarks-buffered to have a separate name from
+ // the bookmarks engine.
+ let engineImpl = lazy.Weave.Service.engineManager.get(name);
+ if (engineImpl && engineImpl.overrideTelemetryName) {
+ this.overrideTelemetryName = engineImpl.overrideTelemetryName;
+ }
+ }
+
+ toJSON() {
+ let result = { name: this.overrideTelemetryName || this.name };
+ let properties = [
+ "took",
+ "status",
+ "failureReason",
+ "incoming",
+ "outgoing",
+ "validation",
+ "steps",
+ ];
+ for (let property of properties) {
+ result[property] = this[property];
+ }
+ return result;
+ }
+
+ finished(error) {
+ let took = timeDeltaFrom(this.startTime);
+ if (took > 0) {
+ this.took = took;
+ }
+ if (error) {
+ this.failureReason = SyncTelemetry.transformError(error);
+ }
+ }
+
+ recordApplied(counts) {
+ if (this.incoming) {
+ log.error(
+ `Incoming records applied multiple times for engine ${this.name}!`
+ );
+ return;
+ }
+ if (this.name === "clients" && !counts.failed) {
+ // ignore successful application of client records
+ // since otherwise they show up every time and are meaningless.
+ return;
+ }
+
+ let incomingData = {};
+
+ if (counts.failedReasons) {
+ // sort the failed reasons in desc by count, then take top 10
+ counts.failedReasons = counts.failedReasons
+ .sort((a, b) => b.count - a.count)
+ .slice(0, 10);
+ }
+ // Counts has extra stuff used for logging, but we only care about a few
+ let properties = ["applied", "failed", "failedReasons"];
+ // Only record non-zero properties and only record incoming at all if
+ // there's at least one property we care about.
+ for (let property of properties) {
+ if (counts[property]) {
+ incomingData[property] = counts[property];
+ this.incoming = incomingData;
+ }
+ }
+ }
+
+ recordStep(stepResult) {
+ let step = {
+ name: stepResult.name,
+ };
+ if (stepResult.took > 0) {
+ step.took = Math.round(stepResult.took);
+ }
+ if (stepResult.counts) {
+ let counts = stepResult.counts.filter(({ count }) => count > 0);
+ if (counts.length) {
+ step.counts = counts;
+ }
+ }
+ if (this.steps) {
+ this.steps.push(step);
+ } else {
+ this.steps = [step];
+ }
+ }
+
+ recordValidation(validationResult) {
+ if (this.validation) {
+ log.error(`Multiple validations occurred for engine ${this.name}!`);
+ return;
+ }
+ let { problems, version, took, checked } = validationResult;
+ let validation = {
+ version: version || 0,
+ checked: checked || 0,
+ };
+ if (took > 0) {
+ validation.took = Math.round(took);
+ }
+ let summarized = problems.filter(({ count }) => count > 0);
+ if (summarized.length) {
+ validation.problems = summarized;
+ }
+ this.validation = validation;
+ }
+
+ recordValidationError(e) {
+ if (this.validation) {
+ log.error(`Multiple validations occurred for engine ${this.name}!`);
+ return;
+ }
+
+ this.validation = {
+ failureReason: SyncTelemetry.transformError(e),
+ };
+ }
+
+ recordUploaded(counts) {
+ if (counts.sent || counts.failed) {
+ if (!this.outgoing) {
+ this.outgoing = [];
+ }
+ if (counts.failedReasons) {
+ // sort the failed reasons in desc by count, then take top 10
+ counts.failedReasons = counts.failedReasons
+ .sort((a, b) => b.count - a.count)
+ .slice(0, 10);
+ }
+ this.outgoing.push({
+ sent: counts.sent || undefined,
+ failed: counts.failed || undefined,
+ failedReasons: counts.failedReasons || undefined,
+ });
+ }
+ }
+}
+
+// The record of a single "sync" - typically many of these are submitted in
+// a single ping (ie, as a 'syncs' array)
+export class SyncRecord {
+ constructor(allowedEngines, why) {
+ this.allowedEngines = allowedEngines;
+ // Our failure reason. This property only exists in the generated ping if an
+ // error actually occurred.
+ this.failureReason = undefined;
+ this.syncNodeType = null;
+ this.when = Date.now();
+ this.startTime = tryGetMonotonicTimestamp();
+ this.took = 0; // will be set later.
+ this.why = why;
+
+ // All engines that have finished (ie, does not include the "current" one)
+ // We omit this from the ping if it's empty.
+ this.engines = [];
+ // The engine that has started but not yet stopped.
+ this.currentEngine = null;
+ }
+
+ toJSON() {
+ let result = {
+ when: this.when,
+ took: this.took,
+ failureReason: this.failureReason,
+ status: this.status,
+ };
+ if (this.why) {
+ result.why = this.why;
+ }
+ let engines = [];
+ for (let engine of this.engines) {
+ engines.push(engine.toJSON());
+ }
+ if (engines.length) {
+ result.engines = engines;
+ }
+ return result;
+ }
+
+ finished(error) {
+ this.took = timeDeltaFrom(this.startTime);
+ if (this.currentEngine != null) {
+ log.error(
+ "Finished called for the sync before the current engine finished"
+ );
+ this.currentEngine.finished(null);
+ this.onEngineStop(this.currentEngine.name);
+ }
+ if (error) {
+ this.failureReason = SyncTelemetry.transformError(error);
+ }
+
+ this.syncNodeType = lazy.Weave.Service.identity.telemetryNodeType;
+
+ // Check for engine statuses. -- We do this now, and not in engine.finished
+ // to make sure any statuses that get set "late" are recorded
+ for (let engine of this.engines) {
+ let status = lazy.Status.engines[engine.name];
+ if (status && status !== constants.ENGINE_SUCCEEDED) {
+ engine.status = status;
+ }
+ }
+
+ let statusObject = {};
+
+ let serviceStatus = lazy.Status.service;
+ if (serviceStatus && serviceStatus !== constants.STATUS_OK) {
+ statusObject.service = serviceStatus;
+ this.status = statusObject;
+ }
+ let syncStatus = lazy.Status.sync;
+ if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) {
+ statusObject.sync = syncStatus;
+ this.status = statusObject;
+ }
+ }
+
+ onEngineStart(engineName) {
+ if (this._shouldIgnoreEngine(engineName, false)) {
+ return;
+ }
+
+ if (this.currentEngine) {
+ log.error(
+ `Being told that engine ${engineName} has started, but current engine ${this.currentEngine.name} hasn't stopped`
+ );
+ // Just discard the current engine rather than making up data for it.
+ }
+ this.currentEngine = new EngineRecord(engineName);
+ }
+
+ onEngineStop(engineName, error) {
+ // We only care if it's the current engine if we have a current engine.
+ if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) {
+ return;
+ }
+ if (!this.currentEngine) {
+ // It's possible for us to get an error before the start message of an engine
+ // (somehow), in which case we still want to record that error.
+ if (!error) {
+ return;
+ }
+ log.error(
+ `Error triggered on ${engineName} when no current engine exists: ${error}`
+ );
+ this.currentEngine = new EngineRecord(engineName);
+ }
+ this.currentEngine.finished(error);
+ this.engines.push(this.currentEngine);
+ this.currentEngine = null;
+ }
+
+ onEngineApplied(engineName, counts) {
+ if (this._shouldIgnoreEngine(engineName)) {
+ return;
+ }
+ this.currentEngine.recordApplied(counts);
+ }
+
+ onEngineStep(engineName, step) {
+ if (this._shouldIgnoreEngine(engineName)) {
+ return;
+ }
+ this.currentEngine.recordStep(step);
+ }
+
+ onEngineValidated(engineName, validationData) {
+ if (this._shouldIgnoreEngine(engineName, false)) {
+ return;
+ }
+ let engine = this.engines.find(e => e.name === engineName);
+ if (
+ !engine &&
+ this.currentEngine &&
+ engineName === this.currentEngine.name
+ ) {
+ engine = this.currentEngine;
+ }
+ if (engine) {
+ engine.recordValidation(validationData);
+ } else {
+ log.warn(
+ `Validation event triggered for engine ${engineName}, which hasn't been synced!`
+ );
+ }
+ }
+
+ onEngineValidateError(engineName, error) {
+ if (this._shouldIgnoreEngine(engineName, false)) {
+ return;
+ }
+ let engine = this.engines.find(e => e.name === engineName);
+ if (
+ !engine &&
+ this.currentEngine &&
+ engineName === this.currentEngine.name
+ ) {
+ engine = this.currentEngine;
+ }
+ if (engine) {
+ engine.recordValidationError(error);
+ } else {
+ log.warn(
+ `Validation failure event triggered for engine ${engineName}, which hasn't been synced!`
+ );
+ }
+ }
+
+ onEngineUploaded(engineName, counts) {
+ if (this._shouldIgnoreEngine(engineName)) {
+ return;
+ }
+ this.currentEngine.recordUploaded(counts);
+ }
+
+ _shouldIgnoreEngine(engineName, shouldBeCurrent = true) {
+ if (!this.allowedEngines.has(engineName)) {
+ log.info(
+ `Notification for engine ${engineName}, but we aren't recording telemetry for it`
+ );
+ return true;
+ }
+ if (shouldBeCurrent) {
+ if (!this.currentEngine || engineName != this.currentEngine.name) {
+ log.info(`Notification for engine ${engineName} but it isn't current`);
+ return true;
+ }
+ }
+ return false;
+ }
+}
+
+// The entire "sync ping" - it includes all the syncs, events etc recorded in
+// the ping.
+class SyncTelemetryImpl {
+ constructor(allowedEngines) {
+ log.manageLevelFromPref("services.sync.log.logger.telemetry");
+ // This is accessible so we can enable custom engines during tests.
+ this.allowedEngines = allowedEngines;
+ this.current = null;
+ this.setupObservers();
+
+ this.payloads = [];
+ this.discarded = 0;
+ this.events = [];
+ this.histograms = {};
+ this.migrations = [];
+ this.maxEventsCount = lazy.Svc.Prefs.get("telemetry.maxEventsCount", 1000);
+ this.maxPayloadCount = lazy.Svc.Prefs.get("telemetry.maxPayloadCount");
+ this.submissionInterval =
+ lazy.Svc.Prefs.get("telemetry.submissionInterval") * 1000;
+ this.lastSubmissionTime = Services.telemetry.msSinceProcessStart();
+ this.lastUID = EMPTY_UID;
+ this.lastSyncNodeType = null;
+ this.currentSyncNodeType = null;
+ // Note that the sessionStartDate is somewhat arbitrary - the telemetry
+ // modules themselves just use `new Date()`. This means that our startDate
+ // isn't going to be the same as the sessionStartDate in the main pings,
+ // but that's OK for now - if it's a problem we'd need to change the
+ // telemetry modules to expose what it thinks the sessionStartDate is.
+ let sessionStartDate = new Date();
+ this.sessionStartDate = lazy.TelemetryUtils.toLocalTimeISOString(
+ lazy.TelemetryUtils.truncateToHours(sessionStartDate)
+ );
+ lazy.TelemetryController.registerSyncPingShutdown(() => this.shutdown());
+ }
+
+ sanitizeFxaDeviceId(deviceId) {
+ return lazy.fxAccounts.telemetry.sanitizeDeviceId(deviceId);
+ }
+
+ prepareFxaDevices(devices) {
+ // For non-sync users, the data per device is limited -- just an id and a
+ // type (and not even the id yet). For sync users, if we can correctly map
+ // the fxaDevice to a sync device, then we can get os and version info,
+ // which would be quite unfortunate to lose.
+ let extraInfoMap = new Map();
+ if (this.syncIsEnabled()) {
+ for (let client of this.getClientsEngineRecords()) {
+ if (client.fxaDeviceId) {
+ extraInfoMap.set(client.fxaDeviceId, {
+ os: client.os,
+ version: client.version,
+ syncID: this.sanitizeFxaDeviceId(client.id),
+ });
+ }
+ }
+ }
+ // Finally, sanitize and convert to the proper format.
+ return devices.map(d => {
+ let { os, version, syncID } = extraInfoMap.get(d.id) || {
+ os: undefined,
+ version: undefined,
+ syncID: undefined,
+ };
+ return {
+ id: this.sanitizeFxaDeviceId(d.id) || EMPTY_UID,
+ type: d.type,
+ os,
+ version,
+ syncID,
+ };
+ });
+ }
+
+ syncIsEnabled() {
+ return lazy.WeaveService.enabled && lazy.WeaveService.ready;
+ }
+
+ // Separate for testing.
+ getClientsEngineRecords() {
+ if (!this.syncIsEnabled()) {
+ throw new Error("Bug: syncIsEnabled() must be true, check it first");
+ }
+ return lazy.Weave.Service.clientsEngine.remoteClients;
+ }
+
+ updateFxaDevices(devices) {
+ if (!devices) {
+ return {};
+ }
+ let me = devices.find(d => d.isCurrentDevice);
+ let id = me ? this.sanitizeFxaDeviceId(me.id) : undefined;
+ let cleanDevices = this.prepareFxaDevices(devices);
+ return { deviceID: id, devices: cleanDevices };
+ }
+
+ getFxaDevices() {
+ return lazy.fxAccounts.device.recentDeviceList;
+ }
+
+ getPingJSON(reason) {
+ let { devices, deviceID } = this.updateFxaDevices(this.getFxaDevices());
+ return {
+ os: lazy.TelemetryEnvironment.currentEnvironment.system.os,
+ why: reason,
+ devices,
+ discarded: this.discarded || undefined,
+ version: PING_FORMAT_VERSION,
+ syncs: this.payloads.slice(),
+ uid: this.lastUID,
+ syncNodeType: this.lastSyncNodeType || undefined,
+ deviceID,
+ sessionStartDate: this.sessionStartDate,
+ events: !this.events.length ? undefined : this.events,
+ migrations: !this.migrations.length ? undefined : this.migrations,
+ histograms: !Object.keys(this.histograms).length
+ ? undefined
+ : this.histograms,
+ };
+ }
+
+ _addMigrationRecord(type, info) {
+ log.debug("Saw telemetry migration info", type, info);
+ // Updates to this need to be documented in `sync-ping.rst`
+ switch (type) {
+ case "webext-storage":
+ this.migrations.push({
+ type: "webext-storage",
+ entries: +info.entries,
+ entriesSuccessful: +info.entries_successful,
+ extensions: +info.extensions,
+ extensionsSuccessful: +info.extensions_successful,
+ openFailure: !!info.open_failure,
+ });
+ break;
+ default:
+ throw new Error("Bug: Unknown migration record type " + type);
+ }
+ }
+
+ finish(reason) {
+ // Note that we might be in the middle of a sync right now, and so we don't
+ // want to touch this.current.
+ let result = this.getPingJSON(reason);
+ this.payloads = [];
+ this.discarded = 0;
+ this.events = [];
+ this.migrations = [];
+ this.histograms = {};
+ this.submit(result);
+ }
+
+ setupObservers() {
+ for (let topic of TOPICS) {
+ lazy.Observers.add(topic, this, this);
+ }
+ }
+
+ shutdown() {
+ this.finish("shutdown");
+ for (let topic of TOPICS) {
+ lazy.Observers.remove(topic, this, this);
+ }
+ }
+
+ submit(record) {
+ if (!this.isProductionSyncUser()) {
+ return false;
+ }
+ // We still call submit() with possibly illegal payloads so that tests can
+ // know that the ping was built. We don't end up submitting them, however.
+ let numEvents = record.events ? record.events.length : 0;
+ let numMigrations = record.migrations ? record.migrations.length : 0;
+ if (record.syncs.length || numEvents || numMigrations) {
+ log.trace(
+ `submitting ${record.syncs.length} sync record(s) and ` +
+ `${numEvents} event(s) to telemetry`
+ );
+ lazy.TelemetryController.submitExternalPing("sync", record, {
+ usePingSender: true,
+ }).catch(err => {
+ log.error("failed to submit ping", err);
+ });
+ return true;
+ }
+ return false;
+ }
+
+ isProductionSyncUser() {
+ // If FxA isn't production then we treat sync as not being production.
+ // Further, there's the deprecated "services.sync.tokenServerURI" pref we
+ // need to consider - fxa doesn't consider that as if that's the only
+ // pref set, they *are* running a production fxa, just not production sync.
+ if (
+ !lazy.FxAccounts.config.isProductionConfig() ||
+ Services.prefs.prefHasUserValue("services.sync.tokenServerURI")
+ ) {
+ log.trace(`Not sending telemetry ping for self-hosted Sync user`);
+ return false;
+ }
+ return true;
+ }
+
+ onSyncStarted(data) {
+ const why = data && JSON.parse(data).why;
+ if (this.current) {
+ log.warn(
+ "Observed weave:service:sync:start, but we're already recording a sync!"
+ );
+ // Just discard the old record, consistent with our handling of engines, above.
+ this.current = null;
+ }
+ this.current = new SyncRecord(this.allowedEngines, why);
+ }
+
+ // We need to ensure that the telemetry `deletion-request` ping always contains the user's
+ // current sync device ID, because if the user opts out of telemetry then the deletion ping
+ // will be immediately triggered for sending, and we won't have a chance to fill it in later.
+ // This keeps the `deletion-ping` up-to-date when the user's account state changes.
+ onAccountInitOrChange() {
+ // We don't submit sync pings for self-hosters, so don't need to collect their device ids either.
+ if (!this.isProductionSyncUser()) {
+ return;
+ }
+ // Awkwardly async, but no need to await. If the user's account state changes while
+ // this promise is in flight, it will reject and we won't record any data in the ping.
+ // (And a new notification will trigger us to try again with the new state).
+ lazy.fxAccounts.device
+ .getLocalId()
+ .then(deviceId => {
+ let sanitizedDeviceId =
+ lazy.fxAccounts.telemetry.sanitizeDeviceId(deviceId);
+ // In the past we did not persist the FxA metrics identifiers to disk,
+ // so this might be missing until we can fetch it from the server for the
+ // first time. There will be a fresh notification tirggered when it's available.
+ if (sanitizedDeviceId) {
+ // Sanitized device ids are 64 characters long, but telemetry limits scalar strings to 50.
+ // The first 32 chars are sufficient to uniquely identify the device, so just send those.
+ // It's hard to change the sync ping itself to only send 32 chars, to b/w compat reasons.
+ sanitizedDeviceId = sanitizedDeviceId.substr(0, 32);
+ Services.telemetry.scalarSet(
+ "deletion.request.sync_device_id",
+ sanitizedDeviceId
+ );
+ }
+ })
+ .catch(err => {
+ log.warn(
+ `Failed to set sync identifiers in the deletion-request ping: ${err}`
+ );
+ });
+ }
+
+ // This keeps the `deletion-request` ping up-to-date when the user signs out,
+ // clearing the now-nonexistent sync device id.
+ onAccountLogout() {
+ Services.telemetry.scalarSet("deletion.request.sync_device_id", "");
+ }
+
+ _checkCurrent(topic) {
+ if (!this.current) {
+ // This is only `info` because it happens when we do a tabs "quick-write"
+ log.info(
+ `Observed notification ${topic} but no current sync is being recorded.`
+ );
+ return false;
+ }
+ return true;
+ }
+
+ _shouldSubmitForDataChange() {
+ let newID = lazy.fxAccounts.telemetry.getSanitizedUID() || EMPTY_UID;
+ let oldID = this.lastUID;
+ if (
+ newID != EMPTY_UID &&
+ oldID != EMPTY_UID &&
+ // Both are "real" uids, so we care if they've changed.
+ newID != oldID
+ ) {
+ log.trace(
+ `shouldSubmitForDataChange - uid from '${oldID}' -> '${newID}'`
+ );
+ return true;
+ }
+ // We've gone from knowing one of the ids to not knowing it (which we
+ // ignore) or we've gone from not knowing it to knowing it (which is fine),
+ // Now check the node type because a change there also means we should
+ // submit.
+ if (
+ this.lastSyncNodeType &&
+ this.currentSyncNodeType != this.lastSyncNodeType
+ ) {
+ log.trace(
+ `shouldSubmitForDataChange - nodeType from '${this.lastSyncNodeType}' -> '${this.currentSyncNodeType}'`
+ );
+ return true;
+ }
+ log.trace("shouldSubmitForDataChange - no need to submit");
+ return false;
+ }
+
+ maybeSubmitForDataChange() {
+ if (this._shouldSubmitForDataChange()) {
+ log.info(
+ "Early submission of sync telemetry due to changed IDs/NodeType"
+ );
+ this.finish("idchange"); // this actually submits.
+ this.lastSubmissionTime = Services.telemetry.msSinceProcessStart();
+ }
+
+ // Only update the last UIDs if we actually know them.
+ let current_uid = lazy.fxAccounts.telemetry.getSanitizedUID();
+ if (current_uid) {
+ this.lastUID = current_uid;
+ }
+ if (this.currentSyncNodeType) {
+ this.lastSyncNodeType = this.currentSyncNodeType;
+ }
+ }
+
+ maybeSubmitForInterval() {
+ // We want to submit the ping every `this.submissionInterval` but only when
+ // there's no current sync in progress, otherwise we may end up submitting
+ // the sync and the events caused by it in different pings.
+ if (
+ this.current == null &&
+ Services.telemetry.msSinceProcessStart() - this.lastSubmissionTime >
+ this.submissionInterval
+ ) {
+ this.finish("schedule");
+ this.lastSubmissionTime = Services.telemetry.msSinceProcessStart();
+ }
+ }
+
+ onSyncFinished(error) {
+ if (!this.current) {
+ log.warn("onSyncFinished but we aren't recording");
+ return;
+ }
+ this.current.finished(error);
+ this.currentSyncNodeType = this.current.syncNodeType;
+ let current = this.current;
+ this.current = null;
+ this.takeTelemetryRecord(current);
+ }
+
+ takeTelemetryRecord(record) {
+ // We check for "data change" before appending the current sync to payloads,
+ // as it is the current sync which has the data with the new data, and thus
+ // must go in the *next* submission.
+ this.maybeSubmitForDataChange();
+ if (this.payloads.length < this.maxPayloadCount) {
+ this.payloads.push(record.toJSON());
+ } else {
+ ++this.discarded;
+ }
+ // If we are submitting due to timing, it's desirable that the most recent
+ // sync is included, so we check after appending the record.
+ this.maybeSubmitForInterval();
+ }
+
+ _addHistogram(hist) {
+ let histogram = Services.telemetry.getHistogramById(hist);
+ let s = histogram.snapshot();
+ this.histograms[hist] = s;
+ }
+
+ _recordEvent(eventDetails) {
+ this.maybeSubmitForDataChange();
+
+ if (this.events.length >= this.maxEventsCount) {
+ log.warn("discarding event - already queued our maximum", eventDetails);
+ return;
+ }
+
+ let { object, method, value, extra } = eventDetails;
+ if (extra) {
+ extra = normalizeExtraTelemetryFields(extra);
+ eventDetails = { object, method, value, extra };
+ }
+
+ if (!validateTelemetryEvent(eventDetails)) {
+ // we've already logged what the problem is...
+ return;
+ }
+ log.debug("recording event", eventDetails);
+
+ if (extra && lazy.Resource.serverTime && !extra.serverTime) {
+ extra.serverTime = String(lazy.Resource.serverTime);
+ }
+ let category = "sync";
+ let ts = Math.floor(tryGetMonotonicTimestamp());
+
+ // An event record is a simple array with at least 4 items.
+ let event = [ts, category, method, object];
+ // It may have up to 6 elements if |extra| is defined
+ if (value) {
+ event.push(value);
+ if (extra) {
+ event.push(extra);
+ }
+ } else if (extra) {
+ event.push(null); // a null for the empty value.
+ event.push(extra);
+ }
+ this.events.push(event);
+ this.maybeSubmitForInterval();
+ }
+
+ observe(subject, topic, data) {
+ log.trace(`observed ${topic} ${data}`);
+
+ switch (topic) {
+ case "weave:service:ready":
+ case "weave:service:login:got-hashed-id":
+ case "fxaccounts:new_device_id":
+ this.onAccountInitOrChange();
+ break;
+
+ case "fxaccounts:onlogout":
+ this.onAccountLogout();
+ break;
+
+ /* sync itself state changes */
+ case "weave:service:sync:start":
+ this.onSyncStarted(data);
+ break;
+
+ case "weave:service:sync:finish":
+ if (this._checkCurrent(topic)) {
+ this.onSyncFinished(null);
+ }
+ break;
+
+ case "weave:service:sync:error":
+ // argument needs to be truthy (this should always be the case)
+ this.onSyncFinished(subject || "Unknown");
+ break;
+
+ /* engine sync state changes */
+ case "weave:engine:sync:start":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineStart(data);
+ }
+ break;
+ case "weave:engine:sync:finish":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineStop(data, null);
+ }
+ break;
+
+ case "weave:engine:sync:error":
+ if (this._checkCurrent(topic)) {
+ // argument needs to be truthy (this should always be the case)
+ this.current.onEngineStop(data, subject || "Unknown");
+ }
+ break;
+
+ /* engine counts */
+ case "weave:engine:sync:applied":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineApplied(data, subject);
+ }
+ break;
+
+ case "weave:engine:sync:step":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineStep(data, subject);
+ }
+ break;
+
+ case "weave:engine:sync:uploaded":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineUploaded(data, subject);
+ }
+ break;
+
+ case "weave:engine:validate:finish":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineValidated(data, subject);
+ }
+ break;
+
+ case "weave:engine:validate:error":
+ if (this._checkCurrent(topic)) {
+ this.current.onEngineValidateError(data, subject || "Unknown");
+ }
+ break;
+
+ case "weave:telemetry:event":
+ case "fxa:telemetry:event":
+ this._recordEvent(subject);
+ break;
+
+ case "weave:telemetry:histogram":
+ this._addHistogram(data);
+ break;
+
+ case "weave:telemetry:migration":
+ this._addMigrationRecord(data, subject);
+ break;
+
+ default:
+ log.warn(`unexpected observer topic ${topic}`);
+ break;
+ }
+ }
+
+ // Transform an exception into a standard description. Exposed here for when
+ // this module isn't directly responsible for knowing the transform should
+ // happen (for example, when including an error in the |extra| field of
+ // event telemetry)
+ transformError(error) {
+ // Certain parts of sync will use this pattern as a way to communicate to
+ // processIncoming to abort the processing. However, there's no guarantee
+ // this can only happen then.
+ if (typeof error == "object" && error.code && error.cause) {
+ error = error.cause;
+ }
+ if (lazy.Async.isShutdownException(error)) {
+ return { name: "shutdownerror" };
+ }
+
+ if (typeof error === "string") {
+ if (error.startsWith("error.")) {
+ // This is hacky, but I can't imagine that it's not also accurate.
+ return { name: "othererror", error };
+ }
+ error = ErrorSanitizer.cleanErrorMessage(error);
+ return { name: "unexpectederror", error };
+ }
+
+ if (error instanceof lazy.AuthenticationError) {
+ return { name: "autherror", from: error.source };
+ }
+
+ if (DOMException.isInstance(error)) {
+ return {
+ name: "unexpectederror",
+ error: ErrorSanitizer.cleanErrorMessage(error.message, error),
+ };
+ }
+
+ let httpCode =
+ error.status || (error.response && error.response.status) || error.code;
+
+ if (httpCode) {
+ return { name: "httperror", code: httpCode };
+ }
+
+ if (error.failureCode) {
+ return { name: "othererror", error: error.failureCode };
+ }
+
+ if (error.result) {
+ // many "nsresult" errors are actually network errors - if they are
+ // associated with the "network" module we assume that's true.
+ // We also assume NS_ERROR_ABORT is such an error - for almost everything
+ // we care about, it acually is (eg, if the connection fails early enough
+ // or if we have a captive portal etc) - we don't lose anything by this
+ // assumption, it's just that the error will no longer be in the "nserror"
+ // category, so our analysis can still find them.
+ if (
+ error.result == Cr.NS_ERROR_ABORT ||
+ NS_ERROR_GET_MODULE(error.result) == NS_ERROR_MODULE_NETWORK
+ ) {
+ return { name: "httperror", code: error.result };
+ }
+ return { name: "nserror", code: error.result };
+ }
+ // It's probably an Error object, but it also could be some
+ // other object that may or may not override toString to do
+ // something useful.
+ let msg = String(error);
+ if (msg.startsWith("[object")) {
+ // Nothing useful in the default, check for a string "message" property.
+ if (typeof error.message == "string") {
+ msg = String(error.message);
+ } else {
+ // Hopefully it won't come to this...
+ msg = JSON.stringify(error);
+ }
+ }
+ return {
+ name: "unexpectederror",
+ error: ErrorSanitizer.cleanErrorMessage(msg),
+ };
+ }
+}
+
+export var SyncTelemetry = new SyncTelemetryImpl(ENGINES);
diff --git a/services/sync/modules/util.sys.mjs b/services/sync/modules/util.sys.mjs
new file mode 100644
index 0000000000..309eddefb2
--- /dev/null
+++ b/services/sync/modules/util.sys.mjs
@@ -0,0 +1,783 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Observers } from "resource://services-common/observers.sys.mjs";
+
+import { CommonUtils } from "resource://services-common/utils.sys.mjs";
+import { CryptoUtils } from "resource://services-crypto/utils.sys.mjs";
+
+import {
+ DEVICE_TYPE_DESKTOP,
+ MAXIMUM_BACKOFF_INTERVAL,
+ PREFS_BRANCH,
+ SYNC_KEY_DECODED_LENGTH,
+ SYNC_KEY_ENCODED_LENGTH,
+ WEAVE_VERSION,
+} from "resource://services-sync/constants.sys.mjs";
+
+import { Preferences } from "resource://gre/modules/Preferences.sys.mjs";
+import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
+
+const lazy = {};
+const FxAccountsCommon = ChromeUtils.import(
+ "resource://gre/modules/FxAccountsCommon.js"
+);
+
+XPCOMUtils.defineLazyServiceGetter(
+ lazy,
+ "cryptoSDR",
+ "@mozilla.org/login-manager/crypto/SDR;1",
+ "nsILoginManagerCrypto"
+);
+
+XPCOMUtils.defineLazyPreferenceGetter(
+ lazy,
+ "localDeviceType",
+ "services.sync.client.type",
+ DEVICE_TYPE_DESKTOP
+);
+
+/*
+ * Custom exception types.
+ */
+class LockException extends Error {
+ constructor(message) {
+ super(message);
+ this.name = "LockException";
+ }
+}
+
+class HMACMismatch extends Error {
+ constructor(message) {
+ super(message);
+ this.name = "HMACMismatch";
+ }
+}
+
+/*
+ * Utility functions
+ */
+export var Utils = {
+ // Aliases from CryptoUtils.
+ generateRandomBytesLegacy: CryptoUtils.generateRandomBytesLegacy,
+ computeHTTPMACSHA1: CryptoUtils.computeHTTPMACSHA1,
+ digestUTF8: CryptoUtils.digestUTF8,
+ digestBytes: CryptoUtils.digestBytes,
+ sha256: CryptoUtils.sha256,
+ hkdfExpand: CryptoUtils.hkdfExpand,
+ pbkdf2Generate: CryptoUtils.pbkdf2Generate,
+ getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header,
+
+ /**
+ * The string to use as the base User-Agent in Sync requests.
+ * This string will look something like
+ *
+ * Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop
+ */
+ _userAgent: null,
+ get userAgent() {
+ if (!this._userAgent) {
+ let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService(
+ Ci.nsIHttpProtocolHandler
+ );
+ /* eslint-disable no-multi-spaces */
+ this._userAgent =
+ Services.appinfo.name +
+ "/" +
+ Services.appinfo.version + // Product.
+ " (" +
+ hph.oscpu +
+ ")" + // (oscpu)
+ " FxSync/" +
+ WEAVE_VERSION +
+ "." + // Sync.
+ Services.appinfo.appBuildID +
+ "."; // Build.
+ /* eslint-enable no-multi-spaces */
+ }
+ return this._userAgent + lazy.localDeviceType;
+ },
+
+ /**
+ * Wrap a [promise-returning] function to catch all exceptions and log them.
+ *
+ * @usage MyObj._catch = Utils.catch;
+ * MyObj.foo = function() { this._catch(func)(); }
+ *
+ * Optionally pass a function which will be called if an
+ * exception occurs.
+ */
+ catch(func, exceptionCallback) {
+ let thisArg = this;
+ return async function WrappedCatch() {
+ try {
+ return await func.call(thisArg);
+ } catch (ex) {
+ thisArg._log.debug(
+ "Exception calling " + (func.name || "anonymous function"),
+ ex
+ );
+ if (exceptionCallback) {
+ return exceptionCallback.call(thisArg, ex);
+ }
+ return null;
+ }
+ };
+ },
+
+ throwLockException(label) {
+ throw new LockException(`Could not acquire lock. Label: "${label}".`);
+ },
+
+ /**
+ * Wrap a [promise-returning] function to call lock before calling the function
+ * then unlock when it finishes executing or if it threw an error.
+ *
+ * @usage MyObj._lock = Utils.lock;
+ * MyObj.foo = async function() { await this._lock(func)(); }
+ */
+ lock(label, func) {
+ let thisArg = this;
+ return async function WrappedLock() {
+ if (!thisArg.lock()) {
+ Utils.throwLockException(label);
+ }
+
+ try {
+ return await func.call(thisArg);
+ } finally {
+ thisArg.unlock();
+ }
+ };
+ },
+
+ isLockException: function isLockException(ex) {
+ return ex instanceof LockException;
+ },
+
+ /**
+ * Wrap [promise-returning] functions to notify when it starts and
+ * finishes executing or if it threw an error.
+ *
+ * The message is a combination of a provided prefix, the local name, and
+ * the event. Possible events are: "start", "finish", "error". The subject
+ * is the function's return value on "finish" or the caught exception on
+ * "error". The data argument is the predefined data value.
+ *
+ * Example:
+ *
+ * @usage function MyObj(name) {
+ * this.name = name;
+ * this._notify = Utils.notify("obj:");
+ * }
+ * MyObj.prototype = {
+ * foo: function() this._notify("func", "data-arg", async function () {
+ * //...
+ * }(),
+ * };
+ */
+ notify(prefix) {
+ return function NotifyMaker(name, data, func) {
+ let thisArg = this;
+ let notify = function (state, subject) {
+ let mesg = prefix + name + ":" + state;
+ thisArg._log.trace("Event: " + mesg);
+ Observers.notify(mesg, subject, data);
+ };
+
+ return async function WrappedNotify() {
+ notify("start", null);
+ try {
+ let ret = await func.call(thisArg);
+ notify("finish", ret);
+ return ret;
+ } catch (ex) {
+ notify("error", ex);
+ throw ex;
+ }
+ };
+ };
+ },
+
+ /**
+ * GUIDs are 9 random bytes encoded with base64url (RFC 4648).
+ * That makes them 12 characters long with 72 bits of entropy.
+ */
+ makeGUID: function makeGUID() {
+ return CommonUtils.encodeBase64URL(Utils.generateRandomBytesLegacy(9));
+ },
+
+ _base64url_regex: /^[-abcdefghijklmnopqrstuvwxyz0123456789_]{12}$/i,
+ checkGUID: function checkGUID(guid) {
+ return !!guid && this._base64url_regex.test(guid);
+ },
+
+ /**
+ * Add a simple getter/setter to an object that defers access of a property
+ * to an inner property.
+ *
+ * @param obj
+ * Object to add properties to defer in its prototype
+ * @param defer
+ * Property of obj to defer to
+ * @param prop
+ * Property name to defer (or an array of property names)
+ */
+ deferGetSet: function Utils_deferGetSet(obj, defer, prop) {
+ if (Array.isArray(prop)) {
+ return prop.map(prop => Utils.deferGetSet(obj, defer, prop));
+ }
+
+ let prot = obj.prototype;
+
+ // Create a getter if it doesn't exist yet
+ if (!prot.__lookupGetter__(prop)) {
+ prot.__defineGetter__(prop, function () {
+ return this[defer][prop];
+ });
+ }
+
+ // Create a setter if it doesn't exist yet
+ if (!prot.__lookupSetter__(prop)) {
+ prot.__defineSetter__(prop, function (val) {
+ this[defer][prop] = val;
+ });
+ }
+ },
+
+ deepEquals: function eq(a, b) {
+ // If they're triple equals, then it must be equals!
+ if (a === b) {
+ return true;
+ }
+
+ // If they weren't equal, they must be objects to be different
+ if (typeof a != "object" || typeof b != "object") {
+ return false;
+ }
+
+ // But null objects won't have properties to compare
+ if (a === null || b === null) {
+ return false;
+ }
+
+ // Make sure all of a's keys have a matching value in b
+ for (let k in a) {
+ if (!eq(a[k], b[k])) {
+ return false;
+ }
+ }
+
+ // Do the same for b's keys but skip those that we already checked
+ for (let k in b) {
+ if (!(k in a) && !eq(a[k], b[k])) {
+ return false;
+ }
+ }
+
+ return true;
+ },
+
+ // Generator and discriminator for HMAC exceptions.
+ // Split these out in case we want to make them richer in future, and to
+ // avoid inevitable confusion if the message changes.
+ throwHMACMismatch: function throwHMACMismatch(shouldBe, is) {
+ throw new HMACMismatch(
+ `Record SHA256 HMAC mismatch: should be ${shouldBe}, is ${is}`
+ );
+ },
+
+ isHMACMismatch: function isHMACMismatch(ex) {
+ return ex instanceof HMACMismatch;
+ },
+
+ /**
+ * Turn RFC 4648 base32 into our own user-friendly version.
+ * ABCDEFGHIJKLMNOPQRSTUVWXYZ234567
+ * becomes
+ * abcdefghijk8mn9pqrstuvwxyz234567
+ */
+ base32ToFriendly: function base32ToFriendly(input) {
+ return input.toLowerCase().replace(/l/g, "8").replace(/o/g, "9");
+ },
+
+ base32FromFriendly: function base32FromFriendly(input) {
+ return input.toUpperCase().replace(/8/g, "L").replace(/9/g, "O");
+ },
+
+ /**
+ * Key manipulation.
+ */
+
+ // Return an octet string in friendly base32 *with no trailing =*.
+ encodeKeyBase32: function encodeKeyBase32(keyData) {
+ return Utils.base32ToFriendly(CommonUtils.encodeBase32(keyData)).slice(
+ 0,
+ SYNC_KEY_ENCODED_LENGTH
+ );
+ },
+
+ decodeKeyBase32: function decodeKeyBase32(encoded) {
+ return CommonUtils.decodeBase32(
+ Utils.base32FromFriendly(Utils.normalizePassphrase(encoded))
+ ).slice(0, SYNC_KEY_DECODED_LENGTH);
+ },
+
+ jsonFilePath(...args) {
+ let [fileName] = args.splice(-1);
+
+ return PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ ...args,
+ `${fileName}.json`
+ );
+ },
+
+ /**
+ * Load a JSON file from disk in the profile directory.
+ *
+ * @param filePath
+ * JSON file path load from profile. Loaded file will be
+ * extension.
+ * @param that
+ * Object to use for logging.
+ *
+ * @return Promise<>
+ * Promise resolved when the write has been performed.
+ */
+ async jsonLoad(filePath, that) {
+ let path;
+ if (Array.isArray(filePath)) {
+ path = Utils.jsonFilePath(...filePath);
+ } else {
+ path = Utils.jsonFilePath(filePath);
+ }
+
+ if (that._log && that._log.trace) {
+ that._log.trace("Loading json from disk: " + path);
+ }
+
+ try {
+ return await IOUtils.readJSON(path);
+ } catch (e) {
+ if (!DOMException.isInstance(e) || e.name !== "NotFoundError") {
+ if (that._log) {
+ that._log.debug("Failed to load json", e);
+ }
+ }
+ return null;
+ }
+ },
+
+ /**
+ * Save a json-able object to disk in the profile directory.
+ *
+ * @param filePath
+ * JSON file path save to <filePath>.json
+ * @param that
+ * Object to use for logging.
+ * @param obj
+ * Function to provide json-able object to save. If this isn't a
+ * function, it'll be used as the object to make a json string.*
+ * Function called when the write has been performed. Optional.
+ *
+ * @return Promise<>
+ * Promise resolved when the write has been performed.
+ */
+ async jsonSave(filePath, that, obj) {
+ let path = PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ ...(filePath + ".json").split("/")
+ );
+ let dir = PathUtils.parent(path);
+
+ await IOUtils.makeDirectory(dir, { createAncestors: true });
+
+ if (that._log) {
+ that._log.trace("Saving json to disk: " + path);
+ }
+
+ let json = typeof obj == "function" ? obj.call(that) : obj;
+
+ return IOUtils.writeJSON(path, json);
+ },
+
+ /**
+ * Helper utility function to fit an array of records so that when serialized,
+ * they will be within payloadSizeMaxBytes. Returns a new array without the
+ * items.
+ *
+ * Note: This shouldn't be used for extremely large record sizes as
+ * it uses JSON.stringify, which could lead to a heavy performance hit.
+ * See Bug 1815151 for more details.
+ *
+ */
+ tryFitItems(records, payloadSizeMaxBytes) {
+ // Copy this so that callers don't have to do it in advance.
+ records = records.slice();
+ let encoder = Utils.utf8Encoder;
+ const computeSerializedSize = () =>
+ encoder.encode(JSON.stringify(records)).byteLength;
+ // Figure out how many records we can pack into a payload.
+ // We use byteLength here because the data is not encrypted in ascii yet.
+ let size = computeSerializedSize();
+ // See bug 535326 comment 8 for an explanation of the estimation
+ const maxSerializedSize = (payloadSizeMaxBytes / 4) * 3 - 1500;
+ if (maxSerializedSize < 0) {
+ // This is probably due to a test, but it causes very bad behavior if a
+ // test causes this accidentally. We could throw, but there's an obvious/
+ // natural way to handle it, so we do that instead (otherwise we'd have a
+ // weird lower bound of ~1125b on the max record payload size).
+ return [];
+ }
+ if (size > maxSerializedSize) {
+ // Estimate a little more than the direct fraction to maximize packing
+ let cutoff = Math.ceil((records.length * maxSerializedSize) / size);
+ records = records.slice(0, cutoff + 1);
+
+ // Keep dropping off the last entry until the data fits.
+ while (computeSerializedSize() > maxSerializedSize) {
+ records.pop();
+ }
+ }
+ return records;
+ },
+
+ /**
+ * Move a json file in the profile directory. Will fail if a file exists at the
+ * destination.
+ *
+ * @returns a promise that resolves to undefined on success, or rejects on failure
+ *
+ * @param aFrom
+ * Current path to the JSON file saved on disk, relative to profileDir/weave
+ * .json will be appended to the file name.
+ * @param aTo
+ * New path to the JSON file saved on disk, relative to profileDir/weave
+ * .json will be appended to the file name.
+ * @param that
+ * Object to use for logging
+ */
+ jsonMove(aFrom, aTo, that) {
+ let pathFrom = PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ ...(aFrom + ".json").split("/")
+ );
+ let pathTo = PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ ...(aTo + ".json").split("/")
+ );
+ if (that._log) {
+ that._log.trace("Moving " + pathFrom + " to " + pathTo);
+ }
+ return IOUtils.move(pathFrom, pathTo, { noOverwrite: true });
+ },
+
+ /**
+ * Removes a json file in the profile directory.
+ *
+ * @returns a promise that resolves to undefined on success, or rejects on failure
+ *
+ * @param filePath
+ * Current path to the JSON file saved on disk, relative to profileDir/weave
+ * .json will be appended to the file name.
+ * @param that
+ * Object to use for logging
+ */
+ jsonRemove(filePath, that) {
+ let path = PathUtils.join(
+ PathUtils.profileDir,
+ "weave",
+ ...(filePath + ".json").split("/")
+ );
+ if (that._log) {
+ that._log.trace("Deleting " + path);
+ }
+ return IOUtils.remove(path, { ignoreAbsent: true });
+ },
+
+ /**
+ * The following are the methods supported for UI use:
+ *
+ * * isPassphrase:
+ * determines whether a string is either a normalized or presentable
+ * passphrase.
+ * * normalizePassphrase:
+ * take a presentable passphrase and reduce it to a normalized
+ * representation for storage. normalizePassphrase can safely be called
+ * on normalized input.
+ */
+
+ isPassphrase(s) {
+ if (s) {
+ return /^[abcdefghijkmnpqrstuvwxyz23456789]{26}$/.test(
+ Utils.normalizePassphrase(s)
+ );
+ }
+ return false;
+ },
+
+ normalizePassphrase: function normalizePassphrase(pp) {
+ // Short var name... have you seen the lines below?!
+ // Allow leading and trailing whitespace.
+ pp = pp.trim().toLowerCase();
+
+ // 20-char sync key.
+ if (pp.length == 23 && [5, 11, 17].every(i => pp[i] == "-")) {
+ return (
+ pp.slice(0, 5) + pp.slice(6, 11) + pp.slice(12, 17) + pp.slice(18, 23)
+ );
+ }
+
+ // "Modern" 26-char key.
+ if (pp.length == 31 && [1, 7, 13, 19, 25].every(i => pp[i] == "-")) {
+ return (
+ pp.slice(0, 1) +
+ pp.slice(2, 7) +
+ pp.slice(8, 13) +
+ pp.slice(14, 19) +
+ pp.slice(20, 25) +
+ pp.slice(26, 31)
+ );
+ }
+
+ // Something else -- just return.
+ return pp;
+ },
+
+ /**
+ * Create an array like the first but without elements of the second. Reuse
+ * arrays if possible.
+ */
+ arraySub: function arraySub(minuend, subtrahend) {
+ if (!minuend.length || !subtrahend.length) {
+ return minuend;
+ }
+ let setSubtrahend = new Set(subtrahend);
+ return minuend.filter(i => !setSubtrahend.has(i));
+ },
+
+ /**
+ * Build the union of two arrays. Reuse arrays if possible.
+ */
+ arrayUnion: function arrayUnion(foo, bar) {
+ if (!foo.length) {
+ return bar;
+ }
+ if (!bar.length) {
+ return foo;
+ }
+ return foo.concat(Utils.arraySub(bar, foo));
+ },
+
+ /**
+ * Add all the items in `items` to the provided Set in-place.
+ *
+ * @return The provided set.
+ */
+ setAddAll(set, items) {
+ for (let item of items) {
+ set.add(item);
+ }
+ return set;
+ },
+
+ /**
+ * Delete every items in `items` to the provided Set in-place.
+ *
+ * @return The provided set.
+ */
+ setDeleteAll(set, items) {
+ for (let item of items) {
+ set.delete(item);
+ }
+ return set;
+ },
+
+ /**
+ * Take the first `size` items from the Set `items`.
+ *
+ * @return A Set of size at most `size`
+ */
+ subsetOfSize(items, size) {
+ let result = new Set();
+ let count = 0;
+ for (let item of items) {
+ if (count++ == size) {
+ return result;
+ }
+ result.add(item);
+ }
+ return result;
+ },
+
+ bind2: function Async_bind2(object, method) {
+ return function innerBind() {
+ return method.apply(object, arguments);
+ };
+ },
+
+ /**
+ * Is there a master password configured and currently locked?
+ */
+ mpLocked() {
+ return !lazy.cryptoSDR.isLoggedIn;
+ },
+
+ // If Master Password is enabled and locked, present a dialog to unlock it.
+ // Return whether the system is unlocked.
+ ensureMPUnlocked() {
+ if (lazy.cryptoSDR.uiBusy) {
+ return false;
+ }
+ try {
+ lazy.cryptoSDR.encrypt("bacon");
+ return true;
+ } catch (e) {}
+ return false;
+ },
+
+ /**
+ * Return a value for a backoff interval. Maximum is eight hours, unless
+ * Status.backoffInterval is higher.
+ *
+ */
+ calculateBackoff: function calculateBackoff(
+ attempts,
+ baseInterval,
+ statusInterval
+ ) {
+ let backoffInterval =
+ attempts * (Math.floor(Math.random() * baseInterval) + baseInterval);
+ return Math.max(
+ Math.min(backoffInterval, MAXIMUM_BACKOFF_INTERVAL),
+ statusInterval
+ );
+ },
+
+ /**
+ * Return a set of hostnames (including the protocol) which may have
+ * credentials for sync itself stored in the login manager.
+ *
+ * In general, these hosts will not have their passwords synced, will be
+ * reset when we drop sync credentials, etc.
+ */
+ getSyncCredentialsHosts() {
+ let result = new Set();
+ // the FxA host
+ result.add(FxAccountsCommon.FXA_PWDMGR_HOST);
+ // We used to include the FxA hosts (hence the Set() result) but we now
+ // don't give them special treatment (hence the Set() with exactly 1 item)
+ return result;
+ },
+
+ /**
+ * Helper to implement a more efficient version of fairly common pattern:
+ *
+ * Utils.defineLazyIDProperty(this, "syncID", "services.sync.client.syncID")
+ *
+ * is equivalent to (but more efficient than) the following:
+ *
+ * Foo.prototype = {
+ * ...
+ * get syncID() {
+ * let syncID = Svc.Prefs.get("client.syncID", "");
+ * return syncID == "" ? this.syncID = Utils.makeGUID() : syncID;
+ * },
+ * set syncID(value) {
+ * Svc.Prefs.set("client.syncID", value);
+ * },
+ * ...
+ * };
+ */
+ defineLazyIDProperty(object, propName, prefName) {
+ // An object that exists to be the target of the lazy pref getter.
+ // We can't use `object` (at least, not using `propName`) since XPCOMUtils
+ // will stomp on any setter we define.
+ const storage = {};
+ XPCOMUtils.defineLazyPreferenceGetter(storage, "value", prefName, "");
+ Object.defineProperty(object, propName, {
+ configurable: true,
+ enumerable: true,
+ get() {
+ let value = storage.value;
+ if (!value) {
+ value = Utils.makeGUID();
+ Services.prefs.setStringPref(prefName, value);
+ }
+ return value;
+ },
+ set(value) {
+ Services.prefs.setStringPref(prefName, value);
+ },
+ });
+ },
+
+ getDeviceType() {
+ return lazy.localDeviceType;
+ },
+
+ formatTimestamp(date) {
+ // Format timestamp as: "%Y-%m-%d %H:%M:%S"
+ let year = String(date.getFullYear());
+ let month = String(date.getMonth() + 1).padStart(2, "0");
+ let day = String(date.getDate()).padStart(2, "0");
+ let hours = String(date.getHours()).padStart(2, "0");
+ let minutes = String(date.getMinutes()).padStart(2, "0");
+ let seconds = String(date.getSeconds()).padStart(2, "0");
+
+ return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
+ },
+
+ *walkTree(tree, parent = null) {
+ if (tree) {
+ // Skip root node
+ if (parent) {
+ yield [tree, parent];
+ }
+ if (tree.children) {
+ for (let child of tree.children) {
+ yield* Utils.walkTree(child, tree);
+ }
+ }
+ }
+ },
+};
+
+/**
+ * A subclass of Set that serializes as an Array when passed to JSON.stringify.
+ */
+export class SerializableSet extends Set {
+ toJSON() {
+ return Array.from(this);
+ }
+}
+
+XPCOMUtils.defineLazyGetter(Utils, "_utf8Converter", function () {
+ let converter = Cc[
+ "@mozilla.org/intl/scriptableunicodeconverter"
+ ].createInstance(Ci.nsIScriptableUnicodeConverter);
+ converter.charset = "UTF-8";
+ return converter;
+});
+
+XPCOMUtils.defineLazyGetter(Utils, "utf8Encoder", () => new TextEncoder());
+
+/*
+ * Commonly-used services
+ */
+export var Svc = {};
+
+Svc.Prefs = new Preferences(PREFS_BRANCH);
+Svc.Obs = Observers;
+
+Svc.Obs.add("xpcom-shutdown", function () {
+ for (let name in Svc) {
+ delete Svc[name];
+ }
+});