/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ const CRYPTO_COLLECTION = "crypto"; const KEYS_WBO = "keys"; import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"; import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; import { Log } from "resource://gre/modules/Log.sys.mjs"; import { Async } from "resource://services-common/async.sys.mjs"; import { CommonUtils } from "resource://services-common/utils.sys.mjs"; import { CLIENT_NOT_CONFIGURED, CREDENTIALS_CHANGED, HMAC_EVENT_INTERVAL, LOGIN_FAILED, LOGIN_FAILED_INVALID_PASSPHRASE, LOGIN_FAILED_NETWORK_ERROR, LOGIN_FAILED_NO_PASSPHRASE, LOGIN_FAILED_NO_USERNAME, LOGIN_FAILED_SERVER_ERROR, LOGIN_SUCCEEDED, MASTER_PASSWORD_LOCKED, METARECORD_DOWNLOAD_FAIL, NO_SYNC_NODE_FOUND, PREFS_BRANCH, STATUS_DISABLED, STATUS_OK, STORAGE_VERSION, VERSION_OUT_OF_DATE, WEAVE_VERSION, kFirefoxShuttingDown, kFirstSyncChoiceNotMade, kSyncBackoffNotMet, kSyncMasterPasswordLocked, kSyncNetworkOffline, kSyncNotConfigured, kSyncWeaveDisabled, } from "resource://services-sync/constants.sys.mjs"; import { EngineManager } from "resource://services-sync/engines.sys.mjs"; import { ClientEngine } from "resource://services-sync/engines/clients.sys.mjs"; import { Weave } from "resource://services-sync/main.sys.mjs"; import { ErrorHandler, SyncScheduler, } from "resource://services-sync/policies.sys.mjs"; import { CollectionKeyManager, CryptoWrapper, RecordManager, WBORecord, } from "resource://services-sync/record.sys.mjs"; import { Resource } from "resource://services-sync/resource.sys.mjs"; import { EngineSynchronizer } from "resource://services-sync/stages/enginesync.sys.mjs"; import { DeclinedEngines } from "resource://services-sync/stages/declined.sys.mjs"; import { Status } from "resource://services-sync/status.sys.mjs"; ChromeUtils.importESModule("resource://services-sync/telemetry.sys.mjs"); import { Svc, Utils } from "resource://services-sync/util.sys.mjs"; import { getFxAccountsSingleton } from "resource://gre/modules/FxAccounts.sys.mjs"; const fxAccounts = getFxAccountsSingleton(); function getEngineModules() { let result = { Addons: { module: "addons.js", symbol: "AddonsEngine" }, Password: { module: "passwords.js", symbol: "PasswordEngine" }, Prefs: { module: "prefs.js", symbol: "PrefsEngine" }, }; if (AppConstants.MOZ_APP_NAME != "thunderbird") { result.Bookmarks = { module: "bookmarks.js", symbol: "BookmarksEngine" }; result.Form = { module: "forms.js", symbol: "FormEngine" }; result.History = { module: "history.js", symbol: "HistoryEngine" }; result.Tab = { module: "tabs.js", symbol: "TabEngine" }; } if (Svc.Prefs.get("engine.addresses.available", false)) { result.Addresses = { module: "resource://autofill/FormAutofillSync.jsm", symbol: "AddressesEngine", }; } if (Svc.Prefs.get("engine.creditcards.available", false)) { result.CreditCards = { module: "resource://autofill/FormAutofillSync.jsm", symbol: "CreditCardsEngine", }; } result["Extension-Storage"] = { module: "extension-storage.js", controllingPref: "webextensions.storage.sync.kinto", whenTrue: "ExtensionStorageEngineKinto", whenFalse: "ExtensionStorageEngineBridge", }; return result; } const lazy = {}; // A unique identifier for this browser session. Used for logging so // we can easily see whether 2 logs are in the same browser session or // after the browser restarted. XPCOMUtils.defineLazyGetter(lazy, "browserSessionID", Utils.makeGUID); function Sync11Service() { this._notify = Utils.notify("weave:service:"); Utils.defineLazyIDProperty(this, "syncID", "services.sync.client.syncID"); } Sync11Service.prototype = { _lock: Utils.lock, _locked: false, _loggedIn: false, infoURL: null, storageURL: null, metaURL: null, cryptoKeyURL: null, // The cluster URL comes via the identity object, which in the FxA // world is ebbedded in the token returned from the token server. _clusterURL: null, get clusterURL() { return this._clusterURL || ""; }, set clusterURL(value) { if (value != null && typeof value != "string") { throw new Error("cluster must be a string, got " + typeof value); } this._clusterURL = value; this._updateCachedURLs(); }, get isLoggedIn() { return this._loggedIn; }, get locked() { return this._locked; }, lock: function lock() { if (this._locked) { return false; } this._locked = true; return true; }, unlock: function unlock() { this._locked = false; }, // A specialized variant of Utils.catch. // This provides a more informative error message when we're already syncing: // see Bug 616568. _catch(func) { function lockExceptions(ex) { if (Utils.isLockException(ex)) { // This only happens if we're syncing already. this._log.info("Cannot start sync: already syncing?"); } } return Utils.catch.call(this, func, lockExceptions); }, get userBaseURL() { // The user URL is the cluster URL. return this.clusterURL; }, _updateCachedURLs: function _updateCachedURLs() { // Nothing to cache yet if we don't have the building blocks if (!this.clusterURL) { // Also reset all other URLs used by Sync to ensure we aren't accidentally // using one cached earlier - if there's no cluster URL any cached ones // are invalid. this.infoURL = undefined; this.storageURL = undefined; this.metaURL = undefined; this.cryptoKeysURL = undefined; return; } this._log.debug( "Caching URLs under storage user base: " + this.userBaseURL ); // Generate and cache various URLs under the storage API for this user this.infoURL = this.userBaseURL + "info/collections"; this.storageURL = this.userBaseURL + "storage/"; this.metaURL = this.storageURL + "meta/global"; this.cryptoKeysURL = this.storageURL + CRYPTO_COLLECTION + "/" + KEYS_WBO; }, _checkCrypto: function _checkCrypto() { let ok = false; try { let iv = Weave.Crypto.generateRandomIV(); if (iv.length == 24) { ok = true; } } catch (e) { this._log.debug("Crypto check failed: " + e); } return ok; }, /** * Here is a disgusting yet reasonable way of handling HMAC errors deep in * the guts of Sync. The astute reader will note that this is a hacky way of * implementing something like continuable conditions. * * A handler function is glued to each engine. If the engine discovers an * HMAC failure, we fetch keys from the server and update our keys, just as * we would on startup. * * If our key collection changed, we signal to the engine (via our return * value) that it should retry decryption. * * If our key collection did not change, it means that we already had the * correct keys... and thus a different client has the wrong ones. Reupload * the bundle that we fetched, which will bump the modified time on the * server and (we hope) prompt a broken client to fix itself. * * We keep track of the time at which we last applied this reasoning, because * thrashing doesn't solve anything. We keep a reasonable interval between * these remedial actions. */ lastHMACEvent: 0, /* * Returns whether to try again. */ async handleHMACEvent() { let now = Date.now(); // Leave a sizable delay between HMAC recovery attempts. This gives us // time for another client to fix themselves if we touch the record. if (now - this.lastHMACEvent < HMAC_EVENT_INTERVAL) { return false; } this._log.info( "Bad HMAC event detected. Attempting recovery " + "or signaling to other clients." ); // Set the last handled time so that we don't act again. this.lastHMACEvent = now; // Fetch keys. let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); try { let cryptoResp = ( await cryptoKeys.fetch(this.resource(this.cryptoKeysURL)) ).response; // Save out the ciphertext for when we reupload. If there's a bug in // CollectionKeyManager, this will prevent us from uploading junk. let cipherText = cryptoKeys.ciphertext; if (!cryptoResp.success) { this._log.warn("Failed to download keys."); return false; } let keysChanged = await this.handleFetchedKeys( this.identity.syncKeyBundle, cryptoKeys, true ); if (keysChanged) { // Did they change? If so, carry on. this._log.info("Suggesting retry."); return true; // Try again. } // If not, reupload them and continue the current sync. cryptoKeys.ciphertext = cipherText; cryptoKeys.cleartext = null; let uploadResp = await this._uploadCryptoKeys( cryptoKeys, cryptoResp.obj.modified ); if (uploadResp.success) { this._log.info("Successfully re-uploaded keys. Continuing sync."); } else { this._log.warn( "Got error response re-uploading keys. " + "Continuing sync; let's try again later." ); } return false; // Don't try again: same keys. } catch (ex) { this._log.warn( "Got exception fetching and handling crypto keys. " + "Will try again later.", ex ); return false; } }, async handleFetchedKeys(syncKey, cryptoKeys, skipReset) { // Don't want to wipe if we're just starting up! let wasBlank = this.collectionKeys.isClear; let keysChanged = await this.collectionKeys.updateContents( syncKey, cryptoKeys ); if (keysChanged && !wasBlank) { this._log.debug("Keys changed: " + JSON.stringify(keysChanged)); if (!skipReset) { this._log.info("Resetting client to reflect key change."); if (keysChanged.length) { // Collection keys only. Reset individual engines. await this.resetClient(keysChanged); } else { // Default key changed: wipe it all. await this.resetClient(); } this._log.info("Downloaded new keys, client reset. Proceeding."); } return true; } return false; }, /** * Prepare to initialize the rest of Weave after waiting a little bit */ async onStartup() { this.status = Status; this.identity = Status._authManager; this.collectionKeys = new CollectionKeyManager(); this.scheduler = new SyncScheduler(this); this.errorHandler = new ErrorHandler(this); this._log = Log.repository.getLogger("Sync.Service"); this._log.manageLevelFromPref("services.sync.log.logger.service.main"); this._log.info("Loading Weave " + WEAVE_VERSION); this.recordManager = new RecordManager(this); this.enabled = true; await this._registerEngines(); let ua = Cc["@mozilla.org/network/protocol;1?name=http"].getService( Ci.nsIHttpProtocolHandler ).userAgent; this._log.info(ua); if (!this._checkCrypto()) { this.enabled = false; this._log.info( "Could not load the Weave crypto component. Disabling " + "Weave, since it will not work correctly." ); } Svc.Obs.add("weave:service:setup-complete", this); Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon Svc.Obs.add("fxaccounts:device_disconnected", this); Services.prefs.addObserver(PREFS_BRANCH + "engine.", this); if (!this.enabled) { this._log.info("Firefox Sync disabled."); } this._updateCachedURLs(); let status = this._checkSetup(); if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) { this._startTracking(); } // Send an event now that Weave service is ready. We don't do this // synchronously so that observers can import this module before // registering an observer. CommonUtils.nextTick(() => { this.status.ready = true; // UI code uses the flag on the XPCOM service so it doesn't have // to load a bunch of modules. let xps = Cc["@mozilla.org/weave/service;1"].getService( Ci.nsISupports ).wrappedJSObject; xps.ready = true; Svc.Obs.notify("weave:service:ready"); }); }, _checkSetup: function _checkSetup() { if (!this.enabled) { return (this.status.service = STATUS_DISABLED); } return this.status.checkSetup(); }, /** * Register the built-in engines for certain applications */ async _registerEngines() { this.engineManager = new EngineManager(this); let engineModules = getEngineModules(); let engines = []; // We allow a pref, which has no default value, to limit the engines // which are registered. We expect only tests will use this. if (Svc.Prefs.has("registerEngines")) { engines = Svc.Prefs.get("registerEngines").split(","); this._log.info("Registering custom set of engines", engines); } else { // default is all engines. engines = Object.keys(engineModules); } let declined = []; let pref = Svc.Prefs.get("declinedEngines"); if (pref) { declined = pref.split(","); } let clientsEngine = new ClientEngine(this); // Ideally clientsEngine should not exist // (or be a promise that calls initialize() before returning the engine) await clientsEngine.initialize(); this.clientsEngine = clientsEngine; for (let name of engines) { if (!(name in engineModules)) { this._log.info("Do not know about engine: " + name); continue; } let modInfo = engineModules[name]; if (!modInfo.module.includes(":")) { modInfo.module = "resource://services-sync/engines/" + modInfo.module; } try { let ns = ChromeUtils.import(modInfo.module); if (modInfo.symbol) { let symbol = modInfo.symbol; if (!(symbol in ns)) { this._log.warn( "Could not find exported engine instance: " + symbol ); continue; } await this.engineManager.register(ns[symbol]); } else { let { whenTrue, whenFalse, controllingPref } = modInfo; if (!(whenTrue in ns) || !(whenFalse in ns)) { this._log.warn("Could not find all exported engine instances", { whenTrue, whenFalse, }); continue; } await this.engineManager.registerAlternatives( name.toLowerCase(), controllingPref, ns[whenTrue], ns[whenFalse] ); } } catch (ex) { this._log.warn("Could not register engine " + name, ex); } } this.engineManager.setDeclined(declined); }, /** * This method updates the local engines state from an existing meta/global * when Sync is disabled. * Running this code if sync is enabled would end up in very weird results * (but we're nice and we check before doing anything!). */ async updateLocalEnginesState() { await this.promiseInitialized; // Sanity check, this method is not meant to be run if Sync is enabled! if (Svc.Prefs.get("username", "")) { throw new Error("Sync is enabled!"); } // For historical reasons the behaviour of setCluster() is bizarre, // so just check what we care about - the meta URL. if (!this.metaURL) { await this.identity.setCluster(); if (!this.metaURL) { this._log.warn("Could not find a cluster."); return; } } // Clear the cache so we always fetch the latest meta/global. this.recordManager.clearCache(); let meta = await this.recordManager.get(this.metaURL); if (!meta) { this._log.info("Meta record is null, aborting engine state update."); return; } const declinedEngines = meta.payload.declined; const allEngines = this.engineManager.getAll().map(e => e.name); // We don't want our observer of the enabled prefs to treat the change as // a user-change, otherwise we will do the wrong thing with declined etc. this._ignorePrefObserver = true; try { for (const engine of allEngines) { Svc.Prefs.set(`engine.${engine}`, !declinedEngines.includes(engine)); } } finally { this._ignorePrefObserver = false; } }, QueryInterface: ChromeUtils.generateQI([ "nsIObserver", "nsISupportsWeakReference", ]), observe(subject, topic, data) { switch (topic) { // Ideally this observer should be in the SyncScheduler, but it would require // some work to know about the sync specific engines. We should move this there once it does. case "sync:collection_changed": // We check if we're running TPS here to avoid TPS failing because it // couldn't get to get the sync lock, due to us currently syncing the // clients engine. if (data.includes("clients") && !Svc.Prefs.get("testing.tps", false)) { // Sync in the background (it's fine not to wait on the returned promise // because sync() has a lock). // [] = clients collection only this.sync({ why: "collection_changed", engines: [] }).catch(e => { this._log.error(e); }); } break; case "fxaccounts:device_disconnected": data = JSON.parse(data); if (!data.isLocalDevice) { // Refresh the known stale clients list in the background. this.clientsEngine.updateKnownStaleClients().catch(e => { this._log.error(e); }); } break; case "weave:service:setup-complete": let status = this._checkSetup(); if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) { this._startTracking(); } break; case "nsPref:changed": if (this._ignorePrefObserver) { return; } const engine = data.slice((PREFS_BRANCH + "engine.").length); if (engine.includes(".")) { // A sub-preference of the engine was changed. For example // `services.sync.engine.bookmarks.validation.percentageChance`. return; } this._handleEngineStatusChanged(engine); break; } }, _handleEngineStatusChanged(engine) { this._log.trace("Status for " + engine + " engine changed."); if (Svc.Prefs.get("engineStatusChanged." + engine, false)) { // The enabled status being changed back to what it was before. Svc.Prefs.reset("engineStatusChanged." + engine); } else { // Remember that the engine status changed locally until the next sync. Svc.Prefs.set("engineStatusChanged." + engine, true); } }, _startTracking() { const engines = [this.clientsEngine, ...this.engineManager.getAll()]; for (let engine of engines) { try { engine.startTracking(); } catch (e) { this._log.error(`Could not start ${engine.name} engine tracker`, e); } } // This is for TPS. We should try to do better. Svc.Obs.notify("weave:service:tracking-started"); }, async _stopTracking() { const engines = [this.clientsEngine, ...this.engineManager.getAll()]; for (let engine of engines) { try { await engine.stopTracking(); } catch (e) { this._log.error(`Could not stop ${engine.name} engine tracker`, e); } } Svc.Obs.notify("weave:service:tracking-stopped"); }, /** * Obtain a Resource instance with authentication credentials. */ resource: function resource(url) { let res = new Resource(url); res.authenticator = this.identity.getResourceAuthenticator(); return res; }, /** * Perform the info fetch as part of a login or key fetch, or * inside engine sync. */ async _fetchInfo(url) { let infoURL = url || this.infoURL; this._log.trace("In _fetchInfo: " + infoURL); let info; try { info = await this.resource(infoURL).get(); } catch (ex) { this.errorHandler.checkServerError(ex); throw ex; } // Always check for errors. this.errorHandler.checkServerError(info); if (!info.success) { this._log.error("Aborting sync: failed to get collections."); throw info; } return info; }, async verifyAndFetchSymmetricKeys(infoResponse) { this._log.debug( "Fetching and verifying -- or generating -- symmetric keys." ); let syncKeyBundle = this.identity.syncKeyBundle; if (!syncKeyBundle) { this.status.login = LOGIN_FAILED_NO_PASSPHRASE; this.status.sync = CREDENTIALS_CHANGED; return false; } try { if (!infoResponse) { infoResponse = await this._fetchInfo(); // Will throw an exception on failure. } // This only applies when the server is already at version 4. if (infoResponse.status != 200) { this._log.warn( "info/collections returned non-200 response. Failing key fetch." ); this.status.login = LOGIN_FAILED_SERVER_ERROR; this.errorHandler.checkServerError(infoResponse); return false; } let infoCollections = infoResponse.obj; this._log.info( "Testing info/collections: " + JSON.stringify(infoCollections) ); if (this.collectionKeys.updateNeeded(infoCollections)) { this._log.info("collection keys reports that a key update is needed."); // Don't always set to CREDENTIALS_CHANGED -- we will probably take care of this. // Fetch storage/crypto/keys. let cryptoKeys; if (infoCollections && CRYPTO_COLLECTION in infoCollections) { try { cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); let cryptoResp = ( await cryptoKeys.fetch(this.resource(this.cryptoKeysURL)) ).response; if (cryptoResp.success) { await this.handleFetchedKeys(syncKeyBundle, cryptoKeys); return true; } else if (cryptoResp.status == 404) { // On failure, ask to generate new keys and upload them. // Fall through to the behavior below. this._log.warn( "Got 404 for crypto/keys, but 'crypto' in info/collections. Regenerating." ); cryptoKeys = null; } else { // Some other problem. this.status.login = LOGIN_FAILED_SERVER_ERROR; this.errorHandler.checkServerError(cryptoResp); this._log.warn( "Got status " + cryptoResp.status + " fetching crypto keys." ); return false; } } catch (ex) { this._log.warn("Got exception fetching cryptoKeys.", ex); // TODO: Um, what exceptions might we get here? Should we re-throw any? // One kind of exception: HMAC failure. if (Utils.isHMACMismatch(ex)) { this.status.login = LOGIN_FAILED_INVALID_PASSPHRASE; this.status.sync = CREDENTIALS_CHANGED; } else { // In the absence of further disambiguation or more precise // failure constants, just report failure. this.status.login = LOGIN_FAILED; } return false; } } else { this._log.info( "... 'crypto' is not a reported collection. Generating new keys." ); } if (!cryptoKeys) { this._log.info("No keys! Generating new ones."); // Better make some and upload them, and wipe the server to ensure // consistency. This is all achieved via _freshStart. // If _freshStart fails to clear the server or upload keys, it will // throw. await this._freshStart(); return true; } // Last-ditch case. return false; } // No update needed: we're good! return true; } catch (ex) { // This means no keys are present, or there's a network error. this._log.debug("Failed to fetch and verify keys", ex); this.errorHandler.checkServerError(ex); return false; } }, getMaxRecordPayloadSize() { let config = this.serverConfiguration; if (!config || !config.max_record_payload_bytes) { this._log.warn( "No config or incomplete config in getMaxRecordPayloadSize." + " Are we running tests?" ); return 256 * 1024; } let payloadMax = config.max_record_payload_bytes; if (config.max_post_bytes && payloadMax <= config.max_post_bytes) { return config.max_post_bytes - 4096; } return payloadMax; }, getMemcacheMaxRecordPayloadSize() { // Collections stored in memcached ("tabs", "clients" or "meta") have a // different max size than ones stored in the normal storage server db. // In practice, the real limit here is 1M (bug 1300451 comment 40), but // there's overhead involved that is hard to calculate on the client, so we // use 512k to be safe (at the recommendation of the server team). Note // that if the server reports a lower limit (via info/configuration), we // respect that limit instead. See also bug 1403052. return Math.min(512 * 1024, this.getMaxRecordPayloadSize()); }, async verifyLogin(allow40XRecovery = true) { // Attaching auth credentials to a request requires access to // passwords, which means that Resource.get can throw MP-related // exceptions! // So we ask the identity to verify the login state after unlocking the // master password (ie, this call is expected to prompt for MP unlock // if necessary) while we still have control. this.status.login = await this.identity.unlockAndVerifyAuthState(); this._log.debug( "Fetching unlocked auth state returned " + this.status.login ); if (this.status.login != STATUS_OK) { return false; } try { // Make sure we have a cluster to verify against. // This is a little weird, if we don't get a node we pretend // to succeed, since that probably means we just don't have storage. if (this.clusterURL == "" && !(await this.identity.setCluster())) { this.status.sync = NO_SYNC_NODE_FOUND; return true; } // Fetch collection info on every startup. let test = await this.resource(this.infoURL).get(); switch (test.status) { case 200: // The user is authenticated. // We have no way of verifying the passphrase right now, // so wait until remoteSetup to do so. // Just make the most trivial checks. if (!this.identity.syncKeyBundle) { this._log.warn("No passphrase in verifyLogin."); this.status.login = LOGIN_FAILED_NO_PASSPHRASE; return false; } // Go ahead and do remote setup, so that we can determine // conclusively that our passphrase is correct. if (await this._remoteSetup(test)) { // Username/password verified. this.status.login = LOGIN_SUCCEEDED; return true; } this._log.warn("Remote setup failed."); // Remote setup must have failed. return false; case 401: this._log.warn("401: login failed."); // Fall through to the 404 case. case 404: // Check that we're verifying with the correct cluster if (allow40XRecovery && (await this.identity.setCluster())) { return await this.verifyLogin(false); } // We must have the right cluster, but the server doesn't expect us. // For FxA this almost certainly means "transient error fetching token". this.status.login = LOGIN_FAILED_NETWORK_ERROR; return false; default: // Server didn't respond with something that we expected this.status.login = LOGIN_FAILED_SERVER_ERROR; this.errorHandler.checkServerError(test); return false; } } catch (ex) { // Must have failed on some network issue this._log.debug("verifyLogin failed", ex); this.status.login = LOGIN_FAILED_NETWORK_ERROR; this.errorHandler.checkServerError(ex); return false; } }, async generateNewSymmetricKeys() { this._log.info("Generating new keys WBO..."); let wbo = await this.collectionKeys.generateNewKeysWBO(); this._log.info("Encrypting new key bundle."); await wbo.encrypt(this.identity.syncKeyBundle); let uploadRes = await this._uploadCryptoKeys(wbo, 0); if (uploadRes.status != 200) { this._log.warn( "Got status " + uploadRes.status + " uploading new keys. What to do? Throw!" ); this.errorHandler.checkServerError(uploadRes); throw new Error("Unable to upload symmetric keys."); } this._log.info("Got status " + uploadRes.status + " uploading keys."); let serverModified = uploadRes.obj; // Modified timestamp according to server. this._log.debug("Server reports crypto modified: " + serverModified); // Now verify that info/collections shows them! this._log.debug("Verifying server collection records."); let info = await this._fetchInfo(); this._log.debug("info/collections is: " + info.data); if (info.status != 200) { this._log.warn("Non-200 info/collections response. Aborting."); throw new Error("Unable to upload symmetric keys."); } info = info.obj; if (!(CRYPTO_COLLECTION in info)) { this._log.error( "Consistency failure: info/collections excludes " + "crypto after successful upload." ); throw new Error("Symmetric key upload failed."); } // Can't check against local modified: clock drift. if (info[CRYPTO_COLLECTION] < serverModified) { this._log.error( "Consistency failure: info/collections crypto entry " + "is stale after successful upload." ); throw new Error("Symmetric key upload failed."); } // Doesn't matter if the timestamp is ahead. // Download and install them. let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO); let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))) .response; if (cryptoResp.status != 200) { this._log.warn("Failed to download keys."); throw new Error("Symmetric key download failed."); } let keysChanged = await this.handleFetchedKeys( this.identity.syncKeyBundle, cryptoKeys, true ); if (keysChanged) { this._log.info("Downloaded keys differed, as expected."); } }, // configures/enabled/turns-on sync. There must be an FxA user signed in. async configure() { // We don't, and must not, throw if sync is already configured, because we // might end up being called as part of a "reconnect" flow. We also want to // avoid checking the FxA user is the same as the pref because the email // address for the FxA account can change - we'd need to use the uid. let user = await fxAccounts.getSignedInUser(); if (!user) { throw new Error("No FxA user is signed in"); } this._log.info("Configuring sync with current FxA user"); Svc.Prefs.set("username", user.email); Svc.Obs.notify("weave:connected"); }, // resets/turns-off sync. async startOver() { this._log.trace("Invoking Service.startOver."); await this._stopTracking(); this.status.resetSync(); // Deletion doesn't make sense if we aren't set up yet! if (this.clusterURL != "") { // Clear client-specific data from the server, including disabled engines. const engines = [this.clientsEngine, ...this.engineManager.getAll()]; for (let engine of engines) { try { await engine.removeClientData(); } catch (ex) { this._log.warn(`Deleting client data for ${engine.name} failed`, ex); } } this._log.debug("Finished deleting client data."); } else { this._log.debug("Skipping client data removal: no cluster URL."); } this.identity.resetCredentials(); this.status.login = LOGIN_FAILED_NO_USERNAME; this.logout(); Svc.Obs.notify("weave:service:start-over"); // Reset all engines and clear keys. await this.resetClient(); this.collectionKeys.clear(); this.status.resetBackoff(); // Reset Weave prefs. this._ignorePrefObserver = true; Svc.Prefs.resetBranch(""); this._ignorePrefObserver = false; this.clusterURL = null; Svc.Prefs.set("lastversion", WEAVE_VERSION); try { this.identity.finalize(); this.status.__authManager = null; this.identity = Status._authManager; Svc.Obs.notify("weave:service:start-over:finish"); } catch (err) { this._log.error( "startOver failed to re-initialize the identity manager", err ); // Still send the observer notification so the current state is // reflected in the UI. Svc.Obs.notify("weave:service:start-over:finish"); } }, async login() { async function onNotify() { this._loggedIn = false; if (this.scheduler.offline) { this.status.login = LOGIN_FAILED_NETWORK_ERROR; throw new Error("Application is offline, login should not be called"); } this._log.info("User logged in successfully - verifying login."); if (!(await this.verifyLogin())) { // verifyLogin sets the failure states here. throw new Error(`Login failed: ${this.status.login}`); } this._updateCachedURLs(); this._loggedIn = true; return true; } let notifier = this._notify("login", "", onNotify.bind(this)); return this._catch(this._lock("service.js: login", notifier))(); }, logout: function logout() { // If we failed during login, we aren't going to have this._loggedIn set, // but we still want to ask the identity to logout, so it doesn't try and // reuse any old credentials next time we sync. this._log.info("Logging out"); this.identity.logout(); this._loggedIn = false; Svc.Obs.notify("weave:service:logout:finish"); }, // Note: returns false if we failed for a reason other than the server not yet // supporting the api. async _fetchServerConfiguration() { // This is similar to _fetchInfo, but with different error handling. let infoURL = this.userBaseURL + "info/configuration"; this._log.debug("Fetching server configuration", infoURL); let configResponse; try { configResponse = await this.resource(infoURL).get(); } catch (ex) { // This is probably a network or similar error. this._log.warn("Failed to fetch info/configuration", ex); this.errorHandler.checkServerError(ex); return false; } if (configResponse.status == 404) { // This server doesn't support the URL yet - that's OK. this._log.debug( "info/configuration returned 404 - using default upload semantics" ); } else if (configResponse.status != 200) { this._log.warn( `info/configuration returned ${configResponse.status} - using default configuration` ); this.errorHandler.checkServerError(configResponse); return false; } else { this.serverConfiguration = configResponse.obj; } this._log.trace( "info/configuration for this server", this.serverConfiguration ); return true; }, // Stuff we need to do after login, before we can really do // anything (e.g. key setup). async _remoteSetup(infoResponse, fetchConfig = true) { if (fetchConfig && !(await this._fetchServerConfiguration())) { return false; } this._log.debug("Fetching global metadata record"); let meta = await this.recordManager.get(this.metaURL); // Checking modified time of the meta record. if ( infoResponse && infoResponse.obj.meta != this.metaModified && (!meta || !meta.isNew) ) { // Delete the cached meta record... this._log.debug( "Clearing cached meta record. metaModified is " + JSON.stringify(this.metaModified) + ", setting to " + JSON.stringify(infoResponse.obj.meta) ); this.recordManager.del(this.metaURL); // ... fetch the current record from the server, and COPY THE FLAGS. let newMeta = await this.recordManager.get(this.metaURL); // If we got a 401, we do not want to create a new meta/global - we // should be able to get the existing meta after we get a new node. if (this.recordManager.response.status == 401) { this._log.debug( "Fetching meta/global record on the server returned 401." ); this.errorHandler.checkServerError(this.recordManager.response); return false; } if (this.recordManager.response.status == 404) { this._log.debug("No meta/global record on the server. Creating one."); try { await this._uploadNewMetaGlobal(); } catch (uploadRes) { this._log.warn( "Unable to upload new meta/global. Failing remote setup." ); this.errorHandler.checkServerError(uploadRes); return false; } } else if (!newMeta) { this._log.warn("Unable to get meta/global. Failing remote setup."); this.errorHandler.checkServerError(this.recordManager.response); return false; } else { // If newMeta, then it stands to reason that meta != null. newMeta.isNew = meta.isNew; newMeta.changed = meta.changed; } // Switch in the new meta object and record the new time. meta = newMeta; this.metaModified = infoResponse.obj.meta; } let remoteVersion = meta && meta.payload.storageVersion ? meta.payload.storageVersion : ""; this._log.debug( [ "Weave Version:", WEAVE_VERSION, "Local Storage:", STORAGE_VERSION, "Remote Storage:", remoteVersion, ].join(" ") ); // Check for cases that require a fresh start. When comparing remoteVersion, // we need to convert it to a number as older clients used it as a string. if ( !meta || !meta.payload.storageVersion || !meta.payload.syncID || STORAGE_VERSION > parseFloat(remoteVersion) ) { this._log.info( "One of: no meta, no meta storageVersion, or no meta syncID. Fresh start needed." ); // abort the server wipe if the GET status was anything other than 404 or 200 let status = this.recordManager.response.status; if (status != 200 && status != 404) { this.status.sync = METARECORD_DOWNLOAD_FAIL; this.errorHandler.checkServerError(this.recordManager.response); this._log.warn( "Unknown error while downloading metadata record. Aborting sync." ); return false; } if (!meta) { this._log.info("No metadata record, server wipe needed"); } if (meta && !meta.payload.syncID) { this._log.warn("No sync id, server wipe needed"); } this._log.info("Wiping server data"); await this._freshStart(); if (status == 404) { this._log.info( "Metadata record not found, server was wiped to ensure " + "consistency." ); } else { // 200 this._log.info("Wiped server; incompatible metadata: " + remoteVersion); } return true; } else if (remoteVersion > STORAGE_VERSION) { this.status.sync = VERSION_OUT_OF_DATE; this._log.warn("Upgrade required to access newer storage version."); return false; } else if (meta.payload.syncID != this.syncID) { this._log.info( "Sync IDs differ. Local is " + this.syncID + ", remote is " + meta.payload.syncID ); await this.resetClient(); this.collectionKeys.clear(); this.syncID = meta.payload.syncID; this._log.debug("Clear cached values and take syncId: " + this.syncID); if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) { this._log.warn("Failed to fetch symmetric keys. Failing remote setup."); return false; } // bug 545725 - re-verify creds and fail sanely if (!(await this.verifyLogin())) { this.status.sync = CREDENTIALS_CHANGED; this._log.info( "Credentials have changed, aborting sync and forcing re-login." ); return false; } return true; } if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) { this._log.warn("Failed to fetch symmetric keys. Failing remote setup."); return false; } return true; }, /** * Return whether we should attempt login at the start of a sync. * * Note that this function has strong ties to _checkSync: callers * of this function should typically use _checkSync to verify that * any necessary login took place. */ _shouldLogin: function _shouldLogin() { return ( this.enabled && !this.scheduler.offline && !this.isLoggedIn && Async.isAppReady() ); }, /** * Determine if a sync should run. * * @param ignore [optional] * array of reasons to ignore when checking * * @return Reason for not syncing; not-truthy if sync should run */ _checkSync: function _checkSync(ignore) { let reason = ""; // Ideally we'd call _checkSetup() here but that has too many side-effects. if (Status.service == CLIENT_NOT_CONFIGURED) { reason = kSyncNotConfigured; } else if (Status.service == STATUS_DISABLED || !this.enabled) { reason = kSyncWeaveDisabled; } else if (this.scheduler.offline) { reason = kSyncNetworkOffline; } else if (this.status.minimumNextSync > Date.now()) { reason = kSyncBackoffNotMet; } else if ( this.status.login == MASTER_PASSWORD_LOCKED && Utils.mpLocked() ) { reason = kSyncMasterPasswordLocked; } else if (Svc.Prefs.get("firstSync") == "notReady") { reason = kFirstSyncChoiceNotMade; } else if (!Async.isAppReady()) { reason = kFirefoxShuttingDown; } if (ignore && ignore.includes(reason)) { return ""; } return reason; }, async sync({ engines, why } = {}) { let dateStr = Utils.formatTimestamp(new Date()); this._log.debug("User-Agent: " + Utils.userAgent); await this.promiseInitialized; this._log.info( `Starting sync at ${dateStr} in browser session ${lazy.browserSessionID}` ); return this._catch(async function () { // Make sure we're logged in. if (this._shouldLogin()) { this._log.debug("In sync: should login."); if (!(await this.login())) { this._log.debug("Not syncing: login returned false."); return; } } else { this._log.trace("In sync: no need to login."); } await this._lockedSync(engines, why); })(); }, /** * Sync up engines with the server. */ async _lockedSync(engineNamesToSync, why) { return this._lock( "service.js: sync", this._notify("sync", JSON.stringify({ why }), async function onNotify() { let histogram = Services.telemetry.getHistogramById("WEAVE_START_COUNT"); histogram.add(1); let synchronizer = new EngineSynchronizer(this); await synchronizer.sync(engineNamesToSync, why); // Might throw! histogram = Services.telemetry.getHistogramById( "WEAVE_COMPLETE_SUCCESS_COUNT" ); histogram.add(1); // We successfully synchronized. // Check if the identity wants to pre-fetch a migration sentinel from // the server. // If we have no clusterURL, we are probably doing a node reassignment // so don't attempt to get it in that case. if (this.clusterURL) { this.identity.prefetchMigrationSentinel(this); } // Now let's update our declined engines await this._maybeUpdateDeclined(); }) )(); }, /** * Update the "declined" information in meta/global if necessary. */ async _maybeUpdateDeclined() { // if Sync failed due to no node we will not have a meta URL, so can't // update anything. if (!this.metaURL) { return; } let meta = await this.recordManager.get(this.metaURL); if (!meta) { this._log.warn("No meta/global; can't update declined state."); return; } let declinedEngines = new DeclinedEngines(this); let didChange = declinedEngines.updateDeclined(meta, this.engineManager); if (!didChange) { this._log.info( "No change to declined engines. Not reuploading meta/global." ); return; } await this.uploadMetaGlobal(meta); }, /** * Upload a fresh meta/global record * @throws the response object if the upload request was not a success */ async _uploadNewMetaGlobal() { let meta = new WBORecord("meta", "global"); meta.payload.syncID = this.syncID; meta.payload.storageVersion = STORAGE_VERSION; meta.payload.declined = this.engineManager.getDeclined(); meta.modified = 0; meta.isNew = true; await this.uploadMetaGlobal(meta); }, /** * Upload meta/global, throwing the response on failure * @param {WBORecord} meta meta/global record * @throws the response object if the request was not a success */ async uploadMetaGlobal(meta) { this._log.debug("Uploading meta/global", meta); let res = this.resource(this.metaURL); res.setHeader("X-If-Unmodified-Since", meta.modified); let response = await res.put(meta); if (!response.success) { throw response; } // From https://docs.services.mozilla.com/storage/apis-1.5.html: // "Successful responses will return the new last-modified time for the collection." meta.modified = response.obj; this.recordManager.set(this.metaURL, meta); }, /** * Upload crypto/keys * @param {WBORecord} cryptoKeys crypto/keys record * @param {Number} lastModified known last modified timestamp (in decimal seconds), * will be used to set the X-If-Unmodified-Since header */ async _uploadCryptoKeys(cryptoKeys, lastModified) { this._log.debug(`Uploading crypto/keys (lastModified: ${lastModified})`); let res = this.resource(this.cryptoKeysURL); res.setHeader("X-If-Unmodified-Since", lastModified); return res.put(cryptoKeys); }, async _freshStart() { this._log.info("Fresh start. Resetting client."); await this.resetClient(); this.collectionKeys.clear(); // Wipe the server. await this.wipeServer(); // Upload a new meta/global record. // _uploadNewMetaGlobal throws on failure -- including race conditions. // If we got into a race condition, we'll abort the sync this way, too. // That's fine. We'll just wait till the next sync. The client that we're // racing is probably busy uploading stuff right now anyway. await this._uploadNewMetaGlobal(); // Wipe everything we know about except meta because we just uploaded it // TODO: there's a bug here. We should be calling resetClient, no? // Generate, upload, and download new keys. Do this last so we don't wipe // them... await this.generateNewSymmetricKeys(); }, /** * Wipe user data from the server. * * @param collections [optional] * Array of collections to wipe. If not given, all collections are * wiped by issuing a DELETE request for `storageURL`. * * @return the server's timestamp of the (last) DELETE. */ async wipeServer(collections) { let response; let histogram = Services.telemetry.getHistogramById( "WEAVE_WIPE_SERVER_SUCCEEDED" ); if (!collections) { // Strip the trailing slash. let res = this.resource(this.storageURL.slice(0, -1)); res.setHeader("X-Confirm-Delete", "1"); try { response = await res.delete(); } catch (ex) { this._log.debug("Failed to wipe server", ex); histogram.add(false); throw ex; } if (response.status != 200 && response.status != 404) { this._log.debug( "Aborting wipeServer. Server responded with " + response.status + " response for " + this.storageURL ); histogram.add(false); throw response; } histogram.add(true); return response.headers["x-weave-timestamp"]; } let timestamp; for (let name of collections) { let url = this.storageURL + name; try { response = await this.resource(url).delete(); } catch (ex) { this._log.debug("Failed to wipe '" + name + "' collection", ex); histogram.add(false); throw ex; } if (response.status != 200 && response.status != 404) { this._log.debug( "Aborting wipeServer. Server responded with " + response.status + " response for " + url ); histogram.add(false); throw response; } if ("x-weave-timestamp" in response.headers) { timestamp = response.headers["x-weave-timestamp"]; } } histogram.add(true); return timestamp; }, /** * Wipe all local user data. * * @param engines [optional] * Array of engine names to wipe. If not given, all engines are used. */ async wipeClient(engines) { // If we don't have any engines, reset the service and wipe all engines if (!engines) { // Clear out any service data await this.resetService(); engines = [this.clientsEngine, ...this.engineManager.getAll()]; } else { // Convert the array of names into engines engines = this.engineManager.get(engines); } // Fully wipe each engine if it's able to decrypt data for (let engine of engines) { if (await engine.canDecrypt()) { await engine.wipeClient(); } } }, /** * Wipe all remote user data by wiping the server then telling each remote * client to wipe itself. * * @param engines * Array of engine names to wipe. */ async wipeRemote(engines) { try { // Make sure stuff gets uploaded. await this.resetClient(engines); // Clear out any server data. await this.wipeServer(engines); // Only wipe the engines provided. let extra = { reason: "wipe-remote" }; for (const e of engines) { await this.clientsEngine.sendCommand("wipeEngine", [e], null, extra); } // Make sure the changed clients get updated. await this.clientsEngine.sync(); } catch (ex) { this.errorHandler.checkServerError(ex); throw ex; } }, /** * Reset local service information like logs, sync times, caches. */ async resetService() { return this._catch(async function reset() { this._log.info("Service reset."); // Pretend we've never synced to the server and drop cached data this.syncID = ""; this.recordManager.clearCache(); })(); }, /** * Reset the client by getting rid of any local server data and client data. * * @param engines [optional] * Array of engine names to reset. If not given, all engines are used. */ async resetClient(engines) { return this._catch(async function doResetClient() { // If we don't have any engines, reset everything including the service if (!engines) { // Clear out any service data await this.resetService(); engines = [this.clientsEngine, ...this.engineManager.getAll()]; } else { // Convert the array of names into engines engines = this.engineManager.get(engines); } // Have each engine drop any temporary meta data for (let engine of engines) { await engine.resetClient(); } })(); }, recordTelemetryEvent(object, method, value, extra = undefined) { Svc.Obs.notify("weave:telemetry:event", { object, method, value, extra }); }, }; export var Service = new Sync11Service(); Service.promiseInitialized = new Promise(resolve => { Service.onStartup().then(resolve); });