diff options
Diffstat (limited to '')
113 files changed, 33127 insertions, 0 deletions
diff --git a/services/sync/tests/unit/addon1-search.json b/services/sync/tests/unit/addon1-search.json new file mode 100644 index 0000000000..55f8af8857 --- /dev/null +++ b/services/sync/tests/unit/addon1-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Non-Restartless Test Extension", + "type": "extension", + "guid": "addon1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/addon1.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/bootstrap1-search.json b/services/sync/tests/unit/bootstrap1-search.json new file mode 100644 index 0000000000..8cd1cf43ed --- /dev/null +++ b/services/sync/tests/unit/bootstrap1-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Restartless Test Extension", + "type": "extension", + "guid": "bootstrap1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/bootstrap1.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/head_appinfo.js b/services/sync/tests/unit/head_appinfo.js new file mode 100644 index 0000000000..b27bb27e61 --- /dev/null +++ b/services/sync/tests/unit/head_appinfo.js @@ -0,0 +1,58 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); + +// Required to avoid failures. +do_get_profile(); + +// Init FormHistoryStartup and pretend we opened a profile. +var fhs = Cc["@mozilla.org/satchel/form-history-startup;1"].getService( + Ci.nsIObserver +); +fhs.observe(null, "profile-after-change", null); + +// An app is going to have some prefs set which xpcshell tests don't. +Services.prefs.setCharPref( + "identity.sync.tokenserver.uri", + "http://token-server" +); + +// Make sure to provide the right OS so crypto loads the right binaries +function getOS() { + switch (mozinfo.os) { + case "win": + return "WINNT"; + case "mac": + return "Darwin"; + default: + return "Linux"; + } +} + +const { updateAppInfo } = ChromeUtils.importESModule( + "resource://testing-common/AppInfo.sys.mjs" +); +updateAppInfo({ + name: "XPCShell", + ID: "xpcshell@tests.mozilla.org", + version: "1", + platformVersion: "", + OS: getOS(), +}); + +// Register resource aliases. Normally done in SyncComponents.manifest. +function addResourceAlias() { + const resProt = Services.io + .getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler); + for (let s of ["common", "sync", "crypto"]) { + let uri = Services.io.newURI("resource://gre/modules/services-" + s + "/"); + resProt.setSubstitution("services-" + s, uri); + } +} +addResourceAlias(); diff --git a/services/sync/tests/unit/head_errorhandler_common.js b/services/sync/tests/unit/head_errorhandler_common.js new file mode 100644 index 0000000000..fcf44ec43b --- /dev/null +++ b/services/sync/tests/unit/head_errorhandler_common.js @@ -0,0 +1,195 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_helpers.js */ +/* import-globals-from head_http_server.js */ + +// This file expects Service to be defined in the global scope when EHTestsCommon +// is used (from service.js). +/* global Service */ + +var { Changeset, EngineManager, Store, SyncEngine, Tracker, LegacyTracker } = + ChromeUtils.importESModule("resource://services-sync/engines.sys.mjs"); +var { + ABORT_SYNC_COMMAND, + CLIENT_NOT_CONFIGURED, + CREDENTIALS_CHANGED, + DEFAULT_DOWNLOAD_BATCH_SIZE, + DEFAULT_GUID_FETCH_BATCH_SIZE, + DEFAULT_KEYBUNDLE_NAME, + DEVICE_TYPE_DESKTOP, + DEVICE_TYPE_MOBILE, + ENGINE_APPLY_FAIL, + ENGINE_BATCH_INTERRUPTED, + ENGINE_DOWNLOAD_FAIL, + ENGINE_SUCCEEDED, + ENGINE_UNKNOWN_FAIL, + ENGINE_UPLOAD_FAIL, + HMAC_EVENT_INTERVAL, + IDLE_OBSERVER_BACK_DELAY, + LOGIN_FAILED, + LOGIN_FAILED_INVALID_PASSPHRASE, + LOGIN_FAILED_LOGIN_REJECTED, + LOGIN_FAILED_NETWORK_ERROR, + LOGIN_FAILED_NO_PASSPHRASE, + LOGIN_FAILED_NO_USERNAME, + LOGIN_FAILED_SERVER_ERROR, + LOGIN_SUCCEEDED, + MASTER_PASSWORD_LOCKED, + MASTER_PASSWORD_LOCKED_RETRY_INTERVAL, + MAXIMUM_BACKOFF_INTERVAL, + MAX_ERROR_COUNT_BEFORE_BACKOFF, + MAX_HISTORY_DOWNLOAD, + MAX_HISTORY_UPLOAD, + METARECORD_DOWNLOAD_FAIL, + MINIMUM_BACKOFF_INTERVAL, + MULTI_DEVICE_THRESHOLD, + NO_SYNC_NODE_FOUND, + NO_SYNC_NODE_INTERVAL, + OVER_QUOTA, + PREFS_BRANCH, + RESPONSE_OVER_QUOTA, + SCORE_INCREMENT_MEDIUM, + SCORE_INCREMENT_SMALL, + SCORE_INCREMENT_XLARGE, + SCORE_UPDATE_DELAY, + SERVER_MAINTENANCE, + SINGLE_USER_THRESHOLD, + SQLITE_MAX_VARIABLE_NUMBER, + STATUS_DISABLED, + STATUS_OK, + STORAGE_VERSION, + SYNC_FAILED, + SYNC_FAILED_PARTIAL, + SYNC_KEY_DECODED_LENGTH, + SYNC_KEY_ENCODED_LENGTH, + SYNC_SUCCEEDED, + URI_LENGTH_MAX, + VERSION_OUT_OF_DATE, + WEAVE_VERSION, + kFirefoxShuttingDown, + kFirstSyncChoiceNotMade, + kSyncBackoffNotMet, + kSyncMasterPasswordLocked, + kSyncNetworkOffline, + kSyncNotConfigured, + kSyncWeaveDisabled, +} = ChromeUtils.importESModule("resource://services-sync/constants.sys.mjs"); +var { BulkKeyBundle, SyncKeyBundle } = ChromeUtils.importESModule( + "resource://services-sync/keys.sys.mjs" +); + +// Common code for test_errorhandler_{1,2}.js -- pulled out to make it less +// monolithic and take less time to execute. +const EHTestsCommon = { + service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "42"); + response.bodyOutputStream.write(body, body.length); + }, + + async sync_httpd_setup() { + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let catapultEngine = Service.engineManager.get("catapult"); + let catapultSyncID = await catapultEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + catapult: { version: catapultEngine.version, syncID: catapultSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let handler_401 = httpd_handler(401, "Unauthorized"); + return httpd_setup({ + // Normal server behaviour. + "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + + // Credentials are wrong or node reallocated. + "/1.1/janedoe/storage/meta/global": handler_401, + "/1.1/janedoe/info/collections": handler_401, + + // Maintenance or overloaded (503 + Retry-After) at info/collections. + "/1.1/broken.info/info/collections": EHTestsCommon.service_unavailable, + + // Maintenance or overloaded (503 + Retry-After) at meta/global. + "/1.1/broken.meta/storage/meta/global": EHTestsCommon.service_unavailable, + "/1.1/broken.meta/info/collections": collectionsHelper.handler, + + // Maintenance or overloaded (503 + Retry-After) at crypto/keys. + "/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()), + "/1.1/broken.keys/info/collections": collectionsHelper.handler, + "/1.1/broken.keys/storage/crypto/keys": EHTestsCommon.service_unavailable, + + // Maintenance or overloaded (503 + Retry-After) at wiping collection. + "/1.1/broken.wipe/info/collections": collectionsHelper.handler, + "/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()), + "/1.1/broken.wipe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/broken.wipe/storage": EHTestsCommon.service_unavailable, + "/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/broken.wipe/storage/catapult": EHTestsCommon.service_unavailable, + }); + }, + + CatapultEngine: (function () { + function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); + } + CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + if (this.exception) { + throw this.exception; + } + }, + }; + Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + + return CatapultEngine; + })(), + + async generateCredentialsChangedFailure() { + // Make sync fail due to changed credentials. We simply re-encrypt + // the keys with a different Sync Key, without changing the local one. + let newSyncKeyBundle = new BulkKeyBundle("crypto"); + await newSyncKeyBundle.generateRandom(); + let keys = Service.collectionKeys.asWBO(); + await keys.encrypt(newSyncKeyBundle); + return keys.upload(Service.resource(Service.cryptoKeysURL)); + }, + + async setUp(server) { + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + return EHTestsCommon.generateAndUploadKeys(); + }, + + async generateAndUploadKeys() { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let response = await serverKeys.upload( + Service.resource(Service.cryptoKeysURL) + ); + return response.success; + }, +}; diff --git a/services/sync/tests/unit/head_helpers.js b/services/sync/tests/unit/head_helpers.js new file mode 100644 index 0000000000..a871debd76 --- /dev/null +++ b/services/sync/tests/unit/head_helpers.js @@ -0,0 +1,704 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_errorhandler_common.js */ +/* import-globals-from head_http_server.js */ + +// This file expects Service to be defined in the global scope when EHTestsCommon +// is used (from service.js). +/* global Service */ + +var { AddonTestUtils, MockAsyncShutdown } = ChromeUtils.importESModule( + "resource://testing-common/AddonTestUtils.sys.mjs" +); +var { Async } = ChromeUtils.importESModule( + "resource://services-common/async.sys.mjs" +); +var { CommonUtils } = ChromeUtils.importESModule( + "resource://services-common/utils.sys.mjs" +); +var { PlacesTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PlacesTestUtils.sys.mjs" +); +var { sinon } = ChromeUtils.importESModule( + "resource://testing-common/Sinon.sys.mjs" +); +var { SerializableSet, Svc, Utils, getChromeWindow } = + ChromeUtils.importESModule("resource://services-sync/util.sys.mjs"); +var { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +var { PlacesUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesUtils.sys.mjs" +); +var { PlacesSyncUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesSyncUtils.sys.mjs" +); +var { ObjectUtils } = ChromeUtils.import( + "resource://gre/modules/ObjectUtils.jsm" +); +var { + MockFxaStorageManager, + SyncTestingInfrastructure, + configureFxAccountIdentity, + configureIdentity, + encryptPayload, + getLoginTelemetryScalar, + makeFxAccountsInternalMock, + makeIdentityConfig, + promiseNamedTimer, + promiseZeroTimer, + sumHistogram, + syncTestLogging, + waitForZeroTimer, +} = ChromeUtils.importESModule( + "resource://testing-common/services/sync/utils.sys.mjs" +); +ChromeUtils.defineESModuleGetters(this, { + AddonManager: "resource://gre/modules/AddonManager.sys.mjs", +}); + +add_setup(async function head_setup() { + // Initialize logging. This will sometimes be reset by a pref reset, + // so it's also called as part of SyncTestingInfrastructure(). + syncTestLogging(); + // If a test imports Service, make sure it is initialized first. + if (typeof Service !== "undefined") { + await Service.promiseInitialized; + } +}); + +XPCOMUtils.defineLazyGetter(this, "SyncPingSchema", function () { + let { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" + ); + let { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm"); + let stream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance( + Ci.nsIFileInputStream + ); + let schema; + try { + let schemaFile = do_get_file("sync_ping_schema.json"); + stream.init(schemaFile, FileUtils.MODE_RDONLY, FileUtils.PERMS_FILE, 0); + + let bytes = NetUtil.readInputStream(stream, stream.available()); + schema = JSON.parse(new TextDecoder().decode(bytes)); + } finally { + stream.close(); + } + + // Allow tests to make whatever engines they want, this shouldn't cause + // validation failure. + schema.definitions.engine.properties.name = { type: "string" }; + return schema; +}); + +XPCOMUtils.defineLazyGetter(this, "SyncPingValidator", function () { + const { JsonSchema } = ChromeUtils.importESModule( + "resource://gre/modules/JsonSchema.sys.mjs" + ); + return new JsonSchema.Validator(SyncPingSchema); +}); + +// This is needed for loadAddonTestFunctions(). +var gGlobalScope = this; + +function ExtensionsTestPath(path) { + if (path[0] != "/") { + throw Error("Path must begin with '/': " + path); + } + + return "../../../../toolkit/mozapps/extensions/test/xpcshell" + path; +} + +function webExtensionsTestPath(path) { + if (path[0] != "/") { + throw Error("Path must begin with '/': " + path); + } + + return "../../../../toolkit/components/extensions/test/xpcshell" + path; +} + +/** + * Loads the WebExtension test functions by importing its test file. + */ +function loadWebExtensionTestFunctions() { + /* import-globals-from ../../../../toolkit/components/extensions/test/xpcshell/head_sync.js */ + const path = webExtensionsTestPath("/head_sync.js"); + let file = do_get_file(path); + let uri = Services.io.newFileURI(file); + Services.scriptloader.loadSubScript(uri.spec, gGlobalScope); +} + +/** + * Installs an add-on from an addonInstall + * + * @param install addonInstall instance to install + */ +async function installAddonFromInstall(install) { + await install.install(); + + Assert.notEqual(null, install.addon); + Assert.notEqual(null, install.addon.syncGUID); + + return install.addon; +} + +/** + * Convenience function to install an add-on from the extensions unit tests. + * + * @param file + * Add-on file to install. + * @param reconciler + * addons reconciler, if passed we will wait on the events to be + * processed before resolving + * @return addon object that was installed + */ +async function installAddon(file, reconciler = null) { + let install = await AddonManager.getInstallForFile(file); + Assert.notEqual(null, install); + const addon = await installAddonFromInstall(install); + if (reconciler) { + await reconciler.queueCaller.promiseCallsComplete(); + } + return addon; +} + +/** + * Convenience function to uninstall an add-on. + * + * @param addon + * Addon instance to uninstall + * @param reconciler + * addons reconciler, if passed we will wait on the events to be + * processed before resolving + */ +async function uninstallAddon(addon, reconciler = null) { + const uninstallPromise = new Promise(res => { + let listener = { + onUninstalled(uninstalled) { + if (uninstalled.id == addon.id) { + AddonManager.removeAddonListener(listener); + res(uninstalled); + } + }, + }; + AddonManager.addAddonListener(listener); + }); + addon.uninstall(); + await uninstallPromise; + if (reconciler) { + await reconciler.queueCaller.promiseCallsComplete(); + } +} + +async function generateNewKeys(collectionKeys, collections = null) { + let wbo = await collectionKeys.generateNewKeysWBO(collections); + let modified = new_timestamp(); + collectionKeys.setContents(wbo.cleartext, modified); +} + +// Helpers for testing open tabs. +// These reflect part of the internal structure of TabEngine, +// and stub part of Service.wm. + +function mockShouldSkipWindow(win) { + return win.closed || win.mockIsPrivate; +} + +function mockGetTabState(tab) { + return tab; +} + +function mockGetWindowEnumerator(urls) { + let elements = []; + + const numWindows = 1; + for (let w = 0; w < numWindows; ++w) { + let tabs = []; + let win = { + closed: false, + mockIsPrivate: false, + gBrowser: { + tabs, + }, + }; + elements.push(win); + + let lastAccessed = 2000; + for (let url of urls) { + tabs.push({ + linkedBrowser: { + currentURI: Services.io.newURI(url), + contentTitle: "title", + }, + lastAccessed, + }); + lastAccessed += 1000; + } + } + + // Always include a closed window and a private window. + elements.push({ + closed: true, + mockIsPrivate: false, + gBrowser: { + tabs: [], + }, + }); + + elements.push({ + closed: false, + mockIsPrivate: true, + gBrowser: { + tabs: [], + }, + }); + + return elements.values(); +} + +// Helper function to get the sync telemetry and add the typically used test +// engine names to its list of allowed engines. +function get_sync_test_telemetry() { + let { SyncTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + SyncTelemetry.tryRefreshDevices = function () {}; + let testEngines = ["rotary", "steam", "sterling", "catapult", "nineties"]; + for (let engineName of testEngines) { + SyncTelemetry.allowedEngines.add(engineName); + } + SyncTelemetry.submissionInterval = -1; + return SyncTelemetry; +} + +function assert_valid_ping(record) { + // Our JSON validator does not like `undefined` values, even though they will + // be skipped when we serialize to JSON. + record = JSON.parse(JSON.stringify(record)); + + // This is called as the test harness tears down due to shutdown. This + // will typically have no recorded syncs, and the validator complains about + // it. So ignore such records (but only ignore when *both* shutdown and + // no Syncs - either of them not being true might be an actual problem) + if (record && (record.why != "shutdown" || !!record.syncs.length)) { + const result = SyncPingValidator.validate(record); + if (!result.valid) { + if (result.errors.length) { + // validation failed - using a simple |deepEqual([], errors)| tends to + // truncate the validation errors in the output and doesn't show that + // the ping actually was - so be helpful. + info("telemetry ping validation failed"); + info("the ping data is: " + JSON.stringify(record, undefined, 2)); + info( + "the validation failures: " + + JSON.stringify(result.errors, undefined, 2) + ); + ok( + false, + "Sync telemetry ping validation failed - see output above for details" + ); + } + } + equal(record.version, 1); + record.syncs.forEach(p => { + lessOrEqual(p.when, Date.now()); + }); + } +} + +// Asserts that `ping` is a ping that doesn't contain any failure information +function assert_success_ping(ping) { + ok(!!ping); + assert_valid_ping(ping); + ping.syncs.forEach(record => { + ok(!record.failureReason, JSON.stringify(record.failureReason)); + equal(undefined, record.status); + greater(record.engines.length, 0); + for (let e of record.engines) { + ok(!e.failureReason); + equal(undefined, e.status); + if (e.validation) { + equal(undefined, e.validation.problems); + equal(undefined, e.validation.failureReason); + } + if (e.outgoing) { + for (let o of e.outgoing) { + equal(undefined, o.failed); + notEqual(undefined, o.sent); + } + } + if (e.incoming) { + equal(undefined, e.incoming.failed); + equal(undefined, e.incoming.newFailed); + notEqual(undefined, e.incoming.applied || e.incoming.reconciled); + } + } + }); +} + +// Hooks into telemetry to validate all pings after calling. +function validate_all_future_pings() { + let telem = get_sync_test_telemetry(); + telem.submit = assert_valid_ping; +} + +function wait_for_pings(expectedPings) { + return new Promise(resolve => { + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + let pings = []; + telem.submit = function (record) { + pings.push(record); + if (pings.length == expectedPings) { + telem.submit = oldSubmit; + resolve(pings); + } + }; + }); +} + +async function wait_for_ping(callback, allowErrorPings, getFullPing = false) { + let pingsPromise = wait_for_pings(1); + await callback(); + let [record] = await pingsPromise; + if (allowErrorPings) { + assert_valid_ping(record); + } else { + assert_success_ping(record); + } + if (getFullPing) { + return record; + } + equal(record.syncs.length, 1); + return record.syncs[0]; +} + +// Perform a sync and validate all telemetry caused by the sync. If fnValidate +// is null, we just check the ping records success. If fnValidate is specified, +// then the sync must have recorded just a single sync, and that sync will be +// passed to the function to be checked. +async function sync_and_validate_telem( + fnValidate = null, + wantFullPing = false +) { + let numErrors = 0; + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + try { + telem.submit = function (record) { + // This is called via an observer, so failures here don't cause the test + // to fail :( + try { + // All pings must be valid. + assert_valid_ping(record); + if (fnValidate) { + // for historical reasons most of these callbacks expect a "sync" + // record, not the entire ping. + if (wantFullPing) { + fnValidate(record); + } else { + Assert.equal(record.syncs.length, 1); + fnValidate(record.syncs[0]); + } + } else { + // no validation function means it must be a "success" ping. + assert_success_ping(record); + } + } catch (ex) { + print("Failure in ping validation callback", ex, "\n", ex.stack); + numErrors += 1; + } + }; + await Service.sync(); + Assert.ok(numErrors == 0, "There were telemetry validation errors"); + } finally { + telem.submit = oldSubmit; + } +} + +// Used for the (many) cases where we do a 'partial' sync, where only a single +// engine is actually synced, but we still want to ensure we're generating a +// valid ping. Returns a promise that resolves to the ping, or rejects with the +// thrown error after calling an optional callback. +async function sync_engine_and_validate_telem( + engine, + allowErrorPings, + onError, + wantFullPing = false +) { + let telem = get_sync_test_telemetry(); + let caughtError = null; + // Clear out status, so failures from previous syncs won't show up in the + // telemetry ping. + let { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" + ); + Status._engines = {}; + Status.partial = false; + // Ideally we'd clear these out like we do with engines, (probably via + // Status.resetSync()), but this causes *numerous* tests to fail, so we just + // assume that if no failureReason or engine failures are set, and the + // status properties are the same as they were initially, that it's just + // a leftover. + // This is only an issue since we're triggering the sync of just one engine, + // without doing any other parts of the sync. + let initialServiceStatus = Status._service; + let initialSyncStatus = Status._sync; + + let oldSubmit = telem.submit; + let submitPromise = new Promise((resolve, reject) => { + telem.submit = function (ping) { + telem.submit = oldSubmit; + ping.syncs.forEach(record => { + if (record && record.status) { + // did we see anything to lead us to believe that something bad actually happened + let realProblem = + record.failureReason || + record.engines.some(e => { + if (e.failureReason || e.status) { + return true; + } + if (e.outgoing && e.outgoing.some(o => o.failed > 0)) { + return true; + } + return e.incoming && e.incoming.failed; + }); + if (!realProblem) { + // no, so if the status is the same as it was initially, just assume + // that its leftover and that we can ignore it. + if (record.status.sync && record.status.sync == initialSyncStatus) { + delete record.status.sync; + } + if ( + record.status.service && + record.status.service == initialServiceStatus + ) { + delete record.status.service; + } + if (!record.status.sync && !record.status.service) { + delete record.status; + } + } + } + }); + if (allowErrorPings) { + assert_valid_ping(ping); + } else { + assert_success_ping(ping); + } + equal(ping.syncs.length, 1); + if (caughtError) { + if (onError) { + onError(ping.syncs[0], ping); + } + reject(caughtError); + } else if (wantFullPing) { + resolve(ping); + } else { + resolve(ping.syncs[0]); + } + }; + }); + // neuter the scheduler as it interacts badly with some of the tests - the + // engine being synced usually isn't the registered engine, so we see + // scored incremented and not removed, which schedules unexpected syncs. + let oldObserve = Service.scheduler.observe; + Service.scheduler.observe = () => {}; + try { + Svc.Obs.notify("weave:service:sync:start"); + try { + await engine.sync(); + } catch (e) { + caughtError = e; + } + if (caughtError) { + Svc.Obs.notify("weave:service:sync:error", caughtError); + } else { + Svc.Obs.notify("weave:service:sync:finish"); + } + } finally { + Service.scheduler.observe = oldObserve; + } + return submitPromise; +} + +// Returns a promise that resolves once the specified observer notification +// has fired. +function promiseOneObserver(topic, callback) { + return new Promise((resolve, reject) => { + let observer = function (subject, data) { + Svc.Obs.remove(topic, observer); + resolve({ subject, data }); + }; + Svc.Obs.add(topic, observer); + }); +} + +async function registerRotaryEngine() { + let { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" + ); + await Service.engineManager.clear(); + + await Service.engineManager.register(RotaryEngine); + let engine = Service.engineManager.get("rotary"); + let syncID = await engine.resetLocalSyncID(); + engine.enabled = true; + + return { engine, syncID, tracker: engine._tracker }; +} + +// Set the validation prefs to attempt validation every time to avoid non-determinism. +function enableValidationPrefs(engines = ["bookmarks"]) { + for (let engine of engines) { + Svc.Prefs.set(`engine.${engine}.validation.interval`, 0); + Svc.Prefs.set(`engine.${engine}.validation.percentageChance`, 100); + Svc.Prefs.set(`engine.${engine}.validation.maxRecords`, -1); + Svc.Prefs.set(`engine.${engine}.validation.enabled`, true); + } +} + +async function serverForEnginesWithKeys(users, engines, callback) { + // Generate and store a fake default key bundle to avoid resetting the client + // before the first sync. + let wbo = await Service.collectionKeys.generateNewKeysWBO(); + let modified = new_timestamp(); + Service.collectionKeys.setContents(wbo.cleartext, modified); + + let allEngines = [Service.clientsEngine].concat(engines); + + let globalEngines = {}; + for (let engine of allEngines) { + let syncID = await engine.resetLocalSyncID(); + globalEngines[engine.name] = { version: engine.version, syncID }; + } + + let contents = { + meta: { + global: { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: globalEngines, + }, + }, + crypto: { + keys: encryptPayload(wbo.cleartext), + }, + }; + for (let engine of allEngines) { + contents[engine.name] = {}; + } + + return serverForUsers(users, contents, callback); +} + +async function serverForFoo(engine, callback) { + // The bookmarks engine *always* tracks changes, meaning we might try + // and sync due to the bookmarks we ourselves create! Worse, because we + // do an engine sync only, there's no locking - so we end up with multiple + // syncs running. Neuter that by making the threshold very large. + Service.scheduler.syncThreshold = 10000000; + return serverForEnginesWithKeys({ foo: "password" }, engine, callback); +} + +// Places notifies history observers asynchronously, so `addVisits` might return +// before the tracker receives the notification. This helper registers an +// observer that resolves once the expected notification fires. +async function promiseVisit(expectedType, expectedURI) { + return new Promise(resolve => { + function done(type, uri) { + if (uri == expectedURI.spec && type == expectedType) { + PlacesObservers.removeListener( + ["page-visited", "page-removed"], + observer.handlePlacesEvents + ); + resolve(); + } + } + let observer = { + handlePlacesEvents(events) { + Assert.equal(events.length, 1); + + if (events[0].type === "page-visited") { + done("added", events[0].url); + } else if (events[0].type === "page-removed") { + Assert.ok(events[0].isRemovedFromStore); + done("removed", events[0].url); + } + }, + }; + PlacesObservers.addListener( + ["page-visited", "page-removed"], + observer.handlePlacesEvents + ); + }); +} + +async function addVisit( + suffix, + referrer = null, + transition = PlacesUtils.history.TRANSITION_LINK +) { + let uriString = "http://getfirefox.com/" + suffix; + let uri = CommonUtils.makeURI(uriString); + _("Adding visit for URI " + uriString); + + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition, + referrer, + }); + await visitAddedPromise; + + return uri; +} + +function bookmarkNodesToInfos(nodes) { + return nodes.map(node => { + let info = { + guid: node.guid, + index: node.index, + }; + if (node.children) { + info.children = bookmarkNodesToInfos(node.children); + } + return info; + }); +} + +async function assertBookmarksTreeMatches(rootGuid, expected, message) { + let root = await PlacesUtils.promiseBookmarksTree(rootGuid, { + includeItemIds: true, + }); + let actual = bookmarkNodesToInfos(root.children); + + if (!ObjectUtils.deepEqual(actual, expected)) { + _(`Expected structure for ${rootGuid}`, JSON.stringify(expected)); + _(`Actual structure for ${rootGuid}`, JSON.stringify(actual)); + throw new Assert.constructor.AssertionError({ actual, expected, message }); + } +} + +function add_bookmark_test(task) { + const { BookmarksEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); + + add_task(async function () { + _(`Running bookmarks test ${task.name}`); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + await engine._resetClient(); + try { + await task(engine); + } finally { + await engine.finalize(); + } + }); +} diff --git a/services/sync/tests/unit/head_http_server.js b/services/sync/tests/unit/head_http_server.js new file mode 100644 index 0000000000..f1be8c3b60 --- /dev/null +++ b/services/sync/tests/unit/head_http_server.js @@ -0,0 +1,1265 @@ +/* import-globals-from head_appinfo.js */ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ +/* import-globals-from head_helpers.js */ + +var Cm = Components.manager; + +// Shared logging for all HTTP server functions. +var { Log } = ChromeUtils.importESModule("resource://gre/modules/Log.sys.mjs"); +var { CommonUtils } = ChromeUtils.importESModule( + "resource://services-common/utils.sys.mjs" +); +var { TestUtils } = ChromeUtils.importESModule( + "resource://testing-common/TestUtils.sys.mjs" +); +var { + MockFxaStorageManager, + SyncTestingInfrastructure, + configureFxAccountIdentity, + configureIdentity, + encryptPayload, + getLoginTelemetryScalar, + makeFxAccountsInternalMock, + makeIdentityConfig, + promiseNamedTimer, + promiseZeroTimer, + sumHistogram, + syncTestLogging, + waitForZeroTimer, +} = ChromeUtils.importESModule( + "resource://testing-common/services/sync/utils.sys.mjs" +); + +const SYNC_HTTP_LOGGER = "Sync.Test.Server"; + +// While the sync code itself uses 1.5, the tests hard-code 1.1, +// so we're sticking with 1.1 here. +const SYNC_API_VERSION = "1.1"; + +// Use the same method that record.js does, which mirrors the server. +// The server returns timestamps with 1/100 sec granularity. Note that this is +// subject to change: see Bug 650435. +function new_timestamp() { + return round_timestamp(Date.now()); +} + +// Rounds a millisecond timestamp `t` to seconds, with centisecond precision. +function round_timestamp(t) { + return Math.round(t / 10) / 100; +} + +function return_timestamp(request, response, timestamp) { + if (!timestamp) { + timestamp = new_timestamp(); + } + let body = "" + timestamp; + response.setHeader("X-Weave-Timestamp", body); + response.setStatusLine(request.httpVersion, 200, "OK"); + writeBytesToOutputStream(response.bodyOutputStream, body); + return timestamp; +} + +function has_hawk_header(req) { + return ( + req.hasHeader("Authorization") && + req.getHeader("Authorization").startsWith("Hawk") + ); +} + +function basic_auth_header(user, password) { + return "Basic " + btoa(user + ":" + CommonUtils.encodeUTF8(password)); +} + +function basic_auth_matches(req, user, password) { + if (!req.hasHeader("Authorization")) { + return false; + } + + let expected = basic_auth_header(user, CommonUtils.encodeUTF8(password)); + return req.getHeader("Authorization") == expected; +} + +function httpd_basic_auth_handler(body, metadata, response) { + if (basic_auth_matches(metadata, "guest", "guest")) { + response.setStatusLine(metadata.httpVersion, 200, "OK, authorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } else { + body = "This path exists and is protected - failed"; + response.setStatusLine(metadata.httpVersion, 401, "Unauthorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } + writeBytesToOutputStream(response.bodyOutputStream, body); +} + +/* + * Represent a WBO on the server + */ +function ServerWBO(id, initialPayload, modified) { + if (!id) { + throw new Error("No ID for ServerWBO!"); + } + this.id = id; + if (!initialPayload) { + return; + } + + if (typeof initialPayload == "object") { + initialPayload = JSON.stringify(initialPayload); + } + this.payload = initialPayload; + this.modified = modified || new_timestamp(); + this.sortindex = 0; +} +ServerWBO.prototype = { + get data() { + return JSON.parse(this.payload); + }, + + get() { + return { id: this.id, modified: this.modified, payload: this.payload }; + }, + + put(input) { + input = JSON.parse(input); + this.payload = input.payload; + this.modified = new_timestamp(); + this.sortindex = input.sortindex || 0; + }, + + delete() { + delete this.payload; + delete this.modified; + delete this.sortindex; + }, + + // This handler sets `newModified` on the response body if the collection + // timestamp has changed. This allows wrapper handlers to extract information + // that otherwise would exist only in the body stream. + handler() { + let self = this; + + return function (request, response) { + var statusCode = 200; + var status = "OK"; + var body; + + switch (request.method) { + case "GET": + if (self.payload) { + body = JSON.stringify(self.get()); + } else { + statusCode = 404; + status = "Not Found"; + body = "Not Found"; + } + break; + + case "PUT": + self.put(readBytesFromInputStream(request.bodyInputStream)); + body = JSON.stringify(self.modified); + response.setHeader("Content-Type", "application/json"); + response.newModified = self.modified; + break; + + case "DELETE": + self.delete(); + let ts = new_timestamp(); + body = JSON.stringify(ts); + response.setHeader("Content-Type", "application/json"); + response.newModified = ts; + break; + } + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + response.setStatusLine(request.httpVersion, statusCode, status); + writeBytesToOutputStream(response.bodyOutputStream, body); + }; + }, + + /** + * Get the cleartext data stored in the payload. + * + * This isn't `get cleartext`, because `x.cleartext.blah = 3;` wouldn't work, + * which seems like a footgun. + */ + getCleartext() { + return JSON.parse(JSON.parse(this.payload).ciphertext); + }, + + /** + * Setter for getCleartext(), but lets you adjust the modified timestamp too. + * Returns this ServerWBO object. + */ + setCleartext(cleartext, modifiedTimestamp = this.modified) { + this.payload = JSON.stringify(encryptPayload(cleartext)); + this.modified = modifiedTimestamp; + return this; + }, +}; + +/** + * Represent a collection on the server. The '_wbos' attribute is a + * mapping of id -> ServerWBO objects. + * + * Note that if you want these records to be accessible individually, + * you need to register their handlers with the server separately, or use a + * containing HTTP server that will do so on your behalf. + * + * @param wbos + * An object mapping WBO IDs to ServerWBOs. + * @param acceptNew + * If true, POSTs to this collection URI will result in new WBOs being + * created and wired in on the fly. + * @param timestamp + * An optional timestamp value to initialize the modified time of the + * collection. This should be in the format returned by new_timestamp(). + * + * @return the new ServerCollection instance. + * + */ +function ServerCollection(wbos, acceptNew, timestamp) { + this._wbos = wbos || {}; + this.acceptNew = acceptNew || false; + + /* + * Track modified timestamp. + * We can't just use the timestamps of contained WBOs: an empty collection + * has a modified time. + */ + this.timestamp = timestamp || new_timestamp(); + this._log = Log.repository.getLogger(SYNC_HTTP_LOGGER); +} +ServerCollection.prototype = { + /** + * Convenience accessor for our WBO keys. + * Excludes deleted items, of course. + * + * @param filter + * A predicate function (applied to the ID and WBO) which dictates + * whether to include the WBO's ID in the output. + * + * @return an array of IDs. + */ + keys: function keys(filter) { + let ids = []; + for (let [id, wbo] of Object.entries(this._wbos)) { + if (wbo.payload && (!filter || filter(id, wbo))) { + ids.push(id); + } + } + return ids; + }, + + /** + * Convenience method to get an array of WBOs. + * Optionally provide a filter function. + * + * @param filter + * A predicate function, applied to the WBO, which dictates whether to + * include the WBO in the output. + * + * @return an array of ServerWBOs. + */ + wbos: function wbos(filter) { + let os = []; + for (let wbo of Object.values(this._wbos)) { + if (wbo.payload) { + os.push(wbo); + } + } + + if (filter) { + return os.filter(filter); + } + return os; + }, + + /** + * Convenience method to get an array of parsed ciphertexts. + * + * @return an array of the payloads of each stored WBO. + */ + payloads() { + return this.wbos().map(wbo => wbo.getCleartext()); + }, + + // Just for syntactic elegance. + wbo: function wbo(id) { + return this._wbos[id]; + }, + + payload: function payload(id) { + return this.wbo(id).payload; + }, + + cleartext(id) { + return this.wbo(id).getCleartext(); + }, + + /** + * Insert the provided WBO under its ID. + * + * @return the provided WBO. + */ + insertWBO: function insertWBO(wbo) { + this.timestamp = Math.max(this.timestamp, wbo.modified); + return (this._wbos[wbo.id] = wbo); + }, + + /** + * Update an existing WBO's cleartext using a callback function that modifies + * the record in place, or returns a new record. + */ + updateRecord(id, updateCallback, optTimestamp) { + let wbo = this.wbo(id); + if (!wbo) { + throw new Error("No record with provided ID"); + } + let curCleartext = wbo.getCleartext(); + // Allow update callback to either return a new cleartext, or modify in place. + let newCleartext = updateCallback(curCleartext) || curCleartext; + wbo.setCleartext(newCleartext, optTimestamp); + // It is already inserted, but we might need to update our timestamp based + // on it's `modified` value, if `optTimestamp` was provided. + return this.insertWBO(wbo); + }, + + /** + * Insert a record, which may either an object with a cleartext property, or + * the cleartext property itself. + */ + insertRecord(record, timestamp = Date.now() / 1000) { + if (typeof timestamp != "number") { + throw new TypeError("insertRecord: Timestamp is not a number."); + } + if (!record.id) { + throw new Error("Attempt to insert record with no id"); + } + // Allow providing either the cleartext directly, or the CryptoWrapper-like. + let cleartext = record.cleartext || record; + return this.insert(record.id, encryptPayload(cleartext), timestamp); + }, + + /** + * Insert the provided payload as part of a new ServerWBO with the provided + * ID. + * + * @param id + * The GUID for the WBO. + * @param payload + * The payload, as provided to the ServerWBO constructor. + * @param modified + * An optional modified time for the ServerWBO. + * + * @return the inserted WBO. + */ + insert: function insert(id, payload, modified) { + return this.insertWBO(new ServerWBO(id, payload, modified)); + }, + + /** + * Removes an object entirely from the collection. + * + * @param id + * (string) ID to remove. + */ + remove: function remove(id) { + delete this._wbos[id]; + }, + + _inResultSet(wbo, options) { + return ( + wbo.payload && + (!options.ids || options.ids.includes(wbo.id)) && + (!options.newer || wbo.modified > options.newer) && + (!options.older || wbo.modified < options.older) + ); + }, + + count(options) { + options = options || {}; + let c = 0; + for (let wbo of Object.values(this._wbos)) { + if (wbo.modified && this._inResultSet(wbo, options)) { + c++; + } + } + return c; + }, + + get(options, request) { + let data = []; + for (let wbo of Object.values(this._wbos)) { + if (wbo.modified && this._inResultSet(wbo, options)) { + data.push(wbo); + } + } + switch (options.sort) { + case "newest": + data.sort((a, b) => b.modified - a.modified); + break; + + case "oldest": + data.sort((a, b) => a.modified - b.modified); + break; + + case "index": + data.sort((a, b) => b.sortindex - a.sortindex); + break; + + default: + if (options.sort) { + this._log.error( + "Error: client requesting unknown sort order", + options.sort + ); + throw new Error("Unknown sort order"); + } + // If the client didn't request a sort order, shuffle the records + // to ensure that we don't accidentally depend on the default order. + TestUtils.shuffle(data); + } + if (options.full) { + data = data.map(wbo => wbo.get()); + let start = options.offset || 0; + if (options.limit) { + let numItemsPastOffset = data.length - start; + data = data.slice(start, start + options.limit); + // use options as a backchannel to set x-weave-next-offset + if (numItemsPastOffset > options.limit) { + options.nextOffset = start + options.limit; + } + } else if (start) { + data = data.slice(start); + } + + if (request && request.getHeader("accept") == "application/newlines") { + this._log.error( + "Error: client requesting application/newlines content" + ); + throw new Error( + "This server should not serve application/newlines content" + ); + } + + // Use options as a backchannel to report count. + options.recordCount = data.length; + } else { + data = data.map(wbo => wbo.id); + let start = options.offset || 0; + if (options.limit) { + data = data.slice(start, start + options.limit); + options.nextOffset = start + options.limit; + } else if (start) { + data = data.slice(start); + } + options.recordCount = data.length; + } + return JSON.stringify(data); + }, + + post(input) { + input = JSON.parse(input); + let success = []; + let failed = {}; + + // This will count records where we have an existing ServerWBO + // registered with us as successful and all other records as failed. + for (let key in input) { + let record = input[key]; + let wbo = this.wbo(record.id); + if (!wbo && this.acceptNew) { + this._log.debug( + "Creating WBO " + JSON.stringify(record.id) + " on the fly." + ); + wbo = new ServerWBO(record.id); + this.insertWBO(wbo); + } + if (wbo) { + wbo.payload = record.payload; + wbo.modified = new_timestamp(); + wbo.sortindex = record.sortindex || 0; + success.push(record.id); + } else { + failed[record.id] = "no wbo configured"; + } + } + return { modified: new_timestamp(), success, failed }; + }, + + delete(options) { + let deleted = []; + for (let wbo of Object.values(this._wbos)) { + if (this._inResultSet(wbo, options)) { + this._log.debug("Deleting " + JSON.stringify(wbo)); + deleted.push(wbo.id); + wbo.delete(); + } + } + return deleted; + }, + + // This handler sets `newModified` on the response body if the collection + // timestamp has changed. + handler() { + let self = this; + + return function (request, response) { + var statusCode = 200; + var status = "OK"; + var body; + + // Parse queryString + let options = {}; + for (let chunk of request.queryString.split("&")) { + if (!chunk) { + continue; + } + chunk = chunk.split("="); + if (chunk.length == 1) { + options[chunk[0]] = ""; + } else { + options[chunk[0]] = chunk[1]; + } + } + // The real servers return 400 if ids= is specified without a list of IDs. + if (options.hasOwnProperty("ids")) { + if (!options.ids) { + response.setStatusLine(request.httpVersion, "400", "Bad Request"); + body = "Bad Request"; + writeBytesToOutputStream(response.bodyOutputStream, body); + return; + } + options.ids = options.ids.split(","); + } + if (options.newer) { + options.newer = parseFloat(options.newer); + } + if (options.older) { + options.older = parseFloat(options.older); + } + if (options.limit) { + options.limit = parseInt(options.limit, 10); + } + if (options.offset) { + options.offset = parseInt(options.offset, 10); + } + + switch (request.method) { + case "GET": + body = self.get(options, request); + // see http://moz-services-docs.readthedocs.io/en/latest/storage/apis-1.5.html + // for description of these headers. + let { recordCount: records, nextOffset } = options; + + self._log.info("Records: " + records + ", nextOffset: " + nextOffset); + if (records != null) { + response.setHeader("X-Weave-Records", "" + records); + } + if (nextOffset) { + response.setHeader("X-Weave-Next-Offset", "" + nextOffset); + } + response.setHeader("X-Last-Modified", "" + self.timestamp); + break; + + case "POST": + let res = self.post( + readBytesFromInputStream(request.bodyInputStream), + request + ); + body = JSON.stringify(res); + response.newModified = res.modified; + break; + + case "DELETE": + self._log.debug("Invoking ServerCollection.DELETE."); + let deleted = self.delete(options, request); + let ts = new_timestamp(); + body = JSON.stringify(ts); + response.newModified = ts; + response.deleted = deleted; + break; + } + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + + // Update the collection timestamp to the appropriate modified time. + // This is either a value set by the handler, or the current time. + if (request.method != "GET") { + self.timestamp = + response.newModified >= 0 ? response.newModified : new_timestamp(); + } + response.setHeader("X-Last-Modified", "" + self.timestamp, false); + + response.setStatusLine(request.httpVersion, statusCode, status); + writeBytesToOutputStream(response.bodyOutputStream, body); + }; + }, +}; + +/* + * Test setup helpers. + */ +function sync_httpd_setup(handlers) { + handlers["/1.1/foo/storage/meta/global"] = new ServerWBO( + "global", + {} + ).handler(); + return httpd_setup(handlers); +} + +/* + * Track collection modified times. Return closures. + * + * XXX - DO NOT USE IN NEW TESTS + * + * This code has very limited and very hacky timestamp support - the test + * server now has more complete and correct support - using this helper + * may cause strangeness wrt timestamp headers and 412 responses. + */ +function track_collections_helper() { + /* + * Our tracking object. + */ + let collections = {}; + + /* + * Update the timestamp of a collection. + */ + function update_collection(coll, ts) { + _("Updating collection " + coll + " to " + ts); + let timestamp = ts || new_timestamp(); + collections[coll] = timestamp; + } + + /* + * Invoke a handler, updating the collection's modified timestamp unless + * it's a GET request. + */ + function with_updated_collection(coll, f) { + return function (request, response) { + f.call(this, request, response); + + // Update the collection timestamp to the appropriate modified time. + // This is either a value set by the handler, or the current time. + if (request.method != "GET") { + update_collection(coll, response.newModified); + } + }; + } + + /* + * Return the info/collections object. + */ + function info_collections(request, response) { + let body = "Error."; + switch (request.method) { + case "GET": + body = JSON.stringify(collections); + break; + default: + throw new Error("Non-GET on info_collections."); + } + + response.setHeader("Content-Type", "application/json"); + response.setHeader("X-Weave-Timestamp", "" + new_timestamp(), false); + response.setStatusLine(request.httpVersion, 200, "OK"); + writeBytesToOutputStream(response.bodyOutputStream, body); + } + + return { + collections, + handler: info_collections, + with_updated_collection, + update_collection, + }; +} + +// ===========================================================================// +// httpd.js-based Sync server. // +// ===========================================================================// + +/** + * In general, the preferred way of using SyncServer is to directly introspect + * it. Callbacks are available for operations which are hard to verify through + * introspection, such as deletions. + * + * One of the goals of this server is to provide enough hooks for test code to + * find out what it needs without monkeypatching. Use this object as your + * prototype, and override as appropriate. + */ +var SyncServerCallback = { + onCollectionDeleted: function onCollectionDeleted(user, collection) {}, + onItemDeleted: function onItemDeleted(user, collection, wboID) {}, + + /** + * Called at the top of every request. + * + * Allows the test to inspect the request. Hooks should be careful not to + * modify or change state of the request or they may impact future processing. + * The response is also passed so the callback can set headers etc - but care + * must be taken to not screw with the response body or headers that may + * conflict with normal operation of this server. + */ + onRequest: function onRequest(request, response) {}, +}; + +/** + * Construct a new test Sync server. Takes a callback object (e.g., + * SyncServerCallback) as input. + */ +function SyncServer(callback) { + this.callback = callback || Object.create(SyncServerCallback); + this.server = new HttpServer(); + this.started = false; + this.users = {}; + this._log = Log.repository.getLogger(SYNC_HTTP_LOGGER); + + // Install our own default handler. This allows us to mess around with the + // whole URL space. + let handler = this.server._handler; + handler._handleDefault = this.handleDefault.bind(this, handler); +} +SyncServer.prototype = { + server: null, // HttpServer. + users: null, // Map of username => {collections, password}. + + /** + * Start the SyncServer's underlying HTTP server. + * + * @param port + * The numeric port on which to start. -1 implies the default, a + * randomly chosen port. + * @param cb + * A callback function (of no arguments) which is invoked after + * startup. + */ + start: function start(port = -1, cb) { + if (this.started) { + this._log.warn("Warning: server already started on " + this.port); + return; + } + try { + this.server.start(port); + let i = this.server.identity; + this.port = i.primaryPort; + this.baseURI = + i.primaryScheme + "://" + i.primaryHost + ":" + i.primaryPort + "/"; + this.started = true; + if (cb) { + cb(); + } + } catch (ex) { + _("=========================================="); + _("Got exception starting Sync HTTP server."); + _("Error: " + Log.exceptionStr(ex)); + _("Is there a process already listening on port " + port + "?"); + _("=========================================="); + do_throw(ex); + } + }, + + /** + * Stop the SyncServer's HTTP server. + * + * @param cb + * A callback function. Invoked after the server has been stopped. + * + */ + stop: function stop(cb) { + if (!this.started) { + this._log.warn( + "SyncServer: Warning: server not running. Can't stop me now!" + ); + return; + } + + this.server.stop(cb); + this.started = false; + }, + + /** + * Return a server timestamp for a record. + * The server returns timestamps with 1/100 sec granularity. Note that this is + * subject to change: see Bug 650435. + */ + timestamp: function timestamp() { + return new_timestamp(); + }, + + /** + * Create a new user, complete with an empty set of collections. + * + * @param username + * The username to use. An Error will be thrown if a user by that name + * already exists. + * @param password + * A password string. + * + * @return a user object, as would be returned by server.user(username). + */ + registerUser: function registerUser(username, password) { + if (username in this.users) { + throw new Error("User already exists."); + } + this.users[username] = { + password, + collections: {}, + }; + return this.user(username); + }, + + userExists: function userExists(username) { + return username in this.users; + }, + + getCollection: function getCollection(username, collection) { + return this.users[username].collections[collection]; + }, + + _insertCollection: function _insertCollection(collections, collection, wbos) { + let coll = new ServerCollection(wbos, true); + coll.collectionHandler = coll.handler(); + collections[collection] = coll; + return coll; + }, + + createCollection: function createCollection(username, collection, wbos) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let collections = this.users[username].collections; + if (collection in collections) { + throw new Error("Collection already exists."); + } + return this._insertCollection(collections, collection, wbos); + }, + + /** + * Accept a map like the following: + * { + * meta: {global: {version: 1, ...}}, + * crypto: {"keys": {}, foo: {bar: 2}}, + * bookmarks: {} + * } + * to cause collections and WBOs to be created. + * If a collection already exists, no error is raised. + * If a WBO already exists, it will be updated to the new contents. + */ + createContents: function createContents(username, collections) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for (let [id, contents] of Object.entries(collections)) { + let coll = + userCollections[id] || this._insertCollection(userCollections, id); + for (let [wboID, payload] of Object.entries(contents)) { + coll.insert(wboID, payload); + } + } + }, + + /** + * Insert a WBO in an existing collection. + */ + insertWBO: function insertWBO(username, collection, wbo) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + if (!(collection in userCollections)) { + throw new Error("Unknown collection."); + } + userCollections[collection].insertWBO(wbo); + return wbo; + }, + + /** + * Delete all of the collections for the named user. + * + * @param username + * The name of the affected user. + * + * @return a timestamp. + */ + deleteCollections: function deleteCollections(username) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for (let name in userCollections) { + let coll = userCollections[name]; + this._log.trace("Bulk deleting " + name + " for " + username + "..."); + coll.delete({}); + } + this.users[username].collections = {}; + return this.timestamp(); + }, + + /** + * Simple accessor to allow collective binding and abbreviation of a bunch of + * methods. Yay! + * Use like this: + * + * let u = server.user("john"); + * u.collection("bookmarks").wbo("abcdefg").payload; // Etc. + * + * @return a proxy for the user data stored in this server. + */ + user: function user(username) { + let collection = this.getCollection.bind(this, username); + let createCollection = this.createCollection.bind(this, username); + let createContents = this.createContents.bind(this, username); + let modified = function (collectionName) { + return collection(collectionName).timestamp; + }; + let deleteCollections = this.deleteCollections.bind(this, username); + return { + collection, + createCollection, + createContents, + deleteCollections, + modified, + }; + }, + + /* + * Regular expressions for splitting up Sync request paths. + * Sync URLs are of the form: + * /$apipath/$version/$user/$further + * where $further is usually: + * storage/$collection/$wbo + * or + * storage/$collection + * or + * info/$op + * We assume for the sake of simplicity that $apipath is empty. + * + * N.B., we don't follow any kind of username spec here, because as far as I + * can tell there isn't one. See Bug 689671. Instead we follow the Python + * server code. + * + * Path: [all, version, username, first, rest] + * Storage: [all, collection?, id?] + */ + pathRE: + /^\/([0-9]+(?:\.[0-9]+)?)\/([-._a-zA-Z0-9]+)(?:\/([^\/]+)(?:\/(.+))?)?$/, + storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/, + + defaultHeaders: {}, + + /** + * HTTP response utility. + */ + respond: function respond(req, resp, code, status, body, headers) { + resp.setStatusLine(req.httpVersion, code, status); + if (!headers) { + headers = this.defaultHeaders; + } + for (let header in headers) { + let value = headers[header]; + resp.setHeader(header, value); + } + resp.setHeader("X-Weave-Timestamp", "" + this.timestamp(), false); + writeBytesToOutputStream(resp.bodyOutputStream, body); + }, + + /** + * This is invoked by the HttpServer. `this` is bound to the SyncServer; + * `handler` is the HttpServer's handler. + * + * TODO: need to use the correct Sync API response codes and errors here. + * TODO: Basic Auth. + * TODO: check username in path against username in BasicAuth. + */ + handleDefault: function handleDefault(handler, req, resp) { + try { + this._handleDefault(handler, req, resp); + } catch (e) { + if (e instanceof HttpError) { + this.respond(req, resp, e.code, e.description, "", {}); + } else { + throw e; + } + } + }, + + _handleDefault: function _handleDefault(handler, req, resp) { + this._log.debug( + "SyncServer: Handling request: " + req.method + " " + req.path + ); + + if (this.callback.onRequest) { + this.callback.onRequest(req, resp); + } + + let parts = this.pathRE.exec(req.path); + if (!parts) { + this._log.debug("SyncServer: Unexpected request: bad URL " + req.path); + throw HTTP_404; + } + + let [, version, username, first, rest] = parts; + // Doing a float compare of the version allows for us to pretend there was + // a node-reassignment - eg, we could re-assign from "1.1/user/" to + // "1.10/user" - this server will then still accept requests with the new + // URL while any code in sync itself which compares URLs will see a + // different URL. + if (parseFloat(version) != parseFloat(SYNC_API_VERSION)) { + this._log.debug("SyncServer: Unknown version."); + throw HTTP_404; + } + + if (!this.userExists(username)) { + this._log.debug("SyncServer: Unknown user."); + throw HTTP_401; + } + + // Hand off to the appropriate handler for this path component. + if (first in this.toplevelHandlers) { + let newHandler = this.toplevelHandlers[first]; + return newHandler.call( + this, + newHandler, + req, + resp, + version, + username, + rest + ); + } + this._log.debug("SyncServer: Unknown top-level " + first); + throw HTTP_404; + }, + + /** + * Compute the object that is returned for an info/collections request. + */ + infoCollections: function infoCollections(username) { + let responseObject = {}; + let colls = this.users[username].collections; + for (let coll in colls) { + responseObject[coll] = colls[coll].timestamp; + } + this._log.trace( + "SyncServer: info/collections returning " + JSON.stringify(responseObject) + ); + return responseObject; + }, + + /** + * Collection of the handler methods we use for top-level path components. + */ + toplevelHandlers: { + storage: function handleStorage( + handler, + req, + resp, + version, + username, + rest + ) { + let respond = this.respond.bind(this, req, resp); + if (!rest || !rest.length) { + this._log.debug( + "SyncServer: top-level storage " + req.method + " request." + ); + + // TODO: verify if this is spec-compliant. + if (req.method != "DELETE") { + respond(405, "Method Not Allowed", "[]", { Allow: "DELETE" }); + return undefined; + } + + // Delete all collections and track the timestamp for the response. + let timestamp = this.user(username).deleteCollections(); + + // Return timestamp and OK for deletion. + respond(200, "OK", JSON.stringify(timestamp)); + return undefined; + } + + let match = this.storageRE.exec(rest); + if (!match) { + this._log.warn("SyncServer: Unknown storage operation " + rest); + throw HTTP_404; + } + let [, collection, wboID] = match; + let coll = this.getCollection(username, collection); + + let checkXIUSFailure = () => { + if (req.hasHeader("x-if-unmodified-since")) { + let xius = parseFloat(req.getHeader("x-if-unmodified-since")); + // Sadly the way our tests are setup, we often end up with xius of + // zero (typically when syncing just one engine, so the date from + // info/collections isn't used) - so we allow that to work. + // Further, the Python server treats non-existing collections as + // having a timestamp of 0. + let collTimestamp = coll ? coll.timestamp : 0; + if (xius && xius < collTimestamp) { + this._log.info( + `x-if-unmodified-since mismatch - request wants ${xius} but our collection has ${collTimestamp}` + ); + respond(412, "precondition failed", "precondition failed"); + return true; + } + } + return false; + }; + + switch (req.method) { + case "GET": { + if (!coll) { + if (wboID) { + respond(404, "Not found", "Not found"); + return undefined; + } + // *cries inside*: - apparently the real sync server returned 200 + // here for some time, then returned 404 for some time (bug 687299), + // and now is back to 200 (bug 963332). + respond(200, "OK", "[]"); + return undefined; + } + if (!wboID) { + return coll.collectionHandler(req, resp); + } + let wbo = coll.wbo(wboID); + if (!wbo) { + respond(404, "Not found", "Not found"); + return undefined; + } + return wbo.handler()(req, resp); + } + case "DELETE": { + if (!coll) { + respond(200, "OK", "{}"); + return undefined; + } + if (checkXIUSFailure()) { + return undefined; + } + if (wboID) { + let wbo = coll.wbo(wboID); + if (wbo) { + wbo.delete(); + this.callback.onItemDeleted(username, collection, wboID); + } + respond(200, "OK", "{}"); + return undefined; + } + coll.collectionHandler(req, resp); + + // Spot if this is a DELETE for some IDs, and don't blow away the + // whole collection! + // + // We already handled deleting the WBOs by invoking the deleted + // collection's handler. However, in the case of + // + // DELETE storage/foobar + // + // we also need to remove foobar from the collections map. This + // clause tries to differentiate the above request from + // + // DELETE storage/foobar?ids=foo,baz + // + // and do the right thing. + // TODO: less hacky method. + if (-1 == req.queryString.indexOf("ids=")) { + // When you delete the entire collection, we drop it. + this._log.debug("Deleting entire collection."); + delete this.users[username].collections[collection]; + this.callback.onCollectionDeleted(username, collection); + } + + // Notify of item deletion. + let deleted = resp.deleted || []; + for (let i = 0; i < deleted.length; ++i) { + this.callback.onItemDeleted(username, collection, deleted[i]); + } + return undefined; + } + case "PUT": + // PUT and POST have slightly different XIUS semantics - for PUT, + // the check is against the item, whereas for POST it is against + // the collection. So first, a special-case for PUT. + if (req.hasHeader("x-if-unmodified-since")) { + let xius = parseFloat(req.getHeader("x-if-unmodified-since")); + // treat and xius of zero as if it wasn't specified - this happens + // in some of our tests for a new collection. + if (xius > 0) { + let wbo = coll.wbo(wboID); + if (xius < wbo.modified) { + this._log.info( + `x-if-unmodified-since mismatch - request wants ${xius} but wbo has ${wbo.modified}` + ); + respond(412, "precondition failed", "precondition failed"); + return undefined; + } + wbo.handler()(req, resp); + coll.timestamp = resp.newModified; + return resp; + } + } + // fall through to post. + case "POST": + if (checkXIUSFailure()) { + return undefined; + } + if (!coll) { + coll = this.createCollection(username, collection); + } + + if (wboID) { + let wbo = coll.wbo(wboID); + if (!wbo) { + this._log.trace( + "SyncServer: creating WBO " + collection + "/" + wboID + ); + wbo = coll.insert(wboID); + } + // Rather than instantiate each WBO's handler function, do it once + // per request. They get hit far less often than do collections. + wbo.handler()(req, resp); + coll.timestamp = resp.newModified; + return resp; + } + return coll.collectionHandler(req, resp); + default: + throw new Error("Request method " + req.method + " not implemented."); + } + }, + + info: function handleInfo(handler, req, resp, version, username, rest) { + switch (rest) { + case "collections": + let body = JSON.stringify(this.infoCollections(username)); + this.respond(req, resp, 200, "OK", body, { + "Content-Type": "application/json", + }); + return; + case "collection_usage": + case "collection_counts": + case "quota": + // TODO: implement additional info methods. + this.respond(req, resp, 200, "OK", "TODO"); + return; + default: + // TODO + this._log.warn("SyncServer: Unknown info operation " + rest); + throw HTTP_404; + } + }, + }, +}; + +/** + * Test helper. + */ +function serverForUsers(users, contents, callback) { + let server = new SyncServer(callback); + for (let [user, pass] of Object.entries(users)) { + server.registerUser(user, pass); + server.createContents(user, contents); + } + server.start(); + return server; +} diff --git a/services/sync/tests/unit/missing-sourceuri.json b/services/sync/tests/unit/missing-sourceuri.json new file mode 100644 index 0000000000..dcd487726a --- /dev/null +++ b/services/sync/tests/unit/missing-sourceuri.json @@ -0,0 +1,20 @@ +{ + "next": null, + "results": [ + { + "name": "Restartless Test Extension", + "type": "extension", + "guid": "missing-sourceuri@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485 + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/missing-xpi-search.json b/services/sync/tests/unit/missing-xpi-search.json new file mode 100644 index 0000000000..55f6432b29 --- /dev/null +++ b/services/sync/tests/unit/missing-xpi-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Non-Restartless Test Extension", + "type": "extension", + "guid": "missing-xpi@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/THIS_DOES_NOT_EXIST.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/prefs_test_prefs_store.js b/services/sync/tests/unit/prefs_test_prefs_store.js new file mode 100644 index 0000000000..63851a6934 --- /dev/null +++ b/services/sync/tests/unit/prefs_test_prefs_store.js @@ -0,0 +1,47 @@ +// This is a "preferences" file used by test_prefs_store.js + +/* global pref, user_pref */ + +// The prefs that control what should be synced. +// Most of these are "default" prefs, so the value itself will not sync. +pref("services.sync.prefs.sync.testing.int", true); +pref("services.sync.prefs.sync.testing.string", true); +pref("services.sync.prefs.sync.testing.bool", true); +pref("services.sync.prefs.sync.testing.dont.change", true); +// This is a default pref, but has the special "sync-seen" pref. +pref("services.sync.prefs.sync.testing.seen", true); +pref("services.sync.prefs.sync-seen.testing.seen", false); + +// this one is a user pref, so it *will* sync. +user_pref("services.sync.prefs.sync.testing.turned.off", false); +pref("services.sync.prefs.sync.testing.nonexistent", true); +pref("services.sync.prefs.sync.testing.default", true); +pref("services.sync.prefs.sync.testing.synced.url", true); +// We shouldn't sync the URL, or the flag that says we should sync the pref +// (otherwise some other client might overwrite our local value). +user_pref("services.sync.prefs.sync.testing.unsynced.url", true); + +// The preference values - these are all user_prefs, otherwise their value +// will not be synced. +user_pref("testing.int", 123); +user_pref("testing.string", "ohai"); +user_pref("testing.bool", true); +user_pref("testing.dont.change", "Please don't change me."); +user_pref("testing.turned.off", "I won't get synced."); +user_pref("testing.not.turned.on", "I won't get synced either!"); +// Some url we don't want to sync +user_pref( + "testing.unsynced.url", + "moz-extension://d5d31b00-b944-4afb-bd3d-d0326551a0ae" +); +user_pref("testing.synced.url", "https://www.example.com"); + +// A pref that exists but still has the default value - will be synced with +// null as the value. +pref("testing.default", "I'm the default value"); + +// A pref that has the default value - it will start syncing as soon as +// we see a change, even if the change is to the default. +pref("testing.seen", "the value"); + +// A pref that shouldn't be synced diff --git a/services/sync/tests/unit/rewrite-search.json b/services/sync/tests/unit/rewrite-search.json new file mode 100644 index 0000000000..740b6f2c30 --- /dev/null +++ b/services/sync/tests/unit/rewrite-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "Rewrite Test Extension", + "type": "extension", + "guid": "rewrite@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 485, + "url": "http://127.0.0.1:8888/require.xpi?src=api" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/sync_ping_schema.json b/services/sync/tests/unit/sync_ping_schema.json new file mode 100644 index 0000000000..a9866e3550 --- /dev/null +++ b/services/sync/tests/unit/sync_ping_schema.json @@ -0,0 +1,262 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "schema for Sync pings, documentation avaliable in toolkit/components/telemetry/docs/sync-ping.rst", + "type": "object", + "additionalProperties": false, + "required": ["version", "syncs", "why", "uid"], + "properties": { + "version": { "type": "integer", "minimum": 0 }, + "os": { "type": "object" }, + "discarded": { "type": "integer", "minimum": 1 }, + "why": { "enum": ["shutdown", "schedule", "idchange"] }, + "uid": { + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "deviceID": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "devices": { + "type": "array", + "items": { "$ref": "#/definitions/device" } + }, + "sessionStartDate": { "type": "string" }, + "syncs": { + "type": "array", + "minItems": 0, + "items": { "$ref": "#/definitions/payload" } + }, + "syncNodeType": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/event" } + }, + "migrations": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/migration" } + }, + "histograms": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "min": { "type": "integer" }, + "max": { "type": "integer" }, + "histogram_type": { "type": "integer" }, + "sum": { "type": "integer" }, + "ranges": { "type": "array" }, + "counts": { "type": "array" } + } + } + } + }, + "definitions": { + "payload": { + "type": "object", + "additionalProperties": false, + "required": ["when", "took"], + "properties": { + "didLogin": { "type": "boolean" }, + "when": { "type": "integer" }, + "status": { + "type": "object", + "anyOf": [{ "required": ["sync"] }, { "required": ["service"] }], + "additionalProperties": false, + "properties": { + "sync": { "type": "string" }, + "service": { "type": "string" } + } + }, + "why": { "type": "string" }, + "took": { "type": "integer", "minimum": -1 }, + "failureReason": { "$ref": "#/definitions/error" }, + "engines": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/engine" } + } + } + }, + "device": { + "required": ["id"], + "additionalProperties": false, + "type": "object", + "properties": { + "id": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "os": { "type": "string" }, + "version": { "type": "string" }, + "type": { "type": "string" }, + "syncID": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + "engine": { + "required": ["name"], + "additionalProperties": false, + "properties": { + "failureReason": { "$ref": "#/definitions/error" }, + "name": { "type": "string" }, + "took": { "type": "integer", "minimum": 1 }, + "status": { "type": "string" }, + "incoming": { + "type": "object", + "additionalProperties": false, + "anyOf": [{ "required": ["applied"] }, { "required": ["failed"] }], + "properties": { + "applied": { "type": "integer", "minimum": 1 }, + "failed": { "type": "integer", "minimum": 1 }, + "failedReasons": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "outgoing": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/outgoingBatch" } + }, + "steps": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/step" + }, + "validation": { + "type": "object", + "additionalProperties": false, + "anyOf": [ + { "required": ["checked"] }, + { "required": ["failureReason"] } + ], + "properties": { + "checked": { "type": "integer", "minimum": 0 }, + "failureReason": { "$ref": "#/definitions/error" }, + "took": { "type": "integer" }, + "version": { "type": "integer" }, + "problems": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + } + } + }, + "outgoingBatch": { + "type": "object", + "additionalProperties": false, + "anyOf": [{ "required": ["sent"] }, { "required": ["failed"] }], + "properties": { + "sent": { "type": "integer", "minimum": 1 }, + "failed": { "type": "integer", "minimum": 1 }, + "failedReasons": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "event": { + "type": "array", + "minItems": 4, + "maxItems": 6 + }, + "migration": { + "oneOf": [{ "$ref": "#/definitions/webextMigration" }] + }, + "webextMigration": { + "required": ["type"], + "properties": { + "type": { "enum": ["webext-storage"] }, + "entries": { "type": "integer" }, + "entriesSuccessful": { "type": "integer" }, + "extensions": { "type": "integer" }, + "extensionsSuccessful": { "type": "integer" }, + "openFailure": { "type": "boolean" } + } + }, + "error": { + "oneOf": [ + { "$ref": "#/definitions/httpError" }, + { "$ref": "#/definitions/nsError" }, + { "$ref": "#/definitions/shutdownError" }, + { "$ref": "#/definitions/authError" }, + { "$ref": "#/definitions/otherError" }, + { "$ref": "#/definitions/unexpectedError" }, + { "$ref": "#/definitions/sqlError" } + ] + }, + "httpError": { + "required": ["name", "code"], + "properties": { + "name": { "enum": ["httperror"] }, + "code": { "type": "integer" } + } + }, + "nsError": { + "required": ["name", "code"], + "properties": { + "name": { "enum": ["nserror"] }, + "code": { "type": "integer" } + } + }, + "shutdownError": { + "required": ["name"], + "properties": { + "name": { "enum": ["shutdownerror"] } + } + }, + "authError": { + "required": ["name"], + "properties": { + "name": { "enum": ["autherror"] }, + "from": { "enum": ["tokenserver", "fxaccounts", "hawkclient"] } + } + }, + "otherError": { + "required": ["name"], + "properties": { + "name": { "enum": ["othererror"] }, + "error": { "type": "string" } + } + }, + "unexpectedError": { + "required": ["name"], + "properties": { + "name": { "enum": ["unexpectederror"] }, + "error": { "type": "string" } + } + }, + "sqlError": { + "required": ["name"], + "properties": { + "name": { "enum": ["sqlerror"] }, + "code": { "type": "integer" } + } + }, + "step": { + "required": ["name"], + "properties": { + "name": { "type": "string" }, + "took": { "type": "integer", "minimum": 1 }, + "counts": { + "type": "array", + "minItems": 1, + "$ref": "#/definitions/namedCount" + } + } + }, + "namedCount": { + "required": ["name", "count"], + "properties": { + "name": { "type": "string" }, + "count": { "type": "integer" } + } + } + } +} diff --git a/services/sync/tests/unit/systemaddon-search.json b/services/sync/tests/unit/systemaddon-search.json new file mode 100644 index 0000000000..a812714918 --- /dev/null +++ b/services/sync/tests/unit/systemaddon-search.json @@ -0,0 +1,21 @@ +{ + "next": null, + "results": [ + { + "name": "System Add-on Test", + "type": "extension", + "guid": "system1@tests.mozilla.org", + "current_version": { + "version": "1.0", + "files": [ + { + "platform": "all", + "size": 999, + "url": "http://127.0.0.1:8888/system.xpi" + } + ] + }, + "last_updated": "2011-09-05T20:42:09Z" + } + ] +} diff --git a/services/sync/tests/unit/test_412.js b/services/sync/tests/unit/test_412.js new file mode 100644 index 0000000000..de0a2c087e --- /dev/null +++ b/services/sync/tests/unit/test_412.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +add_task(async function test_412_not_treated_as_failure() { + await Service.engineManager.register(RotaryEngine); + let engine = Service.engineManager.get("rotary"); + + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + // add an item to the server to the first sync advances lastModified. + let collection = server.getCollection("foo", "rotary"); + let payload = encryptPayload({ + id: "existing", + something: "existing record", + }); + collection.insert("existing", payload); + + let promiseObserved = promiseOneObserver("weave:engine:sync:finish"); + try { + // Do sync. + _("initial sync to initialize the world"); + await Service.sync(); + + // create a new record that should be uploaded and arrange for our lastSync + // timestamp to be wrong so we get a 412. + engine._store.items = { new: "new record" }; + await engine._tracker.addChangedID("new", 0); + + let saw412 = false; + let _uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async () => { + let lastSync = await engine.getLastSync(); + await engine.setLastSync(lastSync - 2); + try { + await _uploadOutgoing.call(engine); + } catch (ex) { + saw412 = ex.status == 412; + throw ex; + } + }; + _("Second sync - expecting a 412"); + await Service.sync(); + await promiseObserved; + ok(saw412, "did see a 412 error"); + // But service status should be OK as the 412 shouldn't be treated as an error. + equal(Service.status.service, STATUS_OK); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_addon_utils.js b/services/sync/tests/unit/test_addon_utils.js new file mode 100644 index 0000000000..30c1824295 --- /dev/null +++ b/services/sync/tests/unit/test_addon_utils.js @@ -0,0 +1,161 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); +const { AddonUtils } = ChromeUtils.importESModule( + "resource://services-sync/addonutils.sys.mjs" +); + +const HTTP_PORT = 8888; +const SERVER_ADDRESS = "http://127.0.0.1:8888"; + +var prefs = new Preferences(); + +prefs.set( + "extensions.getAddons.get.url", + SERVER_ADDRESS + "/search/guid:%IDS%" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); +}); + +function createAndStartHTTPServer(port = HTTP_PORT) { + try { + let server = new HttpServer(); + + server.registerFile( + "/search/guid:missing-sourceuri%40tests.mozilla.org", + do_get_file("missing-sourceuri.json") + ); + + server.registerFile( + "/search/guid:rewrite%40tests.mozilla.org", + do_get_file("rewrite-search.json") + ); + + server.start(port); + + return server; + } catch (ex) { + _("Got exception starting HTTP server on port " + port); + _("Error: " + Log.exceptionStr(ex)); + do_throw(ex); + } + return null; /* not hit, but keeps eslint happy! */ +} + +function run_test() { + syncTestLogging(); + + run_next_test(); +} + +add_task(async function test_handle_empty_source_uri() { + _("Ensure that search results without a sourceURI are properly ignored."); + + let server = createAndStartHTTPServer(); + + const ID = "missing-sourceuri@tests.mozilla.org"; + + const result = await AddonUtils.installAddons([ + { id: ID, requireSecureURI: false }, + ]); + + Assert.ok("installedIDs" in result); + Assert.equal(0, result.installedIDs.length); + + Assert.ok("skipped" in result); + Assert.ok(result.skipped.includes(ID)); + + await promiseStopServer(server); +}); + +add_test(function test_ignore_untrusted_source_uris() { + _("Ensures that source URIs from insecure schemes are rejected."); + + const bad = [ + "http://example.com/foo.xpi", + "ftp://example.com/foo.xpi", + "silly://example.com/foo.xpi", + ]; + + const good = ["https://example.com/foo.xpi"]; + + for (let s of bad) { + let sourceURI = Services.io.newURI(s); + let addon = { sourceURI, name: "bad", id: "bad" }; + + let canInstall = AddonUtils.canInstallAddon(addon); + Assert.ok(!canInstall, "Correctly rejected a bad URL"); + } + + for (let s of good) { + let sourceURI = Services.io.newURI(s); + let addon = { sourceURI, name: "good", id: "good" }; + + let canInstall = AddonUtils.canInstallAddon(addon); + Assert.ok(canInstall, "Correctly accepted a good URL"); + } + run_next_test(); +}); + +add_task(async function test_source_uri_rewrite() { + _("Ensure that a 'src=api' query string is rewritten to 'src=sync'"); + + // This tests for conformance with bug 708134 so server-side metrics aren't + // skewed. + + // We resort to monkeypatching because of the API design. + let oldFunction = + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult; + + let installCalled = false; + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult = + async function testInstallAddon(addon, metadata) { + Assert.equal( + SERVER_ADDRESS + "/require.xpi?src=sync", + addon.sourceURI.spec + ); + + installCalled = true; + + const install = await AddonUtils.getInstallFromSearchResult(addon); + Assert.equal( + SERVER_ADDRESS + "/require.xpi?src=sync", + install.sourceURI.spec + ); + Assert.deepEqual( + install.installTelemetryInfo, + { source: "sync" }, + "Got the expected installTelemetryInfo" + ); + + return { id: addon.id, addon, install }; + }; + + let server = createAndStartHTTPServer(); + + let installOptions = { + id: "rewrite@tests.mozilla.org", + requireSecureURI: false, + }; + await AddonUtils.installAddons([installOptions]); + + Assert.ok(installCalled); + Object.getPrototypeOf(AddonUtils).installAddonFromSearchResult = oldFunction; + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_addons_engine.js b/services/sync/tests/unit/test_addons_engine.js new file mode 100644 index 0000000000..7e1997b25b --- /dev/null +++ b/services/sync/tests/unit/test_addons_engine.js @@ -0,0 +1,279 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonManager } = ChromeUtils.importESModule( + "resource://gre/modules/AddonManager.sys.mjs" +); +const { CHANGE_INSTALLED } = ChromeUtils.importESModule( + "resource://services-sync/addonsreconciler.sys.mjs" +); +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); + +const prefs = new Preferences(); +prefs.set( + "extensions.getAddons.get.url", + "http://localhost:8888/search/guid:%IDS%" +); +prefs.set("extensions.install.requireSecureOrigin", false); + +let engine; +let syncID; +let reconciler; +let tracker; + +AddonTestUtils.init(this); + +const ADDON_ID = "addon1@tests.mozilla.org"; +const XPI = AddonTestUtils.createTempWebExtensionFile({ + manifest: { + name: "Test 1", + description: "Test Description", + browser_specific_settings: { gecko: { id: ADDON_ID } }, + }, +}); + +async function resetReconciler() { + reconciler._addons = {}; + reconciler._changes = []; + + await reconciler.saveState(); + + await tracker.clearChangedIDs(); +} + +add_task(async function setup() { + AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" + ); + AddonTestUtils.overrideCertDB(); + await AddonTestUtils.promiseStartupManager(); + + await Service.engineManager.register(AddonsEngine); + engine = Service.engineManager.get("addons"); + syncID = await engine.resetLocalSyncID(); + reconciler = engine._reconciler; + tracker = engine._tracker; + + reconciler.startListening(); + + // Don't flush to disk in the middle of an event listener! + // This causes test hangs on WinXP. + reconciler._shouldPersist = false; + + await resetReconciler(); +}); + +// This is a basic sanity test for the unit test itself. If this breaks, the +// add-ons API likely changed upstream. +add_task(async function test_addon_install() { + _("Ensure basic add-on APIs work as expected."); + + let install = await AddonManager.getInstallForFile(XPI); + Assert.notEqual(install, null); + Assert.equal(install.type, "extension"); + Assert.equal(install.name, "Test 1"); + + await resetReconciler(); +}); + +add_task(async function test_find_dupe() { + _("Ensure the _findDupe() implementation is sane."); + + // This gets invoked at the top of sync, which is bypassed by this + // test, so we do it manually. + await engine._refreshReconcilerState(); + + let addon = await installAddon(XPI, reconciler); + + let record = { + id: Utils.makeGUID(), + addonID: ADDON_ID, + enabled: true, + applicationID: Services.appinfo.ID, + source: "amo", + }; + + let dupe = await engine._findDupe(record); + Assert.equal(addon.syncGUID, dupe); + + record.id = addon.syncGUID; + dupe = await engine._findDupe(record); + Assert.equal(null, dupe); + + await uninstallAddon(addon, reconciler); + await resetReconciler(); +}); + +add_task(async function test_get_changed_ids() { + let timerPrecision = Preferences.get("privacy.reduceTimerPrecision"); + Preferences.set("privacy.reduceTimerPrecision", false); + + registerCleanupFunction(function () { + Preferences.set("privacy.reduceTimerPrecision", timerPrecision); + }); + + _("Ensure getChangedIDs() has the appropriate behavior."); + + _("Ensure getChangedIDs() returns an empty object by default."); + let changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(0, Object.keys(changes).length); + + _("Ensure tracker changes are populated."); + let now = new Date(); + let changeTime = now.getTime() / 1000; + let guid1 = Utils.makeGUID(); + await tracker.addChangedID(guid1, changeTime); + + changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(guid1 in changes); + Assert.equal(changeTime, changes[guid1]); + + await tracker.clearChangedIDs(); + + _("Ensure reconciler changes are populated."); + let addon = await installAddon(XPI, reconciler); + await tracker.clearChangedIDs(); // Just in case. + changes = await engine.getChangedIDs(); + Assert.equal("object", typeof changes); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(addon.syncGUID in changes); + _( + "Change time: " + changeTime + ", addon change: " + changes[addon.syncGUID] + ); + Assert.ok(changes[addon.syncGUID] >= changeTime); + + let oldTime = changes[addon.syncGUID]; + let guid2 = addon.syncGUID; + await uninstallAddon(addon, reconciler); + changes = await engine.getChangedIDs(); + Assert.equal(1, Object.keys(changes).length); + Assert.ok(guid2 in changes); + Assert.ok(changes[guid2] > oldTime); + + _("Ensure non-syncable add-ons aren't picked up by reconciler changes."); + reconciler._addons = {}; + reconciler._changes = []; + let record = { + id: "DUMMY", + guid: Utils.makeGUID(), + enabled: true, + installed: true, + modified: new Date(), + type: "UNSUPPORTED", + scope: 0, + foreignInstall: false, + }; + reconciler.addons.DUMMY = record; + await reconciler._addChange(record.modified, CHANGE_INSTALLED, record); + + changes = await engine.getChangedIDs(); + _(JSON.stringify(changes)); + Assert.equal(0, Object.keys(changes).length); + + await resetReconciler(); +}); + +add_task(async function test_disabled_install_semantics() { + _("Ensure that syncing a disabled add-on preserves proper state."); + + // This is essentially a test for bug 712542, which snuck into the original + // add-on sync drop. It ensures that when an add-on is installed that the + // disabled state and incoming syncGUID is preserved, even on the next sync. + const USER = "foo"; + const PASSWORD = "password"; + + let server = new SyncServer(); + server.start(); + await SyncTestingInfrastructure(server, USER, PASSWORD); + + await generateNewKeys(Service.collectionKeys); + + let contents = { + meta: { + global: { engines: { addons: { version: engine.version, syncID } } }, + }, + crypto: {}, + addons: {}, + }; + + server.registerUser(USER, "password"); + server.createContents(USER, contents); + + let amoServer = new HttpServer(); + amoServer.registerFile( + "/search/guid:addon1%40tests.mozilla.org", + do_get_file("addon1-search.json") + ); + + amoServer.registerFile("/addon1.xpi", XPI); + amoServer.start(8888); + + // Insert an existing record into the server. + let id = Utils.makeGUID(); + let now = Date.now() / 1000; + + let record = encryptPayload({ + id, + applicationID: Services.appinfo.ID, + addonID: ADDON_ID, + enabled: false, + deleted: false, + source: "amo", + }); + let wbo = new ServerWBO(id, record, now - 2); + server.insertWBO(USER, "addons", wbo); + + _("Performing sync of add-ons engine."); + await engine._sync(); + + // At this point the non-restartless extension should be staged for install. + + // Don't need this server any more. + await promiseStopServer(amoServer); + + // We ensure the reconciler has recorded the proper ID and enabled state. + let addon = reconciler.getAddonStateFromSyncGUID(id); + Assert.notEqual(null, addon); + Assert.equal(false, addon.enabled); + + // We fake an app restart and perform another sync, just to make sure things + // are sane. + await AddonTestUtils.promiseRestartManager(); + + let collection = server.getCollection(USER, "addons"); + engine.lastModified = collection.timestamp; + await engine._sync(); + + // The client should not upload a new record. The old record should be + // retained and unmodified. + Assert.equal(1, collection.count()); + + let payload = collection.payloads()[0]; + Assert.notEqual(null, collection.wbo(id)); + Assert.equal(ADDON_ID, payload.addonID); + Assert.ok(!payload.enabled); + + await promiseStopServer(server); +}); + +add_test(function cleanup() { + // There's an xpcom-shutdown hook for this, but let's give this a shot. + reconciler.stopListening(); + run_next_test(); +}); diff --git a/services/sync/tests/unit/test_addons_reconciler.js b/services/sync/tests/unit/test_addons_reconciler.js new file mode 100644 index 0000000000..9c60e2ad2b --- /dev/null +++ b/services/sync/tests/unit/test_addons_reconciler.js @@ -0,0 +1,209 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonsReconciler, CHANGE_INSTALLED, CHANGE_UNINSTALLED } = + ChromeUtils.importESModule( + "resource://services-sync/addonsreconciler.sys.mjs" + ); +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +const ADDON_ID = "addon1@tests.mozilla.org"; +const XPI = AddonTestUtils.createTempWebExtensionFile({ + manifest: { + name: "Test 1", + description: "Test Description", + browser_specific_settings: { gecko: { id: ADDON_ID } }, + }, +}); + +function makeAddonsReconciler() { + const log = Service.engineManager.get("addons")._log; + const queueCaller = Async.asyncQueueCaller(log); + return new AddonsReconciler(queueCaller); +} + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); + Svc.Prefs.set("engine.addons", true); + await Service.engineManager.register(AddonsEngine); +}); + +add_task(async function test_defaults() { + _("Ensure new objects have reasonable defaults."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + Assert.ok(!reconciler._listening); + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); + Assert.equal(0, reconciler._listeners.length); +}); + +add_task(async function test_load_state_empty_file() { + _("Ensure loading from a missing file results in defaults being set."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + let loaded = await reconciler.loadState(); + Assert.ok(!loaded); + + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); +}); + +add_task(async function test_install_detection() { + _("Ensure that add-on installation results in appropriate side-effects."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler.startListening(); + + let before = new Date(); + let addon = await installAddon(XPI); + let after = new Date(); + + Assert.equal(1, Object.keys(reconciler.addons).length); + Assert.ok(addon.id in reconciler.addons); + let record = reconciler.addons[ADDON_ID]; + + const KEYS = [ + "id", + "guid", + "enabled", + "installed", + "modified", + "type", + "scope", + "foreignInstall", + ]; + for (let key of KEYS) { + Assert.ok(key in record); + Assert.notEqual(null, record[key]); + } + + Assert.equal(addon.id, record.id); + Assert.equal(addon.syncGUID, record.guid); + Assert.ok(record.enabled); + Assert.ok(record.installed); + Assert.ok(record.modified >= before && record.modified <= after); + Assert.equal("extension", record.type); + Assert.ok(!record.foreignInstall); + + Assert.equal(1, reconciler._changes.length); + let change = reconciler._changes[0]; + Assert.ok(change[0] >= before && change[1] <= after); + Assert.equal(CHANGE_INSTALLED, change[1]); + Assert.equal(addon.id, change[2]); + + await uninstallAddon(addon); +}); + +add_task(async function test_uninstall_detection() { + _("Ensure that add-on uninstallation results in appropriate side-effects."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler.startListening(); + + reconciler._addons = {}; + reconciler._changes = []; + + let addon = await installAddon(XPI); + let id = addon.id; + + reconciler._changes = []; + await uninstallAddon(addon, reconciler); + + Assert.equal(1, Object.keys(reconciler.addons).length); + Assert.ok(id in reconciler.addons); + + let record = reconciler.addons[id]; + Assert.ok(!record.installed); + + Assert.equal(1, reconciler._changes.length); + let change = reconciler._changes[0]; + Assert.equal(CHANGE_UNINSTALLED, change[1]); + Assert.equal(id, change[2]); +}); + +add_task(async function test_load_state_future_version() { + _("Ensure loading a file from a future version results in no data loaded."); + + const FILENAME = "TEST_LOAD_STATE_FUTURE_VERSION"; + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + + // First we populate our new file. + let state = { version: 100, addons: { foo: {} }, changes: [[1, 1, "foo"]] }; + + // jsonSave() expects an object with ._log, so we give it a reconciler + // instance. + await Utils.jsonSave(FILENAME, reconciler, state); + + let loaded = await reconciler.loadState(FILENAME); + Assert.ok(!loaded); + + Assert.equal("object", typeof reconciler.addons); + Assert.equal(0, Object.keys(reconciler.addons).length); + Assert.equal(0, reconciler._changes.length); +}); + +add_task(async function test_prune_changes_before_date() { + _("Ensure that old changes are pruned properly."); + + let reconciler = makeAddonsReconciler(); + await reconciler.ensureStateLoaded(); + reconciler._changes = []; + + let now = new Date(); + const HOUR_MS = 1000 * 60 * 60; + + _("Ensure pruning an empty changes array works."); + reconciler.pruneChangesBeforeDate(now); + Assert.equal(0, reconciler._changes.length); + + let old = new Date(now.getTime() - HOUR_MS); + let young = new Date(now.getTime() - 1000); + reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]); + reconciler._changes.push([young, CHANGE_INSTALLED, "bar"]); + Assert.equal(2, reconciler._changes.length); + + _("Ensure pruning with an old time won't delete anything."); + let threshold = new Date(old.getTime() - 1); + reconciler.pruneChangesBeforeDate(threshold); + Assert.equal(2, reconciler._changes.length); + + _("Ensure pruning a single item works."); + threshold = new Date(young.getTime() - 1000); + reconciler.pruneChangesBeforeDate(threshold); + Assert.equal(1, reconciler._changes.length); + Assert.notEqual(undefined, reconciler._changes[0]); + Assert.equal(young, reconciler._changes[0][0]); + Assert.equal("bar", reconciler._changes[0][2]); + + _("Ensure pruning all changes works."); + reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]); + reconciler.pruneChangesBeforeDate(now); + Assert.equal(0, reconciler._changes.length); +}); diff --git a/services/sync/tests/unit/test_addons_store.js b/services/sync/tests/unit/test_addons_store.js new file mode 100644 index 0000000000..444f0e912c --- /dev/null +++ b/services/sync/tests/unit/test_addons_store.js @@ -0,0 +1,753 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +const HTTP_PORT = 8888; + +const prefs = new Preferences(); + +prefs.set( + "extensions.getAddons.get.url", + "http://localhost:8888/search/guid:%IDS%" +); +// Note that all compat-override URLs currently 404, but that's OK - the main +// thing is to avoid us hitting the real AMO. +prefs.set( + "extensions.getAddons.compatOverides.url", + "http://localhost:8888/compat-override/guid:%IDS%" +); +prefs.set("extensions.install.requireSecureOrigin", false); +prefs.set("extensions.checkUpdateSecurity", false); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +Services.prefs.setCharPref("extensions.minCompatibleAppVersion", "0"); +Services.prefs.setCharPref("extensions.minCompatiblePlatformVersion", "0"); +Services.prefs.setBoolPref("extensions.experiments.enabled", true); + +const SYSTEM_ADDON_ID = "system1@tests.mozilla.org"; +add_task(async function setupSystemAddon() { + const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "app0"], true); + AddonTestUtils.registerDirectory("XREAppFeat", distroDir); + + let xpi = await AddonTestUtils.createTempWebExtensionFile({ + manifest: { + browser_specific_settings: { gecko: { id: SYSTEM_ADDON_ID } }, + }, + }); + + xpi.copyTo(distroDir, `${SYSTEM_ADDON_ID}.xpi`); + + await AddonTestUtils.overrideBuiltIns({ system: [SYSTEM_ADDON_ID] }); + await AddonTestUtils.promiseStartupManager(); +}); + +const ID1 = "addon1@tests.mozilla.org"; +const ID2 = "addon2@tests.mozilla.org"; +const ID3 = "addon3@tests.mozilla.org"; + +const ADDONS = { + test_addon1: { + manifest: { + browser_specific_settings: { + gecko: { + id: ID1, + update_url: "http://example.com/data/test_install.json", + }, + }, + }, + }, + + test_addon2: { + manifest: { + browser_specific_settings: { gecko: { id: ID2 } }, + }, + }, + + test_addon3: { + manifest: { + browser_specific_settings: { + gecko: { + id: ID3, + strict_max_version: "0", + }, + }, + }, + }, +}; + +const SEARCH_RESULT = { + next: null, + results: [ + { + name: "Test Extension", + type: "extension", + guid: "addon1@tests.mozilla.org", + current_version: { + version: "1.0", + files: [ + { + platform: "all", + size: 485, + url: "http://localhost:8888/addon1.xpi", + }, + ], + }, + last_updated: "2018-10-27T04:12:00.826Z", + }, + ], +}; + +const MISSING_SEARCH_RESULT = { + next: null, + results: [ + { + name: "Test", + type: "extension", + guid: "missing-xpi@tests.mozilla.org", + current_version: { + version: "1.0", + files: [ + { + platform: "all", + size: 123, + url: "http://localhost:8888/THIS_DOES_NOT_EXIST.xpi", + }, + ], + }, + }, + ], +}; + +const XPIS = {}; +for (let [name, files] of Object.entries(ADDONS)) { + XPIS[name] = AddonTestUtils.createTempWebExtensionFile(files); +} + +let engine; +let store; +let reconciler; + +const proxyService = Cc[ + "@mozilla.org/network/protocol-proxy-service;1" +].getService(Ci.nsIProtocolProxyService); + +const proxyFilter = { + proxyInfo: proxyService.newProxyInfo( + "http", + "localhost", + HTTP_PORT, + "", + "", + 0, + 4096, + null + ), + + applyFilter(channel, defaultProxyInfo, callback) { + if (channel.URI.host === "example.com") { + callback.onProxyFilterResult(this.proxyInfo); + } else { + callback.onProxyFilterResult(defaultProxyInfo); + } + }, +}; + +proxyService.registerChannelFilter(proxyFilter, 0); +registerCleanupFunction(() => { + proxyService.unregisterChannelFilter(proxyFilter); +}); + +/** + * Create a AddonsRec for this application with the fields specified. + * + * @param id Sync GUID of record + * @param addonId ID of add-on + * @param enabled Boolean whether record is enabled + * @param deleted Boolean whether record was deleted + */ +function createRecordForThisApp(id, addonId, enabled, deleted) { + return { + id, + addonID: addonId, + enabled, + deleted: !!deleted, + applicationID: Services.appinfo.ID, + source: "amo", + }; +} + +function createAndStartHTTPServer(port) { + try { + let server = new HttpServer(); + + server.registerPathHandler( + "/search/guid:addon1%40tests.mozilla.org", + (req, resp) => { + resp.setHeader("Content-type", "application/json", true); + resp.write(JSON.stringify(SEARCH_RESULT)); + } + ); + server.registerPathHandler( + "/search/guid:missing-xpi%40tests.mozilla.org", + (req, resp) => { + resp.setHeader("Content-type", "application/json", true); + resp.write(JSON.stringify(MISSING_SEARCH_RESULT)); + } + ); + server.registerFile("/addon1.xpi", XPIS.test_addon1); + + server.start(port); + + return server; + } catch (ex) { + _("Got exception starting HTTP server on port " + port); + _("Error: " + Log.exceptionStr(ex)); + do_throw(ex); + } + return null; /* not hit, but keeps eslint happy! */ +} + +// A helper function to ensure that the reconciler's current view of the addon +// is the same as the addon itself. If it's not, then the reconciler missed a +// change, and is likely to re-upload the addon next sync because of the change +// it missed. +async function checkReconcilerUpToDate(addon) { + let stateBefore = Object.assign({}, store.reconciler.addons[addon.id]); + await store.reconciler.rectifyStateFromAddon(addon); + let stateAfter = store.reconciler.addons[addon.id]; + deepEqual(stateBefore, stateAfter); +} + +add_task(async function setup() { + await Service.engineManager.register(AddonsEngine); + engine = Service.engineManager.get("addons"); + store = engine._store; + reconciler = engine._reconciler; + + reconciler.startListening(); + + // Don't flush to disk in the middle of an event listener! + // This causes test hangs on WinXP. + reconciler._shouldPersist = false; +}); + +add_task(async function test_remove() { + _("Ensure removing add-ons from deleted records works."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + let record = createRecordForThisApp(addon.syncGUID, ID1, true, true); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(null, countTelemetry.failedReasons); + Assert.equal(0, countTelemetry.incomingCounts.failed); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, newAddon); +}); + +add_task(async function test_apply_enabled() { + let countTelemetry = new SyncedRecordsTelemetry(); + _("Ensures that changes to the userEnabled flag apply."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(addon.isActive); + Assert.ok(!addon.userDisabled); + + _("Ensure application of a disable record works as expected."); + let records = []; + records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false)); + + let [failed] = await Promise.all([ + store.applyIncomingBatch(records, countTelemetry), + AddonTestUtils.promiseAddonEvent("onDisabled"), + ]); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enable record works as expected."); + records.push(createRecordForThisApp(addon.syncGUID, ID1, true, false)); + [failed] = await Promise.all([ + store.applyIncomingBatch(records, countTelemetry), + AddonTestUtils.promiseWebExtensionStartup(ID1), + ]); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(!addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enabled state updates don't apply if the ignore pref is set."); + records.push(createRecordForThisApp(addon.syncGUID, ID1, false, false)); + Svc.Prefs.set("addons.ignoreUserEnabledChanges", true); + failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID1); + Assert.ok(!addon.userDisabled); + records = []; + + await uninstallAddon(addon, reconciler); + Svc.Prefs.reset("addons.ignoreUserEnabledChanges"); +}); + +add_task(async function test_apply_enabled_appDisabled() { + _( + "Ensures that changes to the userEnabled flag apply when the addon is appDisabled." + ); + + // this addon is appDisabled by default. + let addon = await installAddon(XPIS.test_addon3); + Assert.ok(addon.appDisabled); + Assert.ok(!addon.isActive); + Assert.ok(!addon.userDisabled); + + _("Ensure application of a disable record works as expected."); + store.reconciler.pruneChangesBeforeDate(Date.now() + 10); + store.reconciler._changes = []; + let records = []; + let countTelemetry = new SyncedRecordsTelemetry(); + records.push(createRecordForThisApp(addon.syncGUID, ID3, false, false)); + let failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID3); + Assert.ok(addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + _("Ensure enable record works as expected."); + records.push(createRecordForThisApp(addon.syncGUID, ID3, true, false)); + failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + addon = await AddonManager.getAddonByID(ID3); + Assert.ok(!addon.userDisabled); + await checkReconcilerUpToDate(addon); + records = []; + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_ignore_different_appid() { + _( + "Ensure that incoming records with a different application ID are ignored." + ); + + // We test by creating a record that should result in an update. + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(!addon.userDisabled); + + let record = createRecordForThisApp(addon.syncGUID, ID1, false, false); + record.applicationID = "FAKE_ID"; + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_ignore_unknown_source() { + _("Ensure incoming records with unknown source are ignored."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let record = createRecordForThisApp(addon.syncGUID, ID1, false, false); + record.source = "DUMMY_SOURCE"; + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(addon, reconciler); +}); + +add_task(async function test_apply_uninstall() { + _("Ensures that uninstalling an add-on from a record works."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let records = []; + let countTelemetry = new SyncedRecordsTelemetry(); + records.push(createRecordForThisApp(addon.syncGUID, ID1, true, true)); + let failed = await store.applyIncomingBatch(records, countTelemetry); + Assert.equal(0, failed.length); + Assert.equal(0, countTelemetry.incomingCounts.failed); + + addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); +}); + +add_task(async function test_addon_syncability() { + _("Ensure isAddonSyncable functions properly."); + + Svc.Prefs.set( + "addons.trustedSourceHostnames", + "addons.mozilla.org,other.example.com" + ); + + Assert.ok(!(await store.isAddonSyncable(null))); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(await store.isAddonSyncable(addon)); + + let dummy = {}; + const KEYS = [ + "id", + "syncGUID", + "type", + "scope", + "foreignInstall", + "isSyncable", + ]; + for (let k of KEYS) { + dummy[k] = addon[k]; + } + + Assert.ok(await store.isAddonSyncable(dummy)); + + dummy.type = "UNSUPPORTED"; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.type = addon.type; + + dummy.scope = 0; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.scope = addon.scope; + + dummy.isSyncable = false; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.isSyncable = addon.isSyncable; + + dummy.foreignInstall = true; + Assert.ok(!(await store.isAddonSyncable(dummy))); + dummy.foreignInstall = false; + + await uninstallAddon(addon, reconciler); + + Assert.ok(!store.isSourceURITrusted(null)); + + let trusted = [ + "https://addons.mozilla.org/foo", + "https://other.example.com/foo", + ]; + + let untrusted = [ + "http://addons.mozilla.org/foo", // non-https + "ftps://addons.mozilla.org/foo", // non-https + "https://untrusted.example.com/foo", // non-trusted hostname` + ]; + + for (let uri of trusted) { + Assert.ok(store.isSourceURITrusted(Services.io.newURI(uri))); + } + + for (let uri of untrusted) { + Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri))); + } + + Svc.Prefs.set("addons.trustedSourceHostnames", ""); + for (let uri of trusted) { + Assert.ok(!store.isSourceURITrusted(Services.io.newURI(uri))); + } + + Svc.Prefs.set("addons.trustedSourceHostnames", "addons.mozilla.org"); + Assert.ok( + store.isSourceURITrusted( + Services.io.newURI("https://addons.mozilla.org/foo") + ) + ); + + Svc.Prefs.reset("addons.trustedSourceHostnames"); +}); + +add_task(async function test_get_all_ids() { + _("Ensures that getAllIDs() returns an appropriate set."); + + _("Installing two addons."); + // XXX - this test seems broken - at this point, before we've installed the + // addons below, store.getAllIDs() returns all addons installed by previous + // tests, even though those tests uninstalled the addon. + // So if any tests above ever add a new addon ID, they are going to need to + // be added here too. + // Assert.equal(0, Object.keys(store.getAllIDs()).length); + let addon1 = await installAddon(XPIS.test_addon1, reconciler); + let addon2 = await installAddon(XPIS.test_addon2, reconciler); + let addon3 = await installAddon(XPIS.test_addon3, reconciler); + + _("Ensure they're syncable."); + Assert.ok(await store.isAddonSyncable(addon1)); + Assert.ok(await store.isAddonSyncable(addon2)); + Assert.ok(await store.isAddonSyncable(addon3)); + + let ids = await store.getAllIDs(); + + Assert.equal("object", typeof ids); + Assert.equal(3, Object.keys(ids).length); + Assert.ok(addon1.syncGUID in ids); + Assert.ok(addon2.syncGUID in ids); + Assert.ok(addon3.syncGUID in ids); + + await uninstallAddon(addon1, reconciler); + await uninstallAddon(addon2, reconciler); + await uninstallAddon(addon3, reconciler); +}); + +add_task(async function test_change_item_id() { + _("Ensures that changeItemID() works properly."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + + let oldID = addon.syncGUID; + let newID = Utils.makeGUID(); + + await store.changeItemID(oldID, newID); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.notEqual(null, newAddon); + Assert.equal(newID, newAddon.syncGUID); + + await uninstallAddon(newAddon, reconciler); +}); + +add_task(async function test_create() { + _("Ensure creating/installing an add-on from a record works."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, ID1, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + let newAddon = await AddonManager.getAddonByID(ID1); + Assert.notEqual(null, newAddon); + Assert.equal(guid, newAddon.syncGUID); + Assert.ok(!newAddon.userDisabled); + + await uninstallAddon(newAddon, reconciler); + + await promiseStopServer(server); +}); + +add_task(async function test_create_missing_search() { + _("Ensures that failed add-on searches are handled gracefully."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + // The handler for this ID is not installed, so a search should 404. + const id = "missing@tests.mozilla.org"; + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, id, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(1, failed.length); + Assert.equal(guid, failed[0]); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "GET <URL> failed (status 404)" + ); + Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1); + + let addon = await AddonManager.getAddonByID(id); + Assert.equal(null, addon); + + await promiseStopServer(server); +}); + +add_task(async function test_create_bad_install() { + _("Ensures that add-ons without a valid install are handled gracefully."); + + let server = createAndStartHTTPServer(HTTP_PORT); + + // The handler returns a search result but the XPI will 404. + const id = "missing-xpi@tests.mozilla.org"; + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, id, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + /* let failed = */ await store.applyIncomingBatch([record], countTelemetry); + // This addon had no source URI so was skipped - but it's not treated as + // failure. + // XXX - this test isn't testing what we thought it was. Previously the addon + // was not being installed due to requireSecureURL checking *before* we'd + // attempted to get the XPI. + // With requireSecureURL disabled we do see a download failure, but the addon + // *does* get added to |failed|. + // FTR: onDownloadFailed() is called with ERROR_NETWORK_FAILURE, so it's going + // to be tricky to distinguish a 404 from other transient network errors + // where we do want the addon to end up in |failed|. + // This is being tracked in bug 1284778. + // Assert.equal(0, failed.length); + + let addon = await AddonManager.getAddonByID(id); + Assert.equal(null, addon); + + await promiseStopServer(server); +}); + +add_task(async function test_ignore_system() { + _("Ensure we ignore system addons"); + // Our system addon should not appear in getAllIDs + await engine._refreshReconcilerState(); + let num = 0; + let ids = await store.getAllIDs(); + for (let guid in ids) { + num += 1; + let addon = reconciler.getAddonStateFromSyncGUID(guid); + Assert.notEqual(addon.id, SYSTEM_ADDON_ID); + } + Assert.greater(num, 1, "should have seen at least one."); +}); + +add_task(async function test_incoming_system() { + _("Ensure we handle incoming records that refer to a system addon"); + // eg, loop initially had a normal addon but it was then "promoted" to be a + // system addon but wanted to keep the same ID. The server record exists due + // to this. + + // before we start, ensure the system addon isn't disabled. + Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled)); + + // Now simulate an incoming record with the same ID as the system addon, + // but flagged as disabled - it should not be applied. + let server = createAndStartHTTPServer(HTTP_PORT); + // We make the incoming record flag the system addon as disabled - it should + // be ignored. + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, SYSTEM_ADDON_ID, false, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + // The system addon should still not be userDisabled. + Assert.ok(!(await AddonManager.getAddonByID(SYSTEM_ADDON_ID).userDisabled)); + + await promiseStopServer(server); +}); + +add_task(async function test_wipe() { + _("Ensures that wiping causes add-ons to be uninstalled."); + + await installAddon(XPIS.test_addon1, reconciler); + + await store.wipe(); + + let addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); +}); + +add_task(async function test_wipe_and_install() { + _("Ensure wipe followed by install works."); + + // This tests the reset sync flow where remote data is replaced by local. The + // receiving client will see a wipe followed by a record which should undo + // the wipe. + let installed = await installAddon(XPIS.test_addon1, reconciler); + + let record = createRecordForThisApp(installed.syncGUID, ID1, true, false); + + await store.wipe(); + + let deleted = await AddonManager.getAddonByID(ID1); + Assert.equal(null, deleted); + + // Re-applying the record can require re-fetching the XPI. + let server = createAndStartHTTPServer(HTTP_PORT); + + await store.applyIncoming(record); + + let fetched = await AddonManager.getAddonByID(record.addonID); + Assert.ok(!!fetched); + + // wipe again to we are left with a clean slate. + await store.wipe(); + + await promiseStopServer(server); +}); + +// STR for what this is testing: +// * Either: +// * Install then remove an addon, then delete addons.json from the profile +// or corrupt it (in which case the addon manager will remove it) +// * Install then remove an addon while addon caching is disabled, then +// re-enable addon caching. +// * Install the same addon in a different profile, sync it. +// * Sync this profile +// Before bug 1467904, the addon would fail to install because this profile +// has a copy of the addon in our addonsreconciler.json, but the addon manager +// does *not* have a copy in its cache, and repopulating that cache would not +// re-add it as the addon is no longer installed locally. +add_task(async function test_incoming_reconciled_but_not_cached() { + _( + "Ensure we handle incoming records our reconciler has but the addon cache does not" + ); + + // Make sure addon is not installed. + let addon = await AddonManager.getAddonByID(ID1); + Assert.equal(null, addon); + + Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", false); + + addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.notEqual(await AddonManager.getAddonByID(ID1), null); + await uninstallAddon(addon, reconciler); + + Services.prefs.setBoolPref("extensions.getAddons.cache.enabled", true); + + // now pretend it is incoming. + let server = createAndStartHTTPServer(HTTP_PORT); + let guid = Utils.makeGUID(); + let record = createRecordForThisApp(guid, ID1, true, false); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch([record], countTelemetry); + Assert.equal(0, failed.length); + + Assert.notEqual(await AddonManager.getAddonByID(ID1), null); + + await promiseStopServer(server); +}); + +// NOTE: The test above must be the last test run due to the addon cache +// being trashed. It is probably possible to fix that by running, eg, +// AddonRespository.backgroundUpdateCheck() to rebuild the cache, but that +// requires implementing more AMO functionality in our test server + +add_task(async function cleanup() { + // There's an xpcom-shutdown hook for this, but let's give this a shot. + reconciler.stopListening(); +}); diff --git a/services/sync/tests/unit/test_addons_tracker.js b/services/sync/tests/unit/test_addons_tracker.js new file mode 100644 index 0000000000..adbb4afa35 --- /dev/null +++ b/services/sync/tests/unit/test_addons_tracker.js @@ -0,0 +1,176 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { AddonsEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +Services.prefs.setCharPref("extensions.minCompatibleAppVersion", "0"); +Services.prefs.setCharPref("extensions.minCompatiblePlatformVersion", "0"); +Services.prefs.setBoolPref("extensions.experiments.enabled", true); + +Svc.Prefs.set("engine.addons", true); + +let reconciler; +let tracker; + +const addon1ID = "addon1@tests.mozilla.org"; + +const ADDONS = { + test_addon1: { + manifest: { + browser_specific_settings: { gecko: { id: addon1ID } }, + }, + }, +}; + +const XPIS = {}; + +async function cleanup() { + tracker.stop(); + + tracker.resetScore(); + await tracker.clearChangedIDs(); + + reconciler._addons = {}; + reconciler._changes = []; + await reconciler.saveState(); +} + +add_task(async function setup() { + await AddonTestUtils.promiseStartupManager(); + for (let [name, data] of Object.entries(ADDONS)) { + XPIS[name] = AddonTestUtils.createTempWebExtensionFile(data); + } + await Service.engineManager.register(AddonsEngine); + let engine = Service.engineManager.get("addons"); + reconciler = engine._reconciler; + tracker = engine._tracker; + + await cleanup(); +}); + +add_task(async function test_empty() { + _("Verify the tracker is empty to start with."); + + Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length); + Assert.equal(0, tracker.score); + + await cleanup(); +}); + +add_task(async function test_not_tracking() { + _("Ensures the tracker doesn't do anything when it isn't tracking."); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + await uninstallAddon(addon, reconciler); + + Assert.equal(0, Object.keys(await tracker.getChangedIDs()).length); + Assert.equal(0, tracker.score); + + await cleanup(); +}); + +add_task(async function test_track_install() { + _("Ensure that installing an add-on notifies tracker."); + + reconciler.startListening(); + + tracker.start(); + + Assert.equal(0, tracker.score); + let addon = await installAddon(XPIS.test_addon1, reconciler); + let changed = await tracker.getChangedIDs(); + + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); + +add_task(async function test_track_uninstall() { + _("Ensure that uninstalling an add-on notifies tracker."); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + let guid = addon.syncGUID; + Assert.equal(0, tracker.score); + + tracker.start(); + + await uninstallAddon(addon, reconciler); + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(guid in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await cleanup(); +}); + +add_task(async function test_track_user_disable() { + _("Ensure that tracker sees disabling of add-on"); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + Assert.ok(!addon.userDisabled); + Assert.ok(!addon.appDisabled); + Assert.ok(addon.isActive); + + tracker.start(); + Assert.equal(0, tracker.score); + + _("Disabling add-on"); + await addon.disable(); + await reconciler.queueCaller.promiseCallsComplete(); + + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); + +add_task(async function test_track_enable() { + _("Ensure that enabling a disabled add-on notifies tracker."); + + reconciler.startListening(); + + let addon = await installAddon(XPIS.test_addon1, reconciler); + await addon.disable(); + await Async.promiseYield(); + + Assert.equal(0, tracker.score); + + tracker.start(); + await addon.enable(); + await Async.promiseYield(); + await reconciler.queueCaller.promiseCallsComplete(); + + let changed = await tracker.getChangedIDs(); + Assert.equal(1, Object.keys(changed).length); + Assert.ok(addon.syncGUID in changed); + Assert.equal(SCORE_INCREMENT_XLARGE, tracker.score); + + await uninstallAddon(addon, reconciler); + await cleanup(); +}); diff --git a/services/sync/tests/unit/test_addons_validator.js b/services/sync/tests/unit/test_addons_validator.js new file mode 100644 index 0000000000..60f2f8bf43 --- /dev/null +++ b/services/sync/tests/unit/test_addons_validator.js @@ -0,0 +1,65 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AddonValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/addons.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "1111", + applicationID: Services.appinfo.ID, + addonID: "synced-addon@example.com", + enabled: true, + source: "amo", + understood: true, + }, + ], + client: [ + { + syncGUID: "1111", + id: "synced-addon@example.com", + type: "extension", + isSystem: false, + isSyncable: true, + }, + { + syncGUID: "2222", + id: "system-addon@example.com", + type: "extension", + isSystem: true, + isSyncable: false, + }, + { + // Plugins don't have a `syncedGUID`, but we don't sync them, so we + // shouldn't report them as client duplicates. + id: "some-plugin", + type: "plugin", + }, + { + id: "another-plugin", + type: "plugin", + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new AddonValidator({ + _findDupe(item) { + return null; + }, + isAddonSyncable(item) { + return item.type != "plugin"; + }, + }); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 4); + equal(records.length, 1); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); diff --git a/services/sync/tests/unit/test_bookmark_batch_fail.js b/services/sync/tests/unit/test_bookmark_batch_fail.js new file mode 100644 index 0000000000..9644d730e4 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_batch_fail.js @@ -0,0 +1,25 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Making sure a failing sync reports a useful error"); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_bookmark_test(async function run_test(engine) { + await engine.initialize(); + engine._syncStartup = async function () { + throw new Error("FAIL!"); + }; + + try { + _("Try calling the sync that should throw right away"); + await engine._sync(); + do_throw("Should have failed sync!"); + } catch (ex) { + _("Making sure what we threw ended up as the exception:", ex); + Assert.equal(ex.message, "FAIL!"); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_decline_undecline.js b/services/sync/tests/unit/test_bookmark_decline_undecline.js new file mode 100644 index 0000000000..12139dd163 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_decline_undecline.js @@ -0,0 +1,48 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// A stored reference to the collection won't be valid after disabling. +function getBookmarkWBO(server, guid) { + let coll = server.user("foo").collection("bookmarks"); + if (!coll) { + return null; + } + return coll.wbo(guid); +} + +add_task(async function test_decline_undecline() { + let engine = Service.engineManager.get("bookmarks"); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let { guid: bzGuid } = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org", + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + title: "bugzilla", + }); + + ok(!getBookmarkWBO(server, bzGuid), "Shouldn't have been uploaded yet"); + await Service.sync(); + ok(getBookmarkWBO(server, bzGuid), "Should be present on server"); + + engine.enabled = false; + await Service.sync(); + ok( + !getBookmarkWBO(server, bzGuid), + "Shouldn't be present on server anymore" + ); + + engine.enabled = true; + await Service.sync(); + ok(getBookmarkWBO(server, bzGuid), "Should be present on server again"); + } finally { + await PlacesSyncUtils.bookmarks.reset(); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_engine.js b/services/sync/tests/unit/test_bookmark_engine.js new file mode 100644 index 0000000000..b84a8d719d --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_engine.js @@ -0,0 +1,1409 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { BookmarkHTMLUtils } = ChromeUtils.importESModule( + "resource://gre/modules/BookmarkHTMLUtils.sys.mjs" +); +const { BookmarkJSONUtils } = ChromeUtils.importESModule( + "resource://gre/modules/BookmarkJSONUtils.sys.mjs" +); +const { Bookmark, BookmarkFolder, BookmarksEngine, Livemark } = + ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +var recordedEvents = []; + +function checkRecordedEvents(object, expected, message) { + // Ignore event telemetry from the merger. + let checkEvents = recordedEvents.filter(event => event.object == object); + deepEqual(checkEvents, expected, message); + // and clear the list so future checks are easier to write. + recordedEvents = []; +} + +async function fetchAllRecordIds() { + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached(` + WITH RECURSIVE + syncedItems(id, guid) AS ( + SELECT b.id, b.guid FROM moz_bookmarks b + WHERE b.guid IN ('menu________', 'toolbar_____', 'unfiled_____', + 'mobile______') + UNION ALL + SELECT b.id, b.guid FROM moz_bookmarks b + JOIN syncedItems s ON b.parent = s.id + ) + SELECT guid FROM syncedItems`); + let recordIds = new Set(); + for (let row of rows) { + let recordId = PlacesSyncUtils.bookmarks.guidToRecordId( + row.getResultByName("guid") + ); + recordIds.add(recordId); + } + return recordIds; +} + +async function cleanupEngine(engine) { + await engine.resetClient(); + await engine._store.wipe(); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + // Note we don't finalize the engine here as add_bookmark_test() does. +} + +async function cleanup(engine, server) { + await promiseStopServer(server); + await cleanupEngine(engine); +} + +add_task(async function setup() { + await generateNewKeys(Service.collectionKeys); + await Service.engineManager.unregister("bookmarks"); + + Service.recordTelemetryEvent = (object, method, value, extra = undefined) => { + recordedEvents.push({ object, method, value, extra }); + }; +}); + +add_task(async function test_buffer_timeout() { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let engine = new BookmarksEngine(Service); + engine._newWatchdog = function () { + // Return an already-aborted watchdog, so that we can abort merges + // immediately. + let watchdog = Async.watchdog(); + watchdog.controller.abort(); + return watchdog; + }; + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + + try { + info("Insert local bookmarks"); + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + guid: "bookmarkAAAA", + url: "http://example.com/a", + title: "A", + }, + { + guid: "bookmarkBBBB", + url: "http://example.com/b", + title: "B", + }, + ], + }); + + info("Insert remote bookmarks"); + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentid: "places", + title: "menu", + children: ["bookmarkCCCC", "bookmarkDDDD"], + }) + ); + collection.insert( + "bookmarkCCCC", + encryptPayload({ + id: "bookmarkCCCC", + type: "bookmark", + parentid: "menu", + bmkUri: "http://example.com/c", + title: "C", + }) + ); + collection.insert( + "bookmarkDDDD", + encryptPayload({ + id: "bookmarkDDDD", + type: "bookmark", + parentid: "menu", + bmkUri: "http://example.com/d", + title: "D", + }) + ); + + info("We expect this sync to fail"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex.name == "InterruptedError" + ); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_maintenance_after_failure(engine) { + _("Ensure we try to run maintenance if the engine fails to sync"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let syncStartup = engine._syncStartup; + let syncError = new Error("Something is rotten in the state of Places"); + engine._syncStartup = function () { + throw syncError; + }; + + Services.prefs.clearUserPref("places.database.lastMaintenance"); + + _("Ensure the sync fails and we run maintenance"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "run", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for first maintenance run" + ); + + _("Sync again, but ensure maintenance doesn't run"); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [], + "Should not record event if maintenance didn't run" + ); + + _("Fast-forward last maintenance pref; ensure maintenance runs"); + Services.prefs.setIntPref( + "places.database.lastMaintenance", + Date.now() / 1000 - 14400 + ); + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex == syncError + ); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "run", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for second maintenance run" + ); + + _("Fix sync failure; ensure we report success after maintenance"); + engine._syncStartup = syncStartup; + await sync_engine_and_validate_telem(engine, false); + checkRecordedEvents( + "maintenance", + [ + { + object: "maintenance", + method: "fix", + value: "bookmarks", + extra: undefined, + }, + ], + "Should record event for successful sync after second maintenance" + ); + + await sync_engine_and_validate_telem(engine, false); + checkRecordedEvents( + "maintenance", + [], + "Should not record maintenance events after successful sync" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_delete_invalid_roots_from_server(engine) { + _("Ensure that we delete the Places and Reading List roots from the server."); + + enableValidationPrefs(); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); + + try { + let placesRecord = await store.createRecord("places"); + collection.insert("places", encryptPayload(placesRecord.cleartext)); + + let listBmk = new Bookmark("bookmarks", Utils.makeGUID()); + listBmk.bmkUri = "https://example.com"; + listBmk.title = "Example reading list entry"; + listBmk.parentName = "Reading List"; + listBmk.parentid = "readinglist"; + collection.insert(listBmk.id, encryptPayload(listBmk.cleartext)); + + let readingList = new BookmarkFolder("bookmarks", "readinglist"); + readingList.title = "Reading List"; + readingList.children = [listBmk.id]; + readingList.parentName = ""; + readingList.parentid = "places"; + collection.insert("readinglist", encryptPayload(readingList.cleartext)); + + // Note that we don't insert a record for the toolbar, so the engine will + // report a parent-child disagreement, since Firefox's `parentid` is + // `toolbar`. + let newBmk = new Bookmark("bookmarks", Utils.makeGUID()); + newBmk.bmkUri = "http://getfirefox.com"; + newBmk.title = "Get Firefox!"; + newBmk.parentName = "Bookmarks Toolbar"; + newBmk.parentid = "toolbar"; + collection.insert(newBmk.id, encryptPayload(newBmk.cleartext)); + + deepEqual( + collection.keys().sort(), + ["places", "readinglist", listBmk.id, newBmk.id].sort(), + "Should store Places root, reading list items, and new bookmark on server" + ); + + let ping = await sync_engine_and_validate_telem(engine, true); + // In a real sync, the engine is named `bookmarks-buffered`. + // However, `sync_engine_and_validate_telem` simulates a sync where + // the engine isn't registered with the engine manager, so the recorder + // doesn't see its `overrideTelemetryName`. + let engineData = ping.engines.find(e => e.name == "bookmarks"); + ok(engineData.validation, "Bookmarks engine should always run validation"); + equal( + engineData.validation.checked, + 6, + "Bookmarks engine should validate all items" + ); + deepEqual( + engineData.validation.problems, + [ + { + name: "parentChildDisagreements", + count: 1, + }, + ], + "Bookmarks engine should report parent-child disagreement" + ); + deepEqual( + engineData.steps.map(step => step.name), + [ + "fetchLocalTree", + "fetchRemoteTree", + "merge", + "apply", + "notifyObservers", + "fetchLocalChangeRecords", + ], + "Bookmarks engine should report all merge steps" + ); + + await Assert.rejects( + PlacesUtils.promiseItemId("readinglist"), + /no item found for the given GUID/, + "Should not apply Reading List root" + ); + await Assert.rejects( + PlacesUtils.promiseItemId(listBmk.id), + /no item found for the given GUID/, + "Should not apply items in Reading List" + ); + ok( + (await PlacesUtils.promiseItemId(newBmk.id)) > 0, + "Should apply new bookmark" + ); + + deepEqual( + collection.keys().sort(), + ["menu", "mobile", "toolbar", "unfiled", newBmk.id].sort(), + "Should remove Places root and reading list items from server; upload local roots" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_processIncoming_error_orderChildren( + engine +) { + _( + "Ensure that _orderChildren() is called even when _processIncoming() throws an error." + ); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + try { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder 1", + }); + + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + let toolbar_record = await store.createRecord("toolbar"); + collection.insert("toolbar", encryptPayload(toolbar_record.cleartext)); + + let bmk1_record = await store.createRecord(bmk1.guid); + collection.insert(bmk1.guid, encryptPayload(bmk1_record.cleartext)); + + let bmk2_record = await store.createRecord(bmk2.guid); + collection.insert(bmk2.guid, encryptPayload(bmk2_record.cleartext)); + + // Create a server record for folder1 where we flip the order of + // the children. + let folder1_record = await store.createRecord(folder1.guid); + let folder1_payload = folder1_record.cleartext; + folder1_payload.children.reverse(); + collection.insert(folder1.guid, encryptPayload(folder1_payload)); + + // Create a bogus record that when synced down will provoke a + // network error which in turn provokes an exception in _processIncoming. + const BOGUS_GUID = "zzzzzzzzzzzz"; + let bogus_record = collection.insert(BOGUS_GUID, "I'm a bogus record!"); + bogus_record.get = function get() { + throw new Error("Sync this!"); + }; + + // Make the 10 minutes old so it will only be synced in the toFetch phase. + bogus_record.modified = new_timestamp() - 60 * 10; + await engine.setLastSync(new_timestamp() - 60); + engine.toFetch = new SerializableSet([BOGUS_GUID]); + + let error; + try { + await sync_engine_and_validate_telem(engine, true); + } catch (ex) { + error = ex; + } + ok(!!error); + + // Verify that the bookmark order has been applied. + folder1_record = await store.createRecord(folder1.guid); + let new_children = folder1_record.children; + Assert.deepEqual( + new_children.sort(), + [folder1_payload.children[0], folder1_payload.children[1]].sort() + ); + + let localChildIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + folder1.guid + ); + Assert.deepEqual(localChildIds.sort(), [bmk2.guid, bmk1.guid].sort()); + } finally { + await cleanup(engine, server); + } +}); + +add_bookmark_test(async function test_restorePromptsReupload(engine) { + await test_restoreOrImport(engine, { replace: true }); +}); + +add_bookmark_test(async function test_importPromptsReupload(engine) { + await test_restoreOrImport(engine, { replace: false }); +}); + +// Test a JSON restore or HTML import. Use JSON if `replace` is `true`, or +// HTML otherwise. +async function test_restoreOrImport(engine, { replace }) { + let verb = replace ? "restore" : "import"; + let verbing = replace ? "restoring" : "importing"; + let bookmarkUtils = replace ? BookmarkJSONUtils : BookmarkHTMLUtils; + + _(`Ensure that ${verbing} from a backup will reupload all records.`); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + try { + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder 1", + }); + + _("Create a single record."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + _(`Get Firefox!: ${bmk1.guid}`); + + let backupFilePath = PathUtils.join( + PathUtils.tempDir, + `t_b_e_${Date.now()}.json` + ); + + _("Make a backup."); + + await bookmarkUtils.exportToFile(backupFilePath); + + _("Create a different record and sync."); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + _(`Get Thunderbird!: ${bmk2.guid}`); + + await PlacesUtils.bookmarks.remove(bmk1.guid); + + let error; + try { + await sync_engine_and_validate_telem(engine, false); + } catch (ex) { + error = ex; + _("Got error: " + Log.exceptionStr(ex)); + } + Assert.ok(!error); + + _( + "Verify that there's only one bookmark on the server, and it's Thunderbird." + ); + // Of course, there's also the Bookmarks Toolbar and Bookmarks Menu... + let wbos = collection.keys(function (id) { + return !["menu", "toolbar", "mobile", "unfiled", folder1.guid].includes( + id + ); + }); + Assert.equal(wbos.length, 1); + Assert.equal(wbos[0], bmk2.guid); + + _(`Now ${verb} from a backup.`); + await bookmarkUtils.importFromFile(backupFilePath, { replace }); + + // If `replace` is `true`, we'll wipe the server on the next sync. + let bookmarksCollection = server.user("foo").collection("bookmarks"); + _("Verify that we didn't wipe the server."); + Assert.ok(!!bookmarksCollection); + + _("Ensure we have the bookmarks we expect locally."); + let recordIds = await fetchAllRecordIds(); + _("GUIDs: " + JSON.stringify([...recordIds])); + + let bookmarkRecordIds = new Map(); + let count = 0; + for (let recordId of recordIds) { + count++; + let info = await PlacesUtils.bookmarks.fetch( + PlacesSyncUtils.bookmarks.recordIdToGuid(recordId) + ); + // Only one bookmark, so _all_ should be Firefox! + if (info.type == PlacesUtils.bookmarks.TYPE_BOOKMARK) { + _(`Found URI ${info.url.href} for record ID ${recordId}`); + bookmarkRecordIds.set(info.url.href, recordId); + } + } + Assert.ok(bookmarkRecordIds.has("http://getfirefox.com/")); + if (!replace) { + Assert.ok(bookmarkRecordIds.has("http://getthunderbird.com/")); + } + + _("Have the correct number of IDs locally, too."); + let expectedResults = [ + "menu", + "toolbar", + "mobile", + "unfiled", + folder1.guid, + bmk1.guid, + ]; + if (!replace) { + expectedResults.push("toolbar", folder1.guid, bmk2.guid); + } + Assert.equal(count, expectedResults.length); + + _("Sync again. This'll wipe bookmarks from the server."); + try { + await sync_engine_and_validate_telem(engine, false); + } catch (ex) { + error = ex; + _("Got error: " + Log.exceptionStr(ex)); + } + Assert.ok(!error); + + _("Verify that there's the right bookmarks on the server."); + // Of course, there's also the Bookmarks Toolbar and Bookmarks Menu... + let payloads = server.user("foo").collection("bookmarks").payloads(); + let bookmarkWBOs = payloads.filter(function (wbo) { + return wbo.type == "bookmark"; + }); + + let folderWBOs = payloads.filter(function (wbo) { + return ( + wbo.type == "folder" && + wbo.id != "menu" && + wbo.id != "toolbar" && + wbo.id != "unfiled" && + wbo.id != "mobile" && + wbo.parentid != "menu" + ); + }); + + let expectedFX = { + id: bookmarkRecordIds.get("http://getfirefox.com/"), + bmkUri: "http://getfirefox.com/", + title: "Get Firefox!", + }; + let expectedTB = { + id: bookmarkRecordIds.get("http://getthunderbird.com/"), + bmkUri: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }; + + let expectedBookmarks; + if (replace) { + expectedBookmarks = [expectedFX]; + } else { + expectedBookmarks = [expectedTB, expectedFX]; + } + + doCheckWBOs(bookmarkWBOs, expectedBookmarks); + + _("Our old friend Folder 1 is still in play."); + let expectedFolder1 = { title: "Folder 1" }; + + let expectedFolders; + if (replace) { + expectedFolders = [expectedFolder1]; + } else { + expectedFolders = [expectedFolder1, expectedFolder1]; + } + + doCheckWBOs(folderWBOs, expectedFolders); + } finally { + await cleanup(engine, server); + } +} + +function doCheckWBOs(WBOs, expected) { + Assert.equal(WBOs.length, expected.length); + for (let i = 0; i < expected.length; i++) { + let lhs = WBOs[i]; + let rhs = expected[i]; + if ("id" in rhs) { + Assert.equal(lhs.id, rhs.id); + } + if ("bmkUri" in rhs) { + Assert.equal(lhs.bmkUri, rhs.bmkUri); + } + if ("title" in rhs) { + Assert.equal(lhs.title, rhs.title); + } + } +} + +function FakeRecord(constructor, r) { + this.defaultCleartext = constructor.prototype.defaultCleartext; + constructor.call(this, "bookmarks", r.id); + for (let x in r) { + this[x] = r[x]; + } + // Borrow the constructor's conversion functions. + this.toSyncBookmark = constructor.prototype.toSyncBookmark; + this.cleartextToString = constructor.prototype.cleartextToString; +} + +// Bug 632287. +// (Note that `test_mismatched_folder_types()` in +// toolkit/components/places/tests/sync/test_bookmark_kinds.js is an exact +// copy of this test, so it's fine to remove it as part of bug 1449730) +add_task(async function test_mismatched_types() { + _( + "Ensure that handling a record that changes type causes deletion " + + "then re-adding." + ); + + let oldRecord = { + id: "l1nZZXfB8nC7", + type: "folder", + parentName: "Bookmarks Toolbar", + title: "Innerst i Sneglehode", + description: null, + parentid: "toolbar", + }; + + let newRecord = { + id: "l1nZZXfB8nC7", + type: "livemark", + siteUri: "http://sneglehode.wordpress.com/", + feedUri: "http://sneglehode.wordpress.com/feed/", + parentName: "Bookmarks Toolbar", + title: "Innerst i Sneglehode", + description: null, + children: [ + "HCRq40Rnxhrd", + "YeyWCV1RVsYw", + "GCceVZMhvMbP", + "sYi2hevdArlF", + "vjbZlPlSyGY8", + "UtjUhVyrpeG6", + "rVq8WMG2wfZI", + "Lx0tcy43ZKhZ", + "oT74WwV8_j4P", + "IztsItWVSo3-", + ], + parentid: "toolbar", + }; + + let engine = new BookmarksEngine(Service); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let oldR = new FakeRecord(BookmarkFolder, oldRecord); + let newR = new FakeRecord(Livemark, newRecord); + oldR.parentid = PlacesUtils.bookmarks.toolbarGuid; + newR.parentid = PlacesUtils.bookmarks.toolbarGuid; + + await store.applyIncoming(oldR); + await engine._apply(); + _("Applied old. It's a folder."); + let oldID = await PlacesUtils.promiseItemId(oldR.id); + _("Old ID: " + oldID); + let oldInfo = await PlacesUtils.bookmarks.fetch(oldR.id); + Assert.equal(oldInfo.type, PlacesUtils.bookmarks.TYPE_FOLDER); + + await store.applyIncoming(newR); + await engine._apply(); + await Assert.rejects( + PlacesUtils.promiseItemId(newR.id), + /no item found for the given GUID/, + "Should not apply Livemark" + ); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_misreconciled_root(engine) { + _("Ensure that we don't reconcile an arbitrary record with a root."); + + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + // Log real hard for this test. + store._log.trace = store._log.debug; + engine._log.trace = engine._log.debug; + + await engine._syncStartup(); + + // Let's find out where the toolbar is right now. + let toolbarBefore = await store.createRecord("toolbar", "bookmarks"); + let toolbarIDBefore = await PlacesUtils.promiseItemId( + PlacesUtils.bookmarks.toolbarGuid + ); + Assert.notEqual(-1, toolbarIDBefore); + + let parentRecordIDBefore = toolbarBefore.parentid; + let parentGUIDBefore = + PlacesSyncUtils.bookmarks.recordIdToGuid(parentRecordIDBefore); + let parentIDBefore = await PlacesUtils.promiseItemId(parentGUIDBefore); + Assert.equal("string", typeof parentGUIDBefore); + + _("Current parent: " + parentGUIDBefore + " (" + parentIDBefore + ")."); + + let to_apply = { + id: "zzzzzzzzzzzz", + type: "folder", + title: "Bookmarks Toolbar", + description: "Now you're for it.", + parentName: "", + parentid: "mobile", // Why not? + children: [], + }; + + let rec = new FakeRecord(BookmarkFolder, to_apply); + + _("Applying record."); + let countTelemetry = new SyncedRecordsTelemetry(); + store.applyIncomingBatch([rec], countTelemetry); + + // Ensure that afterwards, toolbar is still there. + // As of 2012-12-05, this only passes because Places doesn't use "toolbar" as + // the real GUID, instead using a generated one. Sync does the translation. + let toolbarAfter = await store.createRecord("toolbar", "bookmarks"); + let parentRecordIDAfter = toolbarAfter.parentid; + let parentGUIDAfter = + PlacesSyncUtils.bookmarks.recordIdToGuid(parentRecordIDAfter); + let parentIDAfter = await PlacesUtils.promiseItemId(parentGUIDAfter); + Assert.equal( + await PlacesUtils.promiseItemGuid(toolbarIDBefore), + PlacesUtils.bookmarks.toolbarGuid + ); + Assert.equal(parentGUIDBefore, parentGUIDAfter); + Assert.equal(parentIDBefore, parentIDAfter); + + await cleanup(engine, server); +}); + +add_bookmark_test(async function test_sync_dateAdded(engine) { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + // TODO: Avoid random orange (bug 1374599), this is only necessary + // intermittently - reset the last sync date so that we'll get all bookmarks. + await engine.setLastSync(1); + + engine._tracker.start(); // We skip usual startup... + + // Just matters that it's in the past, not how far. + let now = Date.now(); + let oneYearMS = 365 * 24 * 60 * 60 * 1000; + + try { + let toolbar = new BookmarkFolder("bookmarks", "toolbar"); + toolbar.title = "toolbar"; + toolbar.parentName = ""; + toolbar.parentid = "places"; + toolbar.children = [ + "abcdefabcdef", + "aaaaaaaaaaaa", + "bbbbbbbbbbbb", + "cccccccccccc", + "dddddddddddd", + "eeeeeeeeeeee", + ]; + collection.insert("toolbar", encryptPayload(toolbar.cleartext)); + + let item1GUID = "abcdefabcdef"; + let item1 = new Bookmark("bookmarks", item1GUID); + item1.bmkUri = "https://example.com"; + item1.title = "asdf"; + item1.parentName = "Bookmarks Toolbar"; + item1.parentid = "toolbar"; + item1.dateAdded = now - oneYearMS; + collection.insert(item1GUID, encryptPayload(item1.cleartext)); + + let item2GUID = "aaaaaaaaaaaa"; + let item2 = new Bookmark("bookmarks", item2GUID); + item2.bmkUri = "https://example.com/2"; + item2.title = "asdf2"; + item2.parentName = "Bookmarks Toolbar"; + item2.parentid = "toolbar"; + item2.dateAdded = now + oneYearMS; + const item2LastModified = now / 1000 - 100; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + item2LastModified + ); + + let item3GUID = "bbbbbbbbbbbb"; + let item3 = new Bookmark("bookmarks", item3GUID); + item3.bmkUri = "https://example.com/3"; + item3.title = "asdf3"; + item3.parentName = "Bookmarks Toolbar"; + item3.parentid = "toolbar"; + // no dateAdded + collection.insert(item3GUID, encryptPayload(item3.cleartext)); + + let item4GUID = "cccccccccccc"; + let item4 = new Bookmark("bookmarks", item4GUID); + item4.bmkUri = "https://example.com/4"; + item4.title = "asdf4"; + item4.parentName = "Bookmarks Toolbar"; + item4.parentid = "toolbar"; + // no dateAdded, but lastModified in past + const item4LastModified = (now - oneYearMS) / 1000; + collection.insert( + item4GUID, + encryptPayload(item4.cleartext), + item4LastModified + ); + + let item5GUID = "dddddddddddd"; + let item5 = new Bookmark("bookmarks", item5GUID); + item5.bmkUri = "https://example.com/5"; + item5.title = "asdf5"; + item5.parentName = "Bookmarks Toolbar"; + item5.parentid = "toolbar"; + // no dateAdded, lastModified in (near) future. + const item5LastModified = (now + 60000) / 1000; + collection.insert( + item5GUID, + encryptPayload(item5.cleartext), + item5LastModified + ); + + let item6GUID = "eeeeeeeeeeee"; + let item6 = new Bookmark("bookmarks", item6GUID); + item6.bmkUri = "https://example.com/6"; + item6.title = "asdf6"; + item6.parentName = "Bookmarks Toolbar"; + item6.parentid = "toolbar"; + const item6LastModified = (now - oneYearMS) / 1000; + collection.insert( + item6GUID, + encryptPayload(item6.cleartext), + item6LastModified + ); + + await sync_engine_and_validate_telem(engine, false); + + let record1 = await store.createRecord(item1GUID); + let record2 = await store.createRecord(item2GUID); + + equal( + item1.dateAdded, + record1.dateAdded, + "dateAdded in past should be synced" + ); + equal( + record2.dateAdded, + item2LastModified * 1000, + "dateAdded in future should be ignored in favor of last modified" + ); + + let record3 = await store.createRecord(item3GUID); + + ok(record3.dateAdded); + // Make sure it's within 24 hours of the right timestamp... This is a little + // dodgey but we only really care that it's basically accurate and has the + // right day. + ok(Math.abs(Date.now() - record3.dateAdded) < 24 * 60 * 60 * 1000); + + let record4 = await store.createRecord(item4GUID); + equal( + record4.dateAdded, + item4LastModified * 1000, + "If no dateAdded is provided, lastModified should be used" + ); + + let record5 = await store.createRecord(item5GUID); + equal( + record5.dateAdded, + item5LastModified * 1000, + "If no dateAdded is provided, lastModified should be used (even if it's in the future)" + ); + + // Update item2 and try resyncing it. + item2.dateAdded = now - 100000; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + now / 1000 - 50 + ); + + // Also, add a local bookmark and make sure its date added makes it up to the server + let bz = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }); + + // last sync did a POST, which doesn't advance its lastModified value. + // Next sync of the engine doesn't hit info/collections, so lastModified + // remains stale. Setting it to null side-steps that. + engine.lastModified = null; + await sync_engine_and_validate_telem(engine, false); + + let newRecord2 = await store.createRecord(item2GUID); + equal( + newRecord2.dateAdded, + item2.dateAdded, + "dateAdded update should work for earlier date" + ); + + let bzWBO = collection.cleartext(bz.guid); + ok(bzWBO.dateAdded, "Locally added dateAdded lost"); + + let localRecord = await store.createRecord(bz.guid); + equal( + bzWBO.dateAdded, + localRecord.dateAdded, + "dateAdded should not change during upload" + ); + + item2.dateAdded += 10000; + collection.insert( + item2GUID, + encryptPayload(item2.cleartext), + now / 1000 - 10 + ); + + engine.lastModified = null; + await sync_engine_and_validate_telem(engine, false); + + let newerRecord2 = await store.createRecord(item2GUID); + equal( + newerRecord2.dateAdded, + newRecord2.dateAdded, + "dateAdded update should be ignored for later date if we know an earlier one " + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_buffer_hasDupe() { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + engine._tracker.start(); // We skip usual startup... + try { + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + await PlacesUtils.bookmarks.insert({ + guid: guid1, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "https://www.example.com", + title: "example.com", + }); + await PlacesUtils.bookmarks.insert({ + guid: guid2, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "https://www.example.com", + title: "example.com", + }); + + await sync_engine_and_validate_telem(engine, false); + // Make sure we set hasDupe on outgoing records + Assert.ok(collection.payloads().every(payload => payload.hasDupe)); + + await PlacesUtils.bookmarks.remove(guid1); + + await sync_engine_and_validate_telem(engine, false); + + let tombstone = JSON.parse( + JSON.parse(collection.payload(guid1)).ciphertext + ); + // We shouldn't set hasDupe on tombstones. + Assert.ok(tombstone.deleted); + Assert.ok(!tombstone.hasDupe); + + let record = JSON.parse(JSON.parse(collection.payload(guid2)).ciphertext); + // We should set hasDupe on weakly uploaded records. + Assert.ok(!record.deleted); + Assert.ok( + record.hasDupe, + "Bookmarks bookmark engine should set hasDupe for weakly uploaded records." + ); + + await sync_engine_and_validate_telem(engine, false); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +// Bug 890217. +add_bookmark_test(async function test_sync_imap_URLs(engine) { + await Service.recordManager.clearCache(); + await PlacesSyncUtils.bookmarks.reset(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + try { + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentid: "places", + title: "Bookmarks Menu", + children: ["bookmarkAAAA"], + }) + ); + collection.insert( + "bookmarkAAAA", + encryptPayload({ + id: "bookmarkAAAA", + type: "bookmark", + parentid: "menu", + bmkUri: + "imap://vs@eleven.vs.solnicky.cz:993/fetch%3EUID%3E/" + + "INBOX%3E56291?part=1.2&type=image/jpeg&filename=" + + "invalidazPrahy.jpg", + title: + "invalidazPrahy.jpg (JPEG Image, 1280x1024 pixels) - Scaled (71%)", + }) + ); + + await PlacesUtils.bookmarks.insert({ + guid: "bookmarkBBBB", + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: + "imap://eleven.vs.solnicky.cz:993/fetch%3EUID%3E/" + + "CURRENT%3E2433?part=1.2&type=text/html&filename=TomEdwards.html", + title: "TomEdwards.html", + }); + + await sync_engine_and_validate_telem(engine, false); + + let aInfo = await PlacesUtils.bookmarks.fetch("bookmarkAAAA"); + equal( + aInfo.url.href, + "imap://vs@eleven.vs.solnicky.cz:993/" + + "fetch%3EUID%3E/INBOX%3E56291?part=1.2&type=image/jpeg&filename=" + + "invalidazPrahy.jpg", + "Remote bookmark A with IMAP URL should exist locally" + ); + + let bPayload = collection.cleartext("bookmarkBBBB"); + equal( + bPayload.bmkUri, + "imap://eleven.vs.solnicky.cz:993/" + + "fetch%3EUID%3E/CURRENT%3E2433?part=1.2&type=text/html&filename=" + + "TomEdwards.html", + "Local bookmark B with IMAP URL should exist remotely" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_resume_buffer() { + await Service.recordManager.clearCache(); + let engine = new BookmarksEngine(Service); + await engine.initialize(); + await engine._store.wipe(); + await engine.resetClient(); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + + engine._tracker.start(); // We skip usual startup... + + const batchChunkSize = 50; + + engine._store._batchChunkSize = batchChunkSize; + try { + let children = []; + + let timestamp = round_timestamp(Date.now()); + // Add two chunks worth of records to the server + for (let i = 0; i < batchChunkSize * 2; ++i) { + let cleartext = { + id: Utils.makeGUID(), + type: "bookmark", + parentid: "toolbar", + title: `Bookmark ${i}`, + parentName: "Bookmarks Toolbar", + bmkUri: `https://example.com/${i}`, + }; + let wbo = collection.insert( + cleartext.id, + encryptPayload(cleartext), + timestamp + 10 * i + ); + // Something that is effectively random, but deterministic. + // (This is just to ensure we don't accidentally start using the + // sortindex again). + wbo.sortindex = 1000 + Math.round(Math.sin(i / 5) * 100); + children.push(cleartext.id); + } + + // Add the parent of those records, and ensure its timestamp is the most recent. + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + parentid: "places", + title: "Bookmarks Toolbar", + children, + }), + timestamp + 10 * children.length + ); + + // Replace applyIncomingBatch with a custom one that calls the original, + // but forces it to throw on the 2nd chunk. + let origApplyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = function (records) { + if (records.length > batchChunkSize) { + // Hacky way to make reading from the batchChunkSize'th record throw. + delete records[batchChunkSize]; + Object.defineProperty(records, batchChunkSize, { + get() { + throw new Error("D:"); + }, + }); + } + return origApplyIncomingBatch.call(this, records); + }; + + let caughtError; + _("We expect this to fail"); + try { + await sync_engine_and_validate_telem(engine, true); + } catch (e) { + caughtError = e; + } + Assert.ok(caughtError, "Expected engine.sync to throw"); + Assert.equal(caughtError.message, "D:"); + + // The buffer subtracts one second from the actual timestamp. + let lastSync = (await engine.getLastSync()) + 1; + // We poisoned the batchChunkSize'th record, so the last successfully + // applied record will be batchChunkSize - 1. + let expectedLastSync = timestamp + 10 * (batchChunkSize - 1); + Assert.equal(expectedLastSync, lastSync); + + engine._store.applyIncomingBatch = origApplyIncomingBatch; + + await sync_engine_and_validate_telem(engine, false); + + // Check that all the children made it onto the correct record. + let toolbarRecord = await engine._store.createRecord("toolbar"); + Assert.deepEqual(toolbarRecord.children.sort(), children.sort()); + } finally { + await cleanup(engine, server); + await engine.finalize(); + } +}); + +add_bookmark_test(async function test_livemarks(engine) { + _("Ensure we replace new and existing livemarks with tombstones"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.user("foo").collection("bookmarks"); + let now = Date.now(); + + try { + _("Insert existing livemark"); + let modifiedForA = now - 5 * 60 * 1000; + await PlacesUtils.bookmarks.insert({ + guid: "livemarkAAAA", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "A", + lastModified: new Date(modifiedForA), + dateAdded: new Date(modifiedForA), + source: PlacesUtils.bookmarks.SOURCE_SYNC, + }); + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + parentName: "", + title: "menu", + children: ["livemarkAAAA"], + parentid: "places", + }), + round_timestamp(modifiedForA) + ); + collection.insert( + "livemarkAAAA", + encryptPayload({ + id: "livemarkAAAA", + type: "livemark", + feedUri: "http://example.com/a", + parentName: "menu", + title: "A", + parentid: "menu", + }), + round_timestamp(modifiedForA) + ); + + _("Insert remotely updated livemark"); + await PlacesUtils.bookmarks.insert({ + guid: "livemarkBBBB", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "B", + lastModified: new Date(now), + dateAdded: new Date(now), + }); + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + parentName: "", + title: "toolbar", + children: ["livemarkBBBB"], + parentid: "places", + }), + round_timestamp(now) + ); + collection.insert( + "livemarkBBBB", + encryptPayload({ + id: "livemarkBBBB", + type: "livemark", + feedUri: "http://example.com/b", + parentName: "toolbar", + title: "B", + parentid: "toolbar", + }), + round_timestamp(now) + ); + + _("Insert new remote livemark"); + collection.insert( + "unfiled", + encryptPayload({ + id: "unfiled", + type: "folder", + parentName: "", + title: "unfiled", + children: ["livemarkCCCC"], + parentid: "places", + }), + round_timestamp(now) + ); + collection.insert( + "livemarkCCCC", + encryptPayload({ + id: "livemarkCCCC", + type: "livemark", + feedUri: "http://example.com/c", + parentName: "unfiled", + title: "C", + parentid: "unfiled", + }), + round_timestamp(now) + ); + + _("Bump last sync time to ignore A"); + await engine.setLastSync(round_timestamp(now) - 60); + + _("Sync"); + await sync_engine_and_validate_telem(engine, false); + + deepEqual( + collection.keys().sort(), + [ + "livemarkAAAA", + "livemarkBBBB", + "livemarkCCCC", + "menu", + "mobile", + "toolbar", + "unfiled", + ], + "Should store original livemark A and tombstones for B and C on server" + ); + + let payloads = collection.payloads(); + + deepEqual( + payloads.find(payload => payload.id == "menu").children, + ["livemarkAAAA"], + "Should keep A in menu" + ); + ok( + !payloads.find(payload => payload.id == "livemarkAAAA").deleted, + "Should not upload tombstone for A" + ); + + deepEqual( + payloads.find(payload => payload.id == "toolbar").children, + [], + "Should remove B from toolbar" + ); + ok( + payloads.find(payload => payload.id == "livemarkBBBB").deleted, + "Should upload tombstone for B" + ); + + deepEqual( + payloads.find(payload => payload.id == "unfiled").children, + [], + "Should remove C from unfiled" + ); + ok( + payloads.find(payload => payload.id == "livemarkCCCC").deleted, + "Should replace C with tombstone" + ); + + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + children: [ + { + guid: "livemarkAAAA", + index: 0, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "Should keep A and remove B locally" + ); + } finally { + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_order.js b/services/sync/tests/unit/test_bookmark_order.js new file mode 100644 index 0000000000..fc182b81ef --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_order.js @@ -0,0 +1,586 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Making sure after processing incoming bookmarks, they show up in the right order" +); +const { Bookmark, BookmarkFolder } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function serverForFoo(engine) { + await generateNewKeys(Service.collectionKeys); + + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let engineSyncID = await engine.resetLocalSyncID(); + return serverForUsers( + { foo: "password" }, + { + meta: { + global: { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { + version: clientsEngine.version, + syncID: clientsSyncID, + }, + [engine.name]: { + version: engine.version, + syncID: engineSyncID, + }, + }, + }, + }, + crypto: { + keys: encryptPayload({ + id: "keys", + // Generate a fake default key bundle to avoid resetting the client + // before the first sync. + default: [ + await Weave.Crypto.generateRandomKey(), + await Weave.Crypto.generateRandomKey(), + ], + }), + }, + [engine.name]: {}, + } + ); +} + +async function resolveConflict( + engine, + collection, + timestamp, + buildTree, + message +) { + let guids = { + // These items don't exist on the server. + fx: Utils.makeGUID(), + nightly: Utils.makeGUID(), + support: Utils.makeGUID(), + customize: Utils.makeGUID(), + + // These exist on the server, but in a different order, and `res` + // has completely different children. + res: Utils.makeGUID(), + tb: Utils.makeGUID(), + + // These don't exist locally. + bz: Utils.makeGUID(), + irc: Utils.makeGUID(), + mdn: Utils.makeGUID(), + }; + + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.menuGuid, + children: [ + { + guid: guids.fx, + title: "Get Firefox!", + url: "http://getfirefox.com/", + }, + { + guid: guids.res, + title: "Resources", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children: [ + { + guid: guids.nightly, + title: "Nightly", + url: "https://nightly.mozilla.org/", + }, + { + guid: guids.support, + title: "Support", + url: "https://support.mozilla.org/", + }, + { + guid: guids.customize, + title: "Customize", + url: "https://mozilla.org/firefox/customize/", + }, + ], + }, + { + title: "Get Thunderbird!", + guid: guids.tb, + url: "http://getthunderbird.com/", + }, + ], + }); + + let serverRecords = [ + { + id: "menu", + type: "folder", + title: "Bookmarks Menu", + parentid: "places", + children: [guids.tb, guids.res], + }, + { + id: guids.tb, + type: "bookmark", + parentid: "menu", + bmkUri: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }, + { + id: guids.res, + type: "folder", + parentid: "menu", + title: "Resources", + children: [guids.irc, guids.bz, guids.mdn], + }, + { + id: guids.bz, + type: "bookmark", + parentid: guids.res, + bmkUri: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }, + { + id: guids.mdn, + type: "bookmark", + parentid: guids.res, + bmkUri: "https://developer.mozilla.org/", + title: "MDN", + }, + { + id: guids.irc, + type: "bookmark", + parentid: guids.res, + bmkUri: "ircs://irc.mozilla.org/nightly", + title: "IRC", + }, + ]; + for (let record of serverRecords) { + collection.insert(record.id, encryptPayload(record), timestamp); + } + + engine.lastModified = collection.timestamp; + await sync_engine_and_validate_telem(engine, false); + + let expectedTree = buildTree(guids); + await assertBookmarksTreeMatches( + PlacesUtils.bookmarks.menuGuid, + expectedTree, + message + ); +} + +async function get_engine() { + return Service.engineManager.get("bookmarks"); +} + +add_task(async function test_local_order_newer() { + let engine = await get_engine(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let collection = server.user("foo").collection("bookmarks"); + let serverModified = Date.now() / 1000 - 120; + await resolveConflict( + engine, + collection, + serverModified, + guids => [ + { + guid: guids.fx, + index: 0, + }, + { + guid: guids.res, + index: 1, + children: [ + { + guid: guids.nightly, + index: 0, + }, + { + guid: guids.support, + index: 1, + }, + { + guid: guids.customize, + index: 2, + }, + { + guid: guids.irc, + index: 3, + }, + { + guid: guids.bz, + index: 4, + }, + { + guid: guids.mdn, + index: 5, + }, + ], + }, + { + guid: guids.tb, + index: 2, + }, + ], + "Should use local order as base if remote is older" + ); + } finally { + await engine.wipeClient(); + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_remote_order_newer() { + let engine = await get_engine(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + let collection = server.user("foo").collection("bookmarks"); + let serverModified = Date.now() / 1000 + 120; + await resolveConflict( + engine, + collection, + serverModified, + guids => [ + { + guid: guids.tb, + index: 0, + }, + { + guid: guids.res, + index: 1, + children: [ + { + guid: guids.irc, + index: 0, + }, + { + guid: guids.bz, + index: 1, + }, + { + guid: guids.mdn, + index: 2, + }, + { + guid: guids.nightly, + index: 3, + }, + { + guid: guids.support, + index: 4, + }, + { + guid: guids.customize, + index: 5, + }, + ], + }, + { + guid: guids.fx, + index: 2, + }, + ], + "Should use remote order as base if local is older" + ); + } finally { + await engine.wipeClient(); + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bookmark_order() { + let engine = await get_engine(); + let store = engine._store; + _("Starting with a clean slate of no bookmarks"); + await store.wipe(); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + // Index 2 is the tags root. (Root indices depend on the order of the + // `CreateRoot` calls in `Database::CreateBookmarkRoots`). + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "clean slate" + ); + + function bookmark(name, parent) { + let bm = new Bookmark("http://weave.server/my-bookmark"); + bm.id = name; + bm.title = name; + bm.bmkUri = "http://uri/"; + bm.parentid = parent || "unfiled"; + bm.tags = []; + return bm; + } + + function folder(name, parent, children) { + let bmFolder = new BookmarkFolder("http://weave.server/my-bookmark-folder"); + bmFolder.id = name; + bmFolder.title = name; + bmFolder.parentid = parent || "unfiled"; + bmFolder.children = children; + return bmFolder; + } + + async function apply(records) { + for (record of records) { + await store.applyIncoming(record); + } + await engine._apply(); + } + let id10 = "10_aaaaaaaaa"; + _("basic add first bookmark"); + await apply([bookmark(id10, "")]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic add first bookmark" + ); + let id20 = "20_aaaaaaaaa"; + _("basic append behind 10"); + await apply([bookmark(id20, "")]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic append behind 10" + ); + + let id31 = "31_aaaaaaaaa"; + let id30 = "f30_aaaaaaaa"; + _("basic create in folder"); + let b31 = bookmark(id31, id30); + let f30 = folder(id30, "", [id31]); + await apply([b31, f30]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "basic create in folder" + ); + + let id41 = "41_aaaaaaaaa"; + let id40 = "f40_aaaaaaaa"; + _("insert missing parent -> append to unfiled"); + await apply([bookmark(id41, id40)]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + { + guid: id41, + index: 3, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "insert missing parent -> append to unfiled" + ); + + let id42 = "42_aaaaaaaaa"; + + _("insert another missing parent -> append"); + await apply([bookmark(id42, id40)]); + await assertBookmarksTreeMatches( + "", + [ + { + guid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }, + { + guid: PlacesUtils.bookmarks.toolbarGuid, + index: 1, + }, + { + guid: PlacesUtils.bookmarks.unfiledGuid, + index: 3, + children: [ + { + guid: id10, + index: 0, + }, + { + guid: id20, + index: 1, + }, + { + guid: id30, + index: 2, + children: [ + { + guid: id31, + index: 0, + }, + ], + }, + { + guid: id41, + index: 3, + }, + { + guid: id42, + index: 4, + }, + ], + }, + { + guid: PlacesUtils.bookmarks.mobileGuid, + index: 4, + }, + ], + "insert another missing parent -> append" + ); + + await engine.wipeClient(); + await Service.startOver(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_bookmark_places_query_rewriting.js b/services/sync/tests/unit/test_bookmark_places_query_rewriting.js new file mode 100644 index 0000000000..e8dbbb48b1 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_places_query_rewriting.js @@ -0,0 +1,57 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Rewrite place: URIs."); +const { BookmarkQuery, BookmarkFolder } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function makeTagRecord(id, uri) { + let tagRecord = new BookmarkQuery("bookmarks", id); + tagRecord.queryId = "MagicTags"; + tagRecord.parentName = "Bookmarks Toolbar"; + tagRecord.bmkUri = uri; + tagRecord.title = "tagtag"; + tagRecord.folderName = "bar"; + tagRecord.parentid = PlacesUtils.bookmarks.toolbarGuid; + return tagRecord; +} + +add_bookmark_test(async function run_test(engine) { + let store = engine._store; + + let toolbar = new BookmarkFolder("bookmarks", "toolbar"); + toolbar.parentid = "places"; + toolbar.children = ["abcdefabcdef"]; + + let uri = "place:folder=499&type=7&queryType=1"; + let tagRecord = makeTagRecord("abcdefabcdef", uri); + + _("Type: " + tagRecord.type); + _("Folder name: " + tagRecord.folderName); + await store.applyIncoming(toolbar); + await store.applyIncoming(tagRecord); + await engine._apply(); + + let insertedRecord = await store.createRecord("abcdefabcdef", "bookmarks"); + Assert.equal(insertedRecord.bmkUri, "place:tag=bar"); + + _("... but not if the type is wrong."); + let wrongTypeURI = "place:folder=499&type=2&queryType=1"; + let wrongTypeRecord = makeTagRecord("fedcbafedcba", wrongTypeURI); + await store.applyIncoming(wrongTypeRecord); + toolbar.children = ["fedcbafedcba"]; + await store.applyIncoming(toolbar); + let expected = wrongTypeURI; + await engine._apply(); + // the mirror appends a special param to these. + expected += "&excludeItems=1"; + + insertedRecord = await store.createRecord("fedcbafedcba", "bookmarks"); + Assert.equal(insertedRecord.bmkUri, expected); +}); diff --git a/services/sync/tests/unit/test_bookmark_record.js b/services/sync/tests/unit/test_bookmark_record.js new file mode 100644 index 0000000000..c261027ed9 --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_record.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Bookmark, BookmarkQuery, PlacesItem } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function prepareBookmarkItem(collection, id) { + let b = new Bookmark(collection, id); + b.cleartext.stuff = "my payload here"; + return b; +} + +add_task(async function test_bookmark_record() { + await configureIdentity(); + + await generateNewKeys(Service.collectionKeys); + let keyBundle = Service.identity.syncKeyBundle; + + _("Creating a record"); + + let placesItem = new PlacesItem("bookmarks", "foo", "bookmark"); + let bookmarkItem = prepareBookmarkItem("bookmarks", "foo"); + + _("Checking getTypeObject"); + Assert.equal(placesItem.getTypeObject(placesItem.type), Bookmark); + Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark); + + await bookmarkItem.encrypt(keyBundle); + _("Ciphertext is " + bookmarkItem.ciphertext); + Assert.ok(bookmarkItem.ciphertext != null); + + _("Decrypting the record"); + + let payload = await bookmarkItem.decrypt(keyBundle); + Assert.equal(payload.stuff, "my payload here"); + Assert.equal(bookmarkItem.getTypeObject(bookmarkItem.type), Bookmark); + Assert.notEqual(payload, bookmarkItem.payload); // wrap.data.payload is the encrypted one +}); + +add_task(async function test_query_foldername() { + // Bug 1443388 + let checks = [ + ["foo", "foo"], + ["", undefined], + ]; + for (let [inVal, outVal] of checks) { + let bmk1 = new BookmarkQuery("bookmarks", Utils.makeGUID()); + bmk1.fromSyncBookmark({ + url: Services.io.newURI("https://example.com"), + folder: inVal, + }); + Assert.strictEqual(bmk1.folderName, outVal); + + // other direction + let bmk2 = new BookmarkQuery("bookmarks", Utils.makeGUID()); + bmk2.folderName = inVal; + let record = bmk2.toSyncBookmark(); + Assert.strictEqual(record.folder, outVal); + } +}); diff --git a/services/sync/tests/unit/test_bookmark_store.js b/services/sync/tests/unit/test_bookmark_store.js new file mode 100644 index 0000000000..c46c3228fa --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_store.js @@ -0,0 +1,425 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Bookmark, BookmarkFolder, BookmarkQuery, PlacesItem } = + ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" + ); +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const BookmarksToolbarTitle = "toolbar"; + +// apply some test records without going via a test server. +async function apply_records(engine, records) { + for (record of records) { + await engine._store.applyIncoming(record); + } + await engine._apply(); +} + +add_bookmark_test(async function test_ignore_specials(engine) { + _("Ensure that we can't delete bookmark roots."); + let store = engine._store; + + // Belt... + let record = new BookmarkFolder("bookmarks", "toolbar", "folder"); + record.deleted = true; + Assert.notEqual( + null, + await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + + await apply_records(engine, [record]); + + // Ensure that the toolbar exists. + Assert.notEqual( + null, + await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + + await apply_records(engine, [record]); + + Assert.notEqual( + null, + await PlacesUtils.promiseItemId(PlacesUtils.bookmarks.toolbarGuid) + ); + await store.wipe(); +}); + +add_bookmark_test(async function test_bookmark_create(engine) { + let store = engine._store; + + try { + _("Ensure the record isn't present yet."); + let item = await PlacesUtils.bookmarks.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, item); + + _("Let's create a new record."); + let fxrecord = new Bookmark("bookmarks", "get-firefox1"); + fxrecord.bmkUri = "http://getfirefox.com/"; + fxrecord.title = "Get Firefox!"; + fxrecord.tags = ["firefox", "awesome", "browser"]; + fxrecord.keyword = "awesome"; + fxrecord.parentName = BookmarksToolbarTitle; + fxrecord.parentid = "toolbar"; + await apply_records(engine, [fxrecord]); + + _("Verify it has been created correctly."); + item = await PlacesUtils.bookmarks.fetch(fxrecord.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(item.url.href, "http://getfirefox.com/"); + Assert.equal(item.title, fxrecord.title); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + let keyword = await PlacesUtils.keywords.fetch(fxrecord.keyword); + Assert.equal(keyword.url.href, "http://getfirefox.com/"); + + _( + "Have the store create a new record object. Verify that it has the same data." + ); + let newrecord = await store.createRecord(fxrecord.id); + Assert.ok(newrecord instanceof Bookmark); + for (let property of [ + "type", + "bmkUri", + "title", + "keyword", + "parentName", + "parentid", + ]) { + Assert.equal(newrecord[property], fxrecord[property]); + } + Assert.ok(Utils.deepEquals(newrecord.tags.sort(), fxrecord.tags.sort())); + + _("The calculated sort index is based on frecency data."); + Assert.ok(newrecord.sortindex >= 150); + + _("Create a record with some values missing."); + let tbrecord = new Bookmark("bookmarks", "thunderbird1"); + tbrecord.bmkUri = "http://getthunderbird.com/"; + tbrecord.parentName = BookmarksToolbarTitle; + tbrecord.parentid = "toolbar"; + await apply_records(engine, [tbrecord]); + + _("Verify it has been created correctly."); + item = await PlacesUtils.bookmarks.fetch(tbrecord.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_BOOKMARK); + Assert.equal(item.url.href, "http://getthunderbird.com/"); + Assert.equal(item.title, ""); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + keyword = await PlacesUtils.keywords.fetch({ + url: "http://getthunderbird.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_bookmark_update(engine) { + let store = engine._store; + + try { + _("Create a bookmark whose values we'll change."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + await PlacesUtils.keywords.insert({ + url: "http://getfirefox.com/", + keyword: "firefox", + }); + + _("Update the record with some null values."); + let record = await store.createRecord(bmk1.guid); + record.title = null; + record.keyword = null; + record.tags = null; + await apply_records(engine, [record]); + + _("Verify that the values have been cleared."); + let item = await PlacesUtils.bookmarks.fetch(bmk1.guid); + Assert.equal(item.title, ""); + let keyword = await PlacesUtils.keywords.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_bookmark_createRecord(engine) { + let store = engine._store; + + try { + _("Create a bookmark without a title."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + }); + + _("Verify that the record is created accordingly."); + let record = await store.createRecord(bmk1.guid); + Assert.equal(record.title, ""); + Assert.equal(record.keyword, null); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_folder_create(engine) { + let store = engine._store; + + try { + _("Create a folder."); + let folder = new BookmarkFolder("bookmarks", "testfolder-1"); + folder.parentName = BookmarksToolbarTitle; + folder.parentid = "toolbar"; + folder.title = "Test Folder"; + await apply_records(engine, [folder]); + + _("Verify it has been created correctly."); + let item = await PlacesUtils.bookmarks.fetch(folder.id); + Assert.equal(item.type, PlacesUtils.bookmarks.TYPE_FOLDER); + Assert.equal(item.title, folder.title); + Assert.equal(item.parentGuid, PlacesUtils.bookmarks.toolbarGuid); + + _( + "Have the store create a new record object. Verify that it has the same data." + ); + let newrecord = await store.createRecord(folder.id); + Assert.ok(newrecord instanceof BookmarkFolder); + for (let property of ["title", "parentName", "parentid"]) { + Assert.equal(newrecord[property], folder[property]); + } + + _("Folders have high sort index to ensure they're synced first."); + Assert.equal(newrecord.sortindex, 1000000); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_folder_createRecord(engine) { + let store = engine._store; + + try { + _("Create a folder."); + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder1", + }); + + _("Create two bookmarks in that folder without assigning them GUIDs."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + _("Create a record for the folder and verify basic properties."); + let record = await store.createRecord(folder1.guid); + Assert.ok(record instanceof BookmarkFolder); + Assert.equal(record.title, "Folder1"); + Assert.equal(record.parentid, "toolbar"); + Assert.equal(record.parentName, BookmarksToolbarTitle); + + _( + "Verify the folder's children. Ensures that the bookmarks were given GUIDs." + ); + Assert.deepEqual(record.children, [bmk1.guid, bmk2.guid]); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_deleted(engine) { + let store = engine._store; + + try { + _("Create a bookmark that will be deleted."); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + // The engine needs to think we've previously synced it. + await PlacesTestUtils.markBookmarksAsSynced(); + + _("Delete the bookmark through the store."); + let record = new PlacesItem("bookmarks", bmk1.guid); + record.deleted = true; + await apply_records(engine, [record]); + _("Ensure it has been deleted."); + let item = await PlacesUtils.bookmarks.fetch(bmk1.guid); + let newrec = await store.createRecord(bmk1.guid); + Assert.equal(null, item); + Assert.equal(newrec.deleted, true); + _("Verify that the keyword has been cleared."); + let keyword = await PlacesUtils.keywords.fetch({ + url: "http://getfirefox.com/", + }); + Assert.equal(null, keyword); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_move_folder(engine) { + let store = engine._store; + store._childrenToOrder = {}; // *sob* - only needed for legacy. + + try { + _("Create two folders and a bookmark in one of them."); + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder1", + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + type: PlacesUtils.bookmarks.TYPE_FOLDER, + title: "Folder2", + }); + let bmk = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + // add records to the store that represent the current state. + await apply_records(engine, [ + await store.createRecord(folder1.guid), + await store.createRecord(folder2.guid), + await store.createRecord(bmk.guid), + ]); + + _("Now simulate incoming records reparenting it."); + let bmkRecord = await store.createRecord(bmk.guid); + Assert.equal(bmkRecord.parentid, folder1.guid); + bmkRecord.parentid = folder2.guid; + + let folder1Record = await store.createRecord(folder1.guid); + Assert.deepEqual(folder1Record.children, [bmk.guid]); + folder1Record.children = []; + let folder2Record = await store.createRecord(folder2.guid); + Assert.deepEqual(folder2Record.children, []); + folder2Record.children = [bmk.guid]; + + await apply_records(engine, [bmkRecord, folder1Record, folder2Record]); + + _("Verify the new parent."); + let movedBmk = await PlacesUtils.bookmarks.fetch(bmk.guid); + Assert.equal(movedBmk.parentGuid, folder2.guid); + } finally { + _("Clean up."); + await store.wipe(); + } +}); + +add_bookmark_test(async function test_move_order(engine) { + let store = engine._store; + let tracker = engine._tracker; + + // Make sure the tracker is turned on. + tracker.start(); + try { + _("Create two bookmarks"); + let bmk1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + let bmk2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + + _("Verify order."); + let childIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + "toolbar" + ); + Assert.deepEqual(childIds, [bmk1.guid, bmk2.guid]); + let toolbar = await store.createRecord("toolbar"); + Assert.deepEqual(toolbar.children, [bmk1.guid, bmk2.guid]); + + _("Move bookmarks around."); + store._childrenToOrder = {}; + toolbar.children = [bmk2.guid, bmk1.guid]; + await apply_records(engine, [ + toolbar, + await store.createRecord(bmk1.guid), + await store.createRecord(bmk2.guid), + ]); + delete store._childrenToOrder; + + _("Verify new order."); + let newChildIds = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + "toolbar" + ); + Assert.deepEqual(newChildIds, [bmk2.guid, bmk1.guid]); + } finally { + await tracker.stop(); + _("Clean up."); + await store.wipe(); + } +}); + +// Tests Bug 806460, in which query records arrive with empty folder +// names and missing bookmark URIs. +add_bookmark_test(async function test_empty_query_doesnt_die(engine) { + let record = new BookmarkQuery("bookmarks", "8xoDGqKrXf1P"); + record.folderName = ""; + record.queryId = ""; + record.parentName = "Toolbar"; + record.parentid = "toolbar"; + + // These should not throw. + await apply_records(engine, [record]); + + delete record.folderName; + await apply_records(engine, [record]); +}); + +add_bookmark_test(async function test_calculateIndex_for_invalid_url(engine) { + let store = engine._store; + + let folderIndex = await store._calculateIndex({ + type: "folder", + }); + equal(folderIndex, 1000000, "Should use high sort index for folders"); + + let toolbarIndex = await store._calculateIndex({ + parentid: "toolbar", + }); + equal(toolbarIndex, 150, "Should bump sort index for toolbar bookmarks"); + + let validURLIndex = await store._calculateIndex({ + bmkUri: "http://example.com/a", + }); + greaterOrEqual(validURLIndex, 0, "Should use frecency for index"); + + let invalidURLIndex = await store._calculateIndex({ + bmkUri: "!@#$%", + }); + equal(invalidURLIndex, 0, "Should not throw for invalid URLs"); +}); diff --git a/services/sync/tests/unit/test_bookmark_tracker.js b/services/sync/tests/unit/test_bookmark_tracker.js new file mode 100644 index 0000000000..2ac482183d --- /dev/null +++ b/services/sync/tests/unit/test_bookmark_tracker.js @@ -0,0 +1,1275 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { PlacesTransactions } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesTransactions.sys.mjs" +); + +let engine; +let store; +let tracker; + +const DAY_IN_MS = 24 * 60 * 60 * 1000; + +add_task(async function setup() { + await Service.engineManager.switchAlternatives(); + engine = Service.engineManager.get("bookmarks"); + store = engine._store; + tracker = engine._tracker; +}); + +// Test helpers. +async function verifyTrackerEmpty() { + await PlacesTestUtils.promiseAsyncUpdates(); + let changes = await tracker.getChangedIDs(); + deepEqual(changes, {}); + equal(tracker.score, 0); +} + +async function resetTracker() { + await PlacesTestUtils.markBookmarksAsSynced(); + tracker.resetScore(); +} + +async function cleanup() { + await engine.setLastSync(0); + await store.wipe(); + await resetTracker(); + await tracker.stop(); +} + +// startTracking is a signal that the test wants to notice things that happen +// after this is called (ie, things already tracked should be discarded.) +async function startTracking() { + engine._tracker.start(); + await PlacesTestUtils.markBookmarksAsSynced(); +} + +async function verifyTrackedItems(tracked) { + await PlacesTestUtils.promiseAsyncUpdates(); + let changedIDs = await tracker.getChangedIDs(); + let trackedIDs = new Set(Object.keys(changedIDs)); + for (let guid of tracked) { + ok(guid in changedIDs, `${guid} should be tracked`); + ok(changedIDs[guid].modified > 0, `${guid} should have a modified time`); + ok(changedIDs[guid].counter >= -1, `${guid} should have a change counter`); + trackedIDs.delete(guid); + } + equal( + trackedIDs.size, + 0, + `Unhandled tracked IDs: ${JSON.stringify(Array.from(trackedIDs))}` + ); +} + +async function verifyTrackedCount(expected) { + await PlacesTestUtils.promiseAsyncUpdates(); + let changedIDs = await tracker.getChangedIDs(); + do_check_attribute_count(changedIDs, expected); +} + +// A debugging helper that dumps the full bookmarks tree. +// Currently unused, but might come in handy +// eslint-disable-next-line no-unused-vars +async function dumpBookmarks() { + let columns = [ + "id", + "title", + "guid", + "syncStatus", + "syncChangeCounter", + "position", + ]; + return PlacesUtils.promiseDBConnection().then(connection => { + let all = []; + return connection + .executeCached( + `SELECT ${columns.join(", ")} FROM moz_bookmarks;`, + {}, + row => { + let repr = {}; + for (let column of columns) { + repr[column] = row.getResultByName(column); + } + all.push(repr); + } + ) + .then(() => { + dump("All bookmarks:\n"); + dump(JSON.stringify(all, undefined, 2)); + }); + }); +} + +add_task(async function test_tracking() { + _("Test starting and stopping the tracker"); + + // Remove existing tracking information for roots. + await startTracking(); + + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Test Folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // creating the folder should have made 2 changes - the folder itself and + // the parent of the folder. + await verifyTrackedCount(2); + // Reset the changes as the rest of the test doesn't want to see these. + await resetTracker(); + + function createBmk() { + return PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + } + + try { + _("Tell the tracker to start tracking changes."); + await startTracking(); + await createBmk(); + // We expect two changed items because the containing folder + // changed as well (new child). + await verifyTrackedCount(2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + + _("Notifying twice won't do any harm."); + await createBmk(); + await verifyTrackedCount(3); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_tracker_sql_batching() { + _( + "Test tracker does the correct thing when it is forced to batch SQL queries" + ); + + const SQLITE_MAX_VARIABLE_NUMBER = 999; + let numItems = SQLITE_MAX_VARIABLE_NUMBER * 2 + 10; + + await startTracking(); + + let children = []; + for (let i = 0; i < numItems; i++) { + children.push({ + url: "https://example.org/" + i, + title: "Sync Bookmark " + i, + }); + } + let inserted = await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.unfiledGuid, + children: [ + { + type: PlacesUtils.bookmarks.TYPE_FOLDER, + children, + }, + ], + }); + + Assert.equal(children.length, numItems); + Assert.equal(inserted.length, numItems + 1); + await verifyTrackedCount(numItems + 2); // The parent and grandparent are also tracked. + await resetTracker(); + + await PlacesUtils.bookmarks.remove(inserted[0]); + await verifyTrackedCount(numItems + 2); + + await cleanup(); +}); + +add_task(async function test_bookmarkAdded() { + _("Items inserted via the synchronous bookmarks API should be tracked"); + + try { + await startTracking(); + + _("Insert a folder using the sync API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let syncFolder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Sync Folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + await verifyTrackedItems(["menu", syncFolder.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a bookmark using the sync API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let syncBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: syncFolder.guid, + url: "https://example.org/sync", + title: "Sync Bookmark", + }); + await verifyTrackedItems([syncFolder.guid, syncBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_bookmarkAdded() { + _("Items inserted via the asynchronous bookmarks API should be tracked"); + + try { + await startTracking(); + + _("Insert a folder using the async API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Async Folder", + }); + await verifyTrackedItems(["menu", asyncFolder.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a bookmark using the async API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: asyncFolder.guid, + url: "https://example.org/async", + title: "Async Bookmark", + }); + await verifyTrackedItems([asyncFolder.guid, asyncBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + + await resetTracker(); + await startTracking(); + + _("Insert a separator using the async API"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let asyncSep = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_SEPARATOR, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: asyncFolder.index, + }); + await verifyTrackedItems(["menu", asyncSep.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemChanged() { + _("Items updated using the asynchronous bookmarks API should be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark"); + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + + await startTracking(); + + _("Update the bookmark using the async API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: fxBmk.guid, + title: "Download Firefox", + url: "https://www.mozilla.org/firefox", + // PlacesUtils.bookmarks.update rejects last modified dates older than + // the added date. + lastModified: new Date(Date.now() + DAY_IN_MS), + }); + + await verifyTrackedItems([fxBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemChanged_itemDates() { + _("Changes to item dates should be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark"); + let fx_bm = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fx_bm.guid}`); + + await startTracking(); + + _("Reset the bookmark's added date, should not be tracked"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let dateAdded = new Date(Date.now() - DAY_IN_MS); + await PlacesUtils.bookmarks.update({ + guid: fx_bm.guid, + dateAdded, + }); + await verifyTrackedCount(0); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges); + + await resetTracker(); + + _( + "Reset the bookmark's added date and another property, should be tracked" + ); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + dateAdded = new Date(); + await PlacesUtils.bookmarks.update({ + guid: fx_bm.guid, + dateAdded, + title: "test", + }); + await verifyTrackedItems([fx_bm.guid]); + Assert.equal(tracker.score, 2 * SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + + await resetTracker(); + + _("Set the bookmark's last modified date"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + let fx_id = await PlacesUtils.promiseItemId(fx_bm.guid); + let dateModified = Date.now() * 1000; + PlacesUtils.bookmarks.setItemLastModified(fx_id, dateModified); + await verifyTrackedItems([fx_bm.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemTagged() { + _("Items tagged using the synchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Create a folder"); + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Parent", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _("Folder ID: " + folder); + _("Folder GUID: " + folder.guid); + + _("Track changes to tags"); + let uri = CommonUtils.makeURI("http://getfirefox.com"); + let b = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: uri, + title: "Get Firefox!", + }); + _("New item is " + b); + _("GUID: " + b.guid); + + await startTracking(); + + _("Tag the item"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + PlacesUtils.tagging.tagURI(uri, ["foo"]); + + // bookmark should be tracked, folder should not be. + await verifyTrackedItems([b.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 6); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemUntagged() { + _("Items untagged using the synchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert tagged bookmarks"); + let uri = CommonUtils.makeURI("http://getfirefox.com"); + let fx1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: uri, + title: "Get Firefox!", + }); + // Different parent and title; same URL. + let fx2 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: uri, + title: "Download Firefox", + }); + PlacesUtils.tagging.tagURI(uri, ["foo"]); + + await startTracking(); + + _("Remove the tag"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + PlacesUtils.tagging.untagURI(uri, ["foo"]); + + await verifyTrackedItems([fx1.guid, fx2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemUntagged() { + _("Items untagged using the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert tagged bookmarks"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "some tag", + }); + let fxTag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tag.guid, + url: "http://getfirefox.com", + }); + + await startTracking(); + + _("Remove the tag using the async bookmarks API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(fxTag.guid); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemTagged() { + _("Items tagged using the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert untagged bookmarks"); + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Folder 1", + }); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder1.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let folder2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Folder 2", + }); + // Different parent and title; same URL. + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + + await startTracking(); + + // This will change once tags are moved into a separate table (bug 424160). + // We specifically test this case because Bookmarks.jsm updates tagged + // bookmarks and notifies observers. + _("Insert a tag using the async bookmarks API"); + let tag = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.tagsGuid, + title: "some tag", + }); + + _("Tag an item using the async bookmarks API"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: tag.guid, + url: "http://getfirefox.com", + }); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 4); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 5); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemKeywordChanged() { + _("Keyword changes via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert two bookmarks with the same URL"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + + await startTracking(); + + _("Add a keyword for both items"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.keywords.insert({ + keyword: "the_keyword", + url: "http://getfirefox.com", + postData: "postData", + }); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemKeywordDeleted() { + _("Keyword deletions via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert two bookmarks with the same URL and keywords"); + let fxBmk1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let fxBmk2 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com", + title: "Download Firefox", + }); + await PlacesUtils.keywords.insert({ + keyword: "the_keyword", + url: "http://getfirefox.com", + }); + + await startTracking(); + + _("Remove the keyword"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.keywords.remove("the_keyword"); + + await verifyTrackedItems([fxBmk1.guid, fxBmk2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_bookmarkAdded_filtered_root() { + _("Items outside the change roots should not be tracked"); + + try { + await startTracking(); + + _("Create a new root"); + let root = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + title: "New root", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _(`New root GUID: ${root.guid}`); + + _("Insert a bookmark underneath the new root"); + let untrackedBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: root.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`New untracked bookmark GUID: ${untrackedBmk.guid}`); + + _("Insert a bookmark underneath the Places root"); + let rootBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`New Places root bookmark GUID: ${rootBmk.guid}`); + + _("New root and bookmark should be ignored"); + await verifyTrackedItems([]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_filtered_root() { + _("Deleted items outside the change roots should not be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark underneath the Places root"); + let rootBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.rootGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`New Places root bookmark GUID: ${rootBmk.guid}`); + + await startTracking(); + + await PlacesUtils.bookmarks.remove(rootBmk); + + await verifyTrackedItems([]); + // We'll still increment the counter for the removed item. + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onPageAnnoChanged() { + _("Page annotations should not be tracked"); + + try { + await tracker.stop(); + + _("Insert a bookmark without an annotation"); + let pageURI = "http://getfirefox.com"; + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: pageURI, + title: "Get Firefox!", + }); + + await startTracking(); + + _("Add a page annotation"); + await PlacesUtils.history.update({ + url: pageURI, + annotations: new Map([[PlacesUtils.CHARSET_ANNO, "UTF-16"]]), + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + await resetTracker(); + + _("Remove the page annotation"); + await PlacesUtils.history.update({ + url: pageURI, + annotations: new Map([[PlacesUtils.CHARSET_ANNO, null]]), + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onFaviconChanged() { + _("Favicon changes should not be tracked"); + + try { + await tracker.stop(); + + let pageURI = CommonUtils.makeURI("http://getfirefox.com"); + let iconURI = CommonUtils.makeURI("http://getfirefox.com/icon"); + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: pageURI, + title: "Get Firefox!", + }); + + await PlacesTestUtils.addVisits(pageURI); + + await startTracking(); + + _("Favicon annotations should be ignored"); + let iconURL = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAA" + + "AAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg=="; + + PlacesUtils.favicons.replaceFaviconDataFromDataURL( + iconURI, + iconURL, + 0, + Services.scriptSecurityManager.getSystemPrincipal() + ); + + await new Promise(resolve => { + PlacesUtils.favicons.setAndFetchFaviconForPage( + pageURI, + iconURI, + true, + PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + (uri, dataLen, data, mimeType) => { + resolve(); + }, + Services.scriptSecurityManager.getSystemPrincipal() + ); + }); + await verifyTrackedItems([]); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_moveToFolder() { + _("Items moved via `moveToFolder` should be tracked"); + + try { + await tracker.stop(); + + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.menuGuid, + children: [ + { + guid: "bookmarkAAAA", + title: "A", + url: "http://example.com/a", + }, + { + guid: "bookmarkBBBB", + title: "B", + url: "http://example.com/b", + }, + { + guid: "bookmarkCCCC", + title: "C", + url: "http://example.com/c", + }, + { + guid: "bookmarkDDDD", + title: "D", + url: "http://example.com/d", + }, + ], + }); + await PlacesUtils.bookmarks.insertTree({ + guid: PlacesUtils.bookmarks.toolbarGuid, + children: [ + { + guid: "bookmarkEEEE", + title: "E", + url: "http://example.com/e", + }, + ], + }); + + await startTracking(); + + _("Move (A B D) to the toolbar"); + await PlacesUtils.bookmarks.moveToFolder( + ["bookmarkAAAA", "bookmarkBBBB", "bookmarkDDDD"], + PlacesUtils.bookmarks.toolbarGuid, + PlacesUtils.bookmarks.DEFAULT_INDEX + ); + + // Moving multiple bookmarks between two folders should track the old + // folder, new folder, and moved bookmarks. + await verifyTrackedItems([ + "menu", + "toolbar", + "bookmarkAAAA", + "bookmarkBBBB", + "bookmarkDDDD", + ]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Reorder toolbar children: (D A B E)"); + await PlacesUtils.bookmarks.moveToFolder( + ["bookmarkDDDD", "bookmarkAAAA", "bookmarkBBBB"], + PlacesUtils.bookmarks.toolbarGuid, + 0 + ); + + // Reordering bookmarks in a folder should only track the folder, not the + // bookmarks. + await verifyTrackedItems(["toolbar"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_update() { + _("Items moved via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + _("Repositioning a bookmark should track the folder"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: tbBmk.guid, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }); + await verifyTrackedItems(["menu"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + await resetTracker(); + + _("Reparenting a bookmark should track both folders and the bookmark"); + totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: tbBmk.guid, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + }); + await verifyTrackedItems(["menu", "toolbar", tbBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemMoved_reorder() { + _("Items reordered via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + _("Insert out-of-order bookmarks"); + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + + let mozBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://mozilla.org", + title: "Mozilla", + }); + _(`Mozilla GUID: ${mozBmk.guid}`); + + await startTracking(); + + _("Reorder bookmarks"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.reorder(PlacesUtils.bookmarks.menuGuid, [ + mozBmk.guid, + fxBmk.guid, + tbBmk.guid, + ]); + + // We only track the folder if we reorder its children, but we should + // bump the score for every changed item. + await verifyTrackedItems(["menu"]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 1); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_removeFolderTransaction() { + _("Folders removed in a transaction should be tracked"); + + try { + await tracker.stop(); + + _("Create a folder with two children"); + let folder = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "Test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + _(`Folder GUID: ${folder.guid}`); + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fx.guid}`); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: folder.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tb.guid}`); + + await startTracking(); + + let txn = PlacesTransactions.Remove({ guid: folder.guid }); + // We haven't executed the transaction yet. + await verifyTrackerEmpty(); + + _("Execute the remove folder transaction"); + await txn.transact(); + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Undo the remove folder transaction"); + await PlacesTransactions.undo(); + + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + await resetTracker(); + + _("Redo the transaction"); + await PlacesTransactions.redo(); + await verifyTrackedItems(["menu", folder.guid, fx.guid, tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_treeMoved() { + _("Moving an entire tree of bookmarks should track the parents"); + + try { + // Create a couple of parent folders. + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + test: "First test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // A second folder in the first. + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + title: "Second test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // Create a couple of bookmarks in the second folder. + await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Move folder 2 to be a sibling of folder1. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.update({ + guid: folder2.guid, + parentGuid: PlacesUtils.bookmarks.menuGuid, + index: 0, + }); + + // the menu and both folders should be tracked, the children should not be. + await verifyTrackedItems(["menu", folder1.guid, folder2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 3); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted() { + _("Bookmarks deleted via the synchronous API should be tracked"); + + try { + await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Delete the last item - the item and parent should be tracked. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(tb); + + await verifyTrackedItems(["menu", tb.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemDeleted() { + _("Bookmarks deleted via the asynchronous API should be tracked"); + + try { + await tracker.stop(); + + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + _("Delete the first item"); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(fxBmk.guid); + + await verifyTrackedItems(["menu", fxBmk.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 2); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_async_onItemDeleted_eraseEverything() { + _("Erasing everything should track all deleted items"); + + try { + await tracker.stop(); + + let fxBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.mobileGuid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + _(`Firefox GUID: ${fxBmk.guid}`); + let tbBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.mobileGuid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + let mozBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://mozilla.org", + title: "Mozilla", + }); + _(`Mozilla GUID: ${mozBmk.guid}`); + let mdnBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://developer.mozilla.org", + title: "MDN", + }); + _(`MDN GUID: ${mdnBmk.guid}`); + let bugsFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Bugs", + }); + _(`Bugs folder GUID: ${bugsFolder.guid}`); + let bzBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: bugsFolder.guid, + url: "https://bugzilla.mozilla.org", + title: "Bugzilla", + }); + _(`Bugzilla GUID: ${bzBmk.guid}`); + let bugsChildFolder = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: bugsFolder.guid, + title: "Bugs child", + }); + _(`Bugs child GUID: ${bugsChildFolder.guid}`); + let bugsGrandChildBmk = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_BOOKMARK, + parentGuid: bugsChildFolder.guid, + url: "https://example.com", + title: "Bugs grandchild", + }); + _(`Bugs grandchild GUID: ${bugsGrandChildBmk.guid}`); + + await startTracking(); + // Simulate moving a synced item into a new folder. Deleting the folder + // should write a tombstone for the item, but not the folder. + await PlacesTestUtils.setBookmarkSyncFields({ + guid: bugsChildFolder.guid, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NEW, + }); + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.eraseEverything(); + + // bugsChildFolder's sync status is still "NEW", so it shouldn't be + // tracked. bugsGrandChildBmk is "NORMAL", so we *should* write a + // tombstone and track it. + await verifyTrackedItems([ + "menu", + mozBmk.guid, + mdnBmk.guid, + "toolbar", + bugsFolder.guid, + "mobile", + fxBmk.guid, + tbBmk.guid, + "unfiled", + bzBmk.guid, + bugsGrandChildBmk.guid, + ]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 8); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 11); + } finally { + _("Clean up."); + await cleanup(); + } +}); + +add_task(async function test_onItemDeleted_tree() { + _("Deleting a tree of bookmarks should track all items"); + + try { + // Create a couple of parent folders. + let folder1 = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + title: "First test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // A second folder in the first. + let folder2 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + title: "Second test folder", + type: PlacesUtils.bookmarks.TYPE_FOLDER, + }); + + // Create a couple of bookmarks in the second folder. + let fx = await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getfirefox.com", + title: "Get Firefox!", + }); + let tb = await PlacesUtils.bookmarks.insert({ + parentGuid: folder2.guid, + url: "http://getthunderbird.com", + title: "Get Thunderbird!", + }); + + await startTracking(); + + // Delete folder2 - everything we created should be tracked. + let totalSyncChanges = PlacesUtils.bookmarks.totalSyncChanges; + await PlacesUtils.bookmarks.remove(folder2); + + await verifyTrackedItems([fx.guid, tb.guid, folder1.guid, folder2.guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(PlacesUtils.bookmarks.totalSyncChanges, totalSyncChanges + 4); + } finally { + _("Clean up."); + await cleanup(); + } +}); diff --git a/services/sync/tests/unit/test_bridged_engine.js b/services/sync/tests/unit/test_bridged_engine.js new file mode 100644 index 0000000000..25a81f8f69 --- /dev/null +++ b/services/sync/tests/unit/test_bridged_engine.js @@ -0,0 +1,248 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { BridgedEngine, BridgeWrapperXPCOM } = ChromeUtils.importESModule( + "resource://services-sync/bridged_engine.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// Wraps an `object` in a proxy so that its methods are bound to it. This +// simulates how XPCOM class instances have all their methods bound. +function withBoundMethods(object) { + return new Proxy(object, { + get(target, key) { + let value = target[key]; + return typeof value == "function" ? value.bind(target) : value; + }, + }); +} + +add_task(async function test_interface() { + class TestBridge { + constructor() { + this.storageVersion = 2; + this.syncID = "syncID111111"; + this.clear(); + } + + clear() { + this.lastSyncMillis = 0; + this.wasSyncStarted = false; + this.incomingEnvelopes = []; + this.uploadedIDs = []; + this.wasSyncFinished = false; + this.wasReset = false; + this.wasWiped = false; + } + + // `mozIBridgedSyncEngine` methods. + + getLastSync(callback) { + CommonUtils.nextTick(() => callback.handleSuccess(this.lastSyncMillis)); + } + + setLastSync(millis, callback) { + this.lastSyncMillis = millis; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + resetSyncId(callback) { + CommonUtils.nextTick(() => callback.handleSuccess(this.syncID)); + } + + ensureCurrentSyncId(newSyncId, callback) { + equal(newSyncId, this.syncID, "Local and new sync IDs should match"); + CommonUtils.nextTick(() => callback.handleSuccess(this.syncID)); + } + + syncStarted(callback) { + this.wasSyncStarted = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + storeIncoming(envelopes, callback) { + this.incomingEnvelopes.push(...envelopes.map(r => JSON.parse(r))); + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + apply(callback) { + let outgoingEnvelopes = [ + { + id: "hanson", + data: { + plants: ["seed", "flower 💐", "rose"], + canYouTell: false, + }, + }, + { + id: "sheryl-crow", + data: { + today: "winding 🛣", + tomorrow: "winding 🛣", + }, + }, + ].map(cleartext => + JSON.stringify({ + id: cleartext.id, + payload: JSON.stringify(cleartext), + }) + ); + CommonUtils.nextTick(() => callback.handleSuccess(outgoingEnvelopes)); + } + + setUploaded(millis, ids, callback) { + this.uploadedIDs.push(...ids); + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + syncFinished(callback) { + this.wasSyncFinished = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + reset(callback) { + this.clear(); + this.wasReset = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + + wipe(callback) { + this.clear(); + this.wasWiped = true; + CommonUtils.nextTick(() => callback.handleSuccess()); + } + } + + let bridge = new TestBridge(); + let engine = new BridgedEngine("Nineties", Service); + engine._bridge = new BridgeWrapperXPCOM(withBoundMethods(bridge)); + engine.enabled = true; + + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + + info("Add server records"); + let foo = server.user("foo"); + let collection = foo.collection("nineties"); + let now = new_timestamp(); + collection.insert( + "backstreet", + encryptPayload({ + id: "backstreet", + data: { + say: "I want it that way", + when: "never", + }, + }), + now + ); + collection.insert( + "tlc", + encryptPayload({ + id: "tlc", + data: { + forbidden: ["scrubs 🚫"], + numberAvailable: false, + }, + }), + now + 5 + ); + + info("Sync the engine"); + // Advance the last sync time to skip the Backstreet Boys... + bridge.lastSyncMillis = 1000 * (now + 2); + await sync_engine_and_validate_telem(engine, false); + + let metaGlobal = foo.collection("meta").wbo("global").get(); + deepEqual( + JSON.parse(metaGlobal.payload).engines.nineties, + { + version: 2, + syncID: "syncID111111", + }, + "Should write storage version and sync ID to m/g" + ); + + greater(bridge.lastSyncMillis, 0, "Should update last sync time"); + ok( + bridge.wasSyncStarted, + "Should have started sync before storing incoming" + ); + deepEqual( + bridge.incomingEnvelopes + .sort((a, b) => a.id.localeCompare(b.id)) + .map(({ payload, ...envelope }) => ({ + cleartextAsObject: JSON.parse(payload), + ...envelope, + })), + [ + { + id: "tlc", + modified: now + 5, + cleartextAsObject: { + id: "tlc", + data: { + forbidden: ["scrubs 🚫"], + numberAvailable: false, + }, + }, + }, + ], + "Should stage incoming records from server" + ); + deepEqual( + bridge.uploadedIDs.sort(), + ["hanson", "sheryl-crow"], + "Should mark new local records as uploaded" + ); + ok(bridge.wasSyncFinished, "Should have finished sync after uploading"); + + deepEqual( + collection.keys().sort(), + ["backstreet", "hanson", "sheryl-crow", "tlc"], + "Should have all records on server" + ); + let expectedRecords = [ + { + id: "sheryl-crow", + data: { + today: "winding 🛣", + tomorrow: "winding 🛣", + }, + }, + { + id: "hanson", + data: { + plants: ["seed", "flower 💐", "rose"], + canYouTell: false, + }, + }, + ]; + for (let expected of expectedRecords) { + let actual = collection.cleartext(expected.id); + deepEqual( + actual, + expected, + `Should upload record ${expected.id} from bridged engine` + ); + } + + await engine.resetClient(); + ok(bridge.wasReset, "Should reset local storage for bridge"); + + await engine.wipeClient(); + ok(bridge.wasWiped, "Should wipe local storage for bridge"); + + await engine.resetSyncID(); + ok( + !foo.collection("nineties"), + "Should delete server collection after resetting sync ID" + ); + } finally { + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_clients_engine.js b/services/sync/tests/unit/test_clients_engine.js new file mode 100644 index 0000000000..333c145b71 --- /dev/null +++ b/services/sync/tests/unit/test_clients_engine.js @@ -0,0 +1,2103 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { ClientEngine, ClientsRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/clients.sys.mjs" +); +const { CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const MORE_THAN_CLIENTS_TTL_REFRESH = 691200; // 8 days +const LESS_THAN_CLIENTS_TTL_REFRESH = 86400; // 1 day + +let engine; + +/** + * Unpack the record with this ID, and verify that it has the same version that + * we should be putting into records. + */ +async function check_record_version(user, id) { + let payload = user.collection("clients").wbo(id).data; + + let rec = new CryptoWrapper(); + rec.id = id; + rec.collection = "clients"; + rec.ciphertext = payload.ciphertext; + rec.hmac = payload.hmac; + rec.IV = payload.IV; + + let cleartext = await rec.decrypt( + Service.collectionKeys.keyForCollection("clients") + ); + + _("Payload is " + JSON.stringify(cleartext)); + equal(Services.appinfo.version, cleartext.version); + equal(1, cleartext.protocols.length); + equal("1.5", cleartext.protocols[0]); +} + +// compare 2 different command arrays, taking into account that a flowID +// attribute must exist, be unique in the commands, but isn't specified in +// "expected" as the value isn't known. +function compareCommands(actual, expected, description) { + let tweakedActual = JSON.parse(JSON.stringify(actual)); + tweakedActual.map(elt => delete elt.flowID); + deepEqual(tweakedActual, expected, description); + // each item must have a unique flowID. + let allIDs = new Set(actual.map(elt => elt.flowID).filter(fid => !!fid)); + equal(allIDs.size, actual.length, "all items have unique IDs"); +} + +async function syncClientsEngine(server) { + engine._lastFxADevicesFetch = 0; + engine.lastModified = server.getCollection("foo", "clients").timestamp; + await engine._sync(); +} + +add_task(async function setup() { + engine = Service.clientsEngine; +}); + +async function cleanup() { + Svc.Prefs.resetBranch(""); + await engine._tracker.clearChangedIDs(); + await engine._resetClient(); + // un-cleanup the logs (the resetBranch will have reset their levels), since + // not all the tests use SyncTestingInfrastructure, and it's cheap. + syncTestLogging(); + // We don't finalize storage at cleanup, since we use the same clients engine + // instance across all tests. +} + +add_task(async function test_bad_hmac() { + _("Ensure that Clients engine deletes corrupt records."); + let deletedCollections = []; + let deletedItems = []; + let callback = { + onItemDeleted(username, coll, wboID) { + deletedItems.push(coll + "/" + wboID); + }, + onCollectionDeleted(username, coll) { + deletedCollections.push(coll); + }, + }; + Object.setPrototypeOf(callback, SyncServerCallback); + let server = await serverForFoo(engine, callback); + let user = server.user("foo"); + + function check_clients_count(expectedCount) { + let coll = user.collection("clients"); + + // Treat a non-existent collection as empty. + equal(expectedCount, coll ? coll.count() : 0); + } + + function check_client_deleted(id) { + let coll = user.collection("clients"); + let wbo = coll.wbo(id); + return !wbo || !wbo.payload; + } + + async function uploadNewKeys() { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + } + + try { + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + await generateNewKeys(Service.collectionKeys); + + _("First sync, client record is uploaded"); + equal(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + check_clients_count(0); + await syncClientsEngine(server); + check_clients_count(1); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + // Our uploaded record has a version. + await check_record_version(user, engine.localID); + + // Initial setup can wipe the server, so clean up. + deletedCollections = []; + deletedItems = []; + + _("Change our keys and our client ID, reupload keys."); + let oldLocalID = engine.localID; // Preserve to test for deletion! + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + + _("Sync."); + await syncClientsEngine(server); + + _("Old record " + oldLocalID + " was deleted, new one uploaded."); + check_clients_count(1); + check_client_deleted(oldLocalID); + + _( + "Now change our keys but don't upload them. " + + "That means we get an HMAC error but redownload keys." + ); + Service.lastHMACEvent = 0; + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + deletedCollections = []; + deletedItems = []; + check_clients_count(1); + await syncClientsEngine(server); + + _("Old record was not deleted, new one uploaded."); + equal(deletedCollections.length, 0); + equal(deletedItems.length, 0); + check_clients_count(2); + + _( + "Now try the scenario where our keys are wrong *and* there's a bad record." + ); + // Clean up and start fresh. + user.collection("clients")._wbos = {}; + Service.lastHMACEvent = 0; + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + deletedCollections = []; + deletedItems = []; + check_clients_count(0); + + await uploadNewKeys(); + + // Sync once to upload a record. + await syncClientsEngine(server); + check_clients_count(1); + + // Generate and upload new keys, so the old client record is wrong. + await uploadNewKeys(); + + // Create a new client record and new keys. Now our keys are wrong, as well + // as the object on the server. We'll download the new keys and also delete + // the bad client record. + oldLocalID = engine.localID; // Preserve to test for deletion! + engine.localID = Utils.makeGUID(); + await engine.resetClient(); + await generateNewKeys(Service.collectionKeys); + let oldKey = Service.collectionKeys.keyForCollection(); + + equal(deletedCollections.length, 0); + equal(deletedItems.length, 0); + await syncClientsEngine(server); + equal(deletedItems.length, 1); + check_client_deleted(oldLocalID); + check_clients_count(1); + let newKey = Service.collectionKeys.keyForCollection(); + ok(!oldKey.equals(newKey)); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_properties() { + _("Test lastRecordUpload property"); + try { + equal(Svc.Prefs.get("clients.lastRecordUpload"), undefined); + equal(engine.lastRecordUpload, 0); + + let now = Date.now(); + engine.lastRecordUpload = now / 1000; + equal(engine.lastRecordUpload, Math.floor(now / 1000)); + } finally { + await cleanup(); + } +}); + +add_task(async function test_full_sync() { + _("Ensure that Clients engine fetches all records for each sync."); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let activeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let deletedID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: deletedID, + name: "Client to delete", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + let store = engine._store; + + _("First sync. 2 records downloaded; our record uploaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + deepEqual( + user.collection("clients").keys().sort(), + [activeID, deletedID, engine.localID].sort(), + "Our record should be uploaded on first sync" + ); + let ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [activeID, deletedID, engine.localID].sort(), + "Other clients should be downloaded on first sync" + ); + + _("Delete a record, then sync again"); + let collection = server.getCollection("foo", "clients"); + collection.remove(deletedID); + // Simulate a timestamp update in info/collections. + await syncClientsEngine(server); + + _("Record should be updated"); + ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [activeID, engine.localID].sort(), + "Deleted client should be removed on next sync" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_sync() { + _("Ensure that Clients engine uploads a new client record once a week."); + + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + function clientWBO() { + return user.collection("clients").wbo(engine.localID); + } + + try { + _("First sync. Client record is uploaded."); + equal(clientWBO(), undefined); + equal(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(!!clientWBO().payload); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + _( + "Let's time travel more than a week back, new record should've been uploaded." + ); + engine.lastRecordUpload -= MORE_THAN_CLIENTS_TTL_REFRESH; + let lastweek = engine.lastRecordUpload; + clientWBO().payload = undefined; + await syncClientsEngine(server); + ok(!!clientWBO().payload); + ok(engine.lastRecordUpload > lastweek); + ok(!engine.isFirstSync); + + _("Remove client record."); + await engine.removeClientData(); + equal(clientWBO().payload, undefined); + + _("Time travel one day back, no record uploaded."); + engine.lastRecordUpload -= LESS_THAN_CLIENTS_TTL_REFRESH; + let yesterday = engine.lastRecordUpload; + await syncClientsEngine(server); + equal(clientWBO().payload, undefined); + equal(engine.lastRecordUpload, yesterday); + ok(!engine.isFirstSync); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_client_name_change() { + _("Ensure client name change incurs a client record update."); + + let tracker = engine._tracker; + + engine.localID; // Needed to increase the tracker changedIDs count. + let initialName = engine.localName; + + tracker.start(); + _("initial name: " + initialName); + + // Tracker already has data, so clear it. + await tracker.clearChangedIDs(); + + let initialScore = tracker.score; + + let changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 0); + + Services.prefs.setStringPref( + "identity.fxaccounts.account.device.name", + "new name" + ); + await tracker.asyncObserver.promiseObserversComplete(); + + _("new name: " + engine.localName); + notEqual(initialName, engine.localName); + changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 1); + ok(engine.localID in changedIDs); + ok(tracker.score > initialScore); + ok(tracker.score >= SCORE_INCREMENT_XLARGE); + + await tracker.stop(); + + await cleanup(); +}); + +add_task(async function test_fxa_device_id_change() { + _("Ensure an FxA device ID change incurs a client record update."); + + let tracker = engine._tracker; + + engine.localID; // Needed to increase the tracker changedIDs count. + + tracker.start(); + + // Tracker already has data, so clear it. + await tracker.clearChangedIDs(); + + let initialScore = tracker.score; + + let changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 0); + + Services.obs.notifyObservers(null, "fxaccounts:new_device_id"); + await tracker.asyncObserver.promiseObserversComplete(); + + changedIDs = await tracker.getChangedIDs(); + equal(Object.keys(changedIDs).length, 1); + ok(engine.localID in changedIDs); + ok(tracker.score > initialScore); + ok(tracker.score >= SINGLE_USER_THRESHOLD); + + await tracker.stop(); + + await cleanup(); +}); + +add_task(async function test_last_modified() { + _("Ensure that remote records have a sane serverLastModified attribute."); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let activeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + let collection = user.collection("clients"); + + _("Sync to download the record"); + await syncClientsEngine(server); + + equal( + engine._store._remoteClients[activeID].serverLastModified, + now - 10, + "last modified in the local record is correctly the server last-modified" + ); + + _("Modify the record and re-upload it"); + // set a new name to make sure we really did upload. + engine._store._remoteClients[activeID].name = "New name"; + engine._modified.set(activeID, 0); + // The sync above also did a POST, so adjust our lastModified. + engine.lastModified = server.getCollection("foo", "clients").timestamp; + await engine._uploadOutgoing(); + + _("Local record should have updated timestamp"); + ok(engine._store._remoteClients[activeID].serverLastModified >= now); + + _("Record on the server should have new name but not serverLastModified"); + let payload = collection.cleartext(activeID); + equal(payload.name, "New name"); + equal(payload.serverLastModified, undefined); + } finally { + await cleanup(); + server.deleteCollections("foo"); + await promiseStopServer(server); + } +}); + +add_task(async function test_send_command() { + _("Verifies _sendCommandToClient puts commands in the outbound queue."); + + let store = engine._store; + let tracker = engine._tracker; + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + let action = "testCommand"; + let args = ["foo", "bar"]; + let extra = { flowID: "flowy" }; + + await engine._sendCommandToClient(action, args, remoteId, extra); + + let newRecord = store._remoteClients[remoteId]; + let clientCommands = (await engine._readCommands())[remoteId]; + notEqual(newRecord, undefined); + equal(clientCommands.length, 1); + + let command = clientCommands[0]; + equal(command.command, action); + equal(command.args.length, 2); + deepEqual(command.args, args); + ok(command.flowID); + + const changes = await tracker.getChangedIDs(); + notEqual(changes[remoteId], undefined); + + await cleanup(); +}); + +// The browser UI might call _addClientCommand indirectly without awaiting on the returned promise. +// We need to make sure this doesn't result on commands not being saved. +add_task(async function test_add_client_command_race() { + let promises = []; + for (let i = 0; i < 100; i++) { + promises.push( + engine._addClientCommand(`client-${i}`, { command: "cmd", args: [] }) + ); + } + await Promise.all(promises); + + let localCommands = await engine._readCommands(); + for (let i = 0; i < 100; i++) { + equal(localCommands[`client-${i}`].length, 1); + } +}); + +add_task(async function test_command_validation() { + _("Verifies that command validation works properly."); + + let store = engine._store; + + let testCommands = [ + ["resetAll", [], true], + ["resetAll", ["foo"], false], + ["resetEngine", ["tabs"], true], + ["resetEngine", [], false], + ["wipeEngine", ["tabs"], true], + ["wipeEngine", [], false], + ["logout", [], true], + ["logout", ["foo"], false], + ["__UNKNOWN__", [], false], + ]; + + for (let [action, args, expectedResult] of testCommands) { + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + await engine.sendCommand(action, args, remoteId); + + let newRecord = store._remoteClients[remoteId]; + notEqual(newRecord, undefined); + + let clientCommands = (await engine._readCommands())[remoteId]; + + if (expectedResult) { + _("Ensuring command is sent: " + action); + equal(clientCommands.length, 1); + + let command = clientCommands[0]; + equal(command.command, action); + deepEqual(command.args, args); + + notEqual(engine._tracker, undefined); + const changes = await engine._tracker.getChangedIDs(); + notEqual(changes[remoteId], undefined); + } else { + _("Ensuring command is scrubbed: " + action); + equal(clientCommands, undefined); + + if (store._tracker) { + equal(engine._tracker[remoteId], undefined); + } + } + } + await cleanup(); +}); + +add_task(async function test_command_duplication() { + _("Ensures duplicate commands are detected and not added"); + + let store = engine._store; + let remoteId = Utils.makeGUID(); + let rec = new ClientsRec("clients", remoteId); + await store.create(rec); + await store.createRecord(remoteId, "clients"); + + let action = "resetAll"; + let args = []; + + await engine.sendCommand(action, args, remoteId); + await engine.sendCommand(action, args, remoteId); + + let clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 1); + + _("Check variant args length"); + await engine._saveCommands({}); + + action = "resetEngine"; + await engine.sendCommand(action, [{ x: "foo" }], remoteId); + await engine.sendCommand(action, [{ x: "bar" }], remoteId); + + _("Make sure we spot a real dupe argument."); + await engine.sendCommand(action, [{ x: "bar" }], remoteId); + + clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 2); + + await cleanup(); +}); + +add_task(async function test_command_invalid_client() { + _("Ensures invalid client IDs are caught"); + + let id = Utils.makeGUID(); + let error; + + try { + await engine.sendCommand("wipeEngine", ["tabs"], id); + } catch (ex) { + error = ex; + } + + equal(error.message.indexOf("Unknown remote client ID: "), 0); + + await cleanup(); +}); + +add_task(async function test_process_incoming_commands() { + _("Ensures local commands are executed"); + + engine.localCommands = [{ command: "logout", args: [] }]; + + let ev = "weave:service:logout:finish"; + + let logoutPromise = new Promise(resolve => { + var handler = function () { + Svc.Obs.remove(ev, handler); + + resolve(); + }; + + Svc.Obs.add(ev, handler); + }); + + // logout command causes processIncomingCommands to return explicit false. + ok(!(await engine.processIncomingCommands())); + + await logoutPromise; + + await cleanup(); +}); + +add_task(async function test_filter_duplicate_names() { + _( + "Ensure that we exclude clients with identical names that haven't synced in a week." + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + let user = server.user("foo"); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + // Synced recently. + let recentID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: recentID, + name: "My Phone", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + // Dupe of our client, synced more than 1 week ago. + let dupeID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: dupeID, + name: engine.localName, + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 604820 + ); + + // Synced more than 1 week ago, but not a dupe. + let oldID = Utils.makeGUID(); + user.collection("clients").insertRecord( + { + id: oldID, + name: "My old desktop", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 604820 + ); + + try { + let store = engine._store; + + _("First sync"); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + deepEqual( + user.collection("clients").keys().sort(), + [recentID, dupeID, oldID, engine.localID].sort(), + "Our record should be uploaded on first sync" + ); + + let ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [recentID, dupeID, oldID, engine.localID].sort(), + "Duplicate ID should remain in getAllIDs" + ); + ok( + await engine._store.itemExists(dupeID), + "Dupe ID should be considered as existing for Sync methods." + ); + ok( + !engine.remoteClientExists(dupeID), + "Dupe ID should not be considered as existing for external methods." + ); + + // dupe desktop should not appear in .deviceTypes. + equal(engine.deviceTypes.get("desktop"), 2); + equal(engine.deviceTypes.get("mobile"), 1); + + // dupe desktop should not appear in stats + deepEqual(engine.stats, { + hasMobile: 1, + names: [engine.localName, "My Phone", "My old desktop"], + numClients: 3, + }); + + ok(engine.remoteClientExists(oldID), "non-dupe ID should exist."); + ok(!engine.remoteClientExists(dupeID), "dupe ID should not exist"); + equal( + engine.remoteClients.length, + 2, + "dupe should not be in remoteClients" + ); + + // Check that a subsequent Sync doesn't report anything as being processed. + let counts; + Svc.Obs.add("weave:engine:sync:applied", function observe(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", observe); + counts = subject; + }); + + await syncClientsEngine(server); + equal(counts.applied, 0); // We didn't report applying any records. + equal(counts.reconciled, 4); // We reported reconcilliation for all records + equal(counts.succeeded, 0); + equal(counts.failed, 0); + equal(counts.newFailed, 0); + + _("Broadcast logout to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + let collection = server.getCollection("foo", "clients"); + let recentPayload = collection.cleartext(recentID); + compareCommands( + recentPayload.commands, + [{ command: "logout", args: [] }], + "Should send commands to the recent client" + ); + + let oldPayload = collection.cleartext(oldID); + compareCommands( + oldPayload.commands, + [{ command: "logout", args: [] }], + "Should send commands to the week-old client" + ); + + let dupePayload = collection.cleartext(dupeID); + deepEqual( + dupePayload.commands, + [], + "Should not send commands to the dupe client" + ); + + _("Update the dupe client's modified time"); + collection.insertRecord( + { + id: dupeID, + name: engine.localName, + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + _("Second sync."); + await syncClientsEngine(server); + + ids = await store.getAllIDs(); + deepEqual( + Object.keys(ids).sort(), + [recentID, oldID, dupeID, engine.localID].sort(), + "Stale client synced, so it should no longer be marked as a dupe" + ); + + ok( + engine.remoteClientExists(dupeID), + "Dupe ID should appear as it synced." + ); + + // Recently synced dupe desktop should appear in .deviceTypes. + equal(engine.deviceTypes.get("desktop"), 3); + + // Recently synced dupe desktop should now appear in stats + deepEqual(engine.stats, { + hasMobile: 1, + names: [engine.localName, "My Phone", engine.localName, "My old desktop"], + numClients: 4, + }); + + ok( + engine.remoteClientExists(dupeID), + "recently synced dupe ID should now exist" + ); + equal( + engine.remoteClients.length, + 3, + "recently synced dupe should now be in remoteClients" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_command_sync() { + _("Ensure that commands are synced across clients."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let user = server.user("foo"); + let remoteId = Utils.makeGUID(); + + function clientWBO(id) { + return user.collection("clients").wbo(id); + } + + _("Create remote client record"); + user.collection("clients").insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + try { + _("Syncing."); + await syncClientsEngine(server); + + _("Checking remote record was downloaded."); + let clientRecord = engine._store._remoteClients[remoteId]; + notEqual(clientRecord, undefined); + equal(clientRecord.commands.length, 0); + + _("Send a command to the remote client."); + await engine.sendCommand("wipeEngine", ["tabs"]); + let clientCommands = (await engine._readCommands())[remoteId]; + equal(clientCommands.length, 1); + await syncClientsEngine(server); + + _("Checking record was uploaded."); + notEqual(clientWBO(engine.localID).payload, undefined); + ok(engine.lastRecordUpload > 0); + ok(!engine.isFirstSync); + + notEqual(clientWBO(remoteId).payload, undefined); + + Svc.Prefs.set("client.GUID", remoteId); + await engine._resetClient(); + equal(engine.localID, remoteId); + _("Performing sync on resetted client."); + await syncClientsEngine(server); + notEqual(engine.localCommands, undefined); + equal(engine.localCommands.length, 1); + + let command = engine.localCommands[0]; + equal(command.command, "wipeEngine"); + equal(command.args.length, 1); + equal(command.args[0], "tabs"); + } finally { + await cleanup(); + + try { + let collection = server.getCollection("foo", "clients"); + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_clients_not_in_fxa_list() { + _("Ensure that clients not in the FxA devices list are marked as stale."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId, + protocols: ["1.5"], + }); + + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId2, + protocols: ["1.5"], + }); + + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteId }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + + ok(!engine._store._remoteClients[remoteId].stale); + ok(engine._store._remoteClients[remoteId2].stale); + } finally { + engine.fxAccounts = fxAccounts; + await cleanup(); + + try { + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_dupe_device_ids() { + _( + "Ensure that we mark devices with duplicate fxaDeviceIds but older lastModified as stale." + ); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let remoteDeviceId = Utils.makeGUID(); + + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord( + { + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteDeviceId, + protocols: ["1.5"], + }, + new_timestamp() - 3 + ); + collection.insertRecord({ + id: remoteId2, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteDeviceId, + protocols: ["1.5"], + }); + + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteDeviceId }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + + ok(engine._store._remoteClients[remoteId].stale); + ok(!engine._store._remoteClients[remoteId2].stale); + } finally { + engine.fxAccounts = fxAccounts; + await cleanup(); + + try { + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_refresh_fxa_device_list() { + _("Ensure we refresh the fxa device list when we expect to."); + + await engine._store.wipe(); + engine._lastFxaDeviceRefresh = 0; + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let numRefreshes = 0; + let now = Date.now(); + let fxAccounts = engine.fxAccounts; + engine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [], + refreshDeviceList() { + numRefreshes += 1; + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return now; + }, + }, + }; + + try { + _("Syncing."); + await syncClientsEngine(server); + Assert.equal(numRefreshes, 1, "first sync should refresh"); + now += 1000; // a second later. + await syncClientsEngine(server); + Assert.equal(numRefreshes, 1, "next sync should not refresh"); + now += 60 * 60 * 2 * 1000; // 2 hours later + await syncClientsEngine(server); + Assert.equal(numRefreshes, 2, "2 hours later should refresh"); + now += 1000; // a second later. + Assert.equal(numRefreshes, 2, "next sync should not refresh"); + } finally { + await cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_optional_client_fields() { + _("Ensure that we produce records with the fields added in Bug 1097222."); + + const SUPPORTED_PROTOCOL_VERSIONS = ["1.5"]; + let local = await engine._store.createRecord(engine.localID, "clients"); + equal(local.name, engine.localName); + equal(local.type, engine.localType); + equal(local.version, Services.appinfo.version); + deepEqual(local.protocols, SUPPORTED_PROTOCOL_VERSIONS); + + // Optional fields. + // Make sure they're what they ought to be... + equal(local.os, Services.appinfo.OS); + equal(local.appPackage, Services.appinfo.ID); + + // ... and also that they're non-empty. + ok(!!local.os); + ok(!!local.appPackage); + ok(!!local.application); + + // We don't currently populate device or formfactor. + // See Bug 1100722, Bug 1100723. + + await cleanup(); +}); + +add_task(async function test_merge_commands() { + _("Verifies local commands for remote clients are merged with the server's"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let desktopID = Utils.makeGUID(); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let mobileID = Utils.makeGUID(); + collection.insertRecord( + { + id: mobileID, + name: "Mobile client", + type: "mobile", + commands: [ + { + command: "logout", + args: [], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Broadcast logout to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + let desktopPayload = collection.cleartext(desktopID); + compareCommands( + desktopPayload.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + { + command: "logout", + args: [], + }, + ], + "Should send the logout command to the desktop client" + ); + + let mobilePayload = collection.cleartext(mobileID); + compareCommands( + mobilePayload.commands, + [{ command: "logout", args: [] }], + "Should not send a duplicate logout to the mobile client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_duplicate_remote_commands() { + _( + "Verifies local commands for remote clients are sent only once (bug 1289287)" + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let desktopID = Utils.makeGUID(); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 1 record downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Send command to client to wipe history engine"); + await engine.sendCommand("wipeEngine", ["history"]); + await syncClientsEngine(server); + + _( + "Simulate the desktop client consuming the command and syncing to the server" + ); + collection.insertRecord( + { + id: desktopID, + name: "Desktop client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + _("Send another command to the desktop client to wipe tabs engine"); + await engine.sendCommand("wipeEngine", ["tabs"], desktopID); + await syncClientsEngine(server); + + let desktopPayload = collection.cleartext(desktopID); + compareCommands( + desktopPayload.commands, + [ + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should only send the second command to the desktop client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_upload_after_reboot() { + _("Multiple downloads, reboot, then upload (bug 1289287)"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let deviceBID = Utils.makeGUID(); + let deviceCID = Utils.makeGUID(); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceCID, + name: "Device C", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + await syncClientsEngine(server); + + _("Send command to client to wipe tab engine"); + await engine.sendCommand("wipeEngine", ["tabs"], deviceBID); + + const oldUploadOutgoing = SyncEngine.prototype._uploadOutgoing; + SyncEngine.prototype._uploadOutgoing = async () => + engine._onRecordsWritten([], [deviceBID]); + await syncClientsEngine(server); + + let deviceBPayload = collection.cleartext(deviceBID); + compareCommands( + deviceBPayload.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + ], + "Should be the same because the upload failed" + ); + + _("Simulate the client B consuming the command and syncing to the server"); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + // Simulate reboot + SyncEngine.prototype._uploadOutgoing = oldUploadOutgoing; + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + + await syncClientsEngine(server); + + deviceBPayload = collection.cleartext(deviceBID); + compareCommands( + deviceBPayload.commands, + [ + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should only had written our outgoing command" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_keep_cleared_commands_after_reboot() { + _( + "Download commands, fail upload, reboot, then apply new commands (bug 1289287)" + ); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let deviceBID = Utils.makeGUID(); + let deviceCID = Utils.makeGUID(); + collection.insertRecord( + { + id: engine.localID, + name: "Device A", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["tabs"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceBID, + name: "Device B", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + collection.insertRecord( + { + id: deviceCID, + name: "Device C", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. Download remote and our record."); + strictEqual(engine.lastRecordUpload, 0); + ok(engine.isFirstSync); + + const oldUploadOutgoing = SyncEngine.prototype._uploadOutgoing; + SyncEngine.prototype._uploadOutgoing = async () => + engine._onRecordsWritten([], [deviceBID]); + let commandsProcessed = 0; + engine.service.wipeClient = _engine => { + commandsProcessed++; + }; + + await syncClientsEngine(server); + await engine.processIncomingCommands(); // Not called by the engine.sync(), gotta call it ourselves + equal(commandsProcessed, 2, "We processed 2 commands"); + + let localRemoteRecord = collection.cleartext(engine.localID); + compareCommands( + localRemoteRecord.commands, + [ + { + command: "wipeEngine", + args: ["history"], + }, + { + command: "wipeEngine", + args: ["tabs"], + }, + ], + "Should be the same because the upload failed" + ); + + // Another client sends a wipe command + collection.insertRecord( + { + id: engine.localID, + name: "Device A", + type: "desktop", + commands: [ + { + command: "wipeEngine", + args: ["history"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["tabs"], + flowID: Utils.makeGUID(), + }, + { + command: "wipeEngine", + args: ["bookmarks"], + flowID: Utils.makeGUID(), + }, + ], + version: "48", + protocols: ["1.5"], + }, + now - 5 + ); + + // Simulate reboot + SyncEngine.prototype._uploadOutgoing = oldUploadOutgoing; + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + + commandsProcessed = 0; + engine.service.wipeClient = _engine => { + commandsProcessed++; + }; + await syncClientsEngine(server); + await engine.processIncomingCommands(); + equal( + commandsProcessed, + 1, + "We processed one command (the other were cleared)" + ); + + localRemoteRecord = collection.cleartext(deviceBID); + deepEqual(localRemoteRecord.commands, [], "Should be empty"); + } finally { + await cleanup(); + + // Reset service (remove mocks) + engine = Service.clientsEngine = new ClientEngine(Service); + await engine.initialize(); + await engine._resetClient(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_deleted_commands() { + _("Verifies commands for a deleted client are discarded"); + + let now = new_timestamp(); + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let collection = server.getCollection("foo", "clients"); + + let activeID = Utils.makeGUID(); + collection.insertRecord( + { + id: activeID, + name: "Active client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + let deletedID = Utils.makeGUID(); + collection.insertRecord( + { + id: deletedID, + name: "Client to delete", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }, + now - 10 + ); + + try { + _("First sync. 2 records downloaded."); + await syncClientsEngine(server); + + _("Delete a record on the server."); + collection.remove(deletedID); + + _("Broadcast a command to all clients"); + await engine.sendCommand("logout", []); + await syncClientsEngine(server); + + deepEqual( + collection.keys().sort(), + [activeID, engine.localID].sort(), + "Should not reupload deleted clients" + ); + + let activePayload = collection.cleartext(activeID); + compareCommands( + activePayload.commands, + [{ command: "logout", args: [] }], + "Should send the command to the active client" + ); + } finally { + await cleanup(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_command_sync() { + _("Notify other clients when writing their record."); + + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let collection = server.getCollection("foo", "clients"); + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + try { + equal(collection.count(), 2, "2 remote records written"); + await syncClientsEngine(server); + equal( + collection.count(), + 3, + "3 remote records written (+1 for the synced local record)" + ); + + await engine.sendCommand("wipeEngine", ["tabs"]); + await engine._tracker.addChangedID(engine.localID); + const getClientFxaDeviceId = sinon + .stub(engine, "getClientFxaDeviceId") + .callsFake(id => "fxa-" + id); + const engineMock = sinon.mock(engine); + let _notifyCollectionChanged = engineMock + .expects("_notifyCollectionChanged") + .withArgs(["fxa-" + remoteId, "fxa-" + remoteId2]); + _("Syncing."); + await syncClientsEngine(server); + _notifyCollectionChanged.verify(); + + engineMock.restore(); + getClientFxaDeviceId.restore(); + } finally { + await cleanup(); + await engine._tracker.clearChangedIDs(); + + try { + server.deleteCollections("foo"); + } finally { + await promiseStopServer(server); + } + } +}); + +add_task(async function ensureSameFlowIDs() { + let events = []; + let origRecordTelemetryEvent = Service.recordTelemetryEvent; + Service.recordTelemetryEvent = (object, method, value, extra) => { + events.push({ object, method, value, extra }); + }; + + let server = await serverForFoo(engine); + try { + // Setup 2 clients, send them a command, and ensure we get to events + // written, both with the same flowID. + await SyncTestingInfrastructure(server); + let collection = server.getCollection("foo", "clients"); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + await syncClientsEngine(server); + await engine.sendCommand("wipeEngine", ["tabs"]); + await syncClientsEngine(server); + equal(events.length, 2); + // we don't know what the flowID is, but do know it should be the same. + equal(events[0].extra.flowID, events[1].extra.flowID); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + // check it's correctly used when we specify a flow ID + events.length = 0; + let flowID = Utils.makeGUID(); + await engine.sendCommand("wipeEngine", ["tabs"], null, { flowID }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, flowID); + equal(events[1].extra.flowID, flowID); + + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + + // and that it works when something else is in "extra" + events.length = 0; + await engine.sendCommand("wipeEngine", ["tabs"], null, { + reason: "testing", + }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, events[1].extra.flowID); + equal(events[0].extra.reason, "testing"); + equal(events[1].extra.reason, "testing"); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + + // and when both are specified. + events.length = 0; + await engine.sendCommand("wipeEngine", ["tabs"], null, { + reason: "testing", + flowID, + }); + await syncClientsEngine(server); + equal(events.length, 2); + equal(events[0].extra.flowID, flowID); + equal(events[1].extra.flowID, flowID); + equal(events[0].extra.reason, "testing"); + equal(events[1].extra.reason, "testing"); + // Wipe remote clients to ensure deduping doesn't prevent us from adding the command. + for (let client of Object.values(engine._store._remoteClients)) { + client.commands = []; + } + } finally { + Service.recordTelemetryEvent = origRecordTelemetryEvent; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_duplicate_commands_telemetry() { + let events = []; + let origRecordTelemetryEvent = Service.recordTelemetryEvent; + Service.recordTelemetryEvent = (object, method, value, extra) => { + events.push({ object, method, value, extra }); + }; + + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + let collection = server.getCollection("foo", "clients"); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + + _("Create remote client record 1"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + _("Create remote client record 2"); + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "mobile", + commands: [], + version: "48", + protocols: ["1.5"], + }); + + await syncClientsEngine(server); + // Make sure deduping works before syncing + await engine.sendCommand("wipeEngine", ["history"], remoteId); + await engine.sendCommand("wipeEngine", ["history"], remoteId); + equal(events.length, 1); + await syncClientsEngine(server); + // And after syncing. + await engine.sendCommand("wipeEngine", ["history"], remoteId); + equal(events.length, 1); + // Ensure we aren't deduping commands to different clients + await engine.sendCommand("wipeEngine", ["history"], remoteId2); + equal(events.length, 2); + } finally { + Service.recordTelemetryEvent = origRecordTelemetryEvent; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task(async function test_other_clients_notified_on_first_sync() { + _( + "Ensure that other clients are notified when we upload our client record for the first time." + ); + + await engine.resetLastSync(); + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const fxAccounts = engine.fxAccounts; + let calls = 0; + engine.fxAccounts = { + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + }, + notifyDevices() { + calls++; + return Promise.resolve(true); + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + try { + engine.lastRecordUpload = 0; + _("First sync, should notify other clients"); + await syncClientsEngine(server); + equal(calls, 1); + + _("Second sync, should not notify other clients"); + await syncClientsEngine(server); + equal(calls, 1); + } finally { + engine.fxAccounts = fxAccounts; + cleanup(); + await promiseStopServer(server); + } +}); + +add_task( + async function device_disconnected_notification_updates_known_stale_clients() { + const spyUpdate = sinon.spy(engine, "updateKnownStaleClients"); + + Services.obs.notifyObservers( + null, + "fxaccounts:device_disconnected", + JSON.stringify({ isLocalDevice: false }) + ); + ok(spyUpdate.calledOnce, "updateKnownStaleClients should be called"); + spyUpdate.resetHistory(); + + Services.obs.notifyObservers( + null, + "fxaccounts:device_disconnected", + JSON.stringify({ isLocalDevice: true }) + ); + ok(spyUpdate.notCalled, "updateKnownStaleClients should not be called"); + + spyUpdate.restore(); + } +); + +add_task(async function update_known_stale_clients() { + const makeFakeClient = id => ({ id, fxaDeviceId: `fxa-${id}` }); + const clients = [ + makeFakeClient("one"), + makeFakeClient("two"), + makeFakeClient("three"), + ]; + const stubRemoteClients = sinon + .stub(engine._store, "_remoteClients") + .get(() => { + return clients; + }); + const stubFetchFxADevices = sinon + .stub(engine, "_fetchFxADevices") + .callsFake(() => { + engine._knownStaleFxADeviceIds = ["fxa-one", "fxa-two"]; + }); + + engine._knownStaleFxADeviceIds = null; + await engine.updateKnownStaleClients(); + ok(clients[0].stale); + ok(clients[1].stale); + ok(!clients[2].stale); + + stubRemoteClients.restore(); + stubFetchFxADevices.restore(); +}); + +add_task(async function test_create_record_command_limit() { + await engine._store.wipe(); + await generateNewKeys(Service.collectionKeys); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const fakeLimit = 4 * 1024; + + let maxSizeStub = sinon + .stub(Service, "getMemcacheMaxRecordPayloadSize") + .callsFake(() => fakeLimit); + + let user = server.user("foo"); + let remoteId = Utils.makeGUID(); + + _("Create remote client record"); + user.collection("clients").insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "57", + protocols: ["1.5"], + }); + + try { + _("Initial sync."); + await syncClientsEngine(server); + + _("Send a fairly sane number of commands."); + + for (let i = 0; i < 5; ++i) { + await engine.sendCommand("wipeEngine", [`history: ${i}`], remoteId); + } + + await syncClientsEngine(server); + + _("Make sure they all fit and weren't dropped."); + let parsedServerRecord = user.collection("clients").cleartext(remoteId); + + equal(parsedServerRecord.commands.length, 5); + + await engine.sendCommand("wipeEngine", ["history"], remoteId); + + _("Send a not-sane number of commands."); + // Much higher than the maximum number of commands we could actually fit. + for (let i = 0; i < 500; ++i) { + await engine.sendCommand("wipeEngine", [`tabs: ${i}`], remoteId); + } + + await syncClientsEngine(server); + + _("Ensure we didn't overflow the server limit."); + let wbo = user.collection("clients").wbo(remoteId); + less(wbo.payload.length, fakeLimit); + + _( + "And that the data we uploaded is both sane json and containing some commands." + ); + let remoteCommands = wbo.getCleartext().commands; + greater(remoteCommands.length, 2); + let firstCommand = remoteCommands[0]; + _( + "The first command should still be present, since it had a high priority" + ); + equal(firstCommand.command, "wipeEngine"); + _("And the last command in the list should be the last command we sent."); + let lastCommand = remoteCommands[remoteCommands.length - 1]; + equal(lastCommand.command, "wipeEngine"); + deepEqual(lastCommand.args, ["tabs: 499"]); + } finally { + maxSizeStub.restore(); + await cleanup(); + try { + let collection = server.getCollection("foo", "clients"); + collection.remove(remoteId); + } finally { + await promiseStopServer(server); + } + } +}); diff --git a/services/sync/tests/unit/test_clients_escape.js b/services/sync/tests/unit/test_clients_escape.js new file mode 100644 index 0000000000..1c6af9be11 --- /dev/null +++ b/services/sync/tests/unit/test_clients_escape.js @@ -0,0 +1,55 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_clients_escape() { + _("Set up test fixtures."); + + await configureIdentity(); + let keyBundle = Service.identity.syncKeyBundle; + + let engine = Service.clientsEngine; + + try { + _("Test that serializing client records results in uploadable ascii"); + engine.localID = "ascii"; + engine.localName = "wéävê"; + + _("Make sure we have the expected record"); + let record = await engine._createRecord("ascii"); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + + _("Encrypting record..."); + await record.encrypt(keyBundle); + _("Encrypted."); + + let serialized = JSON.stringify(record); + let checkCount = 0; + _("Checking for all ASCII:", serialized); + for (let ch of serialized) { + let code = ch.charCodeAt(0); + _("Checking asciiness of '", ch, "'=", code); + Assert.ok(code < 128); + checkCount++; + } + + _("Processed", checkCount, "characters out of", serialized.length); + Assert.equal(checkCount, serialized.length); + + _("Making sure the record still looks like it did before"); + await record.decrypt(keyBundle); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + + _("Sanity check that creating the record also gives the same"); + record = await engine._createRecord("ascii"); + Assert.equal(record.id, "ascii"); + Assert.equal(record.name, "wéävê"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); diff --git a/services/sync/tests/unit/test_collection_getBatched.js b/services/sync/tests/unit/test_collection_getBatched.js new file mode 100644 index 0000000000..f5425abe92 --- /dev/null +++ b/services/sync/tests/unit/test_collection_getBatched.js @@ -0,0 +1,187 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Collection, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function recordRange(lim, offset, total) { + let res = []; + for (let i = offset; i < Math.min(lim + offset, total); ++i) { + res.push({ id: String(i), payload: "test:" + i }); + } + return res; +} + +function get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + throwAfter = Infinity, + interruptedAfter = Infinity, +}) { + let coll = new Collection("http://example.com/test/", WBORecord, Service); + coll.full = true; + let requests = []; + let responses = []; + coll.get = async function () { + let limit = +this.limit; + let offset = 0; + if (this.offset) { + equal(this.offset.slice(0, 6), "foobar"); + offset = +this.offset.slice(6); + } + requests.push({ + limit, + offset, + spec: this.spec, + headers: Object.assign({}, this.headers), + }); + if (--throwAfter === 0) { + throw new Error("Some Network Error"); + } + let body = recordRange(limit, offset, totalRecords); + let response = { + obj: body, + success: true, + status: 200, + headers: {}, + }; + if (--interruptedAfter === 0) { + response.success = false; + response.status = 412; + response.body = ""; + } else if (offset + limit < totalRecords) { + // Ensure we're treating this as an opaque string, since the docs say + // it might not be numeric. + response.headers["x-weave-next-offset"] = "foobar" + (offset + batchSize); + } + response.headers["x-last-modified"] = lastModified; + responses.push(response); + return response; + }; + return { responses, requests, coll }; +} + +add_task(async function test_success() { + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { responses, requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + }); + let { response, records } = await coll.getBatched(batchSize); + + equal(requests.length, Math.ceil(totalRecords / batchSize)); + + equal(records.length, totalRecords); + checkRecordsOrder(records); + + // ensure we're returning the last response + equal(responses[responses.length - 1], response); + + // check first separately since its a bit of a special case + ok(!requests[0].headers["x-if-unmodified-since"]); + ok(!requests[0].offset); + equal(requests[0].limit, batchSize); + let expectedOffset = 2; + for (let i = 1; i < requests.length; ++i) { + let req = requests[i]; + equal(req.headers["x-if-unmodified-since"], lastModified); + equal(req.limit, batchSize); + if (i !== requests.length - 1) { + equal(req.offset, expectedOffset); + } + + expectedOffset += batchSize; + } + + // ensure we cleaned up anything that would break further + // use of this collection. + ok(!coll._headers["x-if-unmodified-since"]); + ok(!coll.offset); + ok(!coll.limit || coll.limit == Infinity); +}); + +add_task(async function test_total_limit() { + _("getBatched respects the (initial) value of the limit property"); + const totalRecords = 100; + const recordLimit = 11; + const batchSize = 2; + const lastModified = "111111"; + let { requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + }); + coll.limit = recordLimit; + let { records } = await coll.getBatched(batchSize); + checkRecordsOrder(records); + + equal(requests.length, Math.ceil(recordLimit / batchSize)); + equal(records.length, recordLimit); + + for (let i = 0; i < requests.length; ++i) { + let req = requests[i]; + if (i !== requests.length - 1) { + equal(req.limit, batchSize); + } else { + equal(req.limit, recordLimit % batchSize); + } + } + + equal(coll._limit, recordLimit); +}); + +add_task(async function test_412() { + _("We shouldn't record records if we get a 412 in the middle of a batch"); + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { responses, requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + interruptedAfter: 3, + }); + let { response, records } = await coll.getBatched(batchSize); + + equal(requests.length, 3); + equal(records.length, 0); // we should not get any records + + // ensure we're returning the last response + equal(responses[responses.length - 1], response); + + ok(!response.success); + equal(response.status, 412); +}); + +add_task(async function test_get_throws() { + _("getBatched() should throw if a get() throws"); + const totalRecords = 11; + const batchSize = 2; + const lastModified = "111111"; + let { requests, coll } = get_test_collection_info({ + totalRecords, + batchSize, + lastModified, + throwAfter: 3, + }); + + await Assert.rejects(coll.getBatched(batchSize), /Some Network Error/); + + equal(requests.length, 3); +}); + +function checkRecordsOrder(records) { + ok(!!records.length); + for (let i = 0; i < records.length; i++) { + equal(records[i].id, String(i)); + equal(records[i].payload, "test:" + i); + } +} diff --git a/services/sync/tests/unit/test_collections_recovery.js b/services/sync/tests/unit/test_collections_recovery.js new file mode 100644 index 0000000000..22e28ef72f --- /dev/null +++ b/services/sync/tests/unit/test_collections_recovery.js @@ -0,0 +1,93 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Verify that we wipe the server if we have to regenerate keys. +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_missing_crypto_collection() { + enableValidationPrefs(); + + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + let johnColls = johnHelper.collections; + + let empty = false; + function maybe_empty(handler) { + return function (request, response) { + if (empty) { + let body = "{}"; + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + } else { + handler(request, response); + } + }; + } + + let handlers = { + "/1.1/johndoe/info/collections": maybe_empty(johnHelper.handler), + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + }; + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = johnU( + coll, + new ServerCollection({}, true).handler() + ); + } + let server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + + try { + let fresh = 0; + let orig = Service._freshStart; + Service._freshStart = async function () { + _("Called _freshStart."); + await orig.call(Service); + fresh++; + }; + + _("Startup, no meta/global: freshStart called once."); + await sync_and_validate_telem(); + Assert.equal(fresh, 1); + fresh = 0; + + _("Regular sync: no need to freshStart."); + await Service.sync(); + Assert.equal(fresh, 0); + + _("Simulate a bad info/collections."); + delete johnColls.crypto; + await sync_and_validate_telem(); + Assert.equal(fresh, 1); + fresh = 0; + + _("Regular sync: no need to freshStart."); + await sync_and_validate_telem(); + Assert.equal(fresh, 0); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_corrupt_keys.js b/services/sync/tests/unit/test_corrupt_keys.js new file mode 100644 index 0000000000..b62c65011f --- /dev/null +++ b/services/sync/tests/unit/test_corrupt_keys.js @@ -0,0 +1,246 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { CryptoWrapper, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_locally_changed_keys() { + enableValidationPrefs(); + + let hmacErrorCount = 0; + function counting(f) { + return async function () { + hmacErrorCount++; + return f.call(this); + }; + } + + Service.handleHMACEvent = counting(Service.handleHMACEvent); + + let server = new SyncServer(); + let johndoe = server.registerUser("johndoe", "password"); + johndoe.createContents({ + meta: {}, + crypto: {}, + clients: {}, + }); + server.start(); + + try { + Svc.Prefs.set("registerEngines", "Tab"); + + await configureIdentity({ username: "johndoe" }, server); + // We aren't doing a .login yet, so fudge the cluster URL. + Service.clusterURL = Service.identity._token.endpoint; + + await Service.engineManager.register(HistoryEngine); + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); + + async function corrupt_local_keys() { + Service.collectionKeys._default.keyPair = [ + await Weave.Crypto.generateRandomKey(), + await Weave.Crypto.generateRandomKey(), + ]; + } + + _("Setting meta."); + + // Bump version on the server. + let m = new WBORecord("meta", "global"); + m.payload = { + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION, + }; + await m.upload(Service.resource(Service.metaURL)); + + _( + "New meta/global: " + + JSON.stringify(johndoe.collection("meta").wbo("global")) + ); + + // Upload keys. + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + Assert.ok( + (await serverKeys.upload(Service.resource(Service.cryptoKeysURL))).success + ); + + // Check that login works. + Assert.ok(await Service.login()); + Assert.ok(Service.isLoggedIn); + + // Sync should upload records. + await sync_and_validate_telem(); + + // Tabs exist. + _("Tabs modified: " + johndoe.modified("tabs")); + Assert.ok(johndoe.modified("tabs") > 0); + + // Let's create some server side history records. + let liveKeys = Service.collectionKeys.keyForCollection("history"); + _("Keys now: " + liveKeys.keyPair); + let visitType = Ci.nsINavHistoryService.TRANSITION_LINK; + let history = johndoe.createCollection("history"); + for (let i = 0; i < 5; i++) { + let id = "record-no--" + i; + let modified = Date.now() / 1000 - 60 * (i + 10); + + let w = new CryptoWrapper("history", "id"); + w.cleartext = { + id, + histUri: "http://foo/bar?" + id, + title: id, + sortindex: i, + visits: [{ date: (modified - 5) * 1000000, type: visitType }], + deleted: false, + }; + await w.encrypt(liveKeys); + + let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac }; + history.insert(id, payload, modified); + } + + history.timestamp = Date.now() / 1000; + let old_key_time = johndoe.modified("crypto"); + _("Old key time: " + old_key_time); + + // Check that we can decrypt one. + let rec = new CryptoWrapper("history", "record-no--0"); + await rec.fetch( + Service.resource(Service.storageURL + "history/record-no--0") + ); + _(JSON.stringify(rec)); + Assert.ok(!!(await rec.decrypt(liveKeys))); + + Assert.equal(hmacErrorCount, 0); + + // Fill local key cache with bad data. + await corrupt_local_keys(); + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + + Assert.equal(hmacErrorCount, 0); + + _("HMAC error count: " + hmacErrorCount); + // Now syncing should succeed, after one HMAC error. + await sync_and_validate_telem(ping => { + Assert.equal( + ping.engines.find(e => e.name == "history").incoming.applied, + 5 + ); + }); + + Assert.equal(hmacErrorCount, 1); + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + + // And look! We downloaded history! + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--0") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--1") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--2") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--3") + ); + Assert.ok( + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--4") + ); + Assert.equal(hmacErrorCount, 1); + + _("Busting some new server values."); + // Now what happens if we corrupt the HMAC on the server? + for (let i = 5; i < 10; i++) { + let id = "record-no--" + i; + let modified = 1 + Date.now() / 1000; + + let w = new CryptoWrapper("history", "id"); + w.cleartext = { + id, + histUri: "http://foo/bar?" + id, + title: id, + sortindex: i, + visits: [{ date: (modified - 5) * 1000000, type: visitType }], + deleted: false, + }; + await w.encrypt(Service.collectionKeys.keyForCollection("history")); + w.hmac = w.hmac.toUpperCase(); + + let payload = { ciphertext: w.ciphertext, IV: w.IV, hmac: w.hmac }; + history.insert(id, payload, modified); + } + history.timestamp = Date.now() / 1000; + + _("Server key time hasn't changed."); + Assert.equal(johndoe.modified("crypto"), old_key_time); + + _("Resetting HMAC error timer."); + Service.lastHMACEvent = 0; + + _("Syncing..."); + await sync_and_validate_telem(ping => { + Assert.equal( + ping.engines.find(e => e.name == "history").incoming.failed, + 5 + ); + }); + + _( + "Keys now: " + Service.collectionKeys.keyForCollection("history").keyPair + ); + _( + "Server keys have been updated, and we skipped over 5 more HMAC errors without adjusting history." + ); + Assert.ok(johndoe.modified("crypto") > old_key_time); + Assert.equal(hmacErrorCount, 6); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--5") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--6") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--7") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--8") + ); + Assert.equal( + false, + await PlacesUtils.history.hasVisits("http://foo/bar?record-no--9") + ); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + validate_all_future_pings(); + + run_next_test(); +} diff --git a/services/sync/tests/unit/test_declined.js b/services/sync/tests/unit/test_declined.js new file mode 100644 index 0000000000..af7f8eb8c5 --- /dev/null +++ b/services/sync/tests/unit/test_declined.js @@ -0,0 +1,194 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { DeclinedEngines } = ChromeUtils.importESModule( + "resource://services-sync/stages/declined.sys.mjs" +); +const { EngineSynchronizer } = ChromeUtils.importESModule( + "resource://services-sync/stages/enginesync.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); + +function PetrolEngine() {} +PetrolEngine.prototype.name = "petrol"; + +function DieselEngine() {} +DieselEngine.prototype.name = "diesel"; + +function DummyEngine() {} +DummyEngine.prototype.name = "dummy"; + +function ActualEngine() {} +ActualEngine.prototype.name = "actual"; +Object.setPrototypeOf(ActualEngine.prototype, SyncEngine.prototype); + +function getEngineManager() { + let manager = new EngineManager(Service); + Service.engineManager = manager; + manager._engines = { + petrol: new PetrolEngine(), + diesel: new DieselEngine(), + dummy: new DummyEngine(), + actual: new ActualEngine(), + }; + return manager; +} + +/** + * 'Fetch' a meta/global record that doesn't mention declined. + * + * Push it into the EngineSynchronizer to set enabled; verify that those are + * correct. + * + * Then push it into DeclinedEngines to set declined; verify that none are + * declined, and a notification is sent for our locally disabled-but-not- + * declined engines. + */ +add_task(async function testOldMeta() { + let meta = { + payload: { + engines: { + petrol: 1, + diesel: 2, + nonlocal: 3, // Enabled but not supported. + }, + }, + }; + + _("Record: " + JSON.stringify(meta)); + + let manager = getEngineManager(); + + // Update enabled from meta/global. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledFromMeta(meta, 3, manager); + + Assert.ok(manager._engines.petrol.enabled, "'petrol' locally enabled."); + Assert.ok(manager._engines.diesel.enabled, "'diesel' locally enabled."); + Assert.ok( + !("nonlocal" in manager._engines), + "We don't know anything about the 'nonlocal' engine." + ); + Assert.ok(!manager._engines.actual.enabled, "'actual' not locally enabled."); + Assert.ok(!manager.isDeclined("actual"), "'actual' not declined, though."); + + let declinedEngines = new DeclinedEngines(Service); + + function onNotDeclined(subject, topic, data) { + Observers.remove("weave:engines:notdeclined", onNotDeclined); + Assert.ok( + subject.undecided.has("actual"), + "EngineManager observed that 'actual' was undecided." + ); + + let declined = manager.getDeclined(); + _("Declined: " + JSON.stringify(declined)); + + Assert.ok(!meta.changed, "No need to upload a new meta/global."); + run_next_test(); + } + + Observers.add("weave:engines:notdeclined", onNotDeclined); + + declinedEngines.updateDeclined(meta, manager); +}); + +/** + * 'Fetch' a meta/global that declines an engine we don't + * recognize. Ensure that we track that declined engine along + * with any we locally declined, and that the meta/global + * record is marked as changed and includes all declined + * engines. + */ +add_task(async function testDeclinedMeta() { + let meta = { + payload: { + engines: { + petrol: 1, + diesel: 2, + nonlocal: 3, // Enabled but not supported. + }, + declined: ["nonexistent"], // Declined and not supported. + }, + }; + + _("Record: " + JSON.stringify(meta)); + + let manager = getEngineManager(); + manager._engines.petrol.enabled = true; + manager._engines.diesel.enabled = true; + manager._engines.dummy.enabled = true; + manager._engines.actual.enabled = false; // Disabled but not declined. + + manager.decline(["localdecline"]); // Declined and not supported. + + let declinedEngines = new DeclinedEngines(Service); + + function onNotDeclined(subject, topic, data) { + Observers.remove("weave:engines:notdeclined", onNotDeclined); + Assert.ok( + subject.undecided.has("actual"), + "EngineManager observed that 'actual' was undecided." + ); + + let declined = manager.getDeclined(); + _("Declined: " + JSON.stringify(declined)); + + Assert.equal( + declined.indexOf("actual"), + -1, + "'actual' is locally disabled, but not marked as declined." + ); + + Assert.equal( + declined.indexOf("clients"), + -1, + "'clients' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("petrol"), + -1, + "'petrol' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("diesel"), + -1, + "'diesel' is enabled and not remotely declined." + ); + Assert.equal( + declined.indexOf("dummy"), + -1, + "'dummy' is enabled and not remotely declined." + ); + + Assert.ok( + 0 <= declined.indexOf("nonexistent"), + "'nonexistent' was declined on the server." + ); + + Assert.ok( + 0 <= declined.indexOf("localdecline"), + "'localdecline' was declined locally." + ); + + // The meta/global is modified, too. + Assert.ok( + 0 <= meta.payload.declined.indexOf("nonexistent"), + "meta/global's declined contains 'nonexistent'." + ); + Assert.ok( + 0 <= meta.payload.declined.indexOf("localdecline"), + "meta/global's declined contains 'localdecline'." + ); + Assert.strictEqual(true, meta.changed, "meta/global was changed."); + } + + Observers.add("weave:engines:notdeclined", onNotDeclined); + + declinedEngines.updateDeclined(meta, manager); +}); diff --git a/services/sync/tests/unit/test_disconnect_shutdown.js b/services/sync/tests/unit/test_disconnect_shutdown.js new file mode 100644 index 0000000000..cf7536d3de --- /dev/null +++ b/services/sync/tests/unit/test_disconnect_shutdown.js @@ -0,0 +1,92 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { SyncDisconnect, SyncDisconnectInternal } = ChromeUtils.importESModule( + "resource://services-sync/SyncDisconnect.sys.mjs" +); +const { AsyncShutdown } = ChromeUtils.importESModule( + "resource://gre/modules/AsyncShutdown.sys.mjs" +); + +add_task(async function test_shutdown_blocker() { + let spySignout = sinon.stub( + SyncDisconnectInternal, + "doSyncAndAccountDisconnect" + ); + + // We don't need to check for the lock regularly as we end up aborting the wait. + SyncDisconnectInternal.lockRetryInterval = 1000; + // Force the retry count to a very large value - this test should never + // abort due to the retry count and we want the test to fail (aka timeout) + // should our abort code not work. + SyncDisconnectInternal.lockRetryCount = 10000; + // mock the "browser" sanitize function - it should not be called by + // this test. + let spyBrowser = sinon.stub(SyncDisconnectInternal, "doSanitizeBrowserData"); + // mock Sync + let mockEngine1 = { + enabled: true, + name: "Test Engine 1", + wipeClient: sinon.spy(), + }; + let mockEngine2 = { + enabled: false, + name: "Test Engine 2", + wipeClient: sinon.spy(), + }; + + // This weave mock never gives up the lock. + let Weave = { + Service: { + enabled: true, + lock: () => false, // so we never get the lock. + unlock: sinon.spy(), + + engineManager: { + getAll: sinon.stub().returns([mockEngine1, mockEngine2]), + }, + errorHandler: { + resetFileLog: sinon.spy(), + }, + }, + }; + let weaveStub = sinon.stub(SyncDisconnectInternal, "getWeave"); + weaveStub.returns(Weave); + + let promiseDisconnected = SyncDisconnect.disconnect(true); + + // Pretend we hit the shutdown blocker. + info("simulating quitApplicationGranted"); + Services.prefs.setBoolPref("toolkit.asyncshutdown.testing", true); + AsyncShutdown.quitApplicationGranted._trigger(); + Services.prefs.clearUserPref("toolkit.asyncshutdown.testing"); + + info("waiting for disconnect to complete"); + await promiseDisconnected; + + Assert.equal( + Weave.Service.unlock.callCount, + 0, + "should not have unlocked at the end" + ); + Assert.ok(!Weave.Service.enabled, "Weave should be and remain disabled"); + Assert.equal( + Weave.Service.errorHandler.resetFileLog.callCount, + 1, + "should have reset the log" + ); + Assert.equal( + mockEngine1.wipeClient.callCount, + 1, + "enabled engine should have been wiped" + ); + Assert.equal( + mockEngine2.wipeClient.callCount, + 0, + "disabled engine should not have been wiped" + ); + Assert.equal(spyBrowser.callCount, 1, "should not sanitize the browser"); + Assert.equal(spySignout.callCount, 1, "should have signed out of FxA"); +}); diff --git a/services/sync/tests/unit/test_engine.js b/services/sync/tests/unit/test_engine.js new file mode 100644 index 0000000000..b773599304 --- /dev/null +++ b/services/sync/tests/unit/test_engine.js @@ -0,0 +1,247 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function SteamStore(engine) { + Store.call(this, "Steam", engine); + this.wasWiped = false; +} +SteamStore.prototype = { + async wipe() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(SteamStore.prototype, Store.prototype); + +function SteamTracker(name, engine) { + LegacyTracker.call(this, name || "Steam", engine); +} +Object.setPrototypeOf(SteamTracker.prototype, LegacyTracker.prototype); + +function SteamEngine(name, service) { + SyncEngine.call(this, name, service); + this.wasReset = false; + this.wasSynced = false; +} +SteamEngine.prototype = { + _storeObj: SteamStore, + _trackerObj: SteamTracker, + + async _resetClient() { + this.wasReset = true; + }, + + async _sync() { + this.wasSynced = true; + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +var engineObserver = { + topics: [], + + observe(subject, topic, data) { + Assert.equal(data, "steam"); + this.topics.push(topic); + }, + + reset() { + this.topics = []; + }, +}; +Observers.add("weave:engine:reset-client:start", engineObserver); +Observers.add("weave:engine:reset-client:finish", engineObserver); +Observers.add("weave:engine:wipe-client:start", engineObserver); +Observers.add("weave:engine:wipe-client:finish", engineObserver); +Observers.add("weave:engine:sync:start", engineObserver); +Observers.add("weave:engine:sync:finish", engineObserver); + +async function cleanup(engine) { + Svc.Prefs.resetBranch(""); + engine.wasReset = false; + engine.wasSynced = false; + engineObserver.reset(); + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +} + +add_task(async function test_members() { + _("Engine object members"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.equal(engine.Name, "Steam"); + Assert.equal(engine.prefName, "steam"); + Assert.ok(engine._store instanceof SteamStore); + Assert.ok(engine._tracker instanceof SteamTracker); +}); + +add_task(async function test_score() { + _("Engine.score corresponds to tracker.score and is readonly"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.equal(engine.score, 0); + engine._tracker.score += 5; + Assert.equal(engine.score, 5); + + try { + engine.score = 10; + } catch (ex) { + // Setting an attribute that has a getter produces an error in + // Firefox <= 3.6 and is ignored in later versions. Either way, + // the attribute's value won't change. + } + Assert.equal(engine.score, 5); +}); + +add_task(async function test_resetClient() { + _("Engine.resetClient calls _resetClient"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.ok(!engine.wasReset); + + await engine.resetClient(); + Assert.ok(engine.wasReset); + Assert.equal(engineObserver.topics[0], "weave:engine:reset-client:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:finish"); + + await cleanup(engine); +}); + +add_task(async function test_invalidChangedIDs() { + _("Test that invalid changed IDs on disk don't end up live."); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + let tracker = engine._tracker; + + await tracker._beforeSave(); + await IOUtils.writeUTF8(tracker._storage.path, "5", { + tmpPath: tracker._storage.path + ".tmp", + }); + + ok(!tracker._storage.dataReady); + const changes = await tracker.getChangedIDs(); + changes.placeholder = true; + deepEqual( + changes, + { placeholder: true }, + "Accessing changed IDs should load changes from disk as a side effect" + ); + ok(tracker._storage.dataReady); + + Assert.ok(changes.placeholder); + await cleanup(engine); +}); + +add_task(async function test_wipeClient() { + _("Engine.wipeClient calls resetClient, wipes store, clears changed IDs"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + Assert.ok(!engine.wasReset); + Assert.ok(!engine._store.wasWiped); + Assert.ok(await engine._tracker.addChangedID("a-changed-id")); + let changes = await engine._tracker.getChangedIDs(); + Assert.ok("a-changed-id" in changes); + + await engine.wipeClient(); + Assert.ok(engine.wasReset); + Assert.ok(engine._store.wasWiped); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(JSON.stringify(changes), "{}"); + Assert.equal(engineObserver.topics[0], "weave:engine:wipe-client:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:reset-client:start"); + Assert.equal(engineObserver.topics[2], "weave:engine:reset-client:finish"); + Assert.equal(engineObserver.topics[3], "weave:engine:wipe-client:finish"); + + await cleanup(engine); +}); + +add_task(async function test_enabled() { + _("Engine.enabled corresponds to preference"); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + try { + Assert.ok(!engine.enabled); + Svc.Prefs.set("engine.steam", true); + Assert.ok(engine.enabled); + + engine.enabled = false; + Assert.ok(!Svc.Prefs.get("engine.steam")); + } finally { + await cleanup(engine); + } +}); + +add_task(async function test_sync() { + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + try { + _("Engine.sync doesn't call _sync if it's not enabled"); + Assert.ok(!engine.enabled); + Assert.ok(!engine.wasSynced); + await engine.sync(); + + Assert.ok(!engine.wasSynced); + + _("Engine.sync calls _sync if it's enabled"); + engine.enabled = true; + + await engine.sync(); + Assert.ok(engine.wasSynced); + Assert.equal(engineObserver.topics[0], "weave:engine:sync:start"); + Assert.equal(engineObserver.topics[1], "weave:engine:sync:finish"); + } finally { + await cleanup(engine); + } +}); + +add_task(async function test_disabled_no_track() { + _("When an engine is disabled, its tracker is not tracking."); + let engine = new SteamEngine("Steam", Service); + await engine.initialize(); + let tracker = engine._tracker; + Assert.equal(engine, tracker.engine); + + Assert.ok(!engine.enabled); + Assert.ok(!tracker._isTracking); + let changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + Assert.ok(!tracker.engineIsEnabled()); + Assert.ok(!tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + let promisePrefChangeHandled = PromiseUtils.defer(); + const origMethod = tracker.onEngineEnabledChanged; + tracker.onEngineEnabledChanged = async (...args) => { + await origMethod.apply(tracker, args); + promisePrefChangeHandled.resolve(); + }; + + engine.enabled = true; // Also enables the tracker automatically. + await promisePrefChangeHandled.promise; + Assert.ok(tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + await tracker.addChangedID("abcdefghijkl"); + changes = await tracker.getChangedIDs(); + Assert.ok(0 < changes.abcdefghijkl); + promisePrefChangeHandled = PromiseUtils.defer(); + Svc.Prefs.set("engine." + engine.prefName, false); + await promisePrefChangeHandled.promise; + Assert.ok(!tracker._isTracking); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + await cleanup(engine); +}); diff --git a/services/sync/tests/unit/test_engine_abort.js b/services/sync/tests/unit/test_engine_abort.js new file mode 100644 index 0000000000..84a92d5b00 --- /dev/null +++ b/services/sync/tests/unit/test_engine_abort.js @@ -0,0 +1,77 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +add_task(async function test_processIncoming_abort() { + _( + "An abort exception, raised in applyIncoming, will abort _processIncoming." + ); + let engine = new RotaryEngine(Service); + + let collection = new ServerCollection(); + let id = Utils.makeGUID(); + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + _("Create some server data."); + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + _("Fake applyIncoming to abort."); + engine._store.applyIncoming = async function (record) { + let ex = { + code: SyncEngine.prototype.eEngineAbortApplyIncoming, + cause: "Nooo", + }; + _("Throwing: " + JSON.stringify(ex)); + throw ex; + }; + + _("Trying _processIncoming. It will throw after aborting."); + let err; + try { + await engine._syncStartup(); + await engine._processIncoming(); + } catch (ex) { + err = ex; + } + + Assert.equal(err, "Nooo"); + err = undefined; + + _("Trying engine.sync(). It will abort without error."); + try { + // This will quietly fail. + await engine.sync(); + } catch (ex) { + err = ex; + } + + Assert.equal(err, undefined); + + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_engine_changes_during_sync.js b/services/sync/tests/unit/test_engine_changes_during_sync.js new file mode 100644 index 0000000000..8e93ca364e --- /dev/null +++ b/services/sync/tests/unit/test_engine_changes_during_sync.js @@ -0,0 +1,609 @@ +const { FormHistory } = ChromeUtils.importESModule( + "resource://gre/modules/FormHistory.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Bookmark, BookmarkFolder, BookmarkQuery } = ChromeUtils.importESModule( + "resource://services-sync/engines/bookmarks.sys.mjs" +); +const { HistoryRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { FormRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { PrefRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); + +const LoginInfo = Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +/** + * We don't test the clients or tabs engines because neither has conflict + * resolution logic. The clients engine syncs twice per global sync, and + * custom conflict resolution logic for commands that doesn't use + * timestamps. Tabs doesn't have conflict resolution at all, since it's + * read-only. + */ + +async function assertChildGuids(folderGuid, expectedChildGuids, message) { + let tree = await PlacesUtils.promiseBookmarksTree(folderGuid); + let childGuids = tree.children.map(child => child.guid); + deepEqual(childGuids, expectedChildGuids, message); +} + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine._store.wipe(); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function test_history_change_during_sync() { + _("Ensure that we don't bump the score when applying history records."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("history"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("history"); + + // Override `uploadOutgoing` to insert a record while we're applying + // changes. The tracker should ignore this change. + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local history visit"); + await addVisit("during_sync"); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + let remoteRec = new HistoryRec("history", "UrOOuzE5QM-e"); + remoteRec.histUri = "http://getfirefox.com/"; + remoteRec.title = "Get Firefox!"; + remoteRec.visits = [ + { + date: PlacesUtils.toPRTime(Date.now()), + type: PlacesUtils.history.TRANSITION_TYPED, + }, + ]; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for visits added during sync" + ); + + equal( + collection.count(), + 1, + "New local visit should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second history sync" + ); + + equal( + collection.count(), + 2, + "New local visit should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_passwords_change_during_sync() { + _("Ensure that we don't bump the score when applying passwords."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("passwords"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local password"); + let login = new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ); + await Services.logins.addLoginAsync(login); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + let remoteRec = new LoginRec( + "passwords", + "{765e3d6e-071d-d640-a83d-81a7eb62d3ed}" + ); + remoteRec.formSubmitURL = ""; + remoteRec.httpRealm = ""; + remoteRec.hostname = "https://mozilla.org"; + remoteRec.username = "username"; + remoteRec.password = "sekrit"; + remoteRec.timeCreated = Date.now(); + remoteRec.timePasswordChanged = Date.now(); + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for passwords added during first sync" + ); + + equal( + collection.count(), + 1, + "New local password should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second passwords sync" + ); + + equal( + collection.count(), + 2, + "New local password should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_prefs_change_during_sync() { + _("Ensure that we don't bump the score when applying prefs."); + + const TEST_PREF = "test.duringSync"; + // create a "control pref" for the pref we sync. + Services.prefs.setBoolPref("services.sync.prefs.sync.test.duringSync", true); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("prefs"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("prefs"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Updating local pref value"); + // Change the value of a synced pref. + Services.prefs.setCharPref(TEST_PREF, "hello"); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + // All synced prefs are stored in a single record, so we'll only ever + // have one record on the server. This test just checks that we don't + // track or upload prefs changed during the sync. + let guid = CommonUtils.encodeBase64URL(Services.appinfo.ID); + let remoteRec = new PrefRec("prefs", guid); + remoteRec.value = { + [TEST_PREF]: "world", + }; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for prefs added during first sync" + ); + let payloads = collection.payloads(); + equal( + payloads.length, + 1, + "Should not upload multiple prefs records after first sync" + ); + equal( + payloads[0].value[TEST_PREF], + "world", + "Should not upload pref value changed during first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second prefs sync" + ); + payloads = collection.payloads(); + equal( + payloads.length, + 1, + "Should not upload multiple prefs records after second sync" + ); + equal( + payloads[0].value[TEST_PREF], + "hello", + "Should upload changed pref value during second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + Services.prefs.clearUserPref(TEST_PREF); + } +}); + +add_task(async function test_forms_change_during_sync() { + _("Ensure that we don't bump the score when applying form records."); + + enableValidationPrefs(); + + let engine = Service.engineManager.get("forms"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("forms"); + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting local form history entry"); + await FormHistory.update([ + { + op: "add", + fieldname: "favoriteDrink", + value: "cocoa", + }, + ]); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + engine._tracker.start(); + + try { + // Add an existing remote form history entry. We shouldn't bump the score when + // we apply this record. + let remoteRec = new FormRec("forms", "Tl9dHgmJSR6FkyxS"); + remoteRec.name = "name"; + remoteRec.value = "alice"; + collection.insert(remoteRec.id, encryptPayload(remoteRec.cleartext)); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score for forms added during first sync" + ); + + equal( + collection.count(), + 1, + "New local form should not exist on server after first sync" + ); + + await sync_engine_and_validate_telem(engine, true); + strictEqual( + Service.scheduler.globalScore, + 0, + "Should not bump global score during second forms sync" + ); + + equal( + collection.count(), + 2, + "New local form should exist on server after second sync" + ); + } finally { + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); + +add_task(async function test_bookmark_change_during_sync() { + _("Ensure that we track bookmark changes made during a sync."); + + enableValidationPrefs(); + let schedulerProto = Object.getPrototypeOf(Service.scheduler); + let syncThresholdDescriptor = Object.getOwnPropertyDescriptor( + schedulerProto, + "syncThreshold" + ); + Object.defineProperty(Service.scheduler, "syncThreshold", { + // Trigger resync if any changes exist, rather than deciding based on the + // normal sync threshold. + get: () => 0, + }); + + let engine = Service.engineManager.get("bookmarks"); + let server = await serverForEnginesWithKeys({ foo: "password" }, [engine]); + await SyncTestingInfrastructure(server); + + // Already-tracked bookmarks that shouldn't be uploaded during the first sync. + let bzBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.menuGuid, + url: "https://bugzilla.mozilla.org/", + title: "Bugzilla", + }); + _(`Bugzilla GUID: ${bzBmk.guid}`); + + await PlacesTestUtils.setBookmarkSyncFields({ + guid: bzBmk.guid, + syncChangeCounter: 0, + syncStatus: PlacesUtils.bookmarks.SYNC_STATUS.NORMAL, + }); + + let collection = server.user("foo").collection("bookmarks"); + + let bmk3; // New child of Folder 1, created locally during sync. + + let uploadOutgoing = engine._uploadOutgoing; + engine._uploadOutgoing = async function () { + engine._uploadOutgoing = uploadOutgoing; + try { + await uploadOutgoing.call(this); + } finally { + _("Inserting bookmark into local store"); + bmk3 = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "https://mozilla.org/", + title: "Mozilla", + }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + }; + + // New bookmarks that should be uploaded during the first sync. + let folder1 = await PlacesUtils.bookmarks.insert({ + type: PlacesUtils.bookmarks.TYPE_FOLDER, + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + title: "Folder 1", + }); + _(`Folder GUID: ${folder1.guid}`); + + let tbBmk = await PlacesUtils.bookmarks.insert({ + parentGuid: folder1.guid, + url: "http://getthunderbird.com/", + title: "Get Thunderbird!", + }); + _(`Thunderbird GUID: ${tbBmk.guid}`); + + engine._tracker.start(); + + try { + let bmk2_guid = "get-firefox1"; // New child of Folder 1, created remotely. + let folder2_guid = "folder2-1111"; // New folder, created remotely. + let tagQuery_guid = "tag-query111"; // New tag query child of Folder 2, created remotely. + let bmk4_guid = "example-org1"; // New tagged child of Folder 2, created remotely. + { + // An existing record changed on the server that should not trigger + // another sync when applied. + let remoteBzBmk = new Bookmark("bookmarks", bzBmk.guid); + remoteBzBmk.bmkUri = "https://bugzilla.mozilla.org/"; + remoteBzBmk.description = "New description"; + remoteBzBmk.title = "Bugzilla"; + remoteBzBmk.tags = ["new", "tags"]; + remoteBzBmk.parentName = "Bookmarks Menu"; + remoteBzBmk.parentid = "menu"; + collection.insert(bzBmk.guid, encryptPayload(remoteBzBmk.cleartext)); + + let remoteFolder = new BookmarkFolder("bookmarks", folder2_guid); + remoteFolder.title = "Folder 2"; + remoteFolder.children = [bmk4_guid, tagQuery_guid]; + remoteFolder.parentName = "Bookmarks Menu"; + remoteFolder.parentid = "menu"; + collection.insert(folder2_guid, encryptPayload(remoteFolder.cleartext)); + + let remoteFxBmk = new Bookmark("bookmarks", bmk2_guid); + remoteFxBmk.bmkUri = "http://getfirefox.com/"; + remoteFxBmk.description = "Firefox is awesome."; + remoteFxBmk.title = "Get Firefox!"; + remoteFxBmk.tags = ["firefox", "awesome", "browser"]; + remoteFxBmk.keyword = "awesome"; + remoteFxBmk.parentName = "Folder 1"; + remoteFxBmk.parentid = folder1.guid; + collection.insert(bmk2_guid, encryptPayload(remoteFxBmk.cleartext)); + + // A tag query referencing a nonexistent tag folder, which we should + // create locally when applying the record. + let remoteTagQuery = new BookmarkQuery("bookmarks", tagQuery_guid); + remoteTagQuery.bmkUri = "place:type=7&folder=999"; + remoteTagQuery.title = "Taggy tags"; + remoteTagQuery.folderName = "taggy"; + remoteTagQuery.parentName = "Folder 2"; + remoteTagQuery.parentid = folder2_guid; + collection.insert( + tagQuery_guid, + encryptPayload(remoteTagQuery.cleartext) + ); + + // A bookmark that should appear in the results for the tag query. + let remoteTaggedBmk = new Bookmark("bookmarks", bmk4_guid); + remoteTaggedBmk.bmkUri = "https://example.org/"; + remoteTaggedBmk.title = "Tagged bookmark"; + remoteTaggedBmk.tags = ["taggy"]; + remoteTaggedBmk.parentName = "Folder 2"; + remoteTaggedBmk.parentid = folder2_guid; + collection.insert(bmk4_guid, encryptPayload(remoteTaggedBmk.cleartext)); + + collection.insert( + "toolbar", + encryptPayload({ + id: "toolbar", + type: "folder", + title: "toolbar", + children: [folder1.guid], + parentName: "places", + parentid: "places", + }) + ); + + collection.insert( + "menu", + encryptPayload({ + id: "menu", + type: "folder", + title: "menu", + children: [bzBmk.guid, folder2_guid], + parentName: "places", + parentid: "places", + }) + ); + + collection.insert( + folder1.guid, + encryptPayload({ + id: folder1.guid, + type: "folder", + title: "Folder 1", + children: [bmk2_guid], + parentName: "toolbar", + parentid: "toolbar", + }) + ); + } + + await assertChildGuids( + folder1.guid, + [tbBmk.guid], + "Folder should have 1 child before first sync" + ); + + let pingsPromise = wait_for_pings(2); + + let changes = await PlacesSyncUtils.bookmarks.pullChanges(); + deepEqual( + Object.keys(changes).sort(), + [folder1.guid, tbBmk.guid, "menu", "mobile", "toolbar", "unfiled"].sort(), + "Should track bookmark and folder created before first sync" + ); + + // Unlike the tests above, we can't use `sync_engine_and_validate_telem` + // because the bookmarks engine will automatically schedule a follow-up + // sync for us. + _("Perform first sync and immediate follow-up sync"); + Service.sync({ engines: ["bookmarks"] }); + + let pings = await pingsPromise; + equal(pings.length, 2, "Should submit two pings"); + ok( + pings.every(p => { + assert_success_ping(p); + return p.syncs.length == 1; + }), + "Should submit 1 sync per ping" + ); + + strictEqual( + Service.scheduler.globalScore, + 0, + "Should reset global score after follow-up sync" + ); + ok(bmk3, "Should insert bookmark during first sync to simulate change"); + ok( + collection.wbo(bmk3.guid), + "Changed bookmark should be uploaded after follow-up sync" + ); + + let bmk2 = await PlacesUtils.bookmarks.fetch({ + guid: bmk2_guid, + }); + ok(bmk2, "Remote bookmark should be applied during first sync"); + { + // We only check child GUIDs, and not their order, because the exact + // order is an implementation detail. + let folder1Children = await PlacesSyncUtils.bookmarks.fetchChildRecordIds( + folder1.guid + ); + deepEqual( + folder1Children.sort(), + [bmk2_guid, tbBmk.guid, bmk3.guid].sort(), + "Folder 1 should have 3 children after first sync" + ); + } + await assertChildGuids( + folder2_guid, + [bmk4_guid, tagQuery_guid], + "Folder 2 should have 2 children after first sync" + ); + let taggedURIs = []; + await PlacesUtils.bookmarks.fetch({ tags: ["taggy"] }, b => + taggedURIs.push(b.url) + ); + equal(taggedURIs.length, 1, "Should have 1 tagged URI"); + equal( + taggedURIs[0].href, + "https://example.org/", + "Synced tagged bookmark should appear in tagged URI list" + ); + + changes = await PlacesSyncUtils.bookmarks.pullChanges(); + deepEqual( + changes, + {}, + "Should have already uploaded changes in follow-up sync" + ); + + // First ping won't include validation data, since we've changed bookmarks + // and `canValidate` will indicate it can't proceed. + let engineData = pings.map(p => { + return p.syncs[0].engines.find(e => e.name == "bookmarks-buffered"); + }); + ok(engineData[0].validation, "Engine should validate after first sync"); + ok(engineData[1].validation, "Engine should validate after second sync"); + } finally { + Object.defineProperty( + schedulerProto, + "syncThreshold", + syncThresholdDescriptor + ); + engine._uploadOutgoing = uploadOutgoing; + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_enginemanager.js b/services/sync/tests/unit/test_enginemanager.js new file mode 100644 index 0000000000..3e366be54f --- /dev/null +++ b/services/sync/tests/unit/test_enginemanager.js @@ -0,0 +1,232 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function PetrolEngine() {} +PetrolEngine.prototype.name = "petrol"; +PetrolEngine.prototype.finalize = async function () {}; + +function DieselEngine() {} +DieselEngine.prototype.name = "diesel"; +DieselEngine.prototype.finalize = async function () {}; + +function DummyEngine() {} +DummyEngine.prototype.name = "dummy"; +DummyEngine.prototype.finalize = async function () {}; + +class ActualEngine extends SyncEngine { + constructor(service) { + super("Actual", service); + } +} + +add_task(async function test_basics() { + _("We start out with a clean slate"); + + let manager = new EngineManager(Service); + + let engines = await manager.getAll(); + Assert.equal(engines.length, 0); + Assert.equal(await manager.get("dummy"), undefined); + + _("Register an engine"); + await manager.register(DummyEngine); + let dummy = await manager.get("dummy"); + Assert.ok(dummy instanceof DummyEngine); + + engines = await manager.getAll(); + Assert.equal(engines.length, 1); + Assert.equal(engines[0], dummy); + + _("Register an already registered engine is ignored"); + await manager.register(DummyEngine); + Assert.equal(await manager.get("dummy"), dummy); + + _("Register multiple engines in one go"); + await manager.register([PetrolEngine, DieselEngine]); + let petrol = await manager.get("petrol"); + let diesel = await manager.get("diesel"); + Assert.ok(petrol instanceof PetrolEngine); + Assert.ok(diesel instanceof DieselEngine); + + engines = await manager.getAll(); + Assert.equal(engines.length, 3); + Assert.notEqual(engines.indexOf(petrol), -1); + Assert.notEqual(engines.indexOf(diesel), -1); + + _("Retrieve multiple engines in one go"); + engines = await manager.get(["dummy", "diesel"]); + Assert.equal(engines.length, 2); + Assert.notEqual(engines.indexOf(dummy), -1); + Assert.notEqual(engines.indexOf(diesel), -1); + + _("getEnabled() only returns enabled engines"); + engines = await manager.getEnabled(); + Assert.equal(engines.length, 0); + + petrol.enabled = true; + engines = await manager.getEnabled(); + Assert.equal(engines.length, 1); + Assert.equal(engines[0], petrol); + + dummy.enabled = true; + diesel.enabled = true; + engines = await manager.getEnabled(); + Assert.equal(engines.length, 3); + + _("getEnabled() returns enabled engines in sorted order"); + petrol.syncPriority = 1; + dummy.syncPriority = 2; + diesel.syncPriority = 3; + + engines = await manager.getEnabled(); + + Assert.deepEqual(engines, [petrol, dummy, diesel]); + + _("Changing the priorities should change the order in getEnabled()"); + + dummy.syncPriority = 4; + + engines = await manager.getEnabled(); + + Assert.deepEqual(engines, [petrol, diesel, dummy]); + + _("Unregister an engine by name"); + await manager.unregister("dummy"); + Assert.equal(await manager.get("dummy"), undefined); + engines = await manager.getAll(); + Assert.equal(engines.length, 2); + Assert.equal(engines.indexOf(dummy), -1); + + _("Unregister an engine by value"); + // manager.unregister() checks for instanceof Engine, so let's make one: + await manager.register(ActualEngine); + let actual = await manager.get("actual"); + Assert.ok(actual instanceof ActualEngine); + Assert.ok(actual instanceof SyncEngine); + + await manager.unregister(actual); + Assert.equal(await manager.get("actual"), undefined); +}); + +class AutoEngine { + constructor(type) { + this.name = "automobile"; + this.type = type; + this.initializeCalled = false; + this.finalizeCalled = false; + this.isActive = false; + } + + async initialize() { + Assert.ok(!this.initializeCalled); + Assert.equal(AutoEngine.current, undefined); + this.initializeCalled = true; + this.isActive = true; + AutoEngine.current = this; + } + + async finalize() { + Assert.equal(AutoEngine.current, this); + Assert.ok(!this.finalizeCalled); + Assert.ok(this.isActive); + this.finalizeCalled = true; + this.isActive = false; + AutoEngine.current = undefined; + } +} + +class GasolineEngine extends AutoEngine { + constructor() { + super("gasoline"); + } +} + +class ElectricEngine extends AutoEngine { + constructor() { + super("electric"); + } +} + +add_task(async function test_alternates() { + let manager = new EngineManager(Service); + let engines = await manager.getAll(); + Assert.equal(engines.length, 0); + + const prefName = "services.sync.engines.automobile.electric"; + Services.prefs.clearUserPref(prefName); + + await manager.registerAlternatives( + "automobile", + prefName, + ElectricEngine, + GasolineEngine + ); + + let gasEngine = manager.get("automobile"); + Assert.equal(gasEngine.type, "gasoline"); + + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + Assert.equal(AutoEngine.current, gasEngine); + + _("Check that setting the controlling pref to false makes no difference"); + Services.prefs.setBoolPref(prefName, false); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Even after the call to switchAlternatives"); + await manager.switchAlternatives(); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Set the pref to true, we still shouldn't switch yet"); + Services.prefs.setBoolPref(prefName, true); + Assert.equal(manager.get("automobile"), gasEngine); + Assert.ok(gasEngine.isActive); + Assert.ok(gasEngine.initializeCalled); + Assert.ok(!gasEngine.finalizeCalled); + + _("Now we expect to switch from gas to electric"); + await manager.switchAlternatives(); + let elecEngine = manager.get("automobile"); + Assert.equal(elecEngine.type, "electric"); + Assert.ok(elecEngine.isActive); + Assert.ok(elecEngine.initializeCalled); + Assert.ok(!elecEngine.finalizeCalled); + Assert.equal(AutoEngine.current, elecEngine); + + Assert.ok(!gasEngine.isActive); + Assert.ok(gasEngine.finalizeCalled); + + _("Switch back, and ensure we get a new instance that got initialized again"); + Services.prefs.setBoolPref(prefName, false); + await manager.switchAlternatives(); + + // First make sure we deactivated the electric engine as we should + Assert.ok(!elecEngine.isActive); + Assert.ok(elecEngine.initializeCalled); + Assert.ok(elecEngine.finalizeCalled); + + let newGasEngine = manager.get("automobile"); + Assert.notEqual(newGasEngine, gasEngine); + Assert.equal(newGasEngine.type, "gasoline"); + + Assert.ok(newGasEngine.isActive); + Assert.ok(newGasEngine.initializeCalled); + Assert.ok(!newGasEngine.finalizeCalled); + + _("Make sure unregister removes the alt info too"); + await manager.unregister("automobile"); + Assert.equal(manager.get("automobile"), null); + Assert.ok(newGasEngine.finalizeCalled); + Assert.deepEqual(Object.keys(manager._altEngineInfo), []); +}); diff --git a/services/sync/tests/unit/test_errorhandler_1.js b/services/sync/tests/unit/test_errorhandler_1.js new file mode 100644 index 0000000000..baee3bb9b4 --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_1.js @@ -0,0 +1,339 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +const fakeServer = new SyncServer(); +fakeServer.start(); +const fakeServerUrl = "http://localhost:" + fakeServer.port; + +registerCleanupFunction(function () { + return promiseStopServer(fakeServer).finally(() => { + Svc.Prefs.resetBranch(""); + }); +}); + +let engine; +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(EHTestsCommon.CatapultEngine); + engine = Service.engineManager.get("catapult"); +}); + +async function clean() { + let promiseLogReset = promiseOneObserver("weave:service:reset-file-log"); + await Service.startOver(); + await promiseLogReset; + Status.resetSync(); + Status.resetBackoff(); + // Move log levels back to trace (startOver will have reversed this), sicne + syncTestLogging(); +} + +add_task(async function test_401_logout() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await sync_and_validate_telem(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + let promiseErrors = new Promise(res => { + Svc.Obs.add("weave:service:sync:error", onSyncError); + function onSyncError() { + _("Got weave:service:sync:error in first sync."); + Svc.Obs.remove("weave:service:sync:error", onSyncError); + + // Wait for the automatic next sync. + Svc.Obs.add("weave:service:login:error", onLoginError); + function onLoginError() { + _("Got weave:service:login:error in second sync."); + Svc.Obs.remove("weave:service:login:error", onLoginError); + res(); + } + } + }); + + // Make sync fail due to login rejected. + await configureIdentity({ username: "janedoe" }, server); + Service._updateCachedURLs(); + + _("Starting first sync."); + await sync_and_validate_telem(ping => { + deepEqual(ping.failureReason, { name: "httperror", code: 401 }); + }); + _("First sync done."); + + await promiseErrors; + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + Assert.ok(!Service.isLoggedIn); + + // Clean up. + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_credentials_changed_logout() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await sync_and_validate_telem(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, CREDENTIALS_CHANGED); + deepEqual(ping.failureReason, { + name: "unexpectederror", + error: "Error: Aborting sync, remote setup failed", + }); + }); + + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + Assert.ok(!Service.isLoggedIn); + + // Clean up. + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_login_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + // when calling sync + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + Service.identity._syncKeyBundle = null; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_sync_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + // when calling sync + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, CREDENTIALS_CHANGED); + deepEqual(ping.failureReason, { + name: "unexpectederror", + error: "Error: Aborting sync, remote setup failed", + }); + }); + + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + // If we clean this tick, telemetry won't get the right error + await Async.promiseYield(); + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_login_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are reported when calling sync. + await configureIdentity({ username: "broken.wipe" }); + Service.clusterURL = fakeServerUrl; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + await clean(); +}); + +add_task(async function test_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are reported when calling sync. + Services.io.offline = true; + + await Service.sync(); + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_login_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + Service.identity._syncKeyBundle = null; + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_sync_non_network_error() { + enableValidationPrefs(); + + // Test non-network errors are reported + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + // By calling sync, we ensure we're logged in. + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + await EHTestsCommon.generateCredentialsChangedFailure(); + + await Service.sync(); + Assert.equal(Status.sync, CREDENTIALS_CHANGED); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_login_network_error() { + enableValidationPrefs(); + + await configureIdentity({ username: "johndoe" }); + Service.clusterURL = fakeServerUrl; + + // Test network errors are not reported. + + await Service.sync(); + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_sync_network_error() { + enableValidationPrefs(); + + // Test network errors are not reported. + Services.io.offline = true; + + await Service.sync(); + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + + Services.io.offline = false; + await clean(); +}); + +add_task(async function test_sync_server_maintenance_error() { + enableValidationPrefs(); + + // Test server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + const BACKOFF = 42; + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + Assert.equal(Status.service, STATUS_OK); + + await sync_and_validate_telem(ping => { + equal(ping.status.sync, SERVER_MAINTENANCE); + deepEqual(ping.engines.find(e => e.failureReason).failureReason, { + name: "httperror", + code: 503, + }); + }); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_info_collections_login_server_maintenance_error() { + enableValidationPrefs(); + + // Test info/collections server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.info" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_meta_global_login_server_maintenance_error() { + enableValidationPrefs(); + + // Test meta/global server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.meta" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_errorhandler_2.js b/services/sync/tests/unit/test_errorhandler_2.js new file mode 100644 index 0000000000..eaa926777a --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_2.js @@ -0,0 +1,547 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const fakeServer = new SyncServer(); +fakeServer.start(); + +registerCleanupFunction(function () { + return promiseStopServer(fakeServer).finally(() => { + Svc.Prefs.resetBranch(""); + }); +}); + +const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true); + +function removeLogFiles() { + let entries = logsdir.directoryEntries; + while (entries.hasMoreElements()) { + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + logfile.remove(false); + } +} + +function getLogFiles() { + let result = []; + let entries = logsdir.directoryEntries; + while (entries.hasMoreElements()) { + result.push(entries.getNext().QueryInterface(Ci.nsIFile)); + } + return result; +} + +let engine; +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(EHTestsCommon.CatapultEngine); + engine = Service.engineManager.get("catapult"); +}); + +async function clean() { + let promiseLogReset = promiseOneObserver("weave:service:reset-file-log"); + await Service.startOver(); + await promiseLogReset; + Status.resetSync(); + Status.resetBackoff(); + removeLogFiles(); + // Move log levels back to trace (startOver will have reversed this), sicne + syncTestLogging(); +} + +add_task(async function test_crypto_keys_login_server_maintenance_error() { + enableValidationPrefs(); + + Status.resetSync(); + // Test crypto/keys server maintenance errors are not reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.keys" }, server); + + // Force re-download of keys + Service.collectionKeys.clear(); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); +}); + +add_task(async function test_lastSync_not_updated_on_complete_failure() { + enableValidationPrefs(); + + // Test info/collections prolonged server maintenance errors are reported. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "johndoe" }, server); + + // Do an initial sync that we expect to be successful. + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await sync_and_validate_telem(); + await promiseObserved; + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + let lastSync = Svc.Prefs.get("lastSync"); + + Assert.ok(lastSync); + + // Report server maintenance on info/collections requests + server.registerPathHandler( + "/1.1/johndoe/info/collections", + EHTestsCommon.service_unavailable + ); + + promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await sync_and_validate_telem(() => {}); + await promiseObserved; + + Assert.equal(Status.sync, SERVER_MAINTENANCE); + Assert.equal(Status.service, SYNC_FAILED); + + // We shouldn't update lastSync on complete failure. + Assert.equal(lastSync, Svc.Prefs.get("lastSync")); + + await clean(); + await promiseStopServer(server); +}); + +add_task( + async function test_sync_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + const BACKOFF = 42; + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_info_collections_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test info/collections server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.info" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_meta_global_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test meta/global server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.meta" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + await EHTestsCommon.setUp(server); + + await configureIdentity({ username: "broken.keys" }, server); + // Force re-download of keys + Service.collectionKeys.clear(); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + + // Start off with an empty account, do not upload a key. + await configureIdentity({ username: "broken.keys" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test crypto/keys server maintenance errors are reported + // when calling syncAndReportErrors. + let server = await EHTestsCommon.sync_httpd_setup(); + + // Start off with an empty account, do not upload a key. + await configureIdentity({ username: "broken.wipe" }, server); + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, LOGIN_FAILED); + Assert.equal(Status.login, SERVER_MAINTENANCE); + + await clean(); + await promiseStopServer(server); + } +); + +add_task( + async function test_wipeRemote_syncAndReportErrors_server_maintenance_error() { + enableValidationPrefs(); + + // Test that we report prolonged server maintenance errors that occur whilst + // wiping all remote devices. + let server = await EHTestsCommon.sync_httpd_setup(); + + await configureIdentity({ username: "broken.wipe" }, server); + await EHTestsCommon.generateAndUploadKeys(); + + engine.exception = null; + engine.enabled = true; + + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.service, STATUS_OK); + + Svc.Prefs.set("firstSync", "wipeRemote"); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + await Service.sync(); + await promiseObserved; + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Status.service, SYNC_FAILED); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + Assert.equal(Svc.Prefs.get("firstSync"), "wipeRemote"); + + await clean(); + await promiseStopServer(server); + } +); + +add_task(async function test_sync_engine_generic_fail() { + enableValidationPrefs(); + + equal(getLogFiles().length, 0); + + let server = await EHTestsCommon.sync_httpd_setup(); + engine.enabled = true; + engine.sync = async function sync() { + Svc.Obs.notify("weave:engine:sync:error", ENGINE_UNKNOWN_FAIL, "catapult"); + }; + let lastSync = Svc.Prefs.get("lastSync"); + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.Prefs.set("log.appender.file.logOnError", true); + + Assert.equal(Status.engines.catapult, undefined); + + let promiseObserved = new Promise(res => { + Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() { + Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish); + + log.info("Adding reset-file-log observer."); + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + res(); + }); + }); + }); + + Assert.ok(await EHTestsCommon.setUp(server)); + await sync_and_validate_telem(ping => { + deepEqual(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(e => e.status).status, ENGINE_UNKNOWN_FAIL); + }); + + await promiseObserved; + + _("Status.engines: " + JSON.stringify(Status.engines)); + Assert.equal(Status.engines.catapult, ENGINE_UNKNOWN_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // lastSync should update on partial failure. + Assert.notEqual(lastSync, Svc.Prefs.get("lastSync")); + + // Test Error log was written on SYNC_FAILED_PARTIAL. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); + + await promiseStopServer(server); +}); + +add_task(async function test_logs_on_sync_error() { + enableValidationPrefs(); + + _( + "Ensure that an error is still logged when weave:service:sync:error " + + "is notified, despite shouldReportError returning false." + ); + + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.Prefs.set("log.appender.file.logOnError", true); + log.info("TESTING"); + + // Ensure that we report no error. + Status.login = MASTER_PASSWORD_LOCKED; + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + Svc.Obs.notify("weave:service:sync:error", {}); + await promiseObserved; + + // Test that error log was written. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); +}); + +add_task(async function test_logs_on_login_error() { + enableValidationPrefs(); + + _( + "Ensure that an error is still logged when weave:service:login:error " + + "is notified, despite shouldReportError returning false." + ); + + let log = Log.repository.getLogger("Sync.ErrorHandler"); + Svc.Prefs.set("log.appender.file.logOnError", true); + log.info("TESTING"); + + // Ensure that we report no error. + Status.login = MASTER_PASSWORD_LOCKED; + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + Svc.Obs.notify("weave:service:login:error", {}); + await promiseObserved; + + // Test that error log was written. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); +}); + +// This test should be the last one since it monkeypatches the engine object +// and we should only have one engine object throughout the file (bug 629664). +add_task(async function test_engine_applyFailed() { + enableValidationPrefs(); + + let server = await EHTestsCommon.sync_httpd_setup(); + + engine.enabled = true; + delete engine.exception; + engine.sync = async function sync() { + Svc.Obs.notify("weave:engine:sync:applied", { newFailed: 1 }, "catapult"); + }; + + Svc.Prefs.set("log.appender.file.logOnError", true); + + let promiseObserved = promiseOneObserver("weave:service:reset-file-log"); + + Assert.equal(Status.engines.catapult, undefined); + Assert.ok(await EHTestsCommon.setUp(server)); + await Service.sync(); + await promiseObserved; + + Assert.equal(Status.engines.catapult, ENGINE_APPLY_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // Test Error log was written on SYNC_FAILED_PARTIAL. + let logFiles = getLogFiles(); + equal(logFiles.length, 1); + Assert.ok( + logFiles[0].leafName.startsWith("error-sync-"), + logFiles[0].leafName + ); + + await clean(); + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_errorhandler_filelog.js b/services/sync/tests/unit/test_errorhandler_filelog.js new file mode 100644 index 0000000000..5e1e36061f --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_filelog.js @@ -0,0 +1,450 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// `Service` is used as a global in head_helpers.js. +// eslint-disable-next-line no-unused-vars +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { logManager } = ChromeUtils.import( + "resource://gre/modules/FxAccountsCommon.js" +); +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true); + +// Delay to wait before cleanup, to allow files to age. +// This is so large because the file timestamp granularity is per-second, and +// so otherwise we can end up with all of our files -- the ones we want to +// keep, and the ones we want to clean up -- having the same modified time. +const CLEANUP_DELAY = 2000; +const DELAY_BUFFER = 500; // Buffer for timers on different OS platforms. + +function run_test() { + validate_all_future_pings(); + run_next_test(); +} + +add_test(function test_noOutput() { + // Ensure that the log appender won't print anything. + logManager._fileAppender.level = Log.Level.Fatal + 1; + + // Clear log output from startup. + Svc.Prefs.set("log.appender.file.logOnSuccess", false); + Svc.Obs.notify("weave:service:sync:finish"); + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogOuter() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogOuter); + // Clear again without having issued any output. + Svc.Prefs.set("log.appender.file.logOnSuccess", true); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLogInner() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLogInner); + + logManager._fileAppender.level = Log.Level.Trace; + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); + }); +}); + +add_test(function test_logOnSuccess_false() { + Svc.Prefs.set("log.appender.file.logOnSuccess", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); +}); + +function readFile(file, callback) { + NetUtil.asyncFetch( + { + uri: NetUtil.newURI(file), + loadUsingSystemPrincipal: true, + }, + function (inputStream, statusCode, request) { + let data = NetUtil.readInputStreamToString( + inputStream, + inputStream.available() + ); + callback(statusCode, data); + } + ); +} + +add_test(function test_logOnSuccess_true() { + Svc.Prefs.set("log.appender.file.logOnSuccess", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("success-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + }); + + // Fake a successful sync. + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_sync_error_logOnError_false() { + Svc.Prefs.set("log.appender.file.logOnError", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +add_test(function test_sync_error_logOnError_true() { + Svc.Prefs.set("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +add_test(function test_login_error_logOnError_false() { + Svc.Prefs.set("log.appender.file.logOnError", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + log.info("this won't show up"); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + + // Fake an unsuccessful login. + Svc.Obs.notify("weave:service:login:error"); +}); + +add_test(function test_login_error_logOnError_true() { + Svc.Prefs.set("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + }); + + // Fake an unsuccessful login. + Svc.Obs.notify("weave:service:login:error"); +}); + +add_test(function test_noNewFailed_noErrorLog() { + Svc.Prefs.set("log.appender.file.logOnError", true); + Svc.Prefs.set("log.appender.file.logOnSuccess", false); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + // No log file was written. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + // failed is nonzero and newFailed is zero -- shouldn't write a log. + let count = { + applied: 8, + succeeded: 4, + failed: 5, + newFailed: 0, + reconciled: 4, + }; + Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine"); + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_newFailed_errorLog() { + Svc.Prefs.set("log.appender.file.logOnError", true); + Svc.Prefs.set("log.appender.file.logOnSuccess", false); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up 2"; + log.info(MESSAGE); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + // Exactly one log file was written. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure the log message was actually written to file. + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf(MESSAGE), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + }); + // newFailed is nonzero -- should write a log. + let count = { + applied: 8, + succeeded: 4, + failed: 5, + newFailed: 4, + reconciled: 4, + }; + + Svc.Obs.notify("weave:engine:sync:applied", count, "foobar-engine"); + Svc.Obs.notify("weave:service:sync:finish"); +}); + +add_test(function test_errorLog_dumpAddons() { + Svc.Prefs.set("log.logger", "Trace"); + Svc.Prefs.set("log.appender.file.logOnError", true); + + Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() { + Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog); + + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.equal(logfile.leafName.slice(-4), ".txt"); + Assert.ok(logfile.leafName.startsWith("error-sync-"), logfile.leafName); + Assert.ok(!entries.hasMoreElements()); + + // Ensure we logged some addon list (which is probably empty) + readFile(logfile, function (error, data) { + Assert.ok(Components.isSuccessCode(error)); + Assert.notEqual(data.indexOf("Addons installed"), -1); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + }); + }); + + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); +}); + +// Check that error log files are deleted above an age threshold. +add_test(async function test_logErrorCleanup_age() { + _("Beginning test_logErrorCleanup_age."); + let maxAge = CLEANUP_DELAY / 1000; + let oldLogs = []; + let numLogs = 10; + let errString = "some error log\n"; + + Svc.Prefs.set("log.appender.file.logOnError", true); + Svc.Prefs.set("log.appender.file.maxErrorAge", maxAge); + + _("Making some files."); + const logsDir = PathUtils.join(PathUtils.profileDir, "weave", "logs"); + await IOUtils.makeDirectory(logsDir); + for (let i = 0; i < numLogs; i++) { + let now = Date.now(); + let filename = "error-sync-" + now + "" + i + ".txt"; + let newLog = new FileUtils.File(PathUtils.join(logsDir, filename)); + let foStream = FileUtils.openFileOutputStream(newLog); + foStream.write(errString, errString.length); + foStream.close(); + _(" > Created " + filename); + oldLogs.push(newLog.leafName); + } + + Svc.Obs.add( + "services-tests:common:log-manager:cleanup-logs", + function onCleanupLogs() { + Svc.Obs.remove( + "services-tests:common:log-manager:cleanup-logs", + onCleanupLogs + ); + + // Only the newest created log file remains. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + let logfile = entries.getNext().QueryInterface(Ci.nsIFile); + Assert.ok( + oldLogs.every(function (e) { + return e != logfile.leafName; + }) + ); + Assert.ok(!entries.hasMoreElements()); + + // Clean up. + try { + logfile.remove(false); + } catch (ex) { + dump("Couldn't delete file: " + ex.message + "\n"); + // Stupid Windows box. + } + + Svc.Prefs.resetBranch(""); + run_next_test(); + } + ); + + let delay = CLEANUP_DELAY + DELAY_BUFFER; + + _("Cleaning up logs after " + delay + "msec."); + CommonUtils.namedTimer( + function onTimer() { + Svc.Obs.notify("weave:service:sync:error"); + }, + delay, + this, + "cleanup-timer" + ); +}); + +add_task(async function test_remove_log_on_startOver() { + Svc.Prefs.set("log.appender.file.logOnError", true); + + let log = Log.repository.getLogger("Sync.Test.FileLog"); + const MESSAGE = "this WILL show up"; + log.info(MESSAGE); + + let promiseLogWritten = promiseOneObserver("weave:service:reset-file-log"); + // Fake an unsuccessful sync. + Svc.Obs.notify("weave:service:sync:error"); + + await promiseLogWritten; + // Should have at least 1 log file. + let entries = logsdir.directoryEntries; + Assert.ok(entries.hasMoreElements()); + + // Fake a reset. + let promiseRemoved = promiseOneObserver("weave:service:remove-file-log"); + Svc.Obs.notify("weave:service:start-over:finish"); + await promiseRemoved; + + // should be no files left. + Assert.ok(!logsdir.directoryEntries.hasMoreElements()); +}); diff --git a/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js b/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js new file mode 100644 index 0000000000..d73d548cc7 --- /dev/null +++ b/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js @@ -0,0 +1,294 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { FakeCryptoService } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/fakeservices.sys.mjs" +); + +var engineManager = Service.engineManager; + +function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); +} +CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + throw this.exception; + }, +}; +Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + +async function sync_httpd_setup() { + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let catapultEngine = engineManager.get("catapult"); + let syncID = await catapultEngine.resetLocalSyncID(); + let engines = { catapult: { version: catapultEngine.version, syncID } }; + + // Track these using the collections helper, which keeps modified times + // up-to-date. + let clientsColl = new ServerCollection({}, true); + let keysWBO = new ServerWBO("keys"); + let globalWBO = new ServerWBO("global", { + storageVersion: STORAGE_VERSION, + syncID: Utils.makeGUID(), + engines, + }); + + let handlers = { + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/meta/global": upd("meta", globalWBO.handler()), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + }; + return httpd_setup(handlers); +} + +async function setUp(server) { + await configureIdentity({ username: "johndoe" }, server); + new FakeCryptoService(); + syncTestLogging(); +} + +async function generateAndUploadKeys(server) { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let res = Service.resource( + server.baseURI + "/1.1/johndoe/storage/crypto/keys" + ); + return (await serverKeys.upload(res)).success; +} + +add_task(async function setup() { + await engineManager.clear(); + validate_all_future_pings(); + await engineManager.register(CatapultEngine); +}); + +add_task(async function test_backoff500() { + enableValidationPrefs(); + + _("Test: HTTP 500 sets backoff status."); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 500 }; + + try { + Assert.ok(!Status.enforceBackoff); + + // Forcibly create and upload keys here -- otherwise we don't get to the 500! + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + Assert.ok(Status.enforceBackoff); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetBackoff(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_backoff503() { + enableValidationPrefs(); + + _( + "Test: HTTP 503 with Retry-After header leads to backoff notification and sets backoff status." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + const BACKOFF = 42; + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 503, headers: { "retry-after": BACKOFF } }; + + let backoffInterval; + Svc.Obs.add("weave:service:backoff:interval", function (subject) { + backoffInterval = subject; + }); + + try { + Assert.ok(!Status.enforceBackoff); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.equal(backoffInterval, BACKOFF); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(Status.sync, SERVER_MAINTENANCE); + } finally { + Status.resetBackoff(); + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_overQuota() { + enableValidationPrefs(); + + _("Test: HTTP 400 with body error code 14 means over quota."); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { + status: 400, + toString() { + return "14"; + }, + }; + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, OVER_QUOTA); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_service_networkError() { + enableValidationPrefs(); + + _( + "Test: Connection refused error from Service.sync() leads to the right status code." + ); + let server = await sync_httpd_setup(); + await setUp(server); + await promiseStopServer(server); + // Provoke connection refused. + Service.clusterURL = "http://localhost:12345/"; + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Service._loggedIn = true; + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED); + } finally { + Status.resetSync(); + await Service.startOver(); + } +}); + +add_task(async function test_service_offline() { + enableValidationPrefs(); + + _( + "Test: Wanting to sync in offline mode leads to the right status code but does not increment the ignorable error count." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + await promiseStopServer(server); + Services.io.offline = true; + Services.prefs.setBoolPref("network.dns.offline-localhost", false); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Service._loggedIn = true; + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED); + } finally { + Status.resetSync(); + await Service.startOver(); + } + Services.io.offline = false; + Services.prefs.clearUserPref("network.dns.offline-localhost"); +}); + +add_task(async function test_engine_networkError() { + enableValidationPrefs(); + + _( + "Test: Network related exceptions from engine.sync() lead to the right status code." + ); + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + engine.exception = Components.Exception( + "NS_ERROR_UNKNOWN_HOST", + Cr.NS_ERROR_UNKNOWN_HOST + ); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); + +add_task(async function test_resource_timeout() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + let engine = engineManager.get("catapult"); + engine.enabled = true; + // Resource throws this when it encounters a timeout. + engine.exception = Components.Exception( + "Aborting due to channel inactivity.", + Cr.NS_ERROR_NET_TIMEOUT + ); + + try { + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await generateAndUploadKeys(server)); + + await Service.login(); + await Service.sync(); + + Assert.equal(Status.sync, LOGIN_FAILED_NETWORK_ERROR); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + } finally { + Status.resetSync(); + await Service.startOver(); + } + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_extension_storage_engine.js b/services/sync/tests/unit/test_extension_storage_engine.js new file mode 100644 index 0000000000..a061812aca --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_engine.js @@ -0,0 +1,275 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +ChromeUtils.defineESModuleGetters(this, { + Service: "resource://services-sync/service.sys.mjs", + extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.sys.mjs", +}); + +const { ExtensionStorageEngineBridge, ExtensionStorageEngineKinto } = + ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" + ); + +const { BridgeWrapperXPCOM } = ChromeUtils.importESModule( + "resource://services-sync/bridged_engine.sys.mjs" +); + +Services.prefs.setStringPref("webextensions.storage.sync.log.level", "debug"); + +add_task(async function test_switching_between_kinto_and_bridged() { + function assertUsingKinto(message) { + let kintoEngine = Service.engineManager.get("extension-storage"); + Assert.ok(kintoEngine instanceof ExtensionStorageEngineKinto, message); + } + function assertUsingBridged(message) { + let bridgedEngine = Service.engineManager.get("extension-storage"); + Assert.ok(bridgedEngine instanceof ExtensionStorageEngineBridge, message); + } + + let isUsingKinto = Services.prefs.getBoolPref( + "webextensions.storage.sync.kinto", + false + ); + if (isUsingKinto) { + assertUsingKinto("Should use Kinto engine before flipping pref"); + } else { + assertUsingBridged("Should use bridged engine before flipping pref"); + } + + _("Flip pref"); + Services.prefs.setBoolPref("webextensions.storage.sync.kinto", !isUsingKinto); + await Service.engineManager.switchAlternatives(); + + if (isUsingKinto) { + assertUsingBridged("Should use bridged engine after flipping pref"); + } else { + assertUsingKinto("Should use Kinto engine after flipping pref"); + } + + _("Clean up"); + Services.prefs.clearUserPref("webextensions.storage.sync.kinto"); + await Service.engineManager.switchAlternatives(); +}); + +add_task(async function test_enable() { + const PREF = "services.sync.engine.extension-storage.force"; + + let addonsEngine = Service.engineManager.get("addons"); + let extensionStorageEngine = Service.engineManager.get("extension-storage"); + + try { + Assert.ok( + addonsEngine.enabled, + "Add-ons engine should be enabled by default" + ); + Assert.ok( + extensionStorageEngine.enabled, + "Extension storage engine should be enabled by default" + ); + + addonsEngine.enabled = false; + Assert.ok( + !extensionStorageEngine.enabled, + "Disabling add-ons should disable extension storage" + ); + + extensionStorageEngine.enabled = true; + Assert.ok( + !extensionStorageEngine.enabled, + "Enabling extension storage without override pref shouldn't work" + ); + + Services.prefs.setBoolPref(PREF, true); + Assert.ok( + extensionStorageEngine.enabled, + "Setting override pref should enable extension storage" + ); + + extensionStorageEngine.enabled = false; + Assert.ok( + !extensionStorageEngine.enabled, + "Disabling extension storage engine with override pref should work" + ); + + extensionStorageEngine.enabled = true; + Assert.ok( + extensionStorageEngine.enabled, + "Enabling extension storage with override pref should work" + ); + } finally { + addonsEngine.enabled = true; + Services.prefs.clearUserPref(PREF); + } +}); + +add_task(async function test_notifyPendingChanges() { + let engine = new ExtensionStorageEngineBridge(Service); + + let extension = { id: "ext-1" }; + let expectedChange = { + a: "b", + c: "d", + }; + + let lastSync = 0; + let syncID = Utils.makeGUID(); + let error = null; + engine.component = { + QueryInterface: ChromeUtils.generateQI([ + "mozIBridgedSyncEngine", + "mozIExtensionStorageArea", + "mozISyncedExtensionStorageArea", + ]), + ensureCurrentSyncId(id, callback) { + if (syncID != id) { + syncID = id; + lastSync = 0; + } + callback.handleSuccess(id); + }, + resetSyncId(callback) { + callback.handleSuccess(syncID); + }, + syncStarted(callback) { + callback.handleSuccess(); + }, + getLastSync(callback) { + callback.handleSuccess(lastSync); + }, + setLastSync(lastSyncMillis, callback) { + lastSync = lastSyncMillis; + callback.handleSuccess(); + }, + apply(callback) { + callback.handleSuccess([]); + }, + fetchPendingSyncChanges(callback) { + if (error) { + callback.handleError(Cr.NS_ERROR_FAILURE, error.message); + } else { + callback.onChanged(extension.id, JSON.stringify(expectedChange)); + callback.handleSuccess(); + } + }, + setUploaded(modified, ids, callback) { + callback.handleSuccess(); + }, + syncFinished(callback) { + callback.handleSuccess(); + }, + takeMigrationInfo(callback) { + callback.handleSuccess(null); + }, + }; + + engine._bridge = new BridgeWrapperXPCOM(engine.component); + + let server = await serverForFoo(engine); + + let actualChanges = []; + let listener = changes => actualChanges.push(changes); + extensionStorageSync.addOnChangedListener(extension, listener); + + try { + await SyncTestingInfrastructure(server); + + info("Sync engine; notify about changes"); + await sync_engine_and_validate_telem(engine, false); + deepEqual( + actualChanges, + [expectedChange], + "Should notify about changes during sync" + ); + + error = new Error("oops!"); + actualChanges = []; + await sync_engine_and_validate_telem(engine, false); + deepEqual( + actualChanges, + [], + "Should finish syncing even if notifying about changes fails" + ); + } finally { + extensionStorageSync.removeOnChangedListener(extension, listener); + await promiseStopServer(server); + await engine.finalize(); + } +}); + +// It's difficult to know what to test - there's already tests for the bridged +// engine etc - so we just try and check that this engine conforms to the +// mozIBridgedSyncEngine interface guarantees. +add_task(async function test_engine() { + // Forcibly set the bridged engine in the engine manager. the reason we do + // this, unlike the other tests where we just create the engine, is so that + // telemetry can get at the engine's `overrideTelemetryName`, which it gets + // through the engine manager. + await Service.engineManager.unregister("extension-storage"); + await Service.engineManager.register(ExtensionStorageEngineBridge); + let engine = Service.engineManager.get("extension-storage"); + Assert.equal(engine.version, 1); + + Assert.deepEqual(await engine.getSyncID(), null); + await engine.resetLocalSyncID(); + Assert.notEqual(await engine.getSyncID(), null); + + Assert.equal(await engine.getLastSync(), 0); + // lastSync is seconds on this side of the world, but milli-seconds on the other. + await engine.setLastSync(1234.567); + // should have 2 digit precision. + Assert.equal(await engine.getLastSync(), 1234.57); + await engine.setLastSync(0); + + // Set some data. + await extensionStorageSync.set({ id: "ext-2" }, { ext_2_key: "ext_2_value" }); + // Now do a sync with out regular test server. + let server = await serverForFoo(engine); + try { + await SyncTestingInfrastructure(server); + + info("Add server records"); + let foo = server.user("foo"); + let collection = foo.collection("extension-storage"); + let now = new_timestamp(); + + collection.insert( + "fakeguid0000", + encryptPayload({ + id: "fakeguid0000", + extId: "ext-1", + data: JSON.stringify({ foo: "bar" }), + }), + now + ); + + info("Sync the engine"); + + let ping = await sync_engine_and_validate_telem(engine, false); + Assert.ok(ping.engines.find(e => e.name == "rust-webext-storage")); + Assert.equal( + ping.engines.find(e => e.name == "extension-storage"), + null + ); + + // We should have applied the data from the existing collection record. + Assert.deepEqual(await extensionStorageSync.get({ id: "ext-1" }, null), { + foo: "bar", + }); + + // should now be 2 records on the server. + let payloads = collection.payloads(); + Assert.equal(payloads.length, 2); + // find the new one we wrote. + let newPayload = + payloads[0].id == "fakeguid0000" ? payloads[1] : payloads[0]; + Assert.equal(newPayload.data, `{"ext_2_key":"ext_2_value"}`); + // should have updated the timestamp. + greater(await engine.getLastSync(), 0, "Should update last sync time"); + } finally { + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_extension_storage_engine_kinto.js b/services/sync/tests/unit/test_extension_storage_engine_kinto.js new file mode 100644 index 0000000000..b074fe376c --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_engine_kinto.js @@ -0,0 +1,136 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true); + +const { ExtensionStorageEngineKinto: ExtensionStorageEngine } = + ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" + ); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { extensionStorageSyncKinto: extensionStorageSync } = + ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" + ); + +let engine; + +function mock(options) { + let calls = []; + let ret = function () { + calls.push(arguments); + return options.returns; + }; + let proto = { + get calls() { + return calls; + }, + }; + Object.setPrototypeOf(proto, Function.prototype); + Object.setPrototypeOf(ret, proto); + return ret; +} + +function setSkipChance(v) { + Services.prefs.setIntPref( + "services.sync.extension-storage.skipPercentageChance", + v + ); +} + +add_task(async function setup() { + await Service.engineManager.register(ExtensionStorageEngine); + engine = Service.engineManager.get("extension-storage"); + do_get_profile(); // so we can use FxAccounts + loadWebExtensionTestFunctions(); + setSkipChance(0); +}); + +add_task(async function test_calling_sync_calls__sync() { + let oldSync = ExtensionStorageEngine.prototype._sync; + let syncMock = (ExtensionStorageEngine.prototype._sync = mock({ + returns: true, + })); + try { + // I wanted to call the main sync entry point for the entire + // package, but that fails because it tries to sync ClientEngine + // first, which fails. + await engine.sync(); + } finally { + ExtensionStorageEngine.prototype._sync = oldSync; + } + equal(syncMock.calls.length, 1); +}); + +add_task(async function test_sync_skip() { + try { + // Do a few times to ensure we aren't getting "lucky" WRT Math.random() + for (let i = 0; i < 10; ++i) { + setSkipChance(100); + engine._tracker._score = 0; + ok( + !engine.shouldSkipSync("user"), + "Should allow explicitly requested syncs" + ); + ok(!engine.shouldSkipSync("startup"), "Should allow startup syncs"); + ok( + engine.shouldSkipSync("schedule"), + "Should skip scheduled syncs if skipProbability is 100" + ); + engine._tracker._score = MULTI_DEVICE_THRESHOLD; + ok( + !engine.shouldSkipSync("schedule"), + "should allow scheduled syncs if tracker score is high" + ); + engine._tracker._score = 0; + setSkipChance(0); + ok( + !engine.shouldSkipSync("schedule"), + "Should allow scheduled syncs if probability is 0" + ); + } + } finally { + engine._tracker._score = 0; + setSkipChance(0); + } +}); + +add_task(async function test_calling_wipeClient_calls_clearAll() { + let oldClearAll = extensionStorageSync.clearAll; + let clearMock = (extensionStorageSync.clearAll = mock({ + returns: Promise.resolve(), + })); + try { + await engine.wipeClient(); + } finally { + extensionStorageSync.clearAll = oldClearAll; + } + equal(clearMock.calls.length, 1); +}); + +add_task(async function test_calling_sync_calls_ext_storage_sync() { + const extension = { id: "my-extension" }; + let oldSync = extensionStorageSync.syncAll; + let syncMock = (extensionStorageSync.syncAll = mock({ + returns: Promise.resolve(), + })); + try { + await withSyncContext(async function (context) { + // Set something so that everyone knows that we're using storage.sync + await extensionStorageSync.set(extension, { a: "b" }, context); + let ping = await sync_engine_and_validate_telem(engine, false); + Assert.ok(ping.engines.find(e => e.name == "extension-storage")); + Assert.equal( + ping.engines.find(e => e.name == "rust-webext-storage"), + null + ); + }); + } finally { + extensionStorageSync.syncAll = oldSync; + } + Assert.ok(syncMock.calls.length >= 1); +}); diff --git a/services/sync/tests/unit/test_extension_storage_migration_telem.js b/services/sync/tests/unit/test_extension_storage_migration_telem.js new file mode 100644 index 0000000000..a4b4c95f55 --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_migration_telem.js @@ -0,0 +1,81 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Import the rust-based and kinto-based implementations. Not great to grab +// these as they're somewhat private, but we want to run the pings through our +// validation machinery which is here in the sync test code. +const { extensionStorageSync: rustImpl } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSync.sys.mjs" +); +const { extensionStorageSyncKinto: kintoImpl } = ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" +); + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { ExtensionStorageEngineBridge } = ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" +); + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", false); +Services.prefs.setStringPref("webextensions.storage.sync.log.level", "debug"); + +// It's tricky to force error cases here (the databases are opened with +// exclusive locks) and that part of the code has coverage in the vendored +// application-services webext-storage crate. So this just tests that the +// migration data ends up in the ping, and exactly once. +add_task(async function test_sync_migration_telem() { + // Set some stuff using the kinto-based impl prior to fully setting up sync. + let e1 = { id: "test@mozilla.com" }; + let c1 = { extension: e1, callOnClose() {} }; + + let e2 = { id: "test-2@mozilla.com" }; + let c2 = { extension: e2, callOnClose() {} }; + await kintoImpl.set(e1, { foo: "bar" }, c1); + await kintoImpl.set(e1, { baz: "quux" }, c1); + await kintoImpl.set(e2, { second: "2nd" }, c2); + + Assert.deepEqual(await rustImpl.get(e1, "foo", c1), { foo: "bar" }); + Assert.deepEqual(await rustImpl.get(e1, "baz", c1), { baz: "quux" }); + Assert.deepEqual(await rustImpl.get(e2, null, c2), { second: "2nd" }); + + // Explicitly unregister first. It's very possible this isn't needed for this + // case, however it's fairly harmless, we hope to uplift this patch to beta, + // and earlier today we had beta-only problems caused by this (bug 1629116) + await Service.engineManager.unregister("extension-storage"); + await Service.engineManager.register(ExtensionStorageEngineBridge); + let engine = Service.engineManager.get("extension-storage"); + let server = await serverForFoo(engine, undefined); + try { + await SyncTestingInfrastructure(server); + await Service.engineManager.switchAlternatives(); + + _("First sync"); + let ping = await sync_engine_and_validate_telem(engine, false, null, true); + Assert.deepEqual(ping.migrations, [ + { + type: "webext-storage", + entries: 3, + entriesSuccessful: 3, + extensions: 2, + extensionsSuccessful: 2, + openFailure: false, + }, + ]); + + // force another sync + await engine.setLastSync(0); + _("Second sync"); + + ping = await sync_engine_and_validate_telem(engine, false, null, true); + Assert.deepEqual(ping.migrations, undefined); + } finally { + await kintoImpl.clear(e1, c1); + await kintoImpl.clear(e2, c2); + await rustImpl.clear(e1, c1); + await rustImpl.clear(e2, c2); + await promiseStopServer(server); + await engine.finalize(); + } +}); diff --git a/services/sync/tests/unit/test_extension_storage_tracker_kinto.js b/services/sync/tests/unit/test_extension_storage_tracker_kinto.js new file mode 100644 index 0000000000..2de56ae400 --- /dev/null +++ b/services/sync/tests/unit/test_extension_storage_tracker_kinto.js @@ -0,0 +1,44 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Services.prefs.setBoolPref("webextensions.storage.sync.kinto", true); + +const { ExtensionStorageEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/extension-storage.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { extensionStorageSyncKinto: extensionStorageSync } = + ChromeUtils.importESModule( + "resource://gre/modules/ExtensionStorageSyncKinto.sys.mjs" + ); + +let engine; + +add_task(async function setup() { + await Service.engineManager.register(ExtensionStorageEngine); + engine = Service.engineManager.get("extension-storage"); + do_get_profile(); // so we can use FxAccounts + loadWebExtensionTestFunctions(); +}); + +add_task(async function test_changing_extension_storage_changes_score() { + const tracker = engine._tracker; + const extension = { id: "my-extension-id" }; + tracker.start(); + await withSyncContext(async function (context) { + await extensionStorageSync.set(extension, { a: "b" }, context); + }); + Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM); + + tracker.resetScore(); + await withSyncContext(async function (context) { + await extensionStorageSync.remove(extension, "a", context); + }); + Assert.equal(tracker.score, SCORE_INCREMENT_MEDIUM); + + await tracker.stop(); +}); diff --git a/services/sync/tests/unit/test_form_validator.js b/services/sync/tests/unit/test_form_validator.js new file mode 100644 index 0000000000..58ea8b855b --- /dev/null +++ b/services/sync/tests/unit/test_form_validator.js @@ -0,0 +1,86 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FormValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "11111", + guid: "11111", + name: "foo", + fieldname: "foo", + value: "bar", + }, + { + id: "22222", + guid: "22222", + name: "foo2", + fieldname: "foo2", + value: "bar2", + }, + { + id: "33333", + guid: "33333", + name: "foo3", + fieldname: "foo3", + value: "bar3", + }, + ], + client: [ + { + id: "11111", + guid: "11111", + name: "foo", + fieldname: "foo", + value: "bar", + }, + { + id: "22222", + guid: "22222", + name: "foo2", + fieldname: "foo2", + value: "bar2", + }, + { + id: "33333", + guid: "33333", + name: "foo3", + fieldname: "foo3", + value: "bar3", + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new FormValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 3); + equal(records.length, 3); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); + +add_task(async function test_formValidatorIgnoresMissingClients() { + // Since history form records are not deleted from the server, the + // |FormValidator| shouldn't set the |missingClient| flag in |problemData|. + let { server, client } = getDummyServerAndClient(); + client.pop(); + + let validator = new FormValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 2); + equal(records.length, 3); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + deepEqual(problemData, expected); +}); diff --git a/services/sync/tests/unit/test_forms_store.js b/services/sync/tests/unit/test_forms_store.js new file mode 100644 index 0000000000..716487865f --- /dev/null +++ b/services/sync/tests/unit/test_forms_store.js @@ -0,0 +1,176 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Make sure the form store follows the Store api and correctly accesses the backend form storage" +); +const { FormEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +add_task(async function run_test() { + let engine = new FormEngine(Service); + await engine.initialize(); + let store = engine._store; + + async function applyEnsureNoFailures(records) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch(records, countTelemetry)).length, + 0 + ); + } + + _("Remove any existing entries"); + await store.wipe(); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add a form entry"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "name!!", + value: "value??", + }, + ]); + + _("Should have 1 entry now"); + let id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + Assert.ok(store.itemExists(id)); + + _("Should be able to find this entry as a dupe"); + Assert.equal( + await engine._findDupe({ name: "name!!", value: "value??" }), + id + ); + + let rec = await store.createRecord(id); + _("Got record for id", id, rec); + Assert.equal(rec.name, "name!!"); + Assert.equal(rec.value, "value??"); + + _("Create a non-existent id for delete"); + Assert.ok((await store.createRecord("deleted!!")).deleted); + + _("Try updating.. doesn't do anything yet"); + await store.update({}); + + _("Remove all entries"); + await store.wipe(); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add another entry"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "another", + value: "entry", + }, + ]); + id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + + _("Change the id of the new entry to something else"); + await store.changeItemID(id, "newid"); + + _("Make sure it's there"); + Assert.ok(store.itemExists("newid")); + + _("Remove the entry"); + await store.remove({ + id: "newid", + }); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Removing the entry again shouldn't matter"); + await store.remove({ + id: "newid", + }); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add another entry to delete using applyIncomingBatch"); + let toDelete = { + id: Utils.makeGUID(), + name: "todelete", + value: "entry", + }; + await applyEnsureNoFailures([toDelete]); + id = ""; + for (let _id in await store.getAllIDs()) { + if (id == "") { + id = _id; + } else { + do_throw("Should have only gotten one!"); + } + } + Assert.ok(store.itemExists(id)); + // mark entry as deleted + toDelete.id = id; + toDelete.deleted = true; + await applyEnsureNoFailures([toDelete]); + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Add an entry to wipe"); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "towipe", + value: "entry", + }, + ]); + + await store.wipe(); + + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + + _("Ensure we work if formfill is disabled."); + Services.prefs.setBoolPref("browser.formfill.enable", false); + try { + // a search + if ((await store.getAllIDs()).length) { + do_throw("Shouldn't get any ids!"); + } + // an update. + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + name: "some", + value: "entry", + }, + ]); + } finally { + Services.prefs.clearUserPref("browser.formfill.enable"); + await store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_forms_tracker.js b/services/sync/tests/unit/test_forms_tracker.js new file mode 100644 index 0000000000..aee74381ad --- /dev/null +++ b/services/sync/tests/unit/test_forms_tracker.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FormEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/forms.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + _("Verify we've got an empty tracker to work with."); + let engine = new FormEngine(Service); + await engine.initialize(); + let tracker = engine._tracker; + + let changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + async function addEntry(name, value) { + await engine._store.create({ name, value }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + async function removeEntry(name, value) { + let guid = await engine._findDupe({ name, value }); + await engine._store.remove({ id: guid }); + await engine._tracker.asyncObserver.promiseObserversComplete(); + } + + try { + _("Create an entry. Won't show because we haven't started tracking yet"); + await addEntry("name", "John Doe"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + await removeEntry("name", "John Doe"); + await addEntry("email", "john@doe.com"); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 2); + + _("Notifying twice won't do any harm."); + tracker.start(); + await addEntry("address", "Memory Lane"); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 3); + + _("Check that ignoreAll is respected"); + await tracker.clearChangedIDs(); + tracker.score = 0; + tracker.ignoreAll = true; + await addEntry("username", "johndoe123"); + await addEntry("favoritecolor", "green"); + await removeEntry("name", "John Doe"); + tracker.ignoreAll = false; + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + equal(tracker.score, 0); + + _("Let's stop tracking again."); + await tracker.clearChangedIDs(); + await tracker.stop(); + await removeEntry("address", "Memory Lane"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + _("Notifying twice won't do any harm."); + await tracker.stop(); + await removeEntry("email", "john@doe.com"); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + } finally { + _("Clean up."); + await engine._store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_fxa_node_reassignment.js b/services/sync/tests/unit/test_fxa_node_reassignment.js new file mode 100644 index 0000000000..1db460a25f --- /dev/null +++ b/services/sync/tests/unit/test_fxa_node_reassignment.js @@ -0,0 +1,402 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_("Test that node reassignment happens correctly using the FxA identity mgr."); +// The node-reassignment logic is quite different for FxA than for the legacy +// provider. In particular, there's no special request necessary for +// reassignment - it comes from the token server - so we need to ensure the +// Fxa cluster manager grabs a new token. + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); + +add_task(async function setup() { + // Disables all built-in engines. Important for avoiding errors thrown by the + // add-ons engine. + await Service.engineManager.clear(); + + // Setup the sync auth manager. + Status.__authManager = Service.identity = new SyncAuthManager(); +}); + +// API-compatible with SyncServer handler. Bind `handler` to something to use +// as a ServerCollection handler. +function handleReassign(handler, req, resp) { + resp.setStatusLine(req.httpVersion, 401, "Node reassignment"); + resp.setHeader("Content-Type", "application/json"); + let reassignBody = JSON.stringify({ error: "401inator in place" }); + resp.bodyOutputStream.write(reassignBody, reassignBody.length); +} + +var numTokenRequests = 0; + +function prepareServer(cbAfterTokenFetch) { + syncTestLogging(); + let config = makeIdentityConfig({ username: "johndoe" }); + // A server callback to ensure we don't accidentally hit the wrong endpoint + // after a node reassignment. + let callback = { + onRequest(req, resp) { + let full = `${req.scheme}://${req.host}:${req.port}${req.path}`; + let expected = config.fxaccount.token.endpoint; + Assert.ok( + full.startsWith(expected), + `request made to ${full}, expected ${expected}` + ); + }, + }; + Object.setPrototypeOf(callback, SyncServerCallback); + let server = new SyncServer(callback); + server.registerUser("johndoe"); + server.start(); + + // Set the token endpoint for the initial token request that's done implicitly + // via configureIdentity. + config.fxaccount.token.endpoint = server.baseURI + "1.1/johndoe/"; + // And future token fetches will do magic around numReassigns. + let numReassigns = 0; + return configureIdentity(config).then(() => { + Service.identity._tokenServerClient = { + getTokenUsingOAuth() { + return new Promise(res => { + // Build a new URL with trailing zeros for the SYNC_VERSION part - this + // will still be seen as equivalent by the test server, but different + // by sync itself. + numReassigns += 1; + let trailingZeros = new Array(numReassigns + 1).join("0"); + let token = config.fxaccount.token; + token.endpoint = server.baseURI + "1.1" + trailingZeros + "/johndoe"; + token.uid = config.username; + _(`test server saw token fetch - endpoint now ${token.endpoint}`); + numTokenRequests += 1; + res(token); + if (cbAfterTokenFetch) { + cbAfterTokenFetch(); + } + }); + }, + }; + return server; + }); +} + +function getReassigned() { + try { + return Services.prefs.getBoolPref("services.sync.lastSyncReassigned"); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_UNEXPECTED) { + do_throw( + "Got exception retrieving lastSyncReassigned: " + Log.exceptionStr(ex) + ); + } + } + return false; +} + +/** + * Make a test request to `url`, then watch the result of two syncs + * to ensure that a node request was made. + * Runs `between` between the two. This can be used to undo deliberate failure + * setup, detach observers, etc. + */ +async function syncAndExpectNodeReassignment( + server, + firstNotification, + between, + secondNotification, + url +) { + _("Starting syncAndExpectNodeReassignment\n"); + let deferred = PromiseUtils.defer(); + async function onwards() { + let numTokenRequestsBefore; + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // Track whether we fetched a new token. + numTokenRequestsBefore = numTokenRequests; + + // Allow for tests to clean up error conditions. + between(); + } + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Second sync nextTick."); + Assert.equal( + numTokenRequests, + numTokenRequestsBefore + 1, + "fetched a new token" + ); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + await Service.sync(); + } + + // Make sure that we really do get a 401 (but we can only do that if we are + // already logged in, as the login process is what sets up the URLs) + if (Service.isLoggedIn) { + _("Making request to " + url + " which should 401"); + let request = new RESTRequest(url); + await request.get(); + Assert.equal(request.response.status, 401); + CommonUtils.nextTick(onwards); + } else { + _("Skipping preliminary validation check for a 401 as we aren't logged in"); + CommonUtils.nextTick(onwards); + } + await deferred.promise; +} + +// Check that when we sync we don't request a new token by default - our +// test setup has configured the client with a valid token, and that token +// should be used to form the cluster URL. +add_task(async function test_single_token_fetch() { + enableValidationPrefs(); + + _("Test a normal sync only fetches 1 token"); + + let numTokenFetches = 0; + + function afterTokenFetch() { + numTokenFetches++; + } + + // Set the cluster URL to an "old" version - this is to ensure we don't + // use that old cached version for the first sync but prefer the value + // we got from the token (and as above, we are also checking we don't grab + // a new token). If the test actually attempts to connect to this URL + // it will crash. + Service.clusterURL = "http://example.com/"; + + let server = await prepareServer(afterTokenFetch); + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded"); + Assert.equal(numTokenFetches, 0, "didn't fetch a new token"); + // A bit hacky, but given we know how prepareServer works we can deduce + // that clusterURL we expect. + let expectedClusterURL = server.baseURI + "1.1/johndoe/"; + Assert.equal(Service.clusterURL, expectedClusterURL); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_momentary_401_engine() { + enableValidationPrefs(); + + _("Test a failure for engine URLs that's resolved by reassignment."); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // We want to verify that the clusterURL pref has been cleared after a 401 + // inside a sync. Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function between() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Assert.ok(getReassigned()); + } + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:finish", + between, + "weave:service:sync:finish", + Service.storageURL + "rotary" + ); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +// This test ends up being a failing info fetch *after we're already logged in*. +add_task(async function test_momentary_401_info_collections_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for info/collections after login that's resolved by reassignment." + ); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Arrange for info/collections to return a 401."); + let oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.info = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.infoURL + ); +}); + +// This test ends up being a failing info fetch *before we're logged in*. +// In this case we expect to recover during the login phase - so the first +// sync succeeds. +add_task(async function test_momentary_401_info_collections_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for info/collections before login that's resolved by reassignment." + ); + + let oldHandler; + let sawTokenFetch = false; + + function afterTokenFetch() { + // After a single token fetch, we undo our evil handleReassign hack, so + // the next /info request returns the collection instead of a 401 + server.toplevelHandlers.info = oldHandler; + sawTokenFetch = true; + } + + let server = await prepareServer(afterTokenFetch); + + // Return a 401 for the next /info request - it will be reset immediately + // after a new token is fetched. + oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + + await Service.sync(); + Assert.equal(Status.sync, SYNC_SUCCEEDED, "sync succeeded"); + // sync was successful - check we grabbed a new token. + Assert.ok(sawTokenFetch, "a new token was fetched by this test."); + // and we are done. + await Service.startOver(); + await promiseStopServer(server); +}); + +// This test ends up being a failing meta/global fetch *after we're already logged in*. +add_task(async function test_momentary_401_storage_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL after login that's resolved by" + + "reassignment." + ); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Arrange for meta/global to return a 401."); + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +// This test ends up being a failing meta/global fetch *before we've logged in*. +add_task(async function test_momentary_401_storage_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL before login, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(!Service.isLoggedIn, "already logged in"); + + await syncAndExpectNodeReassignment( + server, + "weave:service:login:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); diff --git a/services/sync/tests/unit/test_fxa_service_cluster.js b/services/sync/tests/unit/test_fxa_service_cluster.js new file mode 100644 index 0000000000..0dde508e12 --- /dev/null +++ b/services/sync/tests/unit/test_fxa_service_cluster.js @@ -0,0 +1,56 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { initializeIdentityWithTokenServerResponse } = + ChromeUtils.importESModule( + "resource://testing-common/services/sync/fxa_utils.sys.mjs" + ); + +add_task(async function test_findCluster() { + _("Test FxA _findCluster()"); + + _("_findCluster() throws on 500 errors."); + initializeIdentityWithTokenServerResponse({ + status: 500, + headers: [], + body: "", + }); + + await Assert.rejects( + Service.identity._findCluster(), + /TokenServerClientServerError/ + ); + + _("_findCluster() returns null on authentication errors."); + initializeIdentityWithTokenServerResponse({ + status: 401, + headers: { "content-type": "application/json" }, + body: "{}", + }); + + let cluster = await Service.identity._findCluster(); + Assert.strictEqual(cluster, null); + + _("_findCluster() works with correct tokenserver response."); + let endpoint = "http://example.com/something"; + initializeIdentityWithTokenServerResponse({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + api_endpoint: endpoint, + duration: 300, + id: "id", + key: "key", + uid: "uid", + }), + }); + + cluster = await Service.identity._findCluster(); + // The cluster manager ensures a trailing "/" + Assert.strictEqual(cluster, endpoint + "/"); + + Svc.Prefs.resetBranch(""); +}); diff --git a/services/sync/tests/unit/test_history_engine.js b/services/sync/tests/unit/test_history_engine.js new file mode 100644 index 0000000000..9b11f49c8b --- /dev/null +++ b/services/sync/tests/unit/test_history_engine.js @@ -0,0 +1,327 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); + +// Use only for rawAddVisit. +XPCOMUtils.defineLazyServiceGetter( + this, + "asyncHistory", + "@mozilla.org/browser/history;1", + "mozIAsyncHistory" +); +async function rawAddVisit(id, uri, visitPRTime, transitionType) { + return new Promise((resolve, reject) => { + let results = []; + let handler = { + handleResult(result) { + results.push(result); + }, + handleError(resultCode, placeInfo) { + do_throw(`updatePlaces gave error ${resultCode}!`); + }, + handleCompletion(count) { + resolve({ results, count }); + }, + }; + asyncHistory.updatePlaces( + [ + { + guid: id, + uri: typeof uri == "string" ? CommonUtils.makeURI(uri) : uri, + visits: [{ visitDate: visitPRTime, transitionType }], + }, + ], + handler + ); + }); +} + +add_task(async function test_history_download_limit() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let lastSync = new_timestamp(); + + let collection = server.user("foo").collection("history"); + for (let i = 0; i < 15; i++) { + let id = "place" + i.toString(10).padStart(7, "0"); + let wbo = new ServerWBO( + id, + encryptPayload({ + id, + histUri: "http://example.com/" + i, + title: "Page " + i, + visits: [ + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.TYPED, + }, + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + ], + }), + lastSync + 1 + i + ); + wbo.sortindex = 15 - i; + collection.insertWBO(wbo); + } + + // We have 15 records on the server since the last sync, but our download + // limit is 5 records at a time. We should eventually fetch all 15. + await engine.setLastSync(lastSync); + engine.downloadBatchSize = 4; + engine.downloadLimit = 5; + + // Don't actually fetch any backlogged records, so that we can inspect + // the backlog between syncs. + engine.guidFetchBatchSize = 0; + + let ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + let backlogAfterFirstSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterFirstSync, [ + "place0000000", + "place0000001", + "place0000002", + "place0000003", + "place0000004", + "place0000005", + "place0000006", + "place0000007", + "place0000008", + "place0000009", + ]); + + // We should have fast-forwarded the last sync time. + equal(await engine.getLastSync(), lastSync + 15); + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + ok(!ping.engines[0].incoming); + + // After the second sync, our backlog still contains the same GUIDs: we + // weren't able to make progress on fetching them, since our + // `guidFetchBatchSize` is 0. + let backlogAfterSecondSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterFirstSync, backlogAfterSecondSync); + + // Now add a newer record to the server. + let newWBO = new ServerWBO( + "placeAAAAAAA", + encryptPayload({ + id: "placeAAAAAAA", + histUri: "http://example.com/a", + title: "New Page A", + visits: [ + { + date: Date.now() * 1000, + type: PlacesUtils.history.TRANSITIONS.TYPED, + }, + ], + }), + lastSync + 20 + ); + newWBO.sortindex = -1; + collection.insertWBO(newWBO); + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 1 }); + + // Our backlog should remain the same. + let backlogAfterThirdSync = Array.from(engine.toFetch).sort(); + deepEqual(backlogAfterSecondSync, backlogAfterThirdSync); + + equal(await engine.getLastSync(), lastSync + 20); + + // Bump the fetch batch size to let the backlog make progress. We should + // make 3 requests to fetch 5 backlogged GUIDs. + engine.guidFetchBatchSize = 2; + + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + deepEqual(Array.from(engine.toFetch).sort(), [ + "place0000005", + "place0000006", + "place0000007", + "place0000008", + "place0000009", + ]); + + // Sync again to clear out the backlog. + engine.lastModified = collection.modified; + ping = await sync_engine_and_validate_telem(engine, false); + deepEqual(ping.engines[0].incoming, { applied: 5 }); + + deepEqual(Array.from(engine.toFetch), []); + + await engine.wipeClient(); + await engine.finalize(); +}); + +add_task(async function test_history_visit_roundtrip() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + engine._tracker.start(); + + let id = "aaaaaaaaaaaa"; + let oneHourMS = 60 * 60 * 1000; + // Insert a visit with a non-round microsecond timestamp (e.g. it's not evenly + // divisible by 1000). This will typically be the case for visits that occur + // during normal navigation. + let time = (Date.now() - oneHourMS) * 1000 + 555; + // We use the low level history api since it lets us provide microseconds + let { count } = await rawAddVisit( + id, + "https://www.example.com", + time, + PlacesUtils.history.TRANSITIONS.TYPED + ); + equal(count, 1); + // Check that it was inserted and that we didn't round on the insert. + let visits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(visits.length, 1); + equal(visits[0].date, time); + + let collection = server.user("foo").collection("history"); + + // Sync the visit up to the server. + await sync_engine_and_validate_telem(engine, false); + + collection.updateRecord( + id, + cleartext => { + // Double-check that we didn't round the visit's timestamp to the nearest + // millisecond when uploading. + equal(cleartext.visits[0].date, time); + // Add a remote visit so that we get past the deepEquals check in reconcile + // (otherwise the history engine will skip applying this record). The + // contents of this visit don't matter, beyond the fact that it needs to + // exist. + cleartext.visits.push({ + date: (Date.now() - oneHourMS / 2) * 1000, + type: PlacesUtils.history.TRANSITIONS.LINK, + }); + }, + new_timestamp() + 10 + ); + + // Force a remote sync. + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + // Make sure that we didn't duplicate the visit when inserting. (Prior to bug + // 1423395, we would insert a duplicate visit, where the timestamp was + // effectively `Math.round(microsecondTimestamp / 1000) * 1000`.) + visits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(visits.length, 2); + + await engine.wipeClient(); + await engine.finalize(); +}); + +add_task(async function test_history_visit_dedupe_old() { + let engine = new HistoryEngine(Service); + await engine.initialize(); + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let initialVisits = Array.from({ length: 25 }, (_, index) => ({ + transition: PlacesUtils.history.TRANSITION_LINK, + date: new Date(Date.UTC(2017, 10, 1 + index)), + })); + initialVisits.push({ + transition: PlacesUtils.history.TRANSITION_LINK, + date: new Date(), + }); + await PlacesUtils.history.insert({ + url: "https://www.example.com", + visits: initialVisits, + }); + + let recentVisits = await PlacesSyncUtils.history.fetchVisitsForURL( + "https://www.example.com" + ); + equal(recentVisits.length, 20); + let { visits: allVisits, guid } = await PlacesUtils.history.fetch( + "https://www.example.com", + { + includeVisits: true, + } + ); + equal(allVisits.length, 26); + + let collection = server.user("foo").collection("history"); + + await sync_engine_and_validate_telem(engine, false); + + collection.updateRecord( + guid, + data => { + data.visits.push( + // Add a couple remote visit equivalent to some old visits we have already + { + date: Date.UTC(2017, 10, 1) * 1000, // Nov 1, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + { + date: Date.UTC(2017, 10, 2) * 1000, // Nov 2, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + // Add a couple new visits to make sure we are still applying them. + { + date: Date.UTC(2017, 11, 4) * 1000, // Dec 4, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + }, + { + date: Date.UTC(2017, 11, 5) * 1000, // Dec 5, 2017 + type: PlacesUtils.history.TRANSITIONS.LINK, + } + ); + }, + new_timestamp() + 10 + ); + + await engine.setLastSync(new_timestamp() - 30); + await sync_engine_and_validate_telem(engine, false); + + allVisits = ( + await PlacesUtils.history.fetch("https://www.example.com", { + includeVisits: true, + }) + ).visits; + + equal(allVisits.length, 28); + ok( + allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 4)), + "Should contain the Dec. 4th visit" + ); + ok( + allVisits.find(x => x.date.getTime() === Date.UTC(2017, 11, 5)), + "Should contain the Dec. 5th visit" + ); + + await engine.wipeClient(); + await engine.finalize(); +}); diff --git a/services/sync/tests/unit/test_history_store.js b/services/sync/tests/unit/test_history_store.js new file mode 100644 index 0000000000..f2c072d231 --- /dev/null +++ b/services/sync/tests/unit/test_history_store.js @@ -0,0 +1,574 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +const TIMESTAMP1 = (Date.now() - 103406528) * 1000; +const TIMESTAMP2 = (Date.now() - 6592903) * 1000; +const TIMESTAMP3 = (Date.now() - 123894) * 1000; + +function promiseOnVisitObserved() { + return new Promise(res => { + let listener = new PlacesWeakCallbackWrapper(events => { + PlacesObservers.removeListener(["page-visited"], listener); + res(); + }); + PlacesObservers.addListener(["page-visited"], listener); + }); +} + +function isDateApproximately(actual, expected, skewMillis = 1000) { + let lowerBound = expected - skewMillis; + let upperBound = expected + skewMillis; + return actual >= lowerBound && actual <= upperBound; +} + +let engine, store, fxuri, fxguid, tburi, tbguid; + +async function applyEnsureNoFailures(records) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch(records, countTelemetry)).length, + 0 + ); +} + +add_task(async function setup() { + engine = new HistoryEngine(Service); + await engine.initialize(); + store = engine._store; +}); + +add_task(async function test_store() { + _("Verify that we've got an empty store to work with."); + do_check_empty(await store.getAllIDs()); + + _("Let's create an entry in the database."); + fxuri = CommonUtils.makeURI("http://getfirefox.com/"); + + await PlacesTestUtils.addVisits({ + uri: fxuri, + title: "Get Firefox!", + visitDate: TIMESTAMP1, + }); + _("Verify that the entry exists."); + let ids = Object.keys(await store.getAllIDs()); + Assert.equal(ids.length, 1); + fxguid = ids[0]; + Assert.ok(await store.itemExists(fxguid)); + + _("If we query a non-existent record, it's marked as deleted."); + let record = await store.createRecord("non-existent"); + Assert.ok(record.deleted); + + _("Verify createRecord() returns a complete record."); + record = await store.createRecord(fxguid); + Assert.equal(record.histUri, fxuri.spec); + Assert.equal(record.title, "Get Firefox!"); + Assert.equal(record.visits.length, 1); + Assert.equal(record.visits[0].date, TIMESTAMP1); + Assert.equal(record.visits[0].type, Ci.nsINavHistoryService.TRANSITION_LINK); + + _("Let's modify the record and have the store update the database."); + let secondvisit = { + date: TIMESTAMP2, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }; + let onVisitObserved = promiseOnVisitObserved(); + await applyEnsureNoFailures([ + { + id: fxguid, + histUri: record.histUri, + title: "Hol Dir Firefox!", + visits: [record.visits[0], secondvisit], + }, + ]); + await onVisitObserved; + let queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, "Hol Dir Firefox!"); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP2 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: new Date(TIMESTAMP1 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_LINK, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_store_create() { + _("Create a brand new record through the store."); + tbguid = Utils.makeGUID(); + tburi = CommonUtils.makeURI("http://getthunderbird.com"); + let onVisitObserved = promiseOnVisitObserved(); + await applyEnsureNoFailures([ + { + id: tbguid, + histUri: tburi.spec, + title: "The bird is the word!", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }, + ]); + await onVisitObserved; + Assert.ok(await store.itemExists(tbguid)); + do_check_attribute_count(await store.getAllIDs(), 1); + let queryres = await PlacesUtils.history.fetch(tburi.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, "The bird is the word!"); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP3 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_null_title() { + _( + "Make sure we handle a null title gracefully (it can happen in some cases, e.g. for resource:// URLs)" + ); + let resguid = Utils.makeGUID(); + let resuri = CommonUtils.makeURI("unknown://title"); + await applyEnsureNoFailures([ + { + id: resguid, + histUri: resuri.spec, + title: null, + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }, + ]); + do_check_attribute_count(await store.getAllIDs(), 1); + + let queryres = await PlacesUtils.history.fetch(resuri.spec, { + includeVisits: true, + }); + Assert.equal(queryres.title, ""); + Assert.deepEqual(queryres.visits, [ + { + date: new Date(TIMESTAMP3 / 1000), + transition: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ]); + await PlacesUtils.history.clear(); +}); + +add_task(async function test_invalid_records() { + _("Make sure we handle invalid URLs in places databases gracefully."); + await PlacesUtils.withConnectionWrapper( + "test_invalid_record", + async function (db) { + await db.execute( + "INSERT INTO moz_places " + + "(url, url_hash, title, rev_host, visit_count, last_visit_date) " + + "VALUES ('invalid-uri', hash('invalid-uri'), 'Invalid URI', '.', 1, " + + TIMESTAMP3 + + ")" + ); + // Trigger the update to the moz_origin tables by deleting the added rows + // from moz_updateoriginsinsert_temp + await db.executeCached("DELETE FROM moz_updateoriginsinsert_temp"); + // Add the corresponding visit to retain database coherence. + await db.execute( + "INSERT INTO moz_historyvisits " + + "(place_id, visit_date, visit_type, session) " + + "VALUES ((SELECT id FROM moz_places WHERE url_hash = hash('invalid-uri') AND url = 'invalid-uri'), " + + TIMESTAMP3 + + ", " + + Ci.nsINavHistoryService.TRANSITION_TYPED + + ", 1)" + ); + } + ); + do_check_attribute_count(await store.getAllIDs(), 1); + + _("Make sure we report records with invalid URIs."); + let invalid_uri_guid = Utils.makeGUID(); + let countTelemetry = new SyncedRecordsTelemetry(); + let failed = await store.applyIncomingBatch( + [ + { + id: invalid_uri_guid, + histUri: ":::::::::::::::", + title: "Doesn't have a valid URI", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + Assert.equal(failed.length, 1); + Assert.equal(failed[0], invalid_uri_guid); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "<URL> is not a valid URL." + ); + Assert.equal(countTelemetry.incomingCounts.failedReasons[0].count, 1); + + _("Make sure we handle records with invalid GUIDs gracefully (ignore)."); + await applyEnsureNoFailures([ + { + id: "invalid", + histUri: "http://invalid.guid/", + title: "Doesn't have a valid GUID", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ]); + + _( + "Make sure we handle records with invalid visit codes or visit dates, gracefully ignoring those visits." + ); + let no_date_visit_guid = Utils.makeGUID(); + let no_type_visit_guid = Utils.makeGUID(); + let invalid_type_visit_guid = Utils.makeGUID(); + let non_integer_visit_guid = Utils.makeGUID(); + countTelemetry = new SyncedRecordsTelemetry(); + failed = await store.applyIncomingBatch( + [ + { + id: no_date_visit_guid, + histUri: "http://no.date.visit/", + title: "Visit has no date", + visits: [{ type: Ci.nsINavHistoryService.TRANSITION_EMBED }], + }, + { + id: no_type_visit_guid, + histUri: "http://no.type.visit/", + title: "Visit has no type", + visits: [{ date: TIMESTAMP3 }], + }, + { + id: invalid_type_visit_guid, + histUri: "http://invalid.type.visit/", + title: "Visit has invalid type", + visits: [ + { + date: TIMESTAMP3, + type: Ci.nsINavHistoryService.TRANSITION_LINK - 1, + }, + ], + }, + { + id: non_integer_visit_guid, + histUri: "http://non.integer.visit/", + title: "Visit has non-integer date", + visits: [ + { date: 1234.567, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + Assert.equal(failed.length, 0); + + // Make sure we can apply tombstones (both valid and invalid) + countTelemetry = new SyncedRecordsTelemetry(); + failed = await store.applyIncomingBatch( + [ + { id: no_date_visit_guid, deleted: true }, + { id: "not-a-valid-guid", deleted: true }, + ], + countTelemetry + ); + Assert.deepEqual(failed, ["not-a-valid-guid"]); + Assert.equal( + countTelemetry.incomingCounts.failedReasons[0].name, + "<URL> is not a valid URL." + ); + + _("Make sure we handle records with javascript: URLs gracefully."); + await applyEnsureNoFailures( + [ + { + id: Utils.makeGUID(), + histUri: "javascript:''", + title: "javascript:''", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_EMBED }, + ], + }, + ], + countTelemetry + ); + + _("Make sure we handle records without any visits gracefully."); + await applyEnsureNoFailures([ + { + id: Utils.makeGUID(), + histUri: "http://getfirebug.com", + title: "Get Firebug!", + visits: [], + }, + ]); +}); + +add_task(async function test_unknowingly_invalid_records() { + _("Make sure we handle rejection of records by places gracefully."); + let oldCAU = store._canAddURI; + store._canAddURI = () => true; + try { + _("Make sure that when places rejects this record we record it as failed"); + let guid = Utils.makeGUID(); + let countTelemetry = new SyncedRecordsTelemetry(); + let result = await store.applyIncomingBatch( + [ + { + id: guid, + histUri: "javascript:''", + title: "javascript:''", + visits: [ + { + date: TIMESTAMP3, + type: Ci.nsINavHistoryService.TRANSITION_EMBED, + }, + ], + }, + ], + countTelemetry + ); + deepEqual(result, [guid]); + } finally { + store._canAddURI = oldCAU; + } +}); + +add_task(async function test_clamp_visit_dates() { + let futureVisitTime = Date.now() + 5 * 60 * 1000; + let recentVisitTime = Date.now() - 5 * 60 * 1000; + + await applyEnsureNoFailures([ + { + id: "visitAAAAAAA", + histUri: "http://example.com/a", + title: "A", + visits: [ + { + date: "invalidDate", + type: Ci.nsINavHistoryService.TRANSITION_LINK, + }, + ], + }, + { + id: "visitBBBBBBB", + histUri: "http://example.com/b", + title: "B", + visits: [ + { + date: 100, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: 250, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ], + }, + { + id: "visitCCCCCCC", + histUri: "http://example.com/c", + title: "D", + visits: [ + { + date: futureVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_BOOKMARK, + }, + ], + }, + { + id: "visitDDDDDDD", + histUri: "http://example.com/d", + title: "D", + visits: [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + }, + ], + }, + ]); + + let visitsForA = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/a" + ); + deepEqual(visitsForA, [], "Should ignore visits with invalid dates"); + + let visitsForB = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/b" + ); + deepEqual( + visitsForB, + [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + { + // We should clamp visit dates older than original Mosaic release. + date: PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP * 1000, + type: Ci.nsINavHistoryService.TRANSITION_TYPED, + }, + ], + "Should record clamped visit and valid visit for B" + ); + + let visitsForC = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/c" + ); + equal(visitsForC.length, 1, "Should record clamped future visit for C"); + let visitDateForC = PlacesUtils.toDate(visitsForC[0].date); + ok( + isDateApproximately(visitDateForC, Date.now()), + "Should clamp future visit date for C to now" + ); + + let visitsForD = await PlacesSyncUtils.history.fetchVisitsForURL( + "http://example.com/d" + ); + deepEqual( + visitsForD, + [ + { + date: recentVisitTime * 1000, + type: Ci.nsINavHistoryService.TRANSITION_DOWNLOAD, + }, + ], + "Should not clamp valid visit dates" + ); +}); + +add_task(async function test_remove() { + _("Remove an existent record and a non-existent from the store."); + await applyEnsureNoFailures([ + { id: fxguid, deleted: true }, + { id: Utils.makeGUID(), deleted: true }, + ]); + Assert.equal(false, await store.itemExists(fxguid)); + let queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); + + _("Make sure wipe works."); + await store.wipe(); + do_check_empty(await store.getAllIDs()); + queryres = await PlacesUtils.history.fetch(fxuri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); + queryres = await PlacesUtils.history.fetch(tburi.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); +}); + +add_task(async function test_chunking() { + let mvpi = store.MAX_VISITS_PER_INSERT; + store.MAX_VISITS_PER_INSERT = 3; + let checkChunks = function (input, expected) { + let chunks = Array.from(store._generateChunks(input)); + deepEqual(chunks, expected); + }; + try { + checkChunks([{ visits: ["x"] }], [[{ visits: ["x"] }]]); + + // 3 should still be one chunk. + checkChunks([{ visits: ["x", "x", "x"] }], [[{ visits: ["x", "x", "x"] }]]); + + // 4 should still be one chunk as we don't split individual records. + checkChunks( + [{ visits: ["x", "x", "x", "x"] }], + [[{ visits: ["x", "x", "x", "x"] }]] + ); + + // 4 in the first and 1 in the second should be 2 chunks. + checkChunks( + [{ visits: ["x", "x", "x", "x"] }, { visits: ["x"] }], + // expected + [[{ visits: ["x", "x", "x", "x"] }], [{ visits: ["x"] }]] + ); + + // we put multiple records into chunks + checkChunks( + [ + { visits: ["x", "x"] }, + { visits: ["x"] }, + { visits: ["x"] }, + { visits: ["x", "x"] }, + { visits: ["x", "x", "x", "x"] }, + ], + // expected + [ + [{ visits: ["x", "x"] }, { visits: ["x"] }], + [{ visits: ["x"] }, { visits: ["x", "x"] }], + [{ visits: ["x", "x", "x", "x"] }], + ] + ); + } finally { + store.MAX_VISITS_PER_INSERT = mvpi; + } +}); + +add_task(async function test_getAllIDs_filters_file_uris() { + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition: PlacesUtils.history.TRANSITION_LINK, + }); + await visitAddedPromise; + + do_check_attribute_count(await store.getAllIDs(), 0); + + await PlacesUtils.history.clear(); +}); + +add_task(async function test_applyIncomingBatch_filters_file_uris() { + const guid = Utils.makeGUID(); + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + await applyEnsureNoFailures([ + { + id: guid, + histUri: uri.spec, + title: "TPS CONFIG", + visits: [ + { date: TIMESTAMP3, type: Ci.nsINavHistoryService.TRANSITION_TYPED }, + ], + }, + ]); + Assert.equal(false, await store.itemExists(guid)); + let queryres = await PlacesUtils.history.fetch(uri.spec, { + includeVisits: true, + }); + Assert.equal(null, queryres); +}); + +add_task(async function cleanup() { + _("Clean up."); + await PlacesUtils.history.clear(); +}); diff --git a/services/sync/tests/unit/test_history_tracker.js b/services/sync/tests/unit/test_history_tracker.js new file mode 100644 index 0000000000..6f351d6984 --- /dev/null +++ b/services/sync/tests/unit/test_history_tracker.js @@ -0,0 +1,251 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PlacesDBUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PlacesDBUtils.sys.mjs" +); +const { HistoryEngine } = ChromeUtils.importESModule( + "resource://services-sync/engines/history.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +let tracker; + +add_task(async function setup() { + await Service.engineManager.clear(); + await Service.engineManager.register(HistoryEngine); + engine = Service.engineManager.get("history"); + tracker = engine._tracker; +}); + +async function verifyTrackerEmpty() { + let changes = await engine.pullNewChanges(); + do_check_empty(changes); + equal(tracker.score, 0); +} + +async function verifyTrackedCount(expected) { + let changes = await engine.pullNewChanges(); + do_check_attribute_count(changes, expected); +} + +async function verifyTrackedItems(tracked) { + let changes = await engine.pullNewChanges(); + let trackedIDs = new Set(Object.keys(changes)); + for (let guid of tracked) { + ok(guid in changes, `${guid} should be tracked`); + ok(changes[guid] > 0, `${guid} should have a modified time`); + trackedIDs.delete(guid); + } + equal( + trackedIDs.size, + 0, + `Unhandled tracked IDs: ${JSON.stringify(Array.from(trackedIDs))}` + ); +} + +async function resetTracker() { + await tracker.clearChangedIDs(); + tracker.resetScore(); +} + +async function cleanup() { + await PlacesUtils.history.clear(); + await resetTracker(); + await tracker.stop(); +} + +add_task(async function test_empty() { + _("Verify we've got an empty, disabled tracker to work with."); + await verifyTrackerEmpty(); + Assert.ok(!tracker._isTracking); + + await cleanup(); +}); + +add_task(async function test_not_tracking() { + _("Create history item. Won't show because we haven't started tracking yet"); + await addVisit("not_tracking"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_start_tracking() { + _("Add hook for save completion."); + let savePromise = new Promise((resolve, reject) => { + let save = tracker._storage._save; + tracker._storage._save = async function () { + try { + await save.call(this); + resolve(); + } catch (ex) { + reject(ex); + } finally { + tracker._storage._save = save; + } + }; + }); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await addVisit("start_tracking"); + await scorePromise; + + _("Score updated in test_start_tracking."); + await verifyTrackedCount(1); + Assert.equal(tracker.score, SCORE_INCREMENT_SMALL); + + await savePromise; + + _("changedIDs written to disk. Proceeding."); + await cleanup(); +}); + +add_task(async function test_start_tracking_twice() { + _("Verifying preconditions."); + tracker.start(); + await addVisit("start_tracking_twice1"); + await verifyTrackedCount(1); + Assert.equal(tracker.score, SCORE_INCREMENT_SMALL); + + _("Notifying twice won't do any harm."); + tracker.start(); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await addVisit("start_tracking_twice2"); + await scorePromise; + + _("Score updated in test_start_tracking_twice."); + await verifyTrackedCount(2); + Assert.equal(tracker.score, 2 * SCORE_INCREMENT_SMALL); + + await cleanup(); +}); + +add_task(async function test_track_delete() { + _("Deletions are tracked."); + + // This isn't present because we weren't tracking when it was visited. + await addVisit("track_delete"); + let uri = CommonUtils.makeURI("http://getfirefox.com/track_delete"); + let guid = await engine._store.GUIDForUri(uri.spec); + await verifyTrackerEmpty(); + + tracker.start(); + let visitRemovedPromise = promiseVisit("removed", uri); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + await PlacesUtils.history.remove(uri); + await Promise.all([scorePromise, visitRemovedPromise]); + + await verifyTrackedItems([guid]); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + + await cleanup(); +}); + +add_task(async function test_dont_track_expiration() { + _("Expirations are not tracked."); + let uriToRemove = await addVisit("to_remove"); + let guidToRemove = await engine._store.GUIDForUri(uriToRemove.spec); + + await resetTracker(); + await verifyTrackerEmpty(); + + tracker.start(); + let visitRemovedPromise = promiseVisit("removed", uriToRemove); + let scorePromise = promiseOneObserver("weave:engine:score:updated"); + + // Observe expiration. + Services.obs.addObserver(function onExpiration(aSubject, aTopic, aData) { + Services.obs.removeObserver(onExpiration, aTopic); + // Remove the remaining page to update its score. + PlacesUtils.history.remove(uriToRemove); + }, PlacesUtils.TOPIC_EXPIRATION_FINISHED); + + // Force expiration of 1 entry. + Services.prefs.setIntPref("places.history.expiration.max_pages", 0); + Cc["@mozilla.org/places/expiration;1"] + .getService(Ci.nsIObserver) + .observe(null, "places-debug-start-expiration", 1); + + await Promise.all([scorePromise, visitRemovedPromise]); + await verifyTrackedItems([guidToRemove]); + + await cleanup(); +}); + +add_task(async function test_stop_tracking() { + _("Let's stop tracking again."); + await tracker.stop(); + await addVisit("stop_tracking"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_stop_tracking_twice() { + await tracker.stop(); + await addVisit("stop_tracking_twice1"); + + _("Notifying twice won't do any harm."); + await tracker.stop(); + await addVisit("stop_tracking_twice2"); + await verifyTrackerEmpty(); + + await cleanup(); +}); + +add_task(async function test_filter_file_uris() { + tracker.start(); + + let uri = CommonUtils.makeURI("file:///Users/eoger/tps/config.json"); + let visitAddedPromise = promiseVisit("added", uri); + await PlacesTestUtils.addVisits({ + uri, + visitDate: Date.now() * 1000, + transition: PlacesUtils.history.TRANSITION_LINK, + }); + await visitAddedPromise; + + await verifyTrackerEmpty(); + await tracker.stop(); + await cleanup(); +}); + +add_task(async function test_filter_hidden() { + tracker.start(); + + _("Add visit; should be hidden by the redirect"); + let hiddenURI = await addVisit("hidden"); + let hiddenGUID = await engine._store.GUIDForUri(hiddenURI.spec); + _(`Hidden visit GUID: ${hiddenGUID}`); + + _("Add redirect visit; should be tracked"); + let trackedURI = await addVisit( + "redirect", + hiddenURI.spec, + PlacesUtils.history.TRANSITION_REDIRECT_PERMANENT + ); + let trackedGUID = await engine._store.GUIDForUri(trackedURI.spec); + _(`Tracked visit GUID: ${trackedGUID}`); + + _("Add visit for framed link; should be ignored"); + let embedURI = await addVisit( + "framed_link", + null, + PlacesUtils.history.TRANSITION_FRAMED_LINK + ); + let embedGUID = await engine._store.GUIDForUri(embedURI.spec); + _(`Framed link visit GUID: ${embedGUID}`); + + _("Run Places maintenance to mark redirect visit as hidden"); + await PlacesDBUtils.maintenanceOnIdle(); + + await verifyTrackedItems([trackedGUID]); + + await cleanup(); +}); diff --git a/services/sync/tests/unit/test_hmac_error.js b/services/sync/tests/unit/test_hmac_error.js new file mode 100644 index 0000000000..b190272746 --- /dev/null +++ b/services/sync/tests/unit/test_hmac_error.js @@ -0,0 +1,246 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// Track HMAC error counts. +var hmacErrorCount = 0; +(function () { + let hHE = Service.handleHMACEvent; + Service.handleHMACEvent = async function () { + hmacErrorCount++; + return hHE.call(Service); + }; +})(); + +async function shared_setup() { + enableValidationPrefs(); + syncTestLogging(); + + hmacErrorCount = 0; + + let clientsEngine = Service.clientsEngine; + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + + // Make sure RotaryEngine is the only one we sync. + let { engine, syncID, tracker } = await registerRotaryEngine(); + await engine.setLastSync(123); // Needs to be non-zero so that tracker is queried. + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + await tracker.addChangedID("scotsman", 0); + Assert.equal(1, Service.engineManager.getEnabled().length); + + let engines = { + rotary: { version: engine.version, syncID }, + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }; + + // Common server objects. + let global = new ServerWBO("global", { engines }); + let keysWBO = new ServerWBO("keys"); + let rotaryColl = new ServerCollection({}, true); + let clientsColl = new ServerCollection({}, true); + + return [engine, rotaryColl, clientsColl, keysWBO, global, tracker]; +} + +add_task(async function hmac_error_during_404() { + _("Attempt to replicate the HMAC error setup."); + let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] = + await shared_setup(); + + // Hand out 404s for crypto/keys. + let keysHandler = keysWBO.handler(); + let key404Counter = 0; + let keys404Handler = function (request, response) { + if (key404Counter > 0) { + let body = "Not Found"; + response.setStatusLine(request.httpVersion, 404, body); + response.bodyOutputStream.write(body, body.length); + key404Counter--; + return; + } + keysHandler(request, response); + }; + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let handlers = { + "/1.1/foo/info/collections": collectionsHelper.handler, + "/1.1/foo/storage/meta/global": upd("meta", global.handler()), + "/1.1/foo/storage/crypto/keys": upd("crypto", keys404Handler), + "/1.1/foo/storage/clients": upd("clients", clientsColl.handler()), + "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()), + }; + + let server = sync_httpd_setup(handlers); + // Do not instantiate SyncTestingInfrastructure; we need real crypto. + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + try { + _("Syncing."); + await sync_and_validate_telem(); + + _( + "Partially resetting client, as if after a restart, and forcing redownload." + ); + Service.collectionKeys.clear(); + await engine.setLastSync(0); // So that we redownload records. + key404Counter = 1; + _("---------------------------"); + await sync_and_validate_telem(); + _("---------------------------"); + + // Two rotary items, one client record... no errors. + Assert.equal(hmacErrorCount, 0); + } finally { + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + await promiseStopServer(server); + } +}); + +add_task(async function hmac_error_during_node_reassignment() { + _("Attempt to replicate an HMAC error during node reassignment."); + let [engine, rotaryColl, clientsColl, keysWBO, global, tracker] = + await shared_setup(); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + // We'll provide a 401 mid-way through the sync. This function + // simulates shifting to a node which has no data. + function on401() { + _("Deleting server data..."); + global.delete(); + rotaryColl.delete(); + keysWBO.delete(); + clientsColl.delete(); + delete collectionsHelper.collections.rotary; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.clients; + _("Deleted server data."); + } + + let should401 = false; + function upd401(coll, handler) { + return function (request, response) { + if (should401 && request.method != "DELETE") { + on401(); + should401 = false; + let body = '"reassigned!"'; + response.setStatusLine(request.httpVersion, 401, "Node reassignment."); + response.bodyOutputStream.write(body, body.length); + return; + } + handler(request, response); + }; + } + + let handlers = { + "/1.1/foo/info/collections": collectionsHelper.handler, + "/1.1/foo/storage/meta/global": upd("meta", global.handler()), + "/1.1/foo/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/foo/storage/clients": upd401("clients", clientsColl.handler()), + "/1.1/foo/storage/rotary": upd("rotary", rotaryColl.handler()), + }; + + let server = sync_httpd_setup(handlers); + // Do not instantiate SyncTestingInfrastructure; we need real crypto. + await configureIdentity({ username: "foo" }, server); + + _("Syncing."); + // First hit of clients will 401. This will happen after meta/global and + // keys -- i.e., in the middle of the sync, but before RotaryEngine. + should401 = true; + + // Use observers to perform actions when our sync finishes. + // This allows us to observe the automatic next-tick sync that occurs after + // an abort. + function onSyncError() { + do_throw("Should not get a sync error!"); + } + let onSyncFinished = function () {}; + let obs = { + observe: function observe(subject, topic, data) { + switch (topic) { + case "weave:service:sync:error": + onSyncError(); + break; + case "weave:service:sync:finish": + onSyncFinished(); + break; + } + }, + }; + + Svc.Obs.add("weave:service:sync:finish", obs); + Svc.Obs.add("weave:service:sync:error", obs); + + // This kicks off the actual test. Split into a function here to allow this + // source file to broadly follow actual execution order. + async function onwards() { + _("== Invoking first sync."); + await Service.sync(); + _("We should not simultaneously have data but no keys on the server."); + let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman"); + let hasKeys = keysWBO.modified; + + _("We correctly handle 401s by aborting the sync and starting again."); + Assert.ok(!hasData == !hasKeys); + + _("Be prepared for the second (automatic) sync..."); + } + + _("Make sure that syncing again causes recovery."); + let callbacksPromise = new Promise(resolve => { + onSyncFinished = function () { + _("== First sync done."); + _("---------------------------"); + onSyncFinished = function () { + _("== Second (automatic) sync done."); + let hasData = rotaryColl.wbo("flying") || rotaryColl.wbo("scotsman"); + let hasKeys = keysWBO.modified; + Assert.ok(!hasData == !hasKeys); + + // Kick off another sync. Can't just call it, because we're inside the + // lock... + (async () => { + await Async.promiseYield(); + _("Now a fresh sync will get no HMAC errors."); + _( + "Partially resetting client, as if after a restart, and forcing redownload." + ); + Service.collectionKeys.clear(); + await engine.setLastSync(0); + hmacErrorCount = 0; + + onSyncFinished = async function () { + // Two rotary items, one client record... no errors. + Assert.equal(hmacErrorCount, 0); + + Svc.Obs.remove("weave:service:sync:finish", obs); + Svc.Obs.remove("weave:service:sync:error", obs); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + server.stop(resolve); + }; + + Service.sync(); + })().catch(console.error); + }; + }; + }); + await onwards(); + await callbacksPromise; +}); diff --git a/services/sync/tests/unit/test_httpd_sync_server.js b/services/sync/tests/unit/test_httpd_sync_server.js new file mode 100644 index 0000000000..6ac8ff5e04 --- /dev/null +++ b/services/sync/tests/unit/test_httpd_sync_server.js @@ -0,0 +1,250 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +add_test(function test_creation() { + // Explicit callback for this one. + let server = new SyncServer(Object.create(SyncServerCallback)); + Assert.ok(!!server); // Just so we have a check. + server.start(null, function () { + _("Started on " + server.port); + server.stop(run_next_test); + }); +}); + +add_test(function test_url_parsing() { + let server = new SyncServer(); + + // Check that we can parse a WBO URI. + let parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto/keys"); + let [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage/crypto/keys"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, "crypto/keys"); + Assert.equal(null, server.pathRE.exec("/nothing/else")); + + // Check that we can parse a collection URI. + parts = server.pathRE.exec("/1.1/johnsmith/storage/crypto"); + [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage/crypto"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, "crypto"); + + // We don't allow trailing slash on storage URI. + parts = server.pathRE.exec("/1.1/johnsmith/storage/"); + Assert.equal(parts, undefined); + + // storage alone is a valid request. + parts = server.pathRE.exec("/1.1/johnsmith/storage"); + [all, version, username, first, rest] = parts; + Assert.equal(all, "/1.1/johnsmith/storage"); + Assert.equal(version, "1.1"); + Assert.equal(username, "johnsmith"); + Assert.equal(first, "storage"); + Assert.equal(rest, undefined); + + parts = server.storageRE.exec("storage"); + let collection; + [all, , collection] = parts; + Assert.equal(all, "storage"); + Assert.equal(collection, undefined); + + run_next_test(); +}); + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +function localRequest(server, path) { + _("localRequest: " + path); + let url = server.baseURI.substr(0, server.baseURI.length - 1) + path; + _("url: " + url); + return new RESTRequest(url); +} + +add_task(async function test_basic_http() { + let server = new SyncServer(); + server.registerUser("john", "password"); + Assert.ok(server.userExists("john")); + server.start(); + _("Started on " + server.port); + + let req = localRequest(server, "/1.1/john/storage/crypto/keys"); + _("req is " + req); + // Shouldn't reject, beyond that we don't care. + await req.get(); + + await promiseStopServer(server); +}); + +add_task(async function test_info_collections() { + let server = new SyncServer(Object.create(SyncServerCallback)); + function responseHasCorrectHeaders(r) { + Assert.equal(r.status, 200); + Assert.equal(r.headers["content-type"], "application/json"); + Assert.ok("x-weave-timestamp" in r.headers); + } + + server.registerUser("john", "password"); + server.start(); + + let req = localRequest(server, "/1.1/john/info/collections"); + await req.get(); + responseHasCorrectHeaders(req.response); + Assert.equal(req.response.body, "{}"); + + let putReq = localRequest(server, "/1.1/john/storage/crypto/keys"); + let payload = JSON.stringify({ foo: "bar" }); + let putResp = await putReq.put(payload); + + responseHasCorrectHeaders(putResp); + + let putResponseBody = putResp.body; + _("PUT response body: " + JSON.stringify(putResponseBody)); + + // When we PUT something to crypto/keys, "crypto" appears in the response. + req = localRequest(server, "/1.1/john/info/collections"); + + await req.get(); + responseHasCorrectHeaders(req.response); + let expectedColl = server.getCollection("john", "crypto"); + Assert.ok(!!expectedColl); + let modified = expectedColl.timestamp; + Assert.ok(modified > 0); + Assert.equal(putResponseBody, modified); + Assert.equal(JSON.parse(req.response.body).crypto, modified); + + await promiseStopServer(server); +}); + +add_task(async function test_storage_request() { + let keysURL = "/1.1/john/storage/crypto/keys?foo=bar"; + let foosURL = "/1.1/john/storage/crypto/foos"; + let storageURL = "/1.1/john/storage"; + + let server = new SyncServer(); + let creation = server.timestamp(); + server.registerUser("john", "password"); + + server.createContents("john", { + crypto: { foos: { foo: "bar" } }, + }); + let coll = server.user("john").collection("crypto"); + Assert.ok(!!coll); + + _("We're tracking timestamps."); + Assert.ok(coll.timestamp >= creation); + + async function retrieveWBONotExists() { + let req = localRequest(server, keysURL); + let response = await req.get(); + _("Body is " + response.body); + _("Modified is " + response.newModified); + Assert.equal(response.status, 404); + Assert.equal(response.body, "Not found"); + } + + async function retrieveWBOExists() { + let req = localRequest(server, foosURL); + let response = await req.get(); + _("Body is " + response.body); + _("Modified is " + response.newModified); + let parsedBody = JSON.parse(response.body); + Assert.equal(parsedBody.id, "foos"); + Assert.equal(parsedBody.modified, coll.wbo("foos").modified); + Assert.equal(JSON.parse(parsedBody.payload).foo, "bar"); + } + + async function deleteWBONotExists() { + let req = localRequest(server, keysURL); + server.callback.onItemDeleted = function (username, collection, wboID) { + do_throw("onItemDeleted should not have been called."); + }; + + let response = await req.delete(); + + _("Body is " + response.body); + _("Modified is " + response.newModified); + Assert.equal(response.status, 200); + delete server.callback.onItemDeleted; + } + + async function deleteWBOExists() { + let req = localRequest(server, foosURL); + server.callback.onItemDeleted = function (username, collection, wboID) { + _("onItemDeleted called for " + collection + "/" + wboID); + delete server.callback.onItemDeleted; + Assert.equal(username, "john"); + Assert.equal(collection, "crypto"); + Assert.equal(wboID, "foos"); + }; + await req.delete(); + _("Body is " + req.response.body); + _("Modified is " + req.response.newModified); + Assert.equal(req.response.status, 200); + } + + async function deleteStorage() { + _("Testing DELETE on /storage."); + let now = server.timestamp(); + _("Timestamp: " + now); + let req = localRequest(server, storageURL); + await req.delete(); + + _("Body is " + req.response.body); + _("Modified is " + req.response.newModified); + let parsedBody = JSON.parse(req.response.body); + Assert.ok(parsedBody >= now); + do_check_empty(server.users.john.collections); + } + + async function getStorageFails() { + _("Testing that GET on /storage fails."); + let req = localRequest(server, storageURL); + await req.get(); + Assert.equal(req.response.status, 405); + Assert.equal(req.response.headers.allow, "DELETE"); + } + + async function getMissingCollectionWBO() { + _("Testing that fetching a WBO from an on-existent collection 404s."); + let req = localRequest(server, storageURL + "/foobar/baz"); + await req.get(); + Assert.equal(req.response.status, 404); + } + + server.start(null); + + await retrieveWBONotExists(); + await retrieveWBOExists(); + await deleteWBOExists(); + await deleteWBONotExists(); + await getStorageFails(); + await getMissingCollectionWBO(); + await deleteStorage(); + + await promiseStopServer(server); +}); + +add_task(async function test_x_weave_records() { + let server = new SyncServer(); + server.registerUser("john", "password"); + + server.createContents("john", { + crypto: { foos: { foo: "bar" }, bars: { foo: "baz" } }, + }); + server.start(); + + let wbo = localRequest(server, "/1.1/john/storage/crypto/foos"); + await wbo.get(); + Assert.equal(false, "x-weave-records" in wbo.response.headers); + let col = localRequest(server, "/1.1/john/storage/crypto"); + await col.get(); + // Collection fetches do. + Assert.equal(col.response.headers["x-weave-records"], "2"); + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_interval_triggers.js b/services/sync/tests/unit/test_interval_triggers.js new file mode 100644 index 0000000000..09d8987e1a --- /dev/null +++ b/services/sync/tests/unit/test_interval_triggers.js @@ -0,0 +1,462 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Svc.Prefs.set("registerEngines", ""); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let scheduler; +let clientsEngine; + +async function sync_httpd_setup() { + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + return httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()), + }); +} + +async function setUp(server) { + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)); +} + +add_task(async function setup() { + scheduler = Service.scheduler; + clientsEngine = Service.clientsEngine; + + // Don't remove stale clients when syncing. This is a test-only workaround + // that lets us add clients directly to the store, without losing them on + // the next sync. + clientsEngine._removeRemoteClient = async id => {}; +}); + +add_task(async function test_successful_sync_adjustSyncInterval() { + enableValidationPrefs(); + + _("Test successful sync calling adjustSyncInterval"); + let syncSuccesses = 0; + function onSyncFinish() { + _("Sync success."); + syncSuccesses++; + } + Svc.Obs.add("weave:service:sync:finish", onSyncFinish); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.hasIncomingItems); + + _("Test as long as numClients <= 1 our sync interval is SINGLE_USER."); + // idle == true && numClients <= 1 && hasIncomingItems == false + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncSuccesses, 1); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncSuccesses, 2); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncSuccesses, 3); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == true && numClients <= 1 && hasIncomingItems == true + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncSuccesses, 4); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _( + "Test as long as idle && numClients > 1 our sync interval is idleInterval." + ); + // idle == true && numClients > 1 && hasIncomingItems == true + await Service.clientsEngine._store.create({ + id: "foo", + cleartext: { name: "bar", type: "mobile" }, + }); + await Service.sync(); + Assert.equal(syncSuccesses, 5); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + // idle == true && numClients > 1 && hasIncomingItems == false + scheduler.hasIncomingItems = false; + await Service.sync(); + Assert.equal(syncSuccesses, 6); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + _("Test non-idle, numClients > 1, no incoming items => activeInterval."); + // idle == false && numClients > 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncSuccesses, 7); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + _("Test non-idle, numClients > 1, incoming items => immediateInterval."); + // idle == false && numClients > 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncSuccesses, 8); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); // gets reset to false + Assert.equal(scheduler.syncInterval, scheduler.immediateInterval); + + Svc.Obs.remove("weave:service:sync:finish", onSyncFinish); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_unsuccessful_sync_adjustSyncInterval() { + enableValidationPrefs(); + + _("Test unsuccessful sync calling adjustSyncInterval"); + + let syncFailures = 0; + function onSyncError() { + _("Sync error."); + syncFailures++; + } + Svc.Obs.add("weave:service:sync:error", onSyncError); + + _("Test unsuccessful sync calls adjustSyncInterval"); + // Force sync to fail. + Svc.Prefs.set("firstSync", "notReady"); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.hasIncomingItems); + + _("Test as long as numClients <= 1 our sync interval is SINGLE_USER."); + // idle == true && numClients <= 1 && hasIncomingItems == false + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncFailures, 1); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncFailures, 2); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == false && numClients <= 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncFailures, 3); + Assert.ok(!scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // idle == true && numClients <= 1 && hasIncomingItems == true + scheduler.idle = true; + await Service.sync(); + Assert.equal(syncFailures, 4); + Assert.ok(scheduler.idle); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _( + "Test as long as idle && numClients > 1 our sync interval is idleInterval." + ); + // idle == true && numClients > 1 && hasIncomingItems == true + Svc.Prefs.set("clients.devices.mobile", 2); + scheduler.updateClientMode(); + + await Service.sync(); + Assert.equal(syncFailures, 5); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + // idle == true && numClients > 1 && hasIncomingItems == false + scheduler.hasIncomingItems = false; + await Service.sync(); + Assert.equal(syncFailures, 6); + Assert.ok(scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + _("Test non-idle, numClients > 1, no incoming items => activeInterval."); + // idle == false && numClients > 1 && hasIncomingItems == false + scheduler.idle = false; + await Service.sync(); + Assert.equal(syncFailures, 7); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + _("Test non-idle, numClients > 1, incoming items => immediateInterval."); + // idle == false && numClients > 1 && hasIncomingItems == true + scheduler.hasIncomingItems = true; + await Service.sync(); + Assert.equal(syncFailures, 8); + Assert.ok(!scheduler.idle); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.hasIncomingItems); // gets reset to false + Assert.equal(scheduler.syncInterval, scheduler.immediateInterval); + + await Service.startOver(); + Svc.Obs.remove("weave:service:sync:error", onSyncError); + await promiseStopServer(server); +}); + +add_task(async function test_back_triggers_sync() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Single device: no sync triggered. + scheduler.idle = true; + scheduler.observe(null, "active", Svc.Prefs.get("scheduler.idleTime")); + Assert.ok(!scheduler.idle); + + // Multiple devices: sync is triggered. + Svc.Prefs.set("clients.devices.mobile", 2); + scheduler.updateClientMode(); + + let promiseDone = promiseOneObserver("weave:service:sync:finish"); + + scheduler.idle = true; + scheduler.observe(null, "active", Svc.Prefs.get("scheduler.idleTime")); + Assert.ok(!scheduler.idle); + await promiseDone; + + Service.recordManager.clearCache(); + Svc.Prefs.resetBranch(""); + scheduler.setDefaults(); + await clientsEngine.resetClient(); + + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_adjust_interval_on_sync_error() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + let syncFailures = 0; + function onSyncError() { + _("Sync error."); + syncFailures++; + } + Svc.Obs.add("weave:service:sync:error", onSyncError); + + _("Test unsuccessful sync updates client mode & sync intervals"); + // Force a sync fail. + Svc.Prefs.set("firstSync", "notReady"); + + Assert.equal(syncFailures, 0); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + Svc.Prefs.set("clients.devices.mobile", 2); + await Service.sync(); + + Assert.equal(syncFailures, 1); + Assert.ok(scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + Svc.Obs.remove("weave:service:sync:error", onSyncError); + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_bug671378_scenario() { + enableValidationPrefs(); + + // Test scenario similar to bug 671378. This bug appeared when a score + // update occurred that wasn't large enough to trigger a sync so + // scheduleNextSync() was called without a time interval parameter, + // setting nextSync to a non-zero value and preventing the timer from + // being adjusted in the next call to scheduleNextSync(). + let server = await sync_httpd_setup(); + await setUp(server); + + let syncSuccesses = 0; + function onSyncFinish() { + _("Sync success."); + syncSuccesses++; + } + Svc.Obs.add("weave:service:sync:finish", onSyncFinish); + + // After first sync call, syncInterval & syncTimer are singleDeviceInterval. + await Service.sync(); + Assert.equal(syncSuccesses, 1); + Assert.equal(false, scheduler.numClients > 1); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + + let promiseDone = new Promise(resolve => { + // Wrap scheduleNextSync so we are notified when it is finished. + scheduler._scheduleNextSync = scheduler.scheduleNextSync; + scheduler.scheduleNextSync = function () { + scheduler._scheduleNextSync(); + + // Check on sync:finish scheduleNextSync sets the appropriate + // syncInterval and syncTimer values. + if (syncSuccesses == 2) { + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + scheduler.scheduleNextSync = scheduler._scheduleNextSync; + Svc.Obs.remove("weave:service:sync:finish", onSyncFinish); + Service.startOver().then(() => { + server.stop(resolve); + }); + } + }; + }); + + // Set nextSync != 0 + // syncInterval still hasn't been set by call to updateClientMode. + // Explicitly trying to invoke scheduleNextSync during a sync + // (to immitate a score update that isn't big enough to trigger a sync). + Svc.Obs.add("weave:service:sync:start", function onSyncStart() { + // Wait for other sync:start observers to be called so that + // nextSync is set to 0. + CommonUtils.nextTick(function () { + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + + scheduler.scheduleNextSync(); + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + }); + }); + + await Service.clientsEngine._store.create({ + id: "foo", + cleartext: { name: "bar", type: "mobile" }, + }); + await Service.sync(); + await promiseDone; +}); + +add_task(async function test_adjust_timer_larger_syncInterval() { + _( + "Test syncInterval > current timout period && nextSync != 0, syncInterval is NOT used." + ); + Svc.Prefs.set("clients.devices.mobile", 2); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + scheduler.scheduleNextSync(); + + // Ensure we have a small interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer.delay, scheduler.activeInterval); + + // Make interval large again + await clientsEngine._wipeClient(); + Svc.Prefs.reset("clients.devices.mobile"); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + scheduler.scheduleNextSync(); + + // Ensure timer delay remains as the small interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + // SyncSchedule. + await Service.startOver(); +}); + +add_task(async function test_adjust_timer_smaller_syncInterval() { + _( + "Test current timout > syncInterval period && nextSync != 0, syncInterval is used." + ); + scheduler.scheduleNextSync(); + + // Ensure we have a large interval. + Assert.notEqual(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + + // Make interval smaller + Svc.Prefs.set("clients.devices.mobile", 2); + scheduler.updateClientMode(); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + scheduler.scheduleNextSync(); + + // Ensure smaller timer delay is used. + Assert.notEqual(scheduler.nextSync, 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + // SyncSchedule. + await Service.startOver(); +}); diff --git a/services/sync/tests/unit/test_keys.js b/services/sync/tests/unit/test_keys.js new file mode 100644 index 0000000000..8cc5d4055c --- /dev/null +++ b/services/sync/tests/unit/test_keys.js @@ -0,0 +1,242 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { CollectionKeyManager, CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); + +var collectionKeys = new CollectionKeyManager(); + +function do_check_keypair_eq(a, b) { + Assert.equal(2, a.length); + Assert.equal(2, b.length); + Assert.equal(a[0], b[0]); + Assert.equal(a[1], b[1]); +} + +add_test(function test_set_invalid_values() { + _("Ensure that setting invalid encryption and HMAC key values is caught."); + + let bundle = new BulkKeyBundle("foo"); + + let thrown = false; + try { + bundle.encryptionKey = null; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("Encryption key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.encryptionKey = ["trollololol"]; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("Encryption key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = Utils.generateRandomBytesLegacy(15); + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key must be at least 128"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = null; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key can only be set to string"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = ["trollolol"]; + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key can only be set to"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + try { + bundle.hmacKey = Utils.generateRandomBytesLegacy(15); + } catch (ex) { + thrown = true; + Assert.equal(ex.message.indexOf("HMAC key must be at least 128"), 0); + } finally { + Assert.ok(thrown); + thrown = false; + } + + run_next_test(); +}); + +add_task(async function test_ensureLoggedIn() { + let log = Log.repository.getLogger("Test"); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + await configureIdentity(); + + let keyBundle = Weave.Service.identity.syncKeyBundle; + + /* + * Build a test version of storage/crypto/keys. + * Encrypt it with the sync key. + * Pass it into the CollectionKeyManager. + */ + + log.info("Building storage keys..."); + let storage_keys = new CryptoWrapper("crypto", "keys"); + let default_key64 = await Weave.Crypto.generateRandomKey(); + let default_hmac64 = await Weave.Crypto.generateRandomKey(); + let bookmarks_key64 = await Weave.Crypto.generateRandomKey(); + let bookmarks_hmac64 = await Weave.Crypto.generateRandomKey(); + + storage_keys.cleartext = { + default: [default_key64, default_hmac64], + collections: { bookmarks: [bookmarks_key64, bookmarks_hmac64] }, + }; + storage_keys.modified = Date.now() / 1000; + storage_keys.id = "keys"; + + log.info("Encrypting storage keys..."); + + // Use passphrase (sync key) itself to encrypt the key bundle. + await storage_keys.encrypt(keyBundle); + + // Sanity checking. + Assert.ok(null == storage_keys.cleartext); + Assert.ok(null != storage_keys.ciphertext); + + log.info("Updating collection keys."); + + // updateContents decrypts the object, releasing the payload for us to use. + // Returns true, because the default key has changed. + Assert.ok(await collectionKeys.updateContents(keyBundle, storage_keys)); + let payload = storage_keys.cleartext; + + _("CK: " + JSON.stringify(collectionKeys._collections)); + + // Test that the CollectionKeyManager returns a similar WBO. + let wbo = collectionKeys.asWBO("crypto", "keys"); + + _("WBO: " + JSON.stringify(wbo)); + _("WBO cleartext: " + JSON.stringify(wbo.cleartext)); + + // Check the individual contents. + Assert.equal(wbo.collection, "crypto"); + Assert.equal(wbo.id, "keys"); + Assert.equal(undefined, wbo.modified); + Assert.equal(collectionKeys.lastModified, storage_keys.modified); + Assert.ok(!!wbo.cleartext.default); + do_check_keypair_eq(payload.default, wbo.cleartext.default); + do_check_keypair_eq( + payload.collections.bookmarks, + wbo.cleartext.collections.bookmarks + ); + + Assert.ok("bookmarks" in collectionKeys._collections); + Assert.equal(false, "tabs" in collectionKeys._collections); + + _("Updating contents twice with the same data doesn't proceed."); + await storage_keys.encrypt(keyBundle); + Assert.equal( + false, + await collectionKeys.updateContents(keyBundle, storage_keys) + ); + + /* + * Test that we get the right keys out when we ask for + * a collection's tokens. + */ + let b1 = new BulkKeyBundle("bookmarks"); + b1.keyPairB64 = [bookmarks_key64, bookmarks_hmac64]; + let b2 = collectionKeys.keyForCollection("bookmarks"); + do_check_keypair_eq(b1.keyPair, b2.keyPair); + + // Check key equality. + Assert.ok(b1.equals(b2)); + Assert.ok(b2.equals(b1)); + + b1 = new BulkKeyBundle("[default]"); + b1.keyPairB64 = [default_key64, default_hmac64]; + + Assert.ok(!b1.equals(b2)); + Assert.ok(!b2.equals(b1)); + + b2 = collectionKeys.keyForCollection(null); + do_check_keypair_eq(b1.keyPair, b2.keyPair); + + /* + * Checking for update times. + */ + let info_collections = {}; + Assert.ok(collectionKeys.updateNeeded(info_collections)); + info_collections.crypto = 5000; + Assert.ok(!collectionKeys.updateNeeded(info_collections)); + info_collections.crypto = 1 + Date.now() / 1000; // Add one in case computers are fast! + Assert.ok(collectionKeys.updateNeeded(info_collections)); + + collectionKeys.lastModified = null; + Assert.ok(collectionKeys.updateNeeded({})); + + /* + * Check _compareKeyBundleCollections. + */ + async function newBundle(name) { + let r = new BulkKeyBundle(name); + await r.generateRandom(); + return r; + } + let k1 = await newBundle("k1"); + let k2 = await newBundle("k2"); + let k3 = await newBundle("k3"); + let k4 = await newBundle("k4"); + let k5 = await newBundle("k5"); + let coll1 = { foo: k1, bar: k2 }; + let coll2 = { foo: k1, bar: k2 }; + let coll3 = { foo: k1, bar: k3 }; + let coll4 = { foo: k4 }; + let coll5 = { baz: k5, bar: k2 }; + let coll6 = {}; + + let d1 = collectionKeys._compareKeyBundleCollections(coll1, coll2); // [] + let d2 = collectionKeys._compareKeyBundleCollections(coll1, coll3); // ["bar"] + let d3 = collectionKeys._compareKeyBundleCollections(coll3, coll2); // ["bar"] + let d4 = collectionKeys._compareKeyBundleCollections(coll1, coll4); // ["bar", "foo"] + let d5 = collectionKeys._compareKeyBundleCollections(coll5, coll2); // ["baz", "foo"] + let d6 = collectionKeys._compareKeyBundleCollections(coll6, coll1); // ["bar", "foo"] + let d7 = collectionKeys._compareKeyBundleCollections(coll5, coll5); // [] + let d8 = collectionKeys._compareKeyBundleCollections(coll6, coll6); // [] + + Assert.ok(d1.same); + Assert.ok(!d2.same); + Assert.ok(!d3.same); + Assert.ok(!d4.same); + Assert.ok(!d5.same); + Assert.ok(!d6.same); + Assert.ok(d7.same); + Assert.ok(d8.same); + + Assert.deepEqual(d1.changed, []); + Assert.deepEqual(d2.changed, ["bar"]); + Assert.deepEqual(d3.changed, ["bar"]); + Assert.deepEqual(d4.changed, ["bar", "foo"]); + Assert.deepEqual(d5.changed, ["baz", "foo"]); + Assert.deepEqual(d6.changed, ["bar", "foo"]); +}); diff --git a/services/sync/tests/unit/test_load_modules.js b/services/sync/tests/unit/test_load_modules.js new file mode 100644 index 0000000000..8258b5be1e --- /dev/null +++ b/services/sync/tests/unit/test_load_modules.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +const modules = [ + "addonutils.js", + "addonsreconciler.js", + "constants.js", + "engines/addons.js", + "engines/clients.js", + "engines/extension-storage.js", + "engines/passwords.js", + "engines/prefs.js", + "engines.js", + "keys.js", + "main.js", + "policies.js", + "record.js", + "resource.js", + "service.js", + "stages/declined.js", + "stages/enginesync.js", + "status.js", + "sync_auth.js", + "util.js", +]; + +if (AppConstants.MOZ_APP_NAME != "thunderbird") { + modules.push( + "engines/bookmarks.js", + "engines/forms.js", + "engines/history.js", + "engines/tabs.js" + ); +} + +const testingModules = [ + "fakeservices.js", + "rotaryengine.js", + "utils.js", + "fxa_utils.js", +]; + +function run_test() { + for (let m of modules) { + let res = "resource://services-sync/" + m; + _("Attempting to load " + res); + ChromeUtils.import(res); + } + + for (let m of testingModules) { + let res = "resource://testing-common/services/sync/" + m; + _("Attempting to load " + res); + ChromeUtils.import(res); + } +} diff --git a/services/sync/tests/unit/test_node_reassignment.js b/services/sync/tests/unit/test_node_reassignment.js new file mode 100644 index 0000000000..92fa0b07ed --- /dev/null +++ b/services/sync/tests/unit/test_node_reassignment.js @@ -0,0 +1,515 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +_( + "Test that node reassignment responses are respected on all kinds of " + + "requests." +); + +const { RESTRequest } = ChromeUtils.importESModule( + "resource://services-common/rest.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); + +add_task(async function setup() { + validate_all_future_pings(); +}); + +/** + * Emulate the following Zeus config: + * $draining = data.get($prefix . $host . " draining"); + * if ($draining == "drain.") { + * log.warn($log_host_db_status . " migrating=1 (node-reassignment)" . + * $log_suffix); + * http.sendResponse("401 Node reassignment", $content_type, + * '"server request: node reassignment"', ""); + * } + */ +const reassignBody = '"server request: node reassignment"'; + +// API-compatible with SyncServer handler. Bind `handler` to something to use +// as a ServerCollection handler. +function handleReassign(handler, req, resp) { + resp.setStatusLine(req.httpVersion, 401, "Node reassignment"); + resp.setHeader("Content-Type", "application/json"); + resp.bodyOutputStream.write(reassignBody, reassignBody.length); +} + +async function prepareServer() { + let server = new SyncServer(); + server.registerUser("johndoe"); + server.start(); + syncTestLogging(); + await configureIdentity({ username: "johndoe" }, server); + return server; +} + +function getReassigned() { + try { + return Services.prefs.getBoolPref("services.sync.lastSyncReassigned"); + } catch (ex) { + if (ex.result != Cr.NS_ERROR_UNEXPECTED) { + do_throw( + "Got exception retrieving lastSyncReassigned: " + Log.exceptionStr(ex) + ); + } + } + return false; +} + +/** + * Make a test request to `url`, then watch the result of two syncs + * to ensure that a node request was made. + * Runs `between` between the two. This can be used to undo deliberate failure + * setup, detach observers, etc. + */ +async function syncAndExpectNodeReassignment( + server, + firstNotification, + between, + secondNotification, + url +) { + let deferred = PromiseUtils.defer(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // Make sure that it works! + let request = new RESTRequest(url); + let response = await request.get(); + Assert.equal(response.status, 401); + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // Allow for tests to clean up error conditions. + between(); + } + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Second sync nextTick."); + Assert.equal(getTokenCount, 1); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + await Service.sync(); + + await deferred.promise; +} + +add_task(async function test_momentary_401_engine() { + enableValidationPrefs(); + + _("Test a failure for engine URLs that's resolved by reassignment."); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // We want to verify that the clusterURL pref has been cleared after a 401 + // inside a sync. Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function between() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Assert.ok(getReassigned()); + } + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:finish", + between, + "weave:service:sync:finish", + Service.storageURL + "rotary" + ); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +// This test ends up being a failing fetch *after we're already logged in*. +add_task(async function test_momentary_401_info_collections() { + enableValidationPrefs(); + + _("Test a failure for info/collections that's resolved by reassignment."); + let server = await prepareServer(); + + _("First sync to prepare server contents."); + await Service.sync(); + + // Return a 401 for info requests, particularly info/collections. + let oldHandler = server.toplevelHandlers.info; + server.toplevelHandlers.info = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.info = oldHandler; + } + + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.infoURL + ); +}); + +add_task(async function test_momentary_401_storage_loggedin() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + _("Performing initial sync to ensure we are logged in."); + await Service.sync(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(Service.isLoggedIn, "already logged in"); + await syncAndExpectNodeReassignment( + server, + "weave:service:sync:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +add_task(async function test_momentary_401_storage_loggedout() { + enableValidationPrefs(); + + _( + "Test a failure for any storage URL, not just engine parts. " + + "Resolved by reassignment." + ); + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + function undo() { + _("Undoing test changes."); + server.toplevelHandlers.storage = oldHandler; + } + + Assert.ok(!Service.isLoggedIn, "not already logged in"); + await syncAndExpectNodeReassignment( + server, + "weave:service:login:error", + undo, + "weave:service:sync:finish", + Service.storageURL + "meta/global" + ); +}); + +add_task(async function test_loop_avoidance_storage() { + enableValidationPrefs(); + + _( + "Test that a repeated failure doesn't result in a sync loop " + + "if node reassignment cannot resolve the failure." + ); + + let server = await prepareServer(); + + // Return a 401 for all storage requests. + let oldHandler = server.toplevelHandlers.storage; + server.toplevelHandlers.storage = handleReassign; + + let firstNotification = "weave:service:login:error"; + let secondNotification = "weave:service:login:error"; + let thirdNotification = "weave:service:sync:finish"; + + let deferred = PromiseUtils.defer(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // Track the time. We want to make sure the duration between the first and + // second sync is small, and then that the duration between second and third + // is set to be large. + let now; + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + // We got a 401 mid-sync, and set the pref accordingly. + Assert.ok(Services.prefs.getBoolPref("services.sync.lastSyncReassigned")); + + // Update the timestamp. + now = Date.now(); + } + + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Svc.Obs.add(thirdNotification, onThirdSync); + + // This sync occurred within the backoff interval. + let elapsedTime = Date.now() - now; + Assert.ok(elapsedTime < MINIMUM_BACKOFF_INTERVAL); + + // This pref will be true until a sync completes successfully. + Assert.ok(getReassigned()); + + // The timer will be set for some distant time. + // We store nextSync in prefs, which offers us only limited resolution. + // Include that logic here. + let expectedNextSync = + 1000 * Math.floor((now + MINIMUM_BACKOFF_INTERVAL) / 1000); + _("Next sync scheduled for " + Service.scheduler.nextSync); + _("Expected to be slightly greater than " + expectedNextSync); + + Assert.ok(Service.scheduler.nextSync >= expectedNextSync); + Assert.ok(!!Service.scheduler.syncTimer); + + // Undo our evil scheme. + server.toplevelHandlers.storage = oldHandler; + + // Bring the timer forward to kick off a successful sync, so we can watch + // the pref get cleared. + Service.scheduler.scheduleNextSync(0); + } + function onThirdSync() { + Svc.Obs.remove(thirdNotification, onThirdSync); + + // That'll do for now; no more syncs. + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Third sync nextTick."); + Assert.ok(!getReassigned()); + Assert.equal(getTokenCount, 2); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + + now = Date.now(); + await Service.sync(); + await deferred.promise; +}); + +add_task(async function test_loop_avoidance_engine() { + enableValidationPrefs(); + + _( + "Test that a repeated 401 in an engine doesn't result in a sync loop " + + "if node reassignment cannot resolve the failure." + ); + let server = await prepareServer(); + let john = server.user("johndoe"); + + _("Enabling the Rotary engine."); + let { engine, syncID, tracker } = await registerRotaryEngine(); + let deferred = PromiseUtils.defer(); + + let getTokenCount = 0; + let mockTSC = { + // TokenServerClient + async getTokenUsingOAuth() { + getTokenCount++; + return { endpoint: server.baseURI + "1.1/johndoe/" }; + }, + }; + Service.identity._tokenServerClient = mockTSC; + + // We need the server to be correctly set up prior to experimenting. Do this + // through a sync. + let global = { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + rotary: { version: engine.version, syncID }, + }; + john.createCollection("meta").insert("global", global); + + _("First sync to prepare server contents."); + await Service.sync(); + + _("Setting up Rotary collection to 401."); + let rotary = john.createCollection("rotary"); + let oldHandler = rotary.collectionHandler; + rotary.collectionHandler = handleReassign.bind(this, undefined); + + // Flag the Rotary engine to need syncing. + john.collection("rotary").timestamp += 1000; + + function onLoginStart() { + // lastSyncReassigned shouldn't be cleared until a sync has succeeded. + _("Ensuring that lastSyncReassigned is still set at next sync start."); + Assert.ok(getReassigned()); + } + + function beforeSuccessfulSync() { + _("Undoing test changes."); + rotary.collectionHandler = oldHandler; + } + + let firstNotification = "weave:service:sync:finish"; + let secondNotification = "weave:service:sync:finish"; + let thirdNotification = "weave:service:sync:finish"; + + // Track the time. We want to make sure the duration between the first and + // second sync is small, and then that the duration between second and third + // is set to be large. + let now; + + function onFirstSync() { + _("First sync completed."); + Svc.Obs.remove(firstNotification, onFirstSync); + Svc.Obs.add(secondNotification, onSecondSync); + + Assert.equal(Service.clusterURL, ""); + + _("Adding observer that lastSyncReassigned is still set on login."); + Svc.Obs.add("weave:service:login:start", onLoginStart); + + // We got a 401 mid-sync, and set the pref accordingly. + Assert.ok(Services.prefs.getBoolPref("services.sync.lastSyncReassigned")); + + // Update the timestamp. + now = Date.now(); + } + + function onSecondSync() { + _("Second sync completed."); + Svc.Obs.remove(secondNotification, onSecondSync); + Svc.Obs.add(thirdNotification, onThirdSync); + + // This sync occurred within the backoff interval. + let elapsedTime = Date.now() - now; + Assert.ok(elapsedTime < MINIMUM_BACKOFF_INTERVAL); + + // This pref will be true until a sync completes successfully. + Assert.ok(getReassigned()); + + // The timer will be set for some distant time. + // We store nextSync in prefs, which offers us only limited resolution. + // Include that logic here. + let expectedNextSync = + 1000 * Math.floor((now + MINIMUM_BACKOFF_INTERVAL) / 1000); + _("Next sync scheduled for " + Service.scheduler.nextSync); + _("Expected to be slightly greater than " + expectedNextSync); + + Assert.ok(Service.scheduler.nextSync >= expectedNextSync); + Assert.ok(!!Service.scheduler.syncTimer); + + // Undo our evil scheme. + beforeSuccessfulSync(); + + // Bring the timer forward to kick off a successful sync, so we can watch + // the pref get cleared. + Service.scheduler.scheduleNextSync(0); + } + + function onThirdSync() { + Svc.Obs.remove(thirdNotification, onThirdSync); + + // That'll do for now; no more syncs. + Service.scheduler.clearSyncTriggers(); + + // Make absolutely sure that any event listeners are done with their work + // before we proceed. + waitForZeroTimer(function () { + _("Third sync nextTick."); + Assert.ok(!getReassigned()); + Assert.equal(getTokenCount, 2); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + Service.startOver().then(() => { + server.stop(deferred.resolve); + }); + }); + } + + Svc.Obs.add(firstNotification, onFirstSync); + + now = Date.now(); + await Service.sync(); + await deferred.promise; + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); diff --git a/services/sync/tests/unit/test_password_engine.js b/services/sync/tests/unit/test_password_engine.js new file mode 100644 index 0000000000..e15974b086 --- /dev/null +++ b/services/sync/tests/unit/test_password_engine.js @@ -0,0 +1,587 @@ +const { FXA_PWDMGR_HOST, FXA_PWDMGR_REALM } = ChromeUtils.import( + "resource://gre/modules/FxAccountsCommon.js" +); +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const LoginInfo = Components.Constructor( + "@mozilla.org/login-manager/loginInfo;1", + Ci.nsILoginInfo, + "init" +); + +const PropertyBag = Components.Constructor( + "@mozilla.org/hash-property-bag;1", + Ci.nsIWritablePropertyBag +); + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine.wipeClient(); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + if (server) { + await promiseStopServer(server); + } +} + +add_task(async function setup() { + // Disable addon sync because AddonManager won't be initialized here. + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); +}); + +add_task(async function test_ignored_fields() { + _("Only changes to syncable fields should be tracked"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + enableValidationPrefs(); + + let login = await Services.logins.addLoginAsync( + new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ) + ); + login.QueryInterface(Ci.nsILoginMetaInfo); // For `guid`. + + engine._tracker.start(); + + try { + let nonSyncableProps = new PropertyBag(); + nonSyncableProps.setProperty("timeLastUsed", Date.now()); + nonSyncableProps.setProperty("timesUsed", 3); + Services.logins.modifyLogin(login, nonSyncableProps); + + let noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track non-syncable fields"); + + let syncableProps = new PropertyBag(); + syncableProps.setProperty("username", "newuser"); + Services.logins.modifyLogin(login, syncableProps); + + let changes = await engine.pullNewChanges(); + deepEqual( + Object.keys(changes), + [login.guid], + "Should track syncable fields" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_ignored_sync_credentials() { + _("Sync credentials in login manager should be ignored"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + enableValidationPrefs(); + + engine._tracker.start(); + + try { + let login = await Services.logins.addLoginAsync( + new LoginInfo( + FXA_PWDMGR_HOST, + null, + FXA_PWDMGR_REALM, + "fxa-uid", + "creds", + "", + "" + ) + ); + + let noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track new FxA credentials"); + + let props = new PropertyBag(); + props.setProperty("password", "newcreds"); + Services.logins.modifyLogin(login, props); + + noChanges = await engine.pullNewChanges(); + deepEqual(noChanges, {}, "Should not track changes to FxA credentials"); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_password_engine() { + _("Basic password sync test"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + enableValidationPrefs(); + + _("Add new login to upload during first sync"); + let newLogin; + { + let login = new LoginInfo( + "https://example.com", + "", + null, + "username", + "password", + "", + "" + ); + await Services.logins.addLoginAsync(login); + + let logins = Services.logins.findLogins("https://example.com", "", ""); + equal(logins.length, 1, "Should find new login in login manager"); + newLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + + // Insert a server record that's older, so that we prefer the local one. + let rec = new LoginRec("passwords", newLogin.guid); + rec.formSubmitURL = newLogin.formActionOrigin; + rec.httpRealm = newLogin.httpRealm; + rec.hostname = newLogin.origin; + rec.username = newLogin.username; + rec.password = "sekrit"; + let remotePasswordChangeTime = Date.now() - 1 * 60 * 60 * 24 * 1000; + rec.timeCreated = remotePasswordChangeTime; + rec.timePasswordChanged = remotePasswordChangeTime; + collection.insert( + newLogin.guid, + encryptPayload(rec.cleartext), + remotePasswordChangeTime / 1000 + ); + } + + _("Add login with older password change time to replace during first sync"); + let oldLogin; + { + let login = new LoginInfo( + "https://mozilla.com", + "", + null, + "us3r", + "0ldpa55", + "", + "" + ); + await Services.logins.addLoginAsync(login); + + let props = new PropertyBag(); + let localPasswordChangeTime = Date.now() - 1 * 60 * 60 * 24 * 1000; + props.setProperty("timePasswordChanged", localPasswordChangeTime); + Services.logins.modifyLogin(login, props); + + let logins = Services.logins.findLogins("https://mozilla.com", "", ""); + equal(logins.length, 1, "Should find old login in login manager"); + oldLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + equal(oldLogin.timePasswordChanged, localPasswordChangeTime); + + let rec = new LoginRec("passwords", oldLogin.guid); + rec.hostname = oldLogin.origin; + rec.formSubmitURL = oldLogin.formActionOrigin; + rec.httpRealm = oldLogin.httpRealm; + rec.username = oldLogin.username; + // Change the password and bump the password change time to ensure we prefer + // the remote one during reconciliation. + rec.password = "n3wpa55"; + rec.usernameField = oldLogin.usernameField; + rec.passwordField = oldLogin.usernameField; + rec.timeCreated = oldLogin.timeCreated; + rec.timePasswordChanged = Date.now(); + collection.insert(oldLogin.guid, encryptPayload(rec.cleartext)); + } + + await engine._tracker.stop(); + + try { + await sync_engine_and_validate_telem(engine, false); + + let newRec = collection.cleartext(newLogin.guid); + equal( + newRec.password, + "password", + "Should update remote password for newer login" + ); + + let logins = Services.logins.findLogins("https://mozilla.com", "", ""); + equal( + logins[0].password, + "n3wpa55", + "Should update local password for older login" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_password_dupe() { + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + let guid2 = Utils.makeGUID(); + let cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: "foo", + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Math.round(Date.now()), + timePasswordChanged: Math.round(Date.now()), + }; + rec1.cleartext = cleartext; + + _("Create remote record with same details and guid1"); + collection.insert(guid1, encryptPayload(rec1.cleartext)); + + _("Create remote record with guid2"); + collection.insert(guid2, encryptPayload(cleartext)); + + _("Create local record with same details and guid1"); + await engine._store.create(rec1); + + try { + _("Perform sync"); + await sync_engine_and_validate_telem(engine, true); + + let logins = Services.logins.findLogins("https://www.example.com", "", ""); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid2); + equal(null, collection.payload(guid1)); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_updated_null_password_sync() { + _("Ensure updated null login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + let remoteDetails = { + formSubmitURL: "https://www.nullupdateexample.com", + hostname: "https://www.nullupdateexample.com", + httpRealm: null, + username: null, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + let localDetails = { + formSubmitURL: "https://www.nullupdateexample.com", + hostname: "https://www.nullupdateexample.com", + httpRealm: null, + username: "foo", + password: "foobar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + _("Create remote record with same details and guid1"); + collection.insertRecord(Object.assign({}, remoteDetails, { id: guid1 })); + + try { + _("Create local updated login with null password"); + await engine._store.update(Object.assign({}, localDetails, { id: guid2 })); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = Services.logins.findLogins( + "https://www.nullupdateexample.com", + "", + "" + ); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_updated_undefined_password_sync() { + _("Ensure updated undefined login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + let guid1 = Utils.makeGUID(); + let guid2 = Utils.makeGUID(); + let remoteDetails = { + formSubmitURL: "https://www.undefinedupdateexample.com", + hostname: "https://www.undefinedupdateexample.com", + httpRealm: null, + username: undefined, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + let localDetails = { + formSubmitURL: "https://www.undefinedupdateexample.com", + hostname: "https://www.undefinedupdateexample.com", + httpRealm: null, + username: "foo", + password: "foobar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + _("Create remote record with same details and guid1"); + collection.insertRecord(Object.assign({}, remoteDetails, { id: guid1 })); + + try { + _("Create local updated login with undefined password"); + await engine._store.update(Object.assign({}, localDetails, { id: guid2 })); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = Services.logins.findLogins( + "https://www.undefinedupdateexample.com", + "", + "" + ); + + equal(logins.length, 1); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).guid, guid1); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_new_null_password_sync() { + _("Ensure new null login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + rec1.cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: null, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + _("Create local login with null password"); + await engine._store.create(rec1); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = Services.logins.findLogins("https://www.example.com", "", ""); + + equal(logins.length, 1); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, null); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, undefined); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, ""); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_new_undefined_password_sync() { + _("Ensure new undefined login username is converted to a string"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let guid1 = Utils.makeGUID(); + let rec1 = new LoginRec("passwords", guid1); + rec1.cleartext = { + formSubmitURL: "https://www.example.com", + hostname: "https://www.example.com", + httpRealm: null, + username: undefined, + password: "bar", + usernameField: "username-field", + passwordField: "password-field", + timeCreated: Date.now(), + timePasswordChanged: Date.now(), + }; + + try { + _("Create local login with undefined password"); + await engine._store.create(rec1); + + _("Perform sync"); + await sync_engine_and_validate_telem(engine, false); + + let logins = Services.logins.findLogins("https://www.example.com", "", ""); + + equal(logins.length, 1); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, null); + notEqual(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, undefined); + equal(logins[0].QueryInterface(Ci.nsILoginMetaInfo).username, ""); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_sync_password_validation() { + // This test isn't in test_password_validator to avoid duplicating cleanup. + _("Ensure that if a password validation happens, it ends up in the ping"); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + Svc.Prefs.set("engine.passwords.validation.interval", 0); + Svc.Prefs.set("engine.passwords.validation.percentageChance", 100); + Svc.Prefs.set("engine.passwords.validation.maxRecords", -1); + Svc.Prefs.set("engine.passwords.validation.enabled", true); + + try { + let ping = await wait_for_ping(() => Service.sync()); + + let engineInfo = ping.engines.find(e => e.name == "passwords"); + ok(engineInfo, "Engine should be in ping"); + + let validation = engineInfo.validation; + ok(validation, "Engine should have validation info"); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_roundtrip_unknown_fields() { + _( + "Testing that unknown fields from other clients get roundtripped back to server" + ); + + let engine = Service.engineManager.get("passwords"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("passwords"); + + enableValidationPrefs(); + + _("Add login with older password change time to replace during first sync"); + let oldLogin; + { + let login = new LoginInfo( + "https://mozilla.com", + "", + null, + "us3r", + "0ldpa55", + "", + "" + ); + Services.logins.addLogin(login); + + let props = new PropertyBag(); + let localPasswordChangeTime = Math.round( + Date.now() - 1 * 60 * 60 * 24 * 1000 + ); + props.setProperty("timePasswordChanged", localPasswordChangeTime); + Services.logins.modifyLogin(login, props); + + let logins = Services.logins.findLogins("https://mozilla.com", "", ""); + equal(logins.length, 1, "Should find old login in login manager"); + oldLogin = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + equal(oldLogin.timePasswordChanged, localPasswordChangeTime); + + let rec = new LoginRec("passwords", oldLogin.guid); + rec.hostname = oldLogin.origin; + rec.formSubmitURL = oldLogin.formActionOrigin; + rec.httpRealm = oldLogin.httpRealm; + rec.username = oldLogin.username; + // Change the password and bump the password change time to ensure we prefer + // the remote one during reconciliation. + rec.password = "n3wpa55"; + rec.usernameField = oldLogin.usernameField; + rec.passwordField = oldLogin.usernameField; + rec.timeCreated = oldLogin.timeCreated; + rec.timePasswordChanged = Math.round(Date.now()); + + // pretend other clients have some snazzy new fields + // we don't quite understand yet + rec.cleartext.someStrField = "I am a str"; + rec.cleartext.someObjField = { newField: "I am a new field" }; + collection.insert(oldLogin.guid, encryptPayload(rec.cleartext)); + } + + await engine._tracker.stop(); + + try { + await sync_engine_and_validate_telem(engine, false); + + let logins = Services.logins.findLogins("https://mozilla.com", "", ""); + equal( + logins[0].password, + "n3wpa55", + "Should update local password for older login" + ); + let expectedUnknowns = JSON.stringify({ + someStrField: "I am a str", + someObjField: { newField: "I am a new field" }, + }); + // Check that the local record has all unknown fields properly + // stringified + equal(logins[0].unknownFields, expectedUnknowns); + + // Check that the server has the unknown fields unfurled and on the + // top-level record + let serverRec = collection.cleartext(oldLogin.guid); + equal(serverRec.someStrField, "I am a str"); + equal(serverRec.someObjField.newField, "I am a new field"); + } finally { + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_password_store.js b/services/sync/tests/unit/test_password_store.js new file mode 100644 index 0000000000..e0a4e1343f --- /dev/null +++ b/services/sync/tests/unit/test_password_store.js @@ -0,0 +1,400 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { SyncedRecordsTelemetry } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" +); + +async function checkRecord( + name, + record, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + recordIsUpdated +) { + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + let logins = Services.logins.findLogins( + record.hostname, + record.formSubmitURL, + null + ); + + _("Record" + name + ":" + JSON.stringify(logins)); + _("Count" + name + ":" + logins.length); + + Assert.equal(logins.length, expectedCount); + + if (expectedCount > 0) { + Assert.ok(!!(await store.getAllIDs())[record.id]); + let stored_record = logins[0].QueryInterface(Ci.nsILoginMetaInfo); + + if (timeCreated !== undefined) { + Assert.equal(stored_record.timeCreated, expectedTimeCreated); + } + + if (timePasswordChanged !== undefined) { + if (recordIsUpdated) { + Assert.ok( + stored_record.timePasswordChanged >= expectedTimePasswordChanged + ); + } else { + Assert.equal( + stored_record.timePasswordChanged, + expectedTimePasswordChanged + ); + } + return stored_record.timePasswordChanged; + } + } else { + Assert.ok(!(await store.getAllIDs())[record.id]); + } + return undefined; +} + +async function changePassword( + name, + hostname, + password, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + insert, + recordIsUpdated +) { + const BOGUS_GUID = "zzzzzz" + hostname; + let record = new LoginRec("passwords", BOGUS_GUID); + record.cleartext = { + id: BOGUS_GUID, + hostname, + formSubmitURL: hostname, + username: "john", + password, + usernameField: "username", + passwordField: "password", + }; + + if (timeCreated !== undefined) { + record.timeCreated = timeCreated; + } + + if (timePasswordChanged !== undefined) { + record.timePasswordChanged = timePasswordChanged; + } + + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + if (insert) { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch([record], countTelemetry)).length, + 0 + ); + } + + return checkRecord( + name, + record, + expectedCount, + timeCreated, + expectedTimeCreated, + timePasswordChanged, + expectedTimePasswordChanged, + recordIsUpdated + ); +} + +async function test_apply_records_with_times( + hostname, + timeCreated, + timePasswordChanged +) { + // The following record is going to be inserted in the store and it needs + // to be found there. Then its timestamps are going to be compared to + // the expected values. + await changePassword( + " ", + hostname, + "password", + 1, + timeCreated, + timeCreated, + timePasswordChanged, + timePasswordChanged, + true + ); +} + +async function test_apply_multiple_records_with_times() { + // The following records are going to be inserted in the store and they need + // to be found there. Then their timestamps are going to be compared to + // the expected values. + await changePassword( + "A", + "http://foo.a.com", + "password", + 1, + undefined, + undefined, + undefined, + undefined, + true + ); + await changePassword( + "B", + "http://foo.b.com", + "password", + 1, + 1000, + 1000, + undefined, + undefined, + true + ); + await changePassword( + "C", + "http://foo.c.com", + "password", + 1, + undefined, + undefined, + 1000, + 1000, + true + ); + await changePassword( + "D", + "http://foo.d.com", + "password", + 1, + 1000, + 1000, + 1000, + 1000, + true + ); + + // The following records are not going to be inserted in the store and they + // are not going to be found there. + await changePassword( + "NotInStoreA", + "http://foo.aaaa.com", + "password", + 0, + undefined, + undefined, + undefined, + undefined, + false + ); + await changePassword( + "NotInStoreB", + "http://foo.bbbb.com", + "password", + 0, + 1000, + 1000, + undefined, + undefined, + false + ); + await changePassword( + "NotInStoreC", + "http://foo.cccc.com", + "password", + 0, + undefined, + undefined, + 1000, + 1000, + false + ); + await changePassword( + "NotInStoreD", + "http://foo.dddd.com", + "password", + 0, + 1000, + 1000, + 1000, + 1000, + false + ); +} + +async function test_apply_same_record_with_different_times() { + // The following record is going to be inserted multiple times in the store + // and it needs to be found there. Then its timestamps are going to be + // compared to the expected values. + + /* eslint-disable no-unused-vars */ + /* The eslint linter thinks that timePasswordChanged is unused, even though + it is passed as an argument to changePassword. */ + var timePasswordChanged = 100; + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 100, + 100, + 100, + timePasswordChanged, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 100, + 100, + 800, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password", + 1, + 500, + 100, + 800, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password2", + 1, + 500, + 100, + 1536213005222, + timePasswordChanged, + true, + true + ); + timePasswordChanged = await changePassword( + "A", + "http://a.tn", + "password2", + 1, + 500, + 100, + 800, + timePasswordChanged, + true, + true + ); + /* eslint-enable no-unused-vars */ +} + +async function test_LoginRec_toString(store, recordData) { + let rec = await store.createRecord(recordData.id); + ok(rec); + ok(!rec.toString().includes(rec.password)); +} + +add_task(async function run_test() { + const BOGUS_GUID_A = "zzzzzzzzzzzz"; + const BOGUS_GUID_B = "yyyyyyyyyyyy"; + let recordA = new LoginRec("passwords", BOGUS_GUID_A); + let recordB = new LoginRec("passwords", BOGUS_GUID_B); + recordA.cleartext = { + id: BOGUS_GUID_A, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + httpRealm: "secure", + username: "john", + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordB.cleartext = { + id: BOGUS_GUID_B, + hostname: "http://foo.baz.com", + formSubmitURL: "http://foo.baz.com", + username: "john", + password: "smith", + usernameField: "username", + passwordField: "password", + unknownStr: "an unknown string from another field", + }; + + let engine = Service.engineManager.get("passwords"); + let store = engine._store; + + try { + let countTelemetry = new SyncedRecordsTelemetry(); + Assert.equal( + (await store.applyIncomingBatch([recordA, recordB], countTelemetry)) + .length, + 0 + ); + + // Only the good record makes it to Services.logins. + let badLogins = Services.logins.findLogins( + recordA.hostname, + recordA.formSubmitURL, + recordA.httpRealm + ); + let goodLogins = Services.logins.findLogins( + recordB.hostname, + recordB.formSubmitURL, + null + ); + + _("Bad: " + JSON.stringify(badLogins)); + _("Good: " + JSON.stringify(goodLogins)); + _("Count: " + badLogins.length + ", " + goodLogins.length); + + Assert.equal(goodLogins.length, 1); + Assert.equal(badLogins.length, 0); + + // applyIncoming should've put any unknown fields from the server + // into a catch-all unknownFields field + Assert.equal( + goodLogins[0].unknownFields, + JSON.stringify({ + unknownStr: "an unknown string from another field", + }) + ); + + Assert.ok(!!(await store.getAllIDs())[BOGUS_GUID_B]); + Assert.ok(!(await store.getAllIDs())[BOGUS_GUID_A]); + + await test_LoginRec_toString(store, recordB); + + await test_apply_records_with_times( + "http://afoo.baz.com", + undefined, + undefined + ); + await test_apply_records_with_times("http://bfoo.baz.com", 1000, undefined); + await test_apply_records_with_times("http://cfoo.baz.com", undefined, 2000); + await test_apply_records_with_times("http://dfoo.baz.com", 1000, 2000); + + await test_apply_multiple_records_with_times(); + + await test_apply_same_record_with_different_times(); + } finally { + await store.wipe(); + } +}); diff --git a/services/sync/tests/unit/test_password_tracker.js b/services/sync/tests/unit/test_password_tracker.js new file mode 100644 index 0000000000..eba479da15 --- /dev/null +++ b/services/sync/tests/unit/test_password_tracker.js @@ -0,0 +1,160 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PasswordEngine, LoginRec } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +let store; +let tracker; + +add_task(async function setup() { + await Service.engineManager.register(PasswordEngine); + engine = Service.engineManager.get("passwords"); + store = engine._store; + tracker = engine._tracker; +}); + +add_task(async function test_tracking() { + let recordNum = 0; + + _("Verify we've got an empty tracker to work with."); + let changes = await tracker.getChangedIDs(); + do_check_empty(changes); + + async function createPassword() { + _("RECORD NUM: " + recordNum); + let record = new LoginRec("passwords", "GUID" + recordNum); + record.cleartext = { + id: "GUID" + recordNum, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + username: "john" + recordNum, + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordNum++; + let login = store._nsLoginInfoFromRecord(record); + await Services.logins.addLoginAsync(login); + await tracker.asyncObserver.promiseObserversComplete(); + } + + try { + _( + "Create a password record. Won't show because we haven't started tracking yet" + ); + await createPassword(); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Assert.equal(tracker.score, 0); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + await createPassword(); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 1); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + + _("Starting twice won't do any harm."); + tracker.start(); + await createPassword(); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + + _("Let's stop tracking again."); + await tracker.clearChangedIDs(); + tracker.resetScore(); + await tracker.stop(); + await createPassword(); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Assert.equal(tracker.score, 0); + + _("Stopping twice won't do any harm."); + await tracker.stop(); + await createPassword(); + changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Assert.equal(tracker.score, 0); + } finally { + _("Clean up."); + await store.wipe(); + await tracker.clearChangedIDs(); + tracker.resetScore(); + await tracker.stop(); + } +}); + +add_task(async function test_onWipe() { + _("Verify we've got an empty tracker to work with."); + const changes = await tracker.getChangedIDs(); + do_check_empty(changes); + Assert.equal(tracker.score, 0); + + try { + _("A store wipe should increment the score"); + tracker.start(); + await store.wipe(); + await tracker.asyncObserver.promiseObserversComplete(); + + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + } finally { + tracker.resetScore(); + await tracker.stop(); + } +}); + +add_task(async function test_removeAllLogins() { + let recordNum = 0; + _("Verify that all tracked logins are removed."); + + async function createPassword() { + _("RECORD NUM: " + recordNum); + let record = new LoginRec("passwords", "GUID" + recordNum); + record.cleartext = { + id: "GUID" + recordNum, + hostname: "http://foo.bar.com", + formSubmitURL: "http://foo.bar.com", + username: "john" + recordNum, + password: "smith", + usernameField: "username", + passwordField: "password", + }; + recordNum++; + let login = store._nsLoginInfoFromRecord(record); + await Services.logins.addLoginAsync(login); + await tracker.asyncObserver.promiseObserversComplete(); + } + try { + _("Tell tracker to start tracking changes"); + tracker.start(); + await createPassword(); + await createPassword(); + let changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + + await tracker.clearChangedIDs(); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 0); + + _("Tell sync to remove all logins"); + Services.logins.removeAllUserFacingLogins(); + await tracker.asyncObserver.promiseObserversComplete(); + changes = await tracker.getChangedIDs(); + do_check_attribute_count(changes, 2); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 5); + } finally { + _("Clean up."); + await store.wipe(); + await tracker.clearChangedIDs(); + tracker.resetScore(); + await tracker.stop(); + } +}); diff --git a/services/sync/tests/unit/test_password_validator.js b/services/sync/tests/unit/test_password_validator.js new file mode 100644 index 0000000000..445b119e1d --- /dev/null +++ b/services/sync/tests/unit/test_password_validator.js @@ -0,0 +1,176 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PasswordValidator } = ChromeUtils.importESModule( + "resource://services-sync/engines/passwords.sys.mjs" +); + +function getDummyServerAndClient() { + return { + server: [ + { + id: "11111", + guid: "11111", + hostname: "https://www.11111.com", + formSubmitURL: "https://www.11111.com", + password: "qwerty123", + passwordField: "pass", + username: "foobar", + usernameField: "user", + httpRealm: null, + }, + { + id: "22222", + guid: "22222", + hostname: "https://www.22222.org", + formSubmitURL: "https://www.22222.org", + password: "hunter2", + passwordField: "passwd", + username: "baz12345", + usernameField: "user", + httpRealm: null, + }, + { + id: "33333", + guid: "33333", + hostname: "https://www.33333.com", + formSubmitURL: "https://www.33333.com", + password: "p4ssw0rd", + passwordField: "passwad", + username: "quux", + usernameField: "user", + httpRealm: null, + }, + ], + client: [ + { + id: "11111", + guid: "11111", + hostname: "https://www.11111.com", + formSubmitURL: "https://www.11111.com", + password: "qwerty123", + passwordField: "pass", + username: "foobar", + usernameField: "user", + httpRealm: null, + }, + { + id: "22222", + guid: "22222", + hostname: "https://www.22222.org", + formSubmitURL: "https://www.22222.org", + password: "hunter2", + passwordField: "passwd", + username: "baz12345", + usernameField: "user", + httpRealm: null, + }, + { + id: "33333", + guid: "33333", + hostname: "https://www.33333.com", + formSubmitURL: "https://www.33333.com", + password: "p4ssw0rd", + passwordField: "passwad", + username: "quux", + usernameField: "user", + httpRealm: null, + }, + ], + }; +} + +add_task(async function test_valid() { + let { server, client } = getDummyServerAndClient(); + let validator = new PasswordValidator(); + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + equal(clientRecords.length, 3); + equal(records.length, 3); + equal(deletedRecords.length, 0); + deepEqual(problemData, validator.emptyProblemData()); +}); + +add_task(async function test_missing() { + let validator = new PasswordValidator(); + { + let { server, client } = getDummyServerAndClient(); + + client.pop(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 2); + equal(records.length, 3); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + expected.clientMissing.push("33333"); + deepEqual(problemData, expected); + } + { + let { server, client } = getDummyServerAndClient(); + + server.pop(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 3); + equal(records.length, 2); + equal(deletedRecords.length, 0); + + let expected = validator.emptyProblemData(); + expected.serverMissing.push("33333"); + deepEqual(problemData, expected); + } +}); + +add_task(async function test_deleted() { + let { server, client } = getDummyServerAndClient(); + let deletionRecord = { id: "444444", guid: "444444", deleted: true }; + + server.push(deletionRecord); + let validator = new PasswordValidator(); + + let { problemData, clientRecords, records, deletedRecords } = + await validator.compareClientWithServer(client, server); + + equal(clientRecords.length, 3); + equal(records.length, 4); + deepEqual(deletedRecords, [deletionRecord]); + + let expected = validator.emptyProblemData(); + deepEqual(problemData, expected); +}); + +add_task(async function test_duplicates() { + let validator = new PasswordValidator(); + { + let { server, client } = getDummyServerAndClient(); + client.push(Cu.cloneInto(client[0], {})); + + let { problemData } = await validator.compareClientWithServer( + client, + server + ); + + let expected = validator.emptyProblemData(); + expected.clientDuplicates.push("11111"); + deepEqual(problemData, expected); + } + { + let { server, client } = getDummyServerAndClient(); + server.push(Cu.cloneInto(server[server.length - 1], {})); + + let { problemData } = await validator.compareClientWithServer( + client, + server + ); + + let expected = validator.emptyProblemData(); + expected.duplicates.push("33333"); + deepEqual(problemData, expected); + } +}); diff --git a/services/sync/tests/unit/test_postqueue.js b/services/sync/tests/unit/test_postqueue.js new file mode 100644 index 0000000000..2e687bce11 --- /dev/null +++ b/services/sync/tests/unit/test_postqueue.js @@ -0,0 +1,985 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +let { PostQueue } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); + +function makeRecord(nbytes) { + return { + toJSON: () => ({ payload: "x".repeat(nbytes) }), + }; +} + +// Note: This is 14 bytes. Tests make assumptions about this (even if it's just +// in setting config.max_request_bytes to a specific value). +makeRecord.nonPayloadOverhead = JSON.stringify(makeRecord(0).toJSON()).length; + +// Gives how many encoded bytes a request with the given payload +// sizes will be (assuming the records were created by makeRecord) +// requestBytesFor([20]) => 22, requestBytesFor([20, 20]) => 43 +function requestBytesFor(recordPayloadByteCounts) { + let requestBytes = 1; + for (let size of recordPayloadByteCounts) { + requestBytes += size + 1 + makeRecord.nonPayloadOverhead; + } + return requestBytes; +} + +function makePostQueue(config, lastModTime, responseGenerator) { + let stats = { + posts: [], + batches: [], + }; + let poster = (data, headers, batch, commit) => { + let payloadBytes = 0; + let numRecords = 0; + for (let record of JSON.parse(data)) { + if (config.max_record_payload_bytes) { + less( + record.payload.length, + config.max_record_payload_bytes, + "PostQueue should respect max_record_payload_bytes" + ); + } + payloadBytes += record.payload.length; + ++numRecords; + } + + let thisPost = { + nbytes: data.length, + batch, + commit, + payloadBytes, + numRecords, + }; + + if (headers.length) { + thisPost.headers = headers; + } + + // check that we respected the provided limits for the post + if (config.max_post_records) { + lessOrEqual( + numRecords, + config.max_post_records, + "PostQueue should respect max_post_records" + ); + } + + if (config.max_post_bytes) { + less( + payloadBytes, + config.max_post_bytes, + "PostQueue should respect max_post_bytes" + ); + } + + if (config.max_request_bytes) { + less( + thisPost.nbytes, + config.max_request_bytes, + "PostQueue should respect max_request_bytes" + ); + } + + stats.posts.push(thisPost); + + // Call this now so we can check if there's a batch id in it. + // Kind of cludgey, but allows us to have the correct batch id even + // before the next post is made. + let nextResponse = responseGenerator.next().value; + + // Record info for the batch. + + let curBatch = stats.batches[stats.batches.length - 1]; + // If there's no batch, it committed, or we requested a new one, + // then we need to start a new one. + if (!curBatch || batch == "true" || curBatch.didCommit) { + curBatch = { + posts: 0, + payloadBytes: 0, + numRecords: 0, + didCommit: false, + batch, + serverBatch: false, + }; + if (nextResponse.obj && nextResponse.obj.batch) { + curBatch.batch = nextResponse.obj.batch; + curBatch.serverBatch = true; + } + stats.batches.push(curBatch); + } + + // If we provided a batch id, it must be the same as the current batch + if (batch && batch != "true") { + equal(curBatch.batch, batch); + } + + curBatch.posts += 1; + curBatch.payloadBytes += payloadBytes; + curBatch.numRecords += numRecords; + curBatch.didCommit = commit; + + // if this is an actual server batch (or it's a one-shot batch), check that + // we respected the provided total limits + if (commit && (batch == "true" || curBatch.serverBatch)) { + if (config.max_total_records) { + lessOrEqual( + curBatch.numRecords, + config.max_total_records, + "PostQueue should respect max_total_records" + ); + } + + if (config.max_total_bytes) { + less( + curBatch.payloadBytes, + config.max_total_bytes, + "PostQueue should respect max_total_bytes" + ); + } + } + + return Promise.resolve(nextResponse); + }; + + let done = () => {}; + let pq = new PostQueue(poster, lastModTime, config, getTestLogger(), done); + return { pq, stats }; +} + +add_task(async function test_simple() { + let config = { + max_request_bytes: 1000, + max_record_payload_bytes: 1000, + }; + + const time = 11111111; + + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + await pq.enqueue(makeRecord(10)); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10]), + payloadBytes: 10, + numRecords: 1, + commit: true, // we don't know if we have batch semantics, so committed. + headers: [["x-if-unmodified-since", time]], + batch: "true", + }, + ]); + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 1, + didCommit: true, + batch: "true", + serverBatch: false, + }, + ]); +}); + +// Test we do the right thing when we need to make multiple posts when there +// are no batch semantics +add_task(async function test_max_request_bytes_no_batch() { + let config = { + max_request_bytes: 50, + max_record_payload_bytes: 50, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 200, + "x-last-modified": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + let payloadSize = 20 - makeRecord.nonPayloadOverhead; + await pq.enqueue(makeRecord(payloadSize)); // total size now 22 bytes - "[" + record + "]" + await pq.enqueue(makeRecord(payloadSize)); // total size now 43 bytes - "[" + record + "," + record + "]" + await pq.enqueue(makeRecord(payloadSize)); // this will exceed our byte limit, so will be in the 2nd POST. + await pq.flush(true); + deepEqual(stats.posts, [ + { + nbytes: 43, // 43 for the first part + payloadBytes: payloadSize * 2, + numRecords: 2, + commit: false, + headers: [["x-if-unmodified-since", time]], + batch: "true", + }, + { + nbytes: 22, + payloadBytes: payloadSize, + numRecords: 1, + commit: false, // we know we aren't in a batch, so never commit. + headers: [["x-if-unmodified-since", time + 100]], + batch: null, + }, + ]); + equal(stats.batches.filter(x => x.didCommit).length, 0); + equal(pq.lastModified, time + 200); +}); + +add_task(async function test_max_record_payload_bytes_no_batch() { + let config = { + max_request_bytes: 100, + max_record_payload_bytes: 50, + }; + + const time = 11111111; + + function* responseGenerator() { + yield { + success: true, + status: 200, + headers: { + "x-weave-timestamp": time + 100, + "x-last-modified": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + // Should trigger when the record really is too large to fit + let { enqueued } = await pq.enqueue(makeRecord(51)); + ok(!enqueued); + // Shouldn't trigger when the encoded record is too big + ok( + (await pq.enqueue(makeRecord(50 - makeRecord.nonPayloadOverhead))).enqueued + ); // total size now 52 bytes - "[" + record + "]" + ok( + (await pq.enqueue(makeRecord(46 - makeRecord.nonPayloadOverhead))).enqueued + ); // total size now 99 bytes - "[" + record0 + "," + record1 + "]" + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: 99, + payloadBytes: 50 + 46 - makeRecord.nonPayloadOverhead * 2, + numRecords: 2, + commit: true, // we know we aren't in a batch, so never commit. + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 50 + 46 - makeRecord.nonPayloadOverhead * 2, + numRecords: 2, + didCommit: true, + batch: "true", + serverBatch: false, + }, + ]); + + equal(pq.lastModified, time + 100); +}); + +// Batch tests. + +// Test making a single post when batch semantics are in place. + +add_task(async function test_single_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 100, + max_total_records: 200, + max_record_payload_bytes: 1000, + }; + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10]), + numRecords: 1, + payloadBytes: 10, + commit: true, // we don't know if we have batch semantics, so committed. + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 1, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); +}); + +// Test we do the right thing when we need to make multiple posts due to +// max_post_bytes when there are batch semantics in place. +add_task(async function test_max_post_bytes_batch() { + let config = { + max_post_bytes: 50, + max_post_records: 4, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 50, + max_request_bytes: 4000, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(20))).enqueued); // 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // 40 + // 60 would overflow, so post + ok((await pq.enqueue(makeRecord(20))).enqueued); // 20 + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when we need to make multiple posts due to +// max_request_bytes when there are batch semantics in place. +add_task(async function test_max_request_bytes_batch() { + let config = { + max_post_bytes: 60, + max_post_records: 40, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 500, + max_request_bytes: 100, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 10, request: 26 (10 + 14 + 2) + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 20, request: 51 (10 + 14 + 1) * 2 + 1 + ok((await pq.enqueue(makeRecord(10))).enqueued); // post: 30, request: 76 (10 + 14 + 1) * 3 + 1 + // 1 more would be post: 40 (fine), request: 101, So we should post. + ok((await pq.enqueue(makeRecord(10))).enqueued); + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10, 10, 10]), + payloadBytes: 30, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([10]), + payloadBytes: 10, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 40, + numRecords: 4, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when the batch bytes limit is exceeded. +add_task(async function test_max_total_bytes_batch() { + let config = { + max_post_bytes: 50, + max_post_records: 20, + max_total_bytes: 70, + max_total_records: 100, + max_record_payload_bytes: 50, + max_request_bytes: 500, + }; + + const time0 = 11111111; + const time1 = 22222222; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time0, "x-weave-timestamp": time0 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { + "x-last-modified": time1 + 200, + "x-weave-timestamp": time1 + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time0, responseGenerator()); + + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 40, batch: 40 + + // this will exceed our POST byte limit, so will be in the 2nd POST - but still in the first batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 60 + + // this will exceed our batch byte limit, so will be in a new batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 20 + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 40, batch: 40 + // This will exceed POST byte limit, so will be in the 4th post, part of the 2nd batch. + ok((await pq.enqueue(makeRecord(20))).enqueued); // payloads = post: 20, batch: 60 + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time0]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time0]], + }, + { + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time1]], + }, + { + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 5678, + headers: [["x-if-unmodified-since", time1]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 1234, + serverBatch: true, + }, + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + didCommit: true, + batch: 5678, + serverBatch: true, + }, + ]); + + equal(pq.lastModified, time1 + 200); +}); + +// Test we split up the posts when we exceed the record limit when batch semantics +// are in place. +add_task(async function test_max_post_records_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 2, + max_total_bytes: 5000, + max_total_records: 100, + max_record_payload_bytes: 1000, + max_request_bytes: 1000, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + // will exceed record limit of 2, so will be in 2nd post. + ok((await pq.enqueue(makeRecord(20))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([20, 20]), + numRecords: 2, + payloadBytes: 40, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([20]), + numRecords: 1, + payloadBytes: 20, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 60, + numRecords: 3, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Test we do the right thing when the batch record limit is exceeded. +add_task(async function test_max_records_batch() { + let config = { + max_post_bytes: 1000, + max_post_records: 3, + max_total_bytes: 10000, + max_total_records: 5, + max_record_payload_bytes: 1000, + max_request_bytes: 10000, + }; + + const time0 = 11111111; + const time1 = 22222222; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time0, "x-weave-timestamp": time0 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { "x-last-modified": time1, "x-weave-timestamp": time1 + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 5678 }, + headers: { + "x-last-modified": time1 + 200, + "x-weave-timestamp": time1 + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time0, responseGenerator()); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + ok((await pq.enqueue(makeRecord(20))).enqueued); + + ok((await pq.enqueue(makeRecord(20))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + // 3 records + nbytes: requestBytesFor([20, 20, 20]), + payloadBytes: 60, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time0]], + }, + { + // 2 records -- end batch1 + nbytes: requestBytesFor([20, 20]), + payloadBytes: 40, + numRecords: 2, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time0]], + }, + { + // 3 records + nbytes: requestBytesFor([20, 20, 20]), + payloadBytes: 60, + numRecords: 3, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time1]], + }, + { + // 1 record -- end batch2 + nbytes: requestBytesFor([20]), + payloadBytes: 20, + numRecords: 1, + commit: true, + batch: 5678, + headers: [["x-if-unmodified-since", time1]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 100, + numRecords: 5, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + { + posts: 2, + payloadBytes: 80, + numRecords: 4, + batch: 5678, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time1 + 200); +}); + +// Test we do the right thing when the limits are met but not exceeded. +add_task(async function test_packed_batch() { + let config = { + max_post_bytes: 41, + max_post_records: 4, + + max_total_bytes: 81, + max_total_records: 8, + + max_record_payload_bytes: 20 + makeRecord.nonPayloadOverhead + 1, + max_request_bytes: requestBytesFor([10, 10, 10, 10]) + 1, + }; + + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { "x-last-modified": time, "x-weave-timestamp": time + 100 }, + }; + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 200, + "x-weave-timestamp": time + 200, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + ok((await pq.enqueue(makeRecord(10))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([10, 10, 10, 10]), + numRecords: 4, + payloadBytes: 40, + commit: false, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + { + nbytes: requestBytesFor([10, 10, 10, 10]), + numRecords: 4, + payloadBytes: 40, + commit: true, + batch: 1234, + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 2, + payloadBytes: 80, + numRecords: 8, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 200); +}); + +// Tests that check that a single record fails to enqueue for the provided config +async function test_enqueue_failure_case(failureLimit, config) { + const time = 11111111; + function* responseGenerator() { + yield { + success: true, + status: 202, + obj: { batch: 1234 }, + headers: { + "x-last-modified": time + 100, + "x-weave-timestamp": time + 100, + }, + }; + } + + let { pq, stats } = makePostQueue(config, time, responseGenerator()); + // Check on empty postqueue + let result = await pq.enqueue(makeRecord(failureLimit + 1)); + ok(!result.enqueued); + notEqual(result.error, undefined); + + ok((await pq.enqueue(makeRecord(5))).enqueued); + + // check on nonempty postqueue + result = await pq.enqueue(makeRecord(failureLimit + 1)); + ok(!result.enqueued); + notEqual(result.error, undefined); + + // make sure that we keep working, skipping the bad record entirely + // (handling the error the queue reported is left up to caller) + ok((await pq.enqueue(makeRecord(5))).enqueued); + + await pq.flush(true); + + deepEqual(stats.posts, [ + { + nbytes: requestBytesFor([5, 5]), + numRecords: 2, + payloadBytes: 10, + commit: true, + batch: "true", + headers: [["x-if-unmodified-since", time]], + }, + ]); + + deepEqual(stats.batches, [ + { + posts: 1, + payloadBytes: 10, + numRecords: 2, + batch: 1234, + serverBatch: true, + didCommit: true, + }, + ]); + + equal(pq.lastModified, time + 100); +} + +add_task(async function test_max_post_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 50, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 500, + max_request_bytes: 500, + }); +}); + +add_task(async function test_max_request_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 500, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 500, + max_request_bytes: 50, + }); +}); + +add_task(async function test_max_record_payload_bytes_enqueue_failure() { + await test_enqueue_failure_case(50, { + max_post_bytes: 500, + max_post_records: 100, + + max_total_bytes: 5000, + max_total_records: 100, + + max_record_payload_bytes: 50, + max_request_bytes: 500, + }); +}); diff --git a/services/sync/tests/unit/test_prefs_engine.js b/services/sync/tests/unit/test_prefs_engine.js new file mode 100644 index 0000000000..abea0a1137 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_engine.js @@ -0,0 +1,132 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { getPrefsGUIDForTest } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); +const PREFS_GUID = getPrefsGUIDForTest(); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function cleanup(engine, server) { + await engine._tracker.stop(); + await engine.wipeClient(); + Svc.Prefs.resetBranch(""); + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function test_modified_after_fail() { + let engine = Service.engineManager.get("prefs"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + // The homepage pref is synced by default. + _("Set homepage before first sync"); + Services.prefs.setStringPref("browser.startup.homepage", "about:welcome"); + + _("First sync; create collection and pref record on server"); + await sync_engine_and_validate_telem(engine, false); + + let collection = server.user("foo").collection("prefs"); + equal( + collection.cleartext(PREFS_GUID).value["browser.startup.homepage"], + "about:welcome", + "Should upload homepage in pref record" + ); + ok( + !engine._tracker.modified, + "Tracker shouldn't be modified after first sync" + ); + + // Our tracker only has a `modified` flag that's reset after a + // successful upload. Force it to remain set by failing the + // upload. + _("Second sync; flag tracker as modified and throw on upload"); + Services.prefs.setStringPref("browser.startup.homepage", "about:robots"); + engine._tracker.modified = true; + let oldPost = collection.post; + collection.post = () => { + throw new Error("Sync this!"); + }; + await Assert.rejects( + sync_engine_and_validate_telem(engine, true), + ex => ex.success === false + ); + ok( + engine._tracker.modified, + "Tracker should remain modified after failed sync" + ); + + _("Third sync"); + collection.post = oldPost; + await sync_engine_and_validate_telem(engine, false); + equal( + collection.cleartext(PREFS_GUID).value["browser.startup.homepage"], + "about:robots", + "Should upload new homepage on third sync" + ); + ok( + !engine._tracker.modified, + "Tracker shouldn't be modified again after third sync" + ); + } finally { + await cleanup(engine, server); + } +}); + +add_task(async function test_allow_arbitrary() { + let engine = Service.engineManager.get("prefs"); + + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + try { + _("Create collection and pref record on server"); + await sync_engine_and_validate_telem(engine, false); + + let collection = server.user("foo").collection("prefs"); + + _("Insert arbitrary pref into remote record"); + let cleartext1 = collection.cleartext(PREFS_GUID); + cleartext1.value.let_viruses_take_over = true; + collection.insert( + PREFS_GUID, + encryptPayload(cleartext1), + new_timestamp() + 5 + ); + + _("Sync again; client shouldn't allow pref"); + await sync_engine_and_validate_telem(engine, false); + ok( + !Services.prefs.getBoolPref("let_viruses_take_over", false), + "Shouldn't allow arbitrary remote prefs without control pref" + ); + + _("Sync with control pref set; client should set new pref"); + Services.prefs.setBoolPref( + "services.sync.prefs.sync.let_viruses_take_over_take_two", + true + ); + + let cleartext2 = collection.cleartext(PREFS_GUID); + cleartext2.value.let_viruses_take_over_take_two = true; + collection.insert( + PREFS_GUID, + encryptPayload(cleartext2), + new_timestamp() + 5 + ); + // Reset the last sync time so that the engine fetches the record again. + await engine.setLastSync(0); + await sync_engine_and_validate_telem(engine, false); + ok( + Services.prefs.getBoolPref("let_viruses_take_over_take_two"), + "Should set arbitrary remote pref with control pref" + ); + } finally { + await cleanup(engine, server); + } +}); diff --git a/services/sync/tests/unit/test_prefs_store.js b/services/sync/tests/unit/test_prefs_store.js new file mode 100644 index 0000000000..23ee9f3a72 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_store.js @@ -0,0 +1,414 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PromiseTestUtils } = ChromeUtils.importESModule( + "resource://testing-common/PromiseTestUtils.sys.mjs" +); +PromiseTestUtils.allowMatchingRejectionsGlobally( + /Unable to arm timer, the object has been finalized\./ +); +PromiseTestUtils.allowMatchingRejectionsGlobally( + /IOUtils\.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished/ +); + +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); +const { PrefRec, getPrefsGUIDForTest } = ChromeUtils.importESModule( + "resource://services-sync/engines/prefs.sys.mjs" +); +const PREFS_GUID = getPrefsGUIDForTest(); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const DEFAULT_THEME_ID = "default-theme@mozilla.org"; +const COMPACT_THEME_ID = "firefox-compact-light@mozilla.org"; + +AddonTestUtils.init(this); +AddonTestUtils.createAppInfo( + "xpcshell@tests.mozilla.org", + "XPCShell", + "1", + "1.9.2" +); +AddonTestUtils.overrideCertDB(); + +add_task(async function run_test() { + _("Test fixtures."); + // Part of this test ensures the default theme, via the preference + // extensions.activeThemeID, is synced correctly - so we do a little + // addons initialization to allow this to work. + + // Enable application scopes to ensure the builtin theme is going to + // be installed as part of the the addon manager startup. + Preferences.set("extensions.enabledScopes", AddonManager.SCOPE_APPLICATION); + await AddonTestUtils.promiseStartupManager(); + + // Install another built-in theme. + await AddonManager.installBuiltinAddon("resource://builtin-themes/light/"); + + const defaultThemeAddon = await AddonManager.getAddonByID(DEFAULT_THEME_ID); + ok(defaultThemeAddon, "Got an addon wrapper for the default theme"); + + const otherThemeAddon = await AddonManager.getAddonByID(COMPACT_THEME_ID); + ok(otherThemeAddon, "Got an addon wrapper for the compact theme"); + + await otherThemeAddon.enable(); + + // read our custom prefs file before doing anything. + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + let prefs = new Preferences(); + try { + _("Expect the compact light theme to be active"); + Assert.strictEqual(prefs.get("extensions.activeThemeID"), COMPACT_THEME_ID); + + _("The GUID corresponds to XUL App ID."); + let allIDs = await store.getAllIDs(); + let ids = Object.keys(allIDs); + Assert.equal(ids.length, 1); + Assert.equal(ids[0], PREFS_GUID); + Assert.ok(allIDs[PREFS_GUID]); + + Assert.ok(await store.itemExists(PREFS_GUID)); + Assert.equal(false, await store.itemExists("random-gibberish")); + + _("Unknown prefs record is created as deleted."); + let record = await store.createRecord("random-gibberish", "prefs"); + Assert.ok(record.deleted); + + _("Prefs record contains only prefs that should be synced."); + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual(record.value["testing.int"], 123); + Assert.strictEqual(record.value["testing.string"], "ohai"); + Assert.strictEqual(record.value["testing.bool"], true); + // non-existing prefs get null as the value + Assert.strictEqual(record.value["testing.nonexistent"], null); + // as do prefs that have a default value. + Assert.strictEqual(record.value["testing.default"], null); + Assert.strictEqual(record.value["testing.turned.off"], undefined); + Assert.strictEqual(record.value["testing.not.turned.on"], undefined); + + _("Prefs record contains the correct control prefs."); + // All control prefs which have the default value and where the pref + // itself is synced should appear, but with null as the value. + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.int"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.string"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.bool"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.dont.change"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.nonexistent"], + null + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.default"], + null + ); + + // but this control pref has a non-default value so that value is synced. + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.turned.off"], + false + ); + + _("Unsyncable prefs are treated correctly."); + // Prefs we consider unsyncable (since they are URLs that won't be stable on + // another firefox) shouldn't be included - neither the value nor the + // control pref should appear. + Assert.strictEqual(record.value["testing.unsynced.url"], undefined); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.unsynced.url"], + undefined + ); + // Other URLs with user prefs should be synced, though. + Assert.strictEqual( + record.value["testing.synced.url"], + "https://www.example.com" + ); + Assert.strictEqual( + record.value["services.sync.prefs.sync.testing.synced.url"], + null + ); + + _("Update some prefs, including one that's to be reset/deleted."); + // This pref is not going to be reset or deleted as there's no "control pref" + // in either the incoming record or locally. + prefs.set( + "testing.deleted-without-control-pref", + "I'm deleted-without-control-pref" + ); + // Another pref with only a local control pref. + prefs.set( + "testing.deleted-with-local-control-pref", + "I'm deleted-with-local-control-pref" + ); + prefs.set( + "services.sync.prefs.sync.testing.deleted-with-local-control-pref", + true + ); + // And a pref without a local control pref but one that's incoming. + prefs.set( + "testing.deleted-with-incoming-control-pref", + "I'm deleted-with-incoming-control-pref" + ); + record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "extensions.activeThemeID": DEFAULT_THEME_ID, + "testing.int": 42, + "testing.string": "im in ur prefs", + "testing.bool": false, + "testing.deleted-without-control-pref": null, + "testing.deleted-with-local-control-pref": null, + "testing.deleted-with-incoming-control-pref": null, + "services.sync.prefs.sync.testing.deleted-with-incoming-control-pref": true, + "testing.somepref": "im a new pref from other device", + "services.sync.prefs.sync.testing.somepref": true, + // Pretend some a stale remote client is overwriting it with a value + // we consider unsyncable. + "testing.synced.url": "blob:ebeb707a-502e-40c6-97a5-dd4bda901463", + // Make sure we can replace the unsynced URL with a valid URL. + "testing.unsynced.url": "https://www.example.com/2", + // Make sure our "master control pref" is ignored. + "services.sync.prefs.dangerously_allow_arbitrary": true, + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary": true, + }; + + const onceAddonEnabled = AddonTestUtils.promiseAddonEvent("onEnabled"); + + await store.update(record); + Assert.strictEqual(prefs.get("testing.int"), 42); + Assert.strictEqual(prefs.get("testing.string"), "im in ur prefs"); + Assert.strictEqual(prefs.get("testing.bool"), false); + Assert.strictEqual( + prefs.get("testing.deleted-without-control-pref"), + "I'm deleted-without-control-pref" + ); + Assert.strictEqual( + prefs.get("testing.deleted-with-local-control-pref"), + undefined + ); + Assert.strictEqual( + prefs.get("testing.deleted-with-incoming-control-pref"), + "I'm deleted-with-incoming-control-pref" + ); + Assert.strictEqual( + prefs.get("testing.dont.change"), + "Please don't change me." + ); + Assert.strictEqual(prefs.get("testing.somepref"), undefined); + Assert.strictEqual( + prefs.get("testing.synced.url"), + "https://www.example.com" + ); + Assert.strictEqual( + prefs.get("testing.unsynced.url"), + "https://www.example.com/2" + ); + Assert.strictEqual(Svc.Prefs.get("prefs.sync.testing.somepref"), undefined); + Assert.strictEqual( + prefs.get("services.sync.prefs.dangerously_allow_arbitrary"), + false + ); + Assert.strictEqual( + prefs.get( + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary" + ), + undefined + ); + + await onceAddonEnabled; + ok( + !defaultThemeAddon.userDisabled, + "the default theme should have been enabled" + ); + ok( + otherThemeAddon.userDisabled, + "the compact theme should have been disabled" + ); + + _("Only the current app's preferences are applied."); + record = new PrefRec("prefs", "some-fake-app"); + record.value = { + "testing.int": 98, + }; + await store.update(record); + Assert.equal(prefs.get("testing.int"), 42); + } finally { + prefs.resetBranch(""); + } +}); + +add_task(async function test_dangerously_allow() { + _("services.sync.prefs.dangerously_allow_arbitrary"); + // read our custom prefs file before doing anything. + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + // configure so that arbitrary prefs are synced. + Services.prefs.setBoolPref( + "services.sync.prefs.dangerously_allow_arbitrary", + true + ); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + let prefs = new Preferences(); + try { + _("Update some prefs"); + // This pref is not going to be reset or deleted as there's no "control pref" + // in either the incoming record or locally. + prefs.set( + "testing.deleted-without-control-pref", + "I'm deleted-without-control-pref" + ); + // Another pref with only a local control pref. + prefs.set( + "testing.deleted-with-local-control-pref", + "I'm deleted-with-local-control-pref" + ); + prefs.set( + "services.sync.prefs.sync.testing.deleted-with-local-control-pref", + true + ); + // And a pref without a local control pref but one that's incoming. + prefs.set( + "testing.deleted-with-incoming-control-pref", + "I'm deleted-with-incoming-control-pref" + ); + let record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "testing.deleted-without-control-pref": null, + "testing.deleted-with-local-control-pref": null, + "testing.deleted-with-incoming-control-pref": null, + "services.sync.prefs.sync.testing.deleted-with-incoming-control-pref": true, + "testing.somepref": "im a new pref from other device", + "services.sync.prefs.sync.testing.somepref": true, + // Make sure our "master control pref" is ignored, even when it's already set. + "services.sync.prefs.dangerously_allow_arbitrary": false, + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary": true, + }; + await store.update(record); + Assert.strictEqual( + prefs.get("testing.deleted-without-control-pref"), + "I'm deleted-without-control-pref" + ); + Assert.strictEqual( + prefs.get("testing.deleted-with-local-control-pref"), + undefined + ); + Assert.strictEqual( + prefs.get("testing.deleted-with-incoming-control-pref"), + undefined + ); + Assert.strictEqual( + prefs.get("testing.somepref"), + "im a new pref from other device" + ); + Assert.strictEqual(Svc.Prefs.get("prefs.sync.testing.somepref"), true); + Assert.strictEqual( + prefs.get("services.sync.prefs.dangerously_allow_arbitrary"), + true + ); + Assert.strictEqual( + prefs.get( + "services.sync.prefs.sync.services.sync.prefs.dangerously_allow_arbitrary" + ), + undefined + ); + } finally { + prefs.resetBranch(""); + } +}); + +add_task(async function test_incoming_sets_seen() { + _("Test the sync-seen allow-list"); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + let prefs = new Preferences(); + + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + const defaultValue = "the value"; + Assert.equal(prefs.get("testing.seen"), defaultValue); + + let record = await store.createRecord(PREFS_GUID, "prefs"); + // Haven't seen a non-default value before, so remains null. + Assert.strictEqual(record.value["testing.seen"], null); + + // pretend an incoming record with the default value - it might not be + // the default everywhere, so we treat it specially. + record = new PrefRec("prefs", PREFS_GUID); + record.value = { + "testing.seen": defaultValue, + }; + await store.update(record); + // Our special control value should now be set. + Assert.strictEqual( + prefs.get("services.sync.prefs.sync-seen.testing.seen"), + true + ); + // It's still the default value, so the value is not considered changed + Assert.equal(prefs.isSet("testing.seen"), false); + + // But now that special control value is set, the record always contains the value. + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual(record.value["testing.seen"], defaultValue); +}); + +add_task(async function test_outgoing_when_changed() { + _("Test the 'seen' pref is set first sync of non-default value"); + + let engine = Service.engineManager.get("prefs"); + let store = engine._store; + let prefs = new Preferences(); + prefs.resetBranch(); + + Services.prefs.readDefaultPrefsFromFile( + do_get_file("prefs_test_prefs_store.js") + ); + const defaultValue = "the value"; + Assert.equal(prefs.get("testing.seen"), defaultValue); + + let record = await store.createRecord(PREFS_GUID, "prefs"); + // Haven't seen a non-default value before, so remains null. + Assert.strictEqual(record.value["testing.seen"], null); + + // Change the value. + prefs.set("testing.seen", "new value"); + record = await store.createRecord(PREFS_GUID, "prefs"); + // creating the record toggled that "seen" pref. + Assert.strictEqual( + prefs.get("services.sync.prefs.sync-seen.testing.seen"), + true + ); + Assert.strictEqual(prefs.get("testing.seen"), "new value"); + + // Resetting the pref does not change that seen value. + prefs.reset("testing.seen"); + Assert.strictEqual(prefs.get("testing.seen"), defaultValue); + + record = await store.createRecord(PREFS_GUID, "prefs"); + Assert.strictEqual( + prefs.get("services.sync.prefs.sync-seen.testing.seen"), + true + ); +}); diff --git a/services/sync/tests/unit/test_prefs_tracker.js b/services/sync/tests/unit/test_prefs_tracker.js new file mode 100644 index 0000000000..9840d287b9 --- /dev/null +++ b/services/sync/tests/unit/test_prefs_tracker.js @@ -0,0 +1,92 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Preferences } = ChromeUtils.importESModule( + "resource://gre/modules/Preferences.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + let engine = Service.engineManager.get("prefs"); + let tracker = engine._tracker; + + let prefs = new Preferences(); + + try { + _("tracker.modified corresponds to preference."); + Assert.equal(Svc.Prefs.get("engine.prefs.modified"), undefined); + Assert.ok(!tracker.modified); + + tracker.modified = true; + Assert.equal(Svc.Prefs.get("engine.prefs.modified"), true); + Assert.ok(tracker.modified); + + _("Engine's getChangedID() just returns the one GUID we have."); + let changedIDs = await engine.getChangedIDs(); + let ids = Object.keys(changedIDs); + Assert.equal(ids.length, 1); + Assert.equal(ids[0], CommonUtils.encodeBase64URL(Services.appinfo.ID)); + + Svc.Prefs.set("engine.prefs.modified", false); + Assert.ok(!tracker.modified); + + _("No modified state, so no changed IDs."); + do_check_empty(await engine.getChangedIDs()); + + _("Initial score is 0"); + Assert.equal(tracker.score, 0); + + _("Test fixtures."); + Svc.Prefs.set("prefs.sync.testing.int", true); + + _( + "Test fixtures haven't upped the tracker score yet because it hasn't started tracking yet." + ); + Assert.equal(tracker.score, 0); + + _("Tell the tracker to start tracking changes."); + tracker.start(); + prefs.set("testing.int", 23); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE); + Assert.equal(tracker.modified, true); + + _("Clearing changed IDs reset modified status."); + await tracker.clearChangedIDs(); + Assert.equal(tracker.modified, false); + + _("Resetting a pref ups the score, too."); + prefs.reset("testing.int"); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 2); + Assert.equal(tracker.modified, true); + await tracker.clearChangedIDs(); + + _("So does changing a pref sync pref."); + Svc.Prefs.set("prefs.sync.testing.int", false); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, true); + await tracker.clearChangedIDs(); + + _( + "Now that the pref sync pref has been flipped, changes to it won't be picked up." + ); + prefs.set("testing.int", 42); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, false); + await tracker.clearChangedIDs(); + + _("Changing some other random pref won't do anything."); + prefs.set("testing.other", "blergh"); + await tracker.asyncObserver.promiseObserversComplete(); + Assert.equal(tracker.score, SCORE_INCREMENT_XLARGE * 3); + Assert.equal(tracker.modified, false); + } finally { + await tracker.stop(); + prefs.resetBranch(""); + } +}); diff --git a/services/sync/tests/unit/test_records_crypto.js b/services/sync/tests/unit/test_records_crypto.js new file mode 100644 index 0000000000..8b841653cd --- /dev/null +++ b/services/sync/tests/unit/test_records_crypto.js @@ -0,0 +1,189 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { CollectionKeyManager, CryptoWrapper } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +var cryptoWrap; + +function crypted_resource_handler(metadata, response) { + let obj = { + id: "resource", + modified: cryptoWrap.modified, + payload: JSON.stringify(cryptoWrap.payload), + }; + return httpd_basic_auth_handler(JSON.stringify(obj), metadata, response); +} + +function prepareCryptoWrap(collection, id) { + let w = new CryptoWrapper(); + w.cleartext.stuff = "my payload here"; + w.collection = collection; + w.id = id; + return w; +} + +add_task(async function test_records_crypto() { + let server; + + await configureIdentity({ username: "john@example.com" }); + let keyBundle = Service.identity.syncKeyBundle; + + try { + let log = Log.repository.getLogger("Test"); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + log.info("Setting up server and authenticator"); + + server = httpd_setup({ "/steam/resource": crypted_resource_handler }); + + log.info("Creating a record"); + + cryptoWrap = prepareCryptoWrap("steam", "resource"); + + log.info("cryptoWrap: " + cryptoWrap.toString()); + + log.info("Encrypting a record"); + + await cryptoWrap.encrypt(keyBundle); + log.info("Ciphertext is " + cryptoWrap.ciphertext); + Assert.ok(cryptoWrap.ciphertext != null); + + let firstIV = cryptoWrap.IV; + + log.info("Decrypting the record"); + + let payload = await cryptoWrap.decrypt(keyBundle); + Assert.equal(payload.stuff, "my payload here"); + Assert.notEqual(payload, cryptoWrap.payload); // wrap.data.payload is the encrypted one + + log.info("Make sure multiple decrypts cause failures"); + let error = ""; + try { + payload = await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal(error.message, "No ciphertext: nothing to decrypt?"); + + log.info("Re-encrypting the record with alternate payload"); + + cryptoWrap.cleartext.stuff = "another payload"; + await cryptoWrap.encrypt(keyBundle); + let secondIV = cryptoWrap.IV; + payload = await cryptoWrap.decrypt(keyBundle); + Assert.equal(payload.stuff, "another payload"); + + log.info("Make sure multiple encrypts use different IVs"); + Assert.notEqual(firstIV, secondIV); + + log.info(await "Make sure differing ids cause failures"); + await cryptoWrap.encrypt(keyBundle); + cryptoWrap.data.id = "other"; + error = ""; + try { + await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal(error.message, "Record id mismatch: resource != other"); + + log.info("Make sure wrong hmacs cause failures"); + await cryptoWrap.encrypt(keyBundle); + cryptoWrap.hmac = "foo"; + error = ""; + try { + await cryptoWrap.decrypt(keyBundle); + } catch (ex) { + error = ex; + } + Assert.equal( + error.message.substr(0, 42), + "Record SHA256 HMAC mismatch: should be foo" + ); + + // Checking per-collection keys and default key handling. + + await generateNewKeys(Service.collectionKeys); + let bookmarkItem = prepareCryptoWrap("bookmarks", "foo"); + await bookmarkItem.encrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ); + log.info("Ciphertext is " + bookmarkItem.ciphertext); + Assert.ok(bookmarkItem.ciphertext != null); + log.info("Decrypting the record explicitly with the default key."); + Assert.equal( + (await bookmarkItem.decrypt(Service.collectionKeys._default)).stuff, + "my payload here" + ); + + // Per-collection keys. + // Generate a key for "bookmarks". + await generateNewKeys(Service.collectionKeys, ["bookmarks"]); + bookmarkItem = prepareCryptoWrap("bookmarks", "foo"); + Assert.equal(bookmarkItem.collection, "bookmarks"); + + // Encrypt. This'll use the "bookmarks" encryption key, because we have a + // special key for it. The same key will need to be used for decryption. + await bookmarkItem.encrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ); + Assert.ok(bookmarkItem.ciphertext != null); + + // Attempt to use the default key, because this is a collision that could + // conceivably occur in the real world. Decryption will error, because + // it's not the bookmarks key. + let err; + try { + await bookmarkItem.decrypt(Service.collectionKeys._default); + } catch (ex) { + err = ex; + } + Assert.equal("Record SHA256 HMAC mismatch", err.message.substr(0, 27)); + + // Explicitly check that it's using the bookmarks key. + // This should succeed. + Assert.equal( + ( + await bookmarkItem.decrypt( + Service.collectionKeys.keyForCollection("bookmarks") + ) + ).stuff, + "my payload here" + ); + + Assert.ok(Service.collectionKeys.hasKeysFor(["bookmarks"])); + + // Add a key for some new collection and verify that it isn't the + // default key. + Assert.ok(!Service.collectionKeys.hasKeysFor(["forms"])); + Assert.ok(!Service.collectionKeys.hasKeysFor(["bookmarks", "forms"])); + let oldFormsKey = Service.collectionKeys.keyForCollection("forms"); + Assert.equal(oldFormsKey, Service.collectionKeys._default); + let newKeys = await Service.collectionKeys.ensureKeysFor(["forms"]); + Assert.ok(newKeys.hasKeysFor(["forms"])); + Assert.ok(newKeys.hasKeysFor(["bookmarks", "forms"])); + let newFormsKey = newKeys.keyForCollection("forms"); + Assert.notEqual(newFormsKey, oldFormsKey); + + // Verify that this doesn't overwrite keys + let regetKeys = await newKeys.ensureKeysFor(["forms"]); + Assert.equal(regetKeys.keyForCollection("forms"), newFormsKey); + + const emptyKeys = new CollectionKeyManager(); + payload = { + default: Service.collectionKeys._default.keyPairB64, + collections: {}, + }; + // Verify that not passing `modified` doesn't throw + emptyKeys.setContents(payload, null); + + log.info("Done!"); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_records_wbo.js b/services/sync/tests/unit/test_records_wbo.js new file mode 100644 index 0000000000..61ba33d749 --- /dev/null +++ b/services/sync/tests/unit/test_records_wbo.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_test(function test_toJSON() { + _("Create a record, for now without a TTL."); + let wbo = new WBORecord("coll", "a_record"); + wbo.modified = 12345; + wbo.sortindex = 42; + wbo.payload = {}; + + _( + "Verify that the JSON representation contains the WBO properties, but not TTL." + ); + let json = JSON.parse(JSON.stringify(wbo)); + Assert.equal(json.modified, 12345); + Assert.equal(json.sortindex, 42); + Assert.equal(json.payload, "{}"); + Assert.equal(false, "ttl" in json); + + _("Set a TTL, make sure it's present in the JSON representation."); + wbo.ttl = 30 * 60; + json = JSON.parse(JSON.stringify(wbo)); + Assert.equal(json.ttl, 30 * 60); + run_next_test(); +}); + +add_task(async function test_fetch() { + let record = { + id: "asdf-1234-asdf-1234", + modified: 2454725.98283, + payload: JSON.stringify({ cheese: "roquefort" }), + }; + let record2 = { + id: "record2", + modified: 2454725.98284, + payload: JSON.stringify({ cheese: "gruyere" }), + }; + let coll = [ + { + id: "record2", + modified: 2454725.98284, + payload: JSON.stringify({ cheese: "gruyere" }), + }, + ]; + + _("Setting up server."); + let server = httpd_setup({ + "/record": httpd_handler(200, "OK", JSON.stringify(record)), + "/record2": httpd_handler(200, "OK", JSON.stringify(record2)), + "/coll": httpd_handler(200, "OK", JSON.stringify(coll)), + }); + + try { + _("Fetching a WBO record"); + let rec = new WBORecord("coll", "record"); + await rec.fetch(Service.resource(server.baseURI + "/record")); + Assert.equal(rec.id, "asdf-1234-asdf-1234"); // NOT "record"! + + Assert.equal(rec.modified, 2454725.98283); + Assert.equal(typeof rec.payload, "object"); + Assert.equal(rec.payload.cheese, "roquefort"); + + _("Fetching a WBO record using the record manager"); + let rec2 = await Service.recordManager.get(server.baseURI + "/record2"); + Assert.equal(rec2.id, "record2"); + Assert.equal(rec2.modified, 2454725.98284); + Assert.equal(typeof rec2.payload, "object"); + Assert.equal(rec2.payload.cheese, "gruyere"); + Assert.equal(Service.recordManager.response.status, 200); + + // Testing collection extraction. + _("Extracting collection."); + let rec3 = new WBORecord("tabs", "foo"); // Create through constructor. + Assert.equal(rec3.collection, "tabs"); + } finally { + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_resource.js b/services/sync/tests/unit/test_resource.js new file mode 100644 index 0000000000..a6befe784e --- /dev/null +++ b/services/sync/tests/unit/test_resource.js @@ -0,0 +1,554 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Observers } = ChromeUtils.importESModule( + "resource://services-common/observers.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); + +var fetched = false; +function server_open(metadata, response) { + let body; + if (metadata.method == "GET") { + fetched = true; + body = "This path exists"; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Wrong request method"; + response.setStatusLine(metadata.httpVersion, 405, "Method Not Allowed"); + } + response.bodyOutputStream.write(body, body.length); +} + +function server_protected(metadata, response) { + let body; + + if (has_hawk_header(metadata)) { + body = "This path exists and is protected"; + response.setStatusLine(metadata.httpVersion, 200, "OK, authorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } else { + body = "This path exists and is protected - failed"; + response.setStatusLine(metadata.httpVersion, 401, "Unauthorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } + + response.bodyOutputStream.write(body, body.length); +} + +function server_404(metadata, response) { + let body = "File not found"; + response.setStatusLine(metadata.httpVersion, 404, "Not Found"); + response.bodyOutputStream.write(body, body.length); +} + +var pacFetched = false; +function server_pac(metadata, response) { + _("Invoked PAC handler."); + pacFetched = true; + let body = 'function FindProxyForURL(url, host) { return "DIRECT"; }'; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.setHeader( + "Content-Type", + "application/x-ns-proxy-autoconfig", + false + ); + response.bodyOutputStream.write(body, body.length); +} + +var sample_data = { + some: "sample_data", + injson: "format", + number: 42, +}; + +function server_upload(metadata, response) { + let body; + + let input = readBytesFromInputStream(metadata.bodyInputStream); + if (input == JSON.stringify(sample_data)) { + body = "Valid data upload via " + metadata.method; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Invalid data upload via " + metadata.method + ": " + input; + response.setStatusLine(metadata.httpVersion, 500, "Internal Server Error"); + } + + response.bodyOutputStream.write(body, body.length); +} + +function server_delete(metadata, response) { + let body; + if (metadata.method == "DELETE") { + body = "This resource has been deleted"; + response.setStatusLine(metadata.httpVersion, 200, "OK"); + } else { + body = "Wrong request method"; + response.setStatusLine(metadata.httpVersion, 405, "Method Not Allowed"); + } + response.bodyOutputStream.write(body, body.length); +} + +function server_json(metadata, response) { + let body = JSON.stringify(sample_data); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +const TIMESTAMP = 1274380461; + +function server_timestamp(metadata, response) { + let body = "Thank you for your request"; + response.setHeader("X-Weave-Timestamp", "" + TIMESTAMP, false); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_backoff(metadata, response) { + let body = "Hey, back off!"; + response.setHeader("X-Weave-Backoff", "600", false); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_quota_notice(request, response) { + let body = "You're approaching quota."; + response.setHeader("X-Weave-Quota-Remaining", "1048576", false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_quota_error(request, response) { + let body = "14"; + response.setHeader("X-Weave-Quota-Remaining", "-1024", false); + response.setStatusLine(request.httpVersion, 400, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +function server_headers(metadata, response) { + let ignore_headers = [ + "host", + "user-agent", + "accept-language", + "accept-encoding", + "accept-charset", + "keep-alive", + "connection", + "pragma", + "origin", + "cache-control", + "content-length", + ]; + let headers = metadata.headers; + let header_names = []; + while (headers.hasMoreElements()) { + let header = headers.getNext().toString(); + if (!ignore_headers.includes(header)) { + header_names.push(header); + } + } + header_names = header_names.sort(); + + headers = {}; + for (let header of header_names) { + headers[header] = metadata.getHeader(header); + } + let body = JSON.stringify(headers); + response.setStatusLine(metadata.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); +} + +var quotaValue; +Observers.add("weave:service:quota:remaining", function (subject) { + quotaValue = subject; +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + Svc.Prefs.set("network.numRetries", 1); // speed up test + run_next_test(); +} + +// This apparently has to come first in order for our PAC URL to be hit. +// Don't put any other HTTP requests earlier in the file! +add_task(async function test_proxy_auth_redirect() { + _( + "Ensure that a proxy auth redirect (which switches out our channel) " + + "doesn't break Resource." + ); + let server = httpd_setup({ + "/open": server_open, + "/pac2": server_pac, + }); + + PACSystemSettings.PACURI = server.baseURI + "/pac2"; + installFakePAC(); + let res = new Resource(server.baseURI + "/open"); + let result = await res.get(); + Assert.ok(pacFetched); + Assert.ok(fetched); + Assert.equal("This path exists", result.data); + pacFetched = fetched = false; + uninstallFakePAC(); + await promiseStopServer(server); +}); + +add_task(async function test_new_channel() { + _("Ensure a redirect to a new channel is handled properly."); + + let resourceRequested = false; + function resourceHandler(metadata, response) { + resourceRequested = true; + + let body = "Test"; + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(body, body.length); + } + + let locationURL; + function redirectHandler(metadata, response) { + let body = "Redirecting"; + response.setStatusLine(metadata.httpVersion, 307, "TEMPORARY REDIRECT"); + response.setHeader("Location", locationURL); + response.bodyOutputStream.write(body, body.length); + } + + let server = httpd_setup({ + "/resource": resourceHandler, + "/redirect": redirectHandler, + }); + locationURL = server.baseURI + "/resource"; + + let request = new Resource(server.baseURI + "/redirect"); + let content = await request.get(); + Assert.ok(resourceRequested); + Assert.equal(200, content.status); + Assert.ok("content-type" in content.headers); + Assert.equal("text/plain", content.headers["content-type"]); + + await promiseStopServer(server); +}); + +var server; + +add_test(function setup() { + server = httpd_setup({ + "/open": server_open, + "/protected": server_protected, + "/404": server_404, + "/upload": server_upload, + "/delete": server_delete, + "/json": server_json, + "/timestamp": server_timestamp, + "/headers": server_headers, + "/backoff": server_backoff, + "/pac2": server_pac, + "/quota-notice": server_quota_notice, + "/quota-error": server_quota_error, + }); + + run_next_test(); +}); + +add_test(function test_members() { + _("Resource object members"); + let uri = server.baseURI + "/open"; + let res = new Resource(uri); + Assert.ok(res.uri instanceof Ci.nsIURI); + Assert.equal(res.uri.spec, uri); + Assert.equal(res.spec, uri); + Assert.equal(typeof res.headers, "object"); + Assert.equal(typeof res.authenticator, "object"); + + run_next_test(); +}); + +add_task(async function test_get() { + _("GET a non-password-protected resource"); + let res = new Resource(server.baseURI + "/open"); + let content = await res.get(); + Assert.equal(content.data, "This path exists"); + Assert.equal(content.status, 200); + Assert.ok(content.success); + + // Observe logging messages. + let resLogger = res._log; + let dbg = resLogger.debug; + let debugMessages = []; + resLogger.debug = function (msg, extra) { + debugMessages.push(`${msg}: ${JSON.stringify(extra)}`); + dbg.call(this, msg); + }; + + // Since we didn't receive proper JSON data, accessing content.obj + // will result in a SyntaxError from JSON.parse + let didThrow = false; + try { + content.obj; + } catch (ex) { + didThrow = true; + } + Assert.ok(didThrow); + Assert.equal(debugMessages.length, 1); + Assert.equal( + debugMessages[0], + 'Parse fail: Response body starts: "This path exists"' + ); + resLogger.debug = dbg; +}); + +add_test(function test_basicauth() { + _("Test that the BasicAuthenticator doesn't screw up header case."); + let res1 = new Resource(server.baseURI + "/foo"); + res1.setHeader("Authorization", "Basic foobar"); + Assert.equal(res1._headers.authorization, "Basic foobar"); + Assert.equal(res1.headers.authorization, "Basic foobar"); + + run_next_test(); +}); + +add_task(async function test_get_protected_fail() { + _( + "GET a password protected resource (test that it'll fail w/o pass, no throw)" + ); + let res2 = new Resource(server.baseURI + "/protected"); + let content = await res2.get(); + Assert.equal(content.data, "This path exists and is protected - failed"); + Assert.equal(content.status, 401); + Assert.ok(!content.success); +}); + +add_task(async function test_get_protected_success() { + _("GET a password protected resource"); + let identityConfig = makeIdentityConfig(); + let syncAuthManager = new SyncAuthManager(); + configureFxAccountIdentity(syncAuthManager, identityConfig); + let auth = syncAuthManager.getResourceAuthenticator(); + let res3 = new Resource(server.baseURI + "/protected"); + res3.authenticator = auth; + Assert.equal(res3.authenticator, auth); + let content = await res3.get(); + Assert.equal(content.data, "This path exists and is protected"); + Assert.equal(content.status, 200); + Assert.ok(content.success); +}); + +add_task(async function test_get_404() { + _("GET a non-existent resource (test that it'll fail, but not throw)"); + let res4 = new Resource(server.baseURI + "/404"); + let content = await res4.get(); + Assert.equal(content.data, "File not found"); + Assert.equal(content.status, 404); + Assert.ok(!content.success); + + // Check some headers of the 404 response + Assert.equal(content.headers.connection, "close"); + Assert.equal(content.headers.server, "httpd.js"); + Assert.equal(content.headers["content-length"], 14); +}); + +add_task(async function test_put_string() { + _("PUT to a resource (string)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.put(JSON.stringify(sample_data)); + Assert.equal(content.data, "Valid data upload via PUT"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_put_object() { + _("PUT to a resource (object)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.put(sample_data); + Assert.equal(content.data, "Valid data upload via PUT"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_post_string() { + _("POST to a resource (string)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.post(JSON.stringify(sample_data)); + Assert.equal(content.data, "Valid data upload via POST"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_post_object() { + _("POST to a resource (object)"); + let res_upload = new Resource(server.baseURI + "/upload"); + let content = await res_upload.post(sample_data); + Assert.equal(content.data, "Valid data upload via POST"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_delete() { + _("DELETE a resource"); + let res6 = new Resource(server.baseURI + "/delete"); + let content = await res6.delete(); + Assert.equal(content.data, "This resource has been deleted"); + Assert.equal(content.status, 200); +}); + +add_task(async function test_json_body() { + _("JSON conversion of response body"); + let res7 = new Resource(server.baseURI + "/json"); + let content = await res7.get(); + Assert.equal(content.data, JSON.stringify(sample_data)); + Assert.equal(content.status, 200); + Assert.equal(JSON.stringify(content.obj), JSON.stringify(sample_data)); +}); + +add_task(async function test_weave_timestamp() { + _("X-Weave-Timestamp header updates Resource.serverTime"); + // Before having received any response containing the + // X-Weave-Timestamp header, Resource.serverTime is null. + Assert.equal(Resource.serverTime, null); + let res8 = new Resource(server.baseURI + "/timestamp"); + await res8.get(); + Assert.equal(Resource.serverTime, TIMESTAMP); +}); + +add_task(async function test_get_default_headers() { + _("GET: Accept defaults to application/json"); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); +}); + +add_task(async function test_put_default_headers() { + _( + "PUT: Accept defaults to application/json, Content-Type defaults to text/plain" + ); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.put("data")).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); + Assert.equal(content["content-type"], "text/plain"); +}); + +add_task(async function test_post_default_headers() { + _( + "POST: Accept defaults to application/json, Content-Type defaults to text/plain" + ); + let res_headers = new Resource(server.baseURI + "/headers"); + let content = JSON.parse((await res_headers.post("data")).data); + Assert.equal(content.accept, "application/json;q=0.9,*/*;q=0.2"); + Assert.equal(content["content-type"], "text/plain"); +}); + +add_task(async function test_setHeader() { + _("setHeader(): setting simple header"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("X-What-Is-Weave", "awesome"); + Assert.equal(res_headers.headers["x-what-is-weave"], "awesome"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content["x-what-is-weave"], "awesome"); +}); + +add_task(async function test_setHeader_overwrite() { + _("setHeader(): setting multiple headers, overwriting existing header"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("X-WHAT-is-Weave", "more awesomer"); + res_headers.setHeader("X-Another-Header", "hello world"); + Assert.equal(res_headers.headers["x-what-is-weave"], "more awesomer"); + Assert.equal(res_headers.headers["x-another-header"], "hello world"); + let content = JSON.parse((await res_headers.get()).data); + Assert.equal(content["x-what-is-weave"], "more awesomer"); + Assert.equal(content["x-another-header"], "hello world"); +}); + +add_task(async function test_put_override_content_type() { + _("PUT: override default Content-Type"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("Content-Type", "application/foobar"); + Assert.equal(res_headers.headers["content-type"], "application/foobar"); + let content = JSON.parse((await res_headers.put("data")).data); + Assert.equal(content["content-type"], "application/foobar"); +}); + +add_task(async function test_post_override_content_type() { + _("POST: override default Content-Type"); + let res_headers = new Resource(server.baseURI + "/headers"); + res_headers.setHeader("Content-Type", "application/foobar"); + let content = JSON.parse((await res_headers.post("data")).data); + Assert.equal(content["content-type"], "application/foobar"); +}); + +add_task(async function test_weave_backoff() { + _("X-Weave-Backoff header notifies observer"); + let backoffInterval; + function onBackoff(subject, data) { + backoffInterval = subject; + } + Observers.add("weave:service:backoff:interval", onBackoff); + + let res10 = new Resource(server.baseURI + "/backoff"); + await res10.get(); + Assert.equal(backoffInterval, 600); +}); + +add_task(async function test_quota_error() { + _("X-Weave-Quota-Remaining header notifies observer on successful requests."); + let res10 = new Resource(server.baseURI + "/quota-error"); + let content = await res10.get(); + Assert.equal(content.status, 400); + Assert.equal(quotaValue, undefined); // HTTP 400, so no observer notification. +}); + +add_task(async function test_quota_notice() { + let res10 = new Resource(server.baseURI + "/quota-notice"); + let content = await res10.get(); + Assert.equal(content.status, 200); + Assert.equal(quotaValue, 1048576); +}); + +add_task(async function test_preserve_exceptions() { + _("Error handling preserves exception information"); + let res11 = new Resource("http://localhost:12345/does/not/exist"); + await Assert.rejects(res11.get(), error => { + Assert.notEqual(error, null); + Assert.equal(error.result, Cr.NS_ERROR_CONNECTION_REFUSED); + Assert.equal(error.name, "NS_ERROR_CONNECTION_REFUSED"); + return true; + }); +}); + +add_task(async function test_timeout() { + _("Ensure channel timeouts are thrown appropriately."); + let res19 = new Resource(server.baseURI + "/json"); + res19.ABORT_TIMEOUT = 0; + await Assert.rejects(res19.get(), error => { + Assert.equal(error.result, Cr.NS_ERROR_NET_TIMEOUT); + return true; + }); +}); + +add_test(function test_uri_construction() { + _("Testing URI construction."); + let args = []; + args.push("newer=" + 1234); + args.push("limit=" + 1234); + args.push("sort=" + 1234); + + let query = "?" + args.join("&"); + + let uri1 = CommonUtils.makeURI("http://foo/" + query).QueryInterface( + Ci.nsIURL + ); + let uri2 = CommonUtils.makeURI("http://foo/").QueryInterface(Ci.nsIURL); + uri2 = uri2.mutate().setQuery(query).finalize().QueryInterface(Ci.nsIURL); + Assert.equal(uri1.query, uri2.query); + + run_next_test(); +}); + +/** + * End of tests that rely on a single HTTP server. + * All tests after this point must begin and end their own. + */ +add_test(function eliminate_server() { + server.stop(run_next_test); +}); diff --git a/services/sync/tests/unit/test_resource_header.js b/services/sync/tests/unit/test_resource_header.js new file mode 100644 index 0000000000..2b58d5d8f1 --- /dev/null +++ b/services/sync/tests/unit/test_resource_header.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); + +var httpServer = new HttpServer(); +httpServer.registerPathHandler("/content", contentHandler); +httpServer.start(-1); + +const HTTP_PORT = httpServer.identity.primaryPort; +const TEST_URL = "http://localhost:" + HTTP_PORT + "/content"; +const BODY = "response body"; + +// Keep headers for later inspection. +var auth = null; +var foo = null; +function contentHandler(metadata, response) { + _("Handling request."); + auth = metadata.getHeader("Authorization"); + foo = metadata.getHeader("X-Foo"); + + _("Extracted headers. " + auth + ", " + foo); + + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(BODY, BODY.length); +} + +// Set a proxy function to cause an internal redirect. +function triggerRedirect() { + const PROXY_FUNCTION = + "function FindProxyForURL(url, host) {" + + " return 'PROXY a_non_existent_domain_x7x6c572v:80; " + + "PROXY localhost:" + + HTTP_PORT + + "';" + + "}"; + + let prefs = Services.prefs.getBranch("network.proxy."); + prefs.setIntPref("type", 2); + prefs.setCharPref("autoconfig_url", "data:text/plain," + PROXY_FUNCTION); +} + +add_task(async function test_headers_copied() { + triggerRedirect(); + + _("Issuing request."); + let resource = new Resource(TEST_URL); + resource.setHeader("Authorization", "Basic foobar"); + resource.setHeader("X-Foo", "foofoo"); + + let result = await resource.get(TEST_URL); + _("Result: " + result.data); + + Assert.equal(result.data, BODY); + Assert.equal(auth, "Basic foobar"); + Assert.equal(foo, "foofoo"); + + await promiseStopServer(httpServer); +}); diff --git a/services/sync/tests/unit/test_resource_ua.js b/services/sync/tests/unit/test_resource_ua.js new file mode 100644 index 0000000000..8fb7a85d7d --- /dev/null +++ b/services/sync/tests/unit/test_resource_ua.js @@ -0,0 +1,96 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +var httpProtocolHandler = Cc[ + "@mozilla.org/network/protocol;1?name=http" +].getService(Ci.nsIHttpProtocolHandler); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); + +var meta_global; +var server; + +var expectedUA; +var ua; +function uaHandler(f) { + return function (request, response) { + ua = request.getHeader("User-Agent"); + return f(request, response); + }; +} + +add_task(async function setup() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + meta_global = new ServerWBO("global"); + server = httpd_setup({ + "/1.1/johndoe/info/collections": uaHandler(collectionsHelper.handler), + "/1.1/johndoe/storage/meta/global": uaHandler(meta_global.handler()), + }); + + await configureIdentity({ username: "johndoe" }, server); + _("Server URL: " + server.baseURI); + + // Note this string is missing the trailing ".destkop" as the test + // adjusts the "client.type" pref where that portion comes from. + expectedUA = + Services.appinfo.name + + "/" + + Services.appinfo.version + + " (" + + httpProtocolHandler.oscpu + + ")" + + " FxSync/" + + WEAVE_VERSION + + "." + + Services.appinfo.appBuildID; +}); + +add_task(async function test_fetchInfo() { + _("Testing _fetchInfo."); + await Service.login(); + await Service._fetchInfo(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_desktop_post() { + _("Testing direct Resource POST."); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.post("foo=bar"); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_desktop_get() { + _("Testing async."); + Svc.Prefs.set("client.type", "desktop"); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.get(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".desktop"); + ua = ""; +}); + +add_task(async function test_mobile_get() { + _("Testing mobile."); + Svc.Prefs.set("client.type", "mobile"); + let r = new Resource(server.baseURI + "/1.1/johndoe/storage/meta/global"); + await r.get(); + _("User-Agent: " + ua); + Assert.equal(ua, expectedUA + ".mobile"); + ua = ""; +}); + +add_test(function tear_down() { + server.stop(run_next_test); +}); diff --git a/services/sync/tests/unit/test_score_triggers.js b/services/sync/tests/unit/test_score_triggers.js new file mode 100644 index 0000000000..c6afa06407 --- /dev/null +++ b/services/sync/tests/unit/test_score_triggers.js @@ -0,0 +1,151 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup() { + let handlers = {}; + + handlers["/1.1/johndoe/storage/meta/global"] = new ServerWBO( + "global", + {} + ).handler(); + handlers["/1.1/johndoe/storage/steam"] = new ServerWBO("steam", {}).handler(); + + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp(server) { + let engineInfo = await registerRotaryEngine(); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + return engineInfo; +} + +add_task(async function test_tracker_score_updated() { + enableValidationPrefs(); + let { engine, tracker } = await registerRotaryEngine(); + + let scoreUpdated = 0; + + function onScoreUpdated() { + scoreUpdated++; + } + + Svc.Obs.add("weave:engine:score:updated", onScoreUpdated); + + try { + Assert.equal(engine.score, 0); + + tracker.score += SCORE_INCREMENT_SMALL; + Assert.equal(engine.score, SCORE_INCREMENT_SMALL); + + Assert.equal(scoreUpdated, 1); + } finally { + Svc.Obs.remove("weave:engine:score:updated", onScoreUpdated); + tracker.resetScore(); + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_sync_triggered() { + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + + await Service.login(); + + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + + Assert.equal(Status.login, LOGIN_SUCCEEDED); + tracker.score += SCORE_INCREMENT_XLARGE; + + await promiseOneObserver("weave:service:sync:finish"); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +add_task(async function test_clients_engine_sync_triggered() { + enableValidationPrefs(); + + _("Ensure that client engine score changes trigger a sync."); + + // The clients engine is not registered like other engines. Therefore, + // it needs special treatment throughout the code. Here, we verify the + // global score tracker gives it that treatment. See bug 676042 for more. + + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + await Service.login(); + + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + Assert.equal(Status.login, LOGIN_SUCCEEDED); + Service.clientsEngine._tracker.score += SCORE_INCREMENT_XLARGE; + + await promiseOneObserver("weave:service:sync:finish"); + _("Sync due to clients engine change completed."); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); + +add_task(async function test_incorrect_credentials_sync_not_triggered() { + enableValidationPrefs(); + + _( + "Ensure that score changes don't trigger a sync if Status.login != LOGIN_SUCCEEDED." + ); + let server = sync_httpd_setup(); + let { engine, tracker } = await setUp(server); + + // Ensure we don't actually try to sync. + function onSyncStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:sync:start", onSyncStart); + + // Faking incorrect credentials to prevent score update. + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + tracker.score += SCORE_INCREMENT_XLARGE; + + // First wait >100ms (nsITimers can take up to that much time to fire, so + // we can account for the timer in delayedAutoconnect) and then one event + // loop tick (to account for a possible call to weave:service:sync:start). + await promiseNamedTimer(150, {}, "timer"); + await Async.promiseYield(); + + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED); + + await Service.startOver(); + await promiseStopServer(server); + + await tracker.clearChangedIDs(); + await Service.engineManager.unregister(engine); +}); diff --git a/services/sync/tests/unit/test_service_attributes.js b/services/sync/tests/unit/test_service_attributes.js new file mode 100644 index 0000000000..9006841983 --- /dev/null +++ b/services/sync/tests/unit/test_service_attributes.js @@ -0,0 +1,85 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { FakeGUIDService } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/fakeservices.sys.mjs" +); + +add_task(async function test_urls() { + _("URL related Service properties correspond to preference settings."); + try { + Assert.equal(Service.clusterURL, ""); + Assert.ok(!Service.userBaseURL); + Assert.equal(Service.infoURL, undefined); + Assert.equal(Service.storageURL, undefined); + Assert.equal(Service.metaURL, undefined); + + _("The 'clusterURL' attribute updates preferences and cached URLs."); + + // Since we don't have a cluster URL yet, these will still not be defined. + Assert.equal(Service.infoURL, undefined); + Assert.ok(!Service.userBaseURL); + Assert.equal(Service.storageURL, undefined); + Assert.equal(Service.metaURL, undefined); + + Service.clusterURL = "http://weave.cluster/1.1/johndoe/"; + + Assert.equal(Service.userBaseURL, "http://weave.cluster/1.1/johndoe/"); + Assert.equal( + Service.infoURL, + "http://weave.cluster/1.1/johndoe/info/collections" + ); + Assert.equal( + Service.storageURL, + "http://weave.cluster/1.1/johndoe/storage/" + ); + Assert.equal( + Service.metaURL, + "http://weave.cluster/1.1/johndoe/storage/meta/global" + ); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_test(function test_syncID() { + _("Service.syncID is auto-generated, corresponds to preference."); + new FakeGUIDService(); + + try { + // Ensure pristine environment + Assert.equal(Svc.Prefs.get("client.syncID"), undefined); + + // Performing the first get on the attribute will generate a new GUID. + Assert.equal(Service.syncID, "fake-guid-00"); + Assert.equal(Svc.Prefs.get("client.syncID"), "fake-guid-00"); + + Svc.Prefs.set("client.syncID", Utils.makeGUID()); + Assert.equal(Svc.Prefs.get("client.syncID"), "fake-guid-01"); + Assert.equal(Service.syncID, "fake-guid-01"); + } finally { + Svc.Prefs.resetBranch(""); + new FakeGUIDService(); + run_next_test(); + } +}); + +add_test(function test_locked() { + _("The 'locked' attribute can be toggled with lock() and unlock()"); + + // Defaults to false + Assert.equal(Service.locked, false); + + Assert.equal(Service.lock(), true); + Assert.equal(Service.locked, true); + + // Locking again will return false + Assert.equal(Service.lock(), false); + + Service.unlock(); + Assert.equal(Service.locked, false); + run_next_test(); +}); diff --git a/services/sync/tests/unit/test_service_cluster.js b/services/sync/tests/unit/test_service_cluster.js new file mode 100644 index 0000000000..d7d63e018b --- /dev/null +++ b/services/sync/tests/unit/test_service_cluster.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { PromiseUtils } = ChromeUtils.importESModule( + "resource://gre/modules/PromiseUtils.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_findCluster() { + syncTestLogging(); + _("Test Service._findCluster()"); + try { + let whenReadyToAuthenticate = PromiseUtils.defer(); + Service.identity.whenReadyToAuthenticate = whenReadyToAuthenticate; + whenReadyToAuthenticate.resolve(true); + + Service.identity._ensureValidToken = () => + Promise.reject(new Error("Connection refused")); + + _("_findCluster() throws on network errors (e.g. connection refused)."); + await Assert.rejects(Service.identity._findCluster(), /Connection refused/); + + Service.identity._ensureValidToken = () => + Promise.resolve({ endpoint: "http://weave.user.node" }); + + _("_findCluster() returns the user's cluster node"); + let cluster = await Service.identity._findCluster(); + Assert.equal(cluster, "http://weave.user.node/"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_setCluster() { + syncTestLogging(); + _("Test Service._setCluster()"); + try { + _("Check initial state."); + Assert.equal(Service.clusterURL, ""); + + Service.identity._findCluster = () => "http://weave.user.node/"; + + _("Set the cluster URL."); + Assert.ok(await Service.identity.setCluster()); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + + _("Setting it again won't make a difference if it's the same one."); + Assert.ok(!(await Service.identity.setCluster())); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + + _("A 'null' response won't make a difference either."); + Service.identity._findCluster = () => null; + Assert.ok(!(await Service.identity.setCluster())); + Assert.equal(Service.clusterURL, "http://weave.user.node/"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); diff --git a/services/sync/tests/unit/test_service_detect_upgrade.js b/services/sync/tests/unit/test_service_detect_upgrade.js new file mode 100644 index 0000000000..7d4abf1f67 --- /dev/null +++ b/services/sync/tests/unit/test_service_detect_upgrade.js @@ -0,0 +1,270 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { CryptoWrapper, WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function v4_upgrade() { + enableValidationPrefs(); + + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let collections = collectionsHelper.collections; + + let keysWBO = new ServerWBO("keys"); + let server = httpd_setup({ + // Special. + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/meta/global": upd("meta", meta_global.handler()), + + // Track modified times. + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/tabs": upd("tabs", new ServerCollection().handler()), + + // Just so we don't get 404s in the logs. + "/1.1/johndoe/storage/bookmarks": new ServerCollection().handler(), + "/1.1/johndoe/storage/forms": new ServerCollection().handler(), + "/1.1/johndoe/storage/history": new ServerCollection().handler(), + "/1.1/johndoe/storage/passwords": new ServerCollection().handler(), + "/1.1/johndoe/storage/prefs": new ServerCollection().handler(), + }); + + try { + Service.status.resetSync(); + + _("Logging in."); + + await configureIdentity({ username: "johndoe" }, server); + + await Service.login(); + Assert.ok(Service.isLoggedIn); + await Service.verifyAndFetchSymmetricKeys(); + Assert.ok(await Service._remoteSetup()); + + async function test_out_of_date() { + _("Old meta/global: " + JSON.stringify(meta_global)); + meta_global.payload = JSON.stringify({ + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION + 1, + }); + collections.meta = Date.now() / 1000; + _("New meta/global: " + JSON.stringify(meta_global)); + Service.recordManager.set(Service.metaURL, meta_global); + try { + await Service.sync(); + } catch (ex) {} + Assert.equal(Service.status.sync, VERSION_OUT_OF_DATE); + } + + // See what happens when we bump the storage version. + _("Syncing after server has been upgraded."); + await test_out_of_date(); + + // Same should happen after a wipe. + _("Syncing after server has been upgraded and wiped."); + await Service.wipeServer(); + await test_out_of_date(); + + // Now's a great time to test what happens when keys get replaced. + _("Syncing afresh..."); + Service.logout(); + Service.collectionKeys.clear(); + meta_global.payload = JSON.stringify({ + syncID: "foooooooooooooobbbbbbbbbbbb", + storageVersion: STORAGE_VERSION, + }); + collections.meta = Date.now() / 1000; + Service.recordManager.set(Service.metaURL, meta_global); + await Service.login(); + Assert.ok(Service.isLoggedIn); + await Service.sync(); + Assert.ok(Service.isLoggedIn); + + let serverDecrypted; + let serverKeys; + let serverResp; + + async function retrieve_server_default() { + serverKeys = serverResp = serverDecrypted = null; + + serverKeys = new CryptoWrapper("crypto", "keys"); + serverResp = ( + await serverKeys.fetch(Service.resource(Service.cryptoKeysURL)) + ).response; + Assert.ok(serverResp.success); + + serverDecrypted = await serverKeys.decrypt( + Service.identity.syncKeyBundle + ); + _("Retrieved WBO: " + JSON.stringify(serverDecrypted)); + _("serverKeys: " + JSON.stringify(serverKeys)); + + return serverDecrypted.default; + } + + async function retrieve_and_compare_default(should_succeed) { + let serverDefault = await retrieve_server_default(); + let localDefault = Service.collectionKeys.keyForCollection().keyPairB64; + + _("Retrieved keyBundle: " + JSON.stringify(serverDefault)); + _("Local keyBundle: " + JSON.stringify(localDefault)); + + if (should_succeed) { + Assert.equal( + JSON.stringify(serverDefault), + JSON.stringify(localDefault) + ); + } else { + Assert.notEqual( + JSON.stringify(serverDefault), + JSON.stringify(localDefault) + ); + } + } + + // Uses the objects set above. + async function set_server_keys(pair) { + serverDecrypted.default = pair; + serverKeys.cleartext = serverDecrypted; + await serverKeys.encrypt(Service.identity.syncKeyBundle); + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)); + } + + _("Checking we have the latest keys."); + await retrieve_and_compare_default(true); + + _("Update keys on server."); + await set_server_keys([ + "KaaaaaaaaaaaHAtfmuRY0XEJ7LXfFuqvF7opFdBD/MY=", + "aaaaaaaaaaaapxMO6TEWtLIOv9dj6kBAJdzhWDkkkis=", + ]); + + _("Checking that we no longer have the latest keys."); + await retrieve_and_compare_default(false); + + _("Indeed, they're what we set them to..."); + Assert.equal( + "KaaaaaaaaaaaHAtfmuRY0XEJ7LXfFuqvF7opFdBD/MY=", + (await retrieve_server_default())[0] + ); + + _("Sync. Should download changed keys automatically."); + let oldClientsModified = collections.clients; + let oldTabsModified = collections.tabs; + + await Service.login(); + await Service.sync(); + _("New key should have forced upload of data."); + _("Tabs: " + oldTabsModified + " < " + collections.tabs); + _("Clients: " + oldClientsModified + " < " + collections.clients); + Assert.ok(collections.clients > oldClientsModified); + Assert.ok(collections.tabs > oldTabsModified); + + _("... and keys will now match."); + await retrieve_and_compare_default(true); + + // Clean up. + await Service.startOver(); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); + +add_task(async function v5_upgrade() { + enableValidationPrefs(); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let keysWBO = new ServerWBO("keys"); + let bulkWBO = new ServerWBO("bulk"); + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + let server = httpd_setup({ + // Special. + "/1.1/johndoe/storage/meta/global": upd("meta", meta_global.handler()), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/crypto/bulk": upd("crypto", bulkWBO.handler()), + + // Track modified times. + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/tabs": upd("tabs", new ServerCollection().handler()), + }); + + try { + Service.status.resetSync(); + + Service.clusterURL = server.baseURI + "/"; + + await configureIdentity({ username: "johndoe" }, server); + + // Test an upgrade where the contents of the server would cause us to error + // -- keys decrypted with a different sync key, for example. + _("Testing v4 -> v5 (or similar) upgrade."); + async function update_server_keys(syncKeyBundle, wboName, collWBO) { + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", wboName); + await serverKeys.encrypt(syncKeyBundle); + let res = Service.resource(Service.storageURL + collWBO); + Assert.ok((await serverKeys.upload(res)).success); + } + + _("Bumping version."); + // Bump version on the server. + let m = new WBORecord("meta", "global"); + m.payload = { + syncID: "foooooooooooooooooooooooooo", + storageVersion: STORAGE_VERSION + 1, + }; + await m.upload(Service.resource(Service.metaURL)); + + _("New meta/global: " + JSON.stringify(meta_global)); + + // Fill the keys with bad data. + let badKeys = new BulkKeyBundle("crypto"); + await badKeys.generateRandom(); + await update_server_keys(badKeys, "keys", "crypto/keys"); // v4 + await update_server_keys(badKeys, "bulk", "crypto/bulk"); // v5 + + _("Generating new keys."); + await generateNewKeys(Service.collectionKeys); + + // Now sync and see what happens. It should be a version fail, not a crypto + // fail. + + _("Logging in."); + try { + await Service.login(); + } catch (e) { + _("Exception: " + e); + } + _("Status: " + Service.status); + Assert.ok(!Service.isLoggedIn); + Assert.equal(VERSION_OUT_OF_DATE, Service.status.sync); + + // Clean up. + await Service.startOver(); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + run_next_test(); +} diff --git a/services/sync/tests/unit/test_service_login.js b/services/sync/tests/unit/test_service_login.js new file mode 100644 index 0000000000..9c11f35b12 --- /dev/null +++ b/services/sync/tests/unit/test_service_login.js @@ -0,0 +1,216 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + +function login_handling(handler) { + return function (request, response) { + if (has_hawk_header(request)) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.setHeader("Content-Type", "text/plain"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +add_task(async function test_offline() { + try { + _("The right bits are set when we're offline."); + Services.io.offline = true; + Assert.ok(!(await Service.login())); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + Services.io.offline = false; + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +function setup() { + let janeHelper = track_collections_helper(); + let janeU = janeHelper.with_updated_collection; + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + + let server = httpd_setup({ + "/1.1/johndoe/info/collections": login_handling(johnHelper.handler), + "/1.1/janedoe/info/collections": login_handling(janeHelper.handler), + + // We need these handlers because we test login, and login + // is where keys are generated or fetched. + // TODO: have Jane fetch her keys, not generate them... + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + "/1.1/janedoe/storage/crypto/keys": janeU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/janedoe/storage/meta/global": janeU( + "meta", + new ServerWBO("global").handler() + ), + }); + + return server; +} + +add_task(async function test_not_logged_in() { + let server = setup(); + try { + await Service.login(); + Assert.ok(!Service.isLoggedIn, "no user configured, so can't be logged in"); + Assert.equal(Service._checkSync(), kSyncNotConfigured); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); + +add_task(async function test_login_logout() { + enableValidationPrefs(); + + let server = setup(); + + try { + _("Force the initial state."); + Service.status.service = STATUS_OK; + Assert.equal(Service.status.service, STATUS_OK); + + _("Try logging in. It won't work because we're not configured yet."); + await Service.login(); + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_USERNAME); + Assert.ok(!Service.isLoggedIn); + + _("Try again with a configured account"); + await configureIdentity({ username: "johndoe" }, server); + await Service.login(); + Assert.equal(Service.status.service, STATUS_OK); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + Assert.ok(Service.isLoggedIn); + + _("Logout."); + Service.logout(); + Assert.ok(!Service.isLoggedIn); + + _("Logging out again won't do any harm."); + Service.logout(); + Assert.ok(!Service.isLoggedIn); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); + +add_task(async function test_login_on_sync() { + enableValidationPrefs(); + + let server = setup(); + await configureIdentity({ username: "johndoe" }, server); + + try { + _("Sync calls login."); + let oldLogin = Service.login; + let loginCalled = false; + Service.login = async function () { + loginCalled = true; + Service.status.login = LOGIN_SUCCEEDED; + this._loggedIn = false; // So that sync aborts. + return true; + }; + + await Service.sync(); + + Assert.ok(loginCalled); + Service.login = oldLogin; + + // Stub mpLocked. + let mpLocked = true; + Utils.mpLocked = () => mpLocked; + + // Stub scheduleNextSync. This gets called within checkSyncStatus if we're + // ready to sync, so use it as an indicator. + let scheduleNextSyncF = Service.scheduler.scheduleNextSync; + let scheduleCalled = false; + Service.scheduler.scheduleNextSync = function (wait) { + scheduleCalled = true; + scheduleNextSyncF.call(this, wait); + }; + + // Autoconnect still tries to connect in the background (useful behavior: + // for non-MP users and unlocked MPs, this will detect version expiry + // earlier). + // + // Consequently, non-MP users will be logged in as in the pre-Bug 543784 world, + // and checkSyncStatus reflects that by waiting for login. + // + // This process doesn't apply if your MP is still locked, so we make + // checkSyncStatus accept a locked MP in place of being logged in. + // + // This test exercises these two branches. + + _("We're ready to sync if locked."); + Service.enabled = true; + Services.io.offline = false; + Service.scheduler.checkSyncStatus(); + Assert.ok(scheduleCalled); + + _("... and also if we're not locked."); + scheduleCalled = false; + mpLocked = false; + Service.scheduler.checkSyncStatus(); + Assert.ok(scheduleCalled); + Service.scheduler.scheduleNextSync = scheduleNextSyncF; + + // TODO: need better tests around master password prompting. See Bug 620583. + + mpLocked = true; + + // Testing exception handling if master password dialog is canceled. + // Do this by monkeypatching. + Service.identity.unlockAndVerifyAuthState = () => + Promise.resolve(MASTER_PASSWORD_LOCKED); + + let cSTCalled = false; + let lockedSyncCalled = false; + + Service.scheduler.clearSyncTriggers = function () { + cSTCalled = true; + }; + Service._lockedSync = async function () { + lockedSyncCalled = true; + }; + + _("If master password is canceled, login fails and we report lockage."); + Assert.ok(!(await Service.login())); + Assert.equal(Service.status.login, MASTER_PASSWORD_LOCKED); + Assert.equal(Service.status.service, LOGIN_FAILED); + _("Locked? " + Utils.mpLocked()); + _("checkSync reports the correct term."); + Assert.equal(Service._checkSync(), kSyncMasterPasswordLocked); + + _("Sync doesn't proceed and clears triggers if MP is still locked."); + await Service.sync(); + + Assert.ok(cSTCalled); + Assert.ok(!lockedSyncCalled); + + // N.B., a bunch of methods are stubbed at this point. Be careful putting + // new tests after this point! + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_startOver.js b/services/sync/tests/unit/test_service_startOver.js new file mode 100644 index 0000000000..15487ef1d7 --- /dev/null +++ b/services/sync/tests/unit/test_service_startOver.js @@ -0,0 +1,88 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function BlaEngine() { + SyncEngine.call(this, "Bla", Service); +} +BlaEngine.prototype = { + removed: false, + async removeClientData() { + this.removed = true; + }, +}; +Object.setPrototypeOf(BlaEngine.prototype, SyncEngine.prototype); + +add_task(async function setup() { + await Service.engineManager.register(BlaEngine); +}); + +add_task(async function test_resetLocalData() { + await configureIdentity(); + Service.status.enforceBackoff = true; + Service.status.backoffInterval = 42; + Service.status.minimumNextSync = 23; + + // Verify set up. + Assert.equal(Service.status.checkSetup(), STATUS_OK); + + // Verify state that the observer sees. + let observerCalled = false; + Svc.Obs.add("weave:service:start-over", function onStartOver() { + Svc.Obs.remove("weave:service:start-over", onStartOver); + observerCalled = true; + + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + }); + + await Service.startOver(); + Assert.ok(observerCalled); + + // Verify the site was nuked from orbit. + Assert.equal(Svc.Prefs.get("username"), undefined); + + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.ok(!Service.status.enforceBackoff); + Assert.equal(Service.status.backoffInterval, 0); + Assert.equal(Service.status.minimumNextSync, 0); +}); + +add_task(async function test_removeClientData() { + let engine = Service.engineManager.get("bla"); + + // No cluster URL = no removal. + Assert.ok(!engine.removed); + await Service.startOver(); + Assert.ok(!engine.removed); + + Service.clusterURL = "https://localhost/"; + + Assert.ok(!engine.removed); + await Service.startOver(); + Assert.ok(engine.removed); +}); + +add_task(async function test_reset_SyncScheduler() { + // Some non-default values for SyncScheduler's attributes. + Service.scheduler.idle = true; + Service.scheduler.hasIncomingItems = true; + Svc.Prefs.set("clients.devices.desktop", 42); + Service.scheduler.nextSync = Date.now(); + Service.scheduler.syncThreshold = MULTI_DEVICE_THRESHOLD; + Service.scheduler.syncInterval = Service.scheduler.activeInterval; + + await Service.startOver(); + + Assert.ok(!Service.scheduler.idle); + Assert.ok(!Service.scheduler.hasIncomingItems); + Assert.equal(Service.scheduler.numClients, 0); + Assert.equal(Service.scheduler.nextSync, 0); + Assert.equal(Service.scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal( + Service.scheduler.syncInterval, + Service.scheduler.singleDeviceInterval + ); +}); diff --git a/services/sync/tests/unit/test_service_startup.js b/services/sync/tests/unit/test_service_startup.js new file mode 100644 index 0000000000..beb4d1d18a --- /dev/null +++ b/services/sync/tests/unit/test_service_startup.js @@ -0,0 +1,58 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +// Svc.Prefs.set("services.sync.log.appender.dump", "All"); +Svc.Prefs.set("registerEngines", "Tab,Bookmarks,Form,History"); + +add_task(async function run_test() { + validate_all_future_pings(); + _("When imported, Service.onStartup is called"); + + let xps = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + Assert.ok(!xps.enabled); + + // Test fixtures + let { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" + ); + Services.prefs.setStringPref("services.sync.username", "johndoe"); + Assert.ok(xps.enabled); + + _("Service is enabled."); + Assert.equal(Service.enabled, true); + + _("Observers are notified of startup"); + Assert.ok(!Service.status.ready); + Assert.ok(!xps.ready); + + await promiseOneObserver("weave:service:ready"); + + Assert.ok(Service.status.ready); + Assert.ok(xps.ready); + + _("Engines are registered."); + let engines = Service.engineManager.getAll(); + if (AppConstants.MOZ_APP_NAME == "thunderbird") { + // Thunderbird's engines are registered later, so they're not here yet. + Assert.deepEqual( + engines.map(engine => engine.name), + [] + ); + } else { + Assert.deepEqual( + engines.map(engine => engine.name), + ["tabs", "bookmarks", "forms", "history"] + ); + } + + // Clean up. + Svc.Prefs.resetBranch(""); + + do_test_finished(); +}); diff --git a/services/sync/tests/unit/test_service_sync_401.js b/services/sync/tests/unit/test_service_sync_401.js new file mode 100644 index 0000000000..b96c2235dd --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_401.js @@ -0,0 +1,88 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function login_handling(handler) { + return function (request, response) { + if ( + request.hasHeader("Authorization") && + request.getHeader("Authorization").includes('Hawk id="id"') + ) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +add_task(async function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + let server = httpd_setup({ + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + "/1.1/johndoe/info/collections": login_handling(collectionsHelper.handler), + }); + + const GLOBAL_SCORE = 42; + + try { + _("Set up test fixtures."); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + Service.scheduler.globalScore = GLOBAL_SCORE; + // Avoid daily ping + Svc.Prefs.set("lastPing", Math.floor(Date.now() / 1000)); + + let threw = false; + Svc.Obs.add("weave:service:sync:error", function (subject, data) { + threw = true; + }); + + _("Initial state: We're successfully logged in."); + await Service.login(); + Assert.ok(Service.isLoggedIn); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + + _("Simulate having changed the password somewhere else."); + Service.identity._token.id = "somethingelse"; + Service.identity.unlockAndVerifyAuthState = () => + Promise.resolve(LOGIN_FAILED_LOGIN_REJECTED); + + _("Let's try to sync."); + await Service.sync(); + + _("Verify that sync() threw an exception."); + Assert.ok(threw); + + _("We're no longer logged in."); + Assert.ok(!Service.isLoggedIn); + + _("Sync status won't have changed yet, because we haven't tried again."); + + _("globalScore is reset upon starting a sync."); + Assert.equal(Service.scheduler.globalScore, 0); + + _("Our next sync will fail appropriately."); + try { + await Service.sync(); + } catch (ex) {} + Assert.equal(Service.status.login, LOGIN_FAILED_LOGIN_REJECTED); + } finally { + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_locked.js b/services/sync/tests/unit/test_service_sync_locked.js new file mode 100644 index 0000000000..1fe72eb092 --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_locked.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + validate_all_future_pings(); + let debug = []; + let info = []; + + function augmentLogger(old) { + let d = old.debug; + let i = old.info; + // For the purposes of this test we don't need to do full formatting + // of the 2nd param, as the ones we care about are always strings. + old.debug = function (m, p) { + debug.push(p ? m + ": " + (p.message || p) : m); + d.call(old, m, p); + }; + old.info = function (m, p) { + info.push(p ? m + ": " + (p.message || p) : m); + i.call(old, m, p); + }; + return old; + } + + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + augmentLogger(Service._log); + + // Avoid daily ping + Svc.Prefs.set("lastPing", Math.floor(Date.now() / 1000)); + + _("Check that sync will log appropriately if already in 'progress'."); + Service._locked = true; + await Service.sync(); + Service._locked = false; + + Assert.ok( + debug[debug.length - 2].startsWith( + 'Exception calling WrappedLock: Could not acquire lock. Label: "service.js: login".' + ) + ); + Assert.equal(info[info.length - 1], "Cannot start sync: already syncing?"); +}); diff --git a/services/sync/tests/unit/test_service_sync_remoteSetup.js b/services/sync/tests/unit/test_service_sync_remoteSetup.js new file mode 100644 index 0000000000..ec9fcce20c --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_remoteSetup.js @@ -0,0 +1,239 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// This sucks, but this test fails if this engine is enabled, due to dumb +// things that aren't related to this engine. In short: +// * Because the addon manager isn't initialized, the addons engine fails to +// initialize. So we end up writing a meta/global with `extension-storage` +// but not addons. +// * After we sync, we discover 'addons' is locally enabled, but because it's +// not in m/g, we decide it's been remotely declined (and it decides this +// without even considering `declined`). So we disable 'addons'. +// * Disabling 'addons' means 'extension-storage' is disabled - but because +// that *is* in meta/global we re-update meta/global to remove it. +// * This test fails due to the extra, unexpected update of m/g. +// +// Another option would be to ensure the addons manager is initialized, but +// that's a larger patch and still isn't strictly relevant to what's being +// tested here, so... +Services.prefs.setBoolPref( + "services.sync.engine.extension-storage.force", + false +); + +add_task(async function run_test() { + enableValidationPrefs(); + + validate_all_future_pings(); + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + let clients = new ServerCollection(); + let meta_global = new ServerWBO("global"); + + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + let collections = collectionsHelper.collections; + + function wasCalledHandler(wbo) { + let handler = wbo.handler(); + return function () { + wbo.wasCalled = true; + handler.apply(this, arguments); + }; + } + + let keysWBO = new ServerWBO("keys"); + let cryptoColl = new ServerCollection({ keys: keysWBO }); + let metaColl = new ServerCollection({ global: meta_global }); + do_test_pending(); + + /** + * Handle the bulk DELETE request sent by wipeServer. + */ + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + + _("Wiping out all collections."); + cryptoColl.delete({}); + clients.delete({}); + metaColl.delete({}); + + let ts = new_timestamp(); + collectionsHelper.update_collection("crypto", ts); + collectionsHelper.update_collection("clients", ts); + collectionsHelper.update_collection("meta", ts); + return_timestamp(request, response, ts); + } + + const GLOBAL_PATH = "/1.1/johndoe/storage/meta/global"; + + let handlers = { + "/1.1/johndoe/storage": storageHandler, + "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()), + "/1.1/johndoe/storage/crypto": upd("crypto", cryptoColl.handler()), + "/1.1/johndoe/storage/clients": upd("clients", clients.handler()), + "/1.1/johndoe/storage/meta": upd("meta", wasCalledHandler(metaColl)), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + wasCalledHandler(meta_global) + ), + "/1.1/johndoe/info/collections": collectionsHelper.handler, + }; + + function mockHandler(path, mock) { + server.registerPathHandler(path, mock(handlers[path])); + return { + restore() { + server.registerPathHandler(path, handlers[path]); + }, + }; + } + + let server = httpd_setup(handlers); + + try { + _("Checking Status.sync with no credentials."); + await Service.verifyAndFetchSymmetricKeys(); + Assert.equal(Service.status.sync, CREDENTIALS_CHANGED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_PASSPHRASE); + + await configureIdentity({ username: "johndoe" }, server); + + await Service.login(); + _("Checking that remoteSetup returns true when credentials have changed."); + (await Service.recordManager.get(Service.metaURL)).payload.syncID = + "foobar"; + Assert.ok(await Service._remoteSetup()); + + let returnStatusCode = (method, code) => oldMethod => (req, res) => { + if (req.method === method) { + res.setStatusLine(req.httpVersion, code, ""); + } else { + oldMethod(req, res); + } + }; + + let mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 401)); + Service.recordManager.del(Service.metaURL); + _( + "Checking that remoteSetup returns false on 401 on first get /meta/global." + ); + Assert.equal(false, await Service._remoteSetup()); + mock.restore(); + + await Service.login(); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 503)); + Service.recordManager.del(Service.metaURL); + _( + "Checking that remoteSetup returns false on 503 on first get /meta/global." + ); + Assert.equal(false, await Service._remoteSetup()); + Assert.equal(Service.status.sync, METARECORD_DOWNLOAD_FAIL); + mock.restore(); + + await Service.login(); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 404)); + Service.recordManager.del(Service.metaURL); + _("Checking that remoteSetup recovers on 404 on first get /meta/global."); + Assert.ok(await Service._remoteSetup()); + mock.restore(); + + let makeOutdatedMeta = async () => { + Service.metaModified = 0; + let infoResponse = await Service._fetchInfo(); + return { + status: infoResponse.status, + obj: { + crypto: infoResponse.obj.crypto, + clients: infoResponse.obj.clients, + meta: 1, + }, + }; + }; + + _( + "Checking that remoteSetup recovers on 404 on get /meta/global after clear cached one." + ); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 404)); + Service.recordManager.set(Service.metaURL, { isNew: false }); + Assert.ok(await Service._remoteSetup(await makeOutdatedMeta())); + mock.restore(); + + _( + "Checking that remoteSetup returns false on 503 on get /meta/global after clear cached one." + ); + mock = mockHandler(GLOBAL_PATH, returnStatusCode("GET", 503)); + Service.status.sync = ""; + Service.recordManager.set(Service.metaURL, { isNew: false }); + Assert.equal(false, await Service._remoteSetup(await makeOutdatedMeta())); + Assert.equal(Service.status.sync, ""); + mock.restore(); + + metaColl.delete({}); + + _("Do an initial sync."); + await Service.sync(); + + _("Checking that remoteSetup returns true."); + Assert.ok(await Service._remoteSetup()); + + _("Verify that the meta record was uploaded."); + Assert.equal(meta_global.data.syncID, Service.syncID); + Assert.equal(meta_global.data.storageVersion, STORAGE_VERSION); + Assert.equal( + meta_global.data.engines.clients.version, + Service.clientsEngine.version + ); + Assert.equal( + meta_global.data.engines.clients.syncID, + await Service.clientsEngine.getSyncID() + ); + + _( + "Set the collection info hash so that sync() will remember the modified times for future runs." + ); + let lastSync = await Service.clientsEngine.getLastSync(); + collections.meta = lastSync; + collections.clients = lastSync; + await Service.sync(); + + _("Sync again and verify that meta/global wasn't downloaded again"); + meta_global.wasCalled = false; + await Service.sync(); + Assert.ok(!meta_global.wasCalled); + + _( + "Fake modified records. This will cause a redownload, but not reupload since it hasn't changed." + ); + collections.meta += 42; + meta_global.wasCalled = false; + + let metaModified = meta_global.modified; + + await Service.sync(); + Assert.ok(meta_global.wasCalled); + Assert.equal(metaModified, meta_global.modified); + + // Try to screw up HMAC calculation. + // Re-encrypt keys with a new random keybundle, and upload them to the + // server, just as might happen with a second client. + _("Attempting to screw up HMAC by re-encrypting keys."); + let keys = Service.collectionKeys.asWBO(); + let b = new BulkKeyBundle("hmacerror"); + await b.generateRandom(); + collections.crypto = keys.modified = 100 + Date.now() / 1000; // Future modification time. + await keys.encrypt(b); + await keys.upload(Service.resource(Service.cryptoKeysURL)); + + Assert.equal(false, await Service.verifyAndFetchSymmetricKeys()); + Assert.equal(Service.status.login, LOGIN_FAILED_INVALID_PASSPHRASE); + } finally { + Svc.Prefs.resetBranch(""); + server.stop(do_test_finished); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_specified.js b/services/sync/tests/unit/test_service_sync_specified.js new file mode 100644 index 0000000000..845cdb3669 --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_specified.js @@ -0,0 +1,150 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let syncedEngines = []; + +function SteamEngine() { + SyncEngine.call(this, "Steam", Service); +} +SteamEngine.prototype = { + async _sync() { + syncedEngines.push(this.name); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function StirlingEngine() { + SyncEngine.call(this, "Stirling", Service); +} +StirlingEngine.prototype = { + async _sync() { + syncedEngines.push(this.name); + }, +}; +Object.setPrototypeOf(StirlingEngine.prototype, SteamEngine.prototype); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup(handlers) { + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp() { + syncedEngines = []; + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + engine.syncPriority = 1; + + engine = Service.engineManager.get("stirling"); + engine.enabled = true; + engine.syncPriority = 2; + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": new ServerWBO("global", {}).handler(), + }); + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + return server; +} + +add_task(async function setup() { + await Service.engineManager.clear(); + validate_all_future_pings(); + + await Service.engineManager.register(SteamEngine); + await Service.engineManager.register(StirlingEngine); +}); + +add_task(async function test_noEngines() { + enableValidationPrefs(); + + _("Test: An empty array of engines to sync does nothing."); + let server = await setUp(); + + try { + _("Sync with no engines specified."); + await Service.sync({ engines: [] }); + deepEqual(syncedEngines, [], "no engines were synced"); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_oneEngine() { + enableValidationPrefs(); + + _("Test: Only one engine is synced."); + let server = await setUp(); + + try { + _("Sync with 1 engine specified."); + await Service.sync({ engines: ["steam"] }); + deepEqual(syncedEngines, ["steam"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesSpecified() { + enableValidationPrefs(); + + _("Test: All engines are synced when specified in the correct order (1)."); + let server = await setUp(); + + try { + _("Sync with both engines specified."); + await Service.sync({ engines: ["steam", "stirling"] }); + deepEqual(syncedEngines, ["steam", "stirling"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesSpecified() { + enableValidationPrefs(); + + _("Test: All engines are synced when specified in the correct order (2)."); + let server = await setUp(); + + try { + _("Sync with both engines specified."); + await Service.sync({ engines: ["stirling", "steam"] }); + deepEqual(syncedEngines, ["stirling", "steam"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_bothEnginesDefault() { + enableValidationPrefs(); + + _("Test: All engines are synced when nothing is specified."); + let server = await setUp(); + + try { + await Service.sync(); + deepEqual(syncedEngines, ["steam", "stirling"]); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); diff --git a/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js b/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js new file mode 100644 index 0000000000..fd8b3f71bc --- /dev/null +++ b/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js @@ -0,0 +1,587 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { EngineSynchronizer } = ChromeUtils.importESModule( + "resource://services-sync/stages/enginesync.sys.mjs" +); + +function QuietStore() { + Store.call("Quiet"); +} +QuietStore.prototype = { + async getAllIDs() { + return []; + }, +}; + +function SteamEngine() { + SyncEngine.call(this, "Steam", Service); +} +SteamEngine.prototype = { + // We're not interested in engine sync but what the service does. + _storeObj: QuietStore, + + _sync: async function _sync() { + await this._syncStartup(); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function StirlingEngine() { + SyncEngine.call(this, "Stirling", Service); +} +StirlingEngine.prototype = { + // This engine's enabled state is the same as the SteamEngine's. + get prefName() { + return "steam"; + }, +}; +Object.setPrototypeOf(StirlingEngine.prototype, SteamEngine.prototype); + +// Tracking info/collections. +var collectionsHelper = track_collections_helper(); +var upd = collectionsHelper.with_updated_collection; + +function sync_httpd_setup(handlers) { + handlers["/1.1/johndoe/info/collections"] = collectionsHelper.handler; + delete collectionsHelper.collections.crypto; + delete collectionsHelper.collections.meta; + + let cr = new ServerWBO("keys"); + handlers["/1.1/johndoe/storage/crypto/keys"] = upd("crypto", cr.handler()); + + let cl = new ServerCollection(); + handlers["/1.1/johndoe/storage/clients"] = upd("clients", cl.handler()); + + return httpd_setup(handlers); +} + +async function setUp(server) { + await SyncTestingInfrastructure(server, "johndoe", "ilovejane"); + // Ensure that the server has valid keys so that logging in will work and not + // result in a server wipe, rendering many of these tests useless. + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let { success } = await serverKeys.upload( + Service.resource(Service.cryptoKeysURL) + ); + ok(success); +} + +const PAYLOAD = 42; + +add_task(async function setup() { + await Service.engineManager.clear(); + validate_all_future_pings(); + + await Service.engineManager.register(SteamEngine); + await Service.engineManager.register(StirlingEngine); +}); + +add_task(async function test_newAccount() { + enableValidationPrefs(); + + _("Test: New account does not disable locally enabled engines."); + let engine = Service.engineManager.get("steam"); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": new ServerWBO("global", {}).handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Engine is enabled from the beginning."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Engine continues to be enabled."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_enabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is disabled on remote clients and enabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally."); + engine.enabled = true; + + _("Sync."); + await Service.sync(); + + _("Meta record now contains the new engine."); + Assert.ok(!!metaWBO.data.engines.steam); + + _("Engine continues to be enabled."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is enabled on remote clients and disabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + let steamCollection = new ServerWBO("steam", PAYLOAD); + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": steamCollection.handler(), + }); + await setUp(server); + + try { + _("Disable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + engine.enabled = false; + + _("Sync."); + await Service.sync(); + + _("Meta record no longer contains engine."); + Assert.ok(!metaWBO.data.engines.steam); + + _("Server records are wiped."); + Assert.equal(steamCollection.payload, undefined); + + _("Engine continues to be disabled."); + Assert.ok(!engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledLocally_wipe503() { + enableValidationPrefs(); + + _("Test: Engine is enabled on remote clients and disabled locally"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + + function service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "23"); + response.bodyOutputStream.write(body, body.length); + } + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": service_unavailable, + }); + await setUp(server); + + _("Disable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + engine.enabled = false; + + _("Sync."); + await Service.sync(); + Assert.equal(Service.status.sync, SERVER_MAINTENANCE); + + await Service.startOver(); + await promiseStopServer(server); +}); + +add_task(async function test_enabledRemotely() { + enableValidationPrefs(); + + _("Test: Engine is disabled locally and enabled on a remote client"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let syncID = await engine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { steam: { syncID, version: engine.version } }, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", metaWBO.handler()), + + "/1.1/johndoe/storage/steam": upd( + "steam", + new ServerWBO("steam", {}).handler() + ), + }); + await setUp(server); + + // We need to be very careful how we do this, so that we don't trigger a + // fresh start! + try { + _("Upload some keys to avoid a fresh start."); + let wbo = await Service.collectionKeys.generateNewKeysWBO(); + await wbo.encrypt(Service.identity.syncKeyBundle); + Assert.equal( + 200, + (await wbo.upload(Service.resource(Service.cryptoKeysURL))).status + ); + + _("Engine is disabled."); + Assert.ok(!engine.enabled); + + _("Sync."); + await Service.sync(); + + _("Engine is enabled."); + Assert.ok(engine.enabled); + + _("Meta record still present."); + Assert.equal(metaWBO.data.engines.steam.syncID, await engine.getSyncID()); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledRemotelyTwoClients() { + enableValidationPrefs(); + + _( + "Test: Engine is enabled locally and disabled on a remote client... with two clients." + ); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": upd("meta", metaWBO.handler()), + + "/1.1/johndoe/storage/steam": upd( + "steam", + new ServerWBO("steam", {}).handler() + ), + }); + await setUp(server); + + try { + _("Enable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Disable engine by deleting from meta/global."); + let d = metaWBO.data; + delete d.engines.steam; + metaWBO.payload = JSON.stringify(d); + metaWBO.modified = Date.now() / 1000; + + _("Add a second client and verify that the local pref is changed."); + Service.clientsEngine._store._remoteClients.foobar = { + name: "foobar", + type: "desktop", + }; + await Service.sync(); + + _("Engine is disabled."); + Assert.ok(!engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_disabledRemotely() { + enableValidationPrefs(); + + _("Test: Engine is enabled locally and disabled on a remote client"); + Service.syncID = "abcdefghij"; + let engine = Service.engineManager.get("steam"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally."); + Service._ignorePrefObserver = true; + engine.enabled = true; + Service._ignorePrefObserver = false; + + _("Sync."); + await Service.sync(); + + _("Engine is not disabled: only one client."); + Assert.ok(engine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_dependentEnginesEnabledLocally() { + enableValidationPrefs(); + + _("Test: Engine is disabled on remote clients and enabled locally"); + Service.syncID = "abcdefghij"; + let steamEngine = Service.engineManager.get("steam"); + let stirlingEngine = Service.engineManager.get("stirling"); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: {}, + }); + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler(), + "/1.1/johndoe/storage/stirling": new ServerWBO("stirling", {}).handler(), + }); + await setUp(server); + + try { + _("Enable engine locally. Doing it on one is enough."); + steamEngine.enabled = true; + + _("Sync."); + await Service.sync(); + + _("Meta record now contains the new engines."); + Assert.ok(!!metaWBO.data.engines.steam); + Assert.ok(!!metaWBO.data.engines.stirling); + + _("Engines continue to be enabled."); + Assert.ok(steamEngine.enabled); + Assert.ok(stirlingEngine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_dependentEnginesDisabledLocally() { + enableValidationPrefs(); + + _( + "Test: Two dependent engines are enabled on remote clients and disabled locally" + ); + Service.syncID = "abcdefghij"; + let steamEngine = Service.engineManager.get("steam"); + let steamSyncID = await steamEngine.resetLocalSyncID(); + let stirlingEngine = Service.engineManager.get("stirling"); + let stirlingSyncID = await stirlingEngine.resetLocalSyncID(); + let metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + steam: { syncID: steamSyncID, version: steamEngine.version }, + stirling: { syncID: stirlingSyncID, version: stirlingEngine.version }, + }, + }); + + let steamCollection = new ServerWBO("steam", PAYLOAD); + let stirlingCollection = new ServerWBO("stirling", PAYLOAD); + + let server = sync_httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + "/1.1/johndoe/storage/steam": steamCollection.handler(), + "/1.1/johndoe/storage/stirling": stirlingCollection.handler(), + }); + await setUp(server); + + try { + _("Disable engines locally. Doing it on one is enough."); + Service._ignorePrefObserver = true; + steamEngine.enabled = true; + Assert.ok(stirlingEngine.enabled); + Service._ignorePrefObserver = false; + steamEngine.enabled = false; + Assert.ok(!stirlingEngine.enabled); + + _("Sync."); + await Service.sync(); + + _("Meta record no longer contains engines."); + Assert.ok(!metaWBO.data.engines.steam); + Assert.ok(!metaWBO.data.engines.stirling); + + _("Server records are wiped."); + Assert.equal(steamCollection.payload, undefined); + Assert.equal(stirlingCollection.payload, undefined); + + _("Engines continue to be disabled."); + Assert.ok(!steamEngine.enabled); + Assert.ok(!stirlingEngine.enabled); + } finally { + await Service.startOver(); + await promiseStopServer(server); + } +}); + +add_task(async function test_service_updateLocalEnginesState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: ["steam"], + engines: {}, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service._ignorePrefObserver = true; + // Steam engine is enabled on our machine. + engine.enabled = true; + Service._ignorePrefObserver = false; + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now disabled. + Assert.ok(!engine.enabled); +}); + +add_task(async function test_service_enableAfterUpdateState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: ["steam"], + engines: { someengine: {} }, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now disabled, reflecting what's on the server. + Assert.ok(!engine.enabled); + // Enable the engine, as though the user selected it via CWTS. + engine.enabled = true; + + // Do the "reconcile local and remote states" dance. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledEngines(); + await Service._maybeUpdateDeclined(); + // engine should remain enabled. + Assert.ok(engine.enabled); + // engine should no longer appear in declined on the server. + Assert.deepEqual(metaWBO.data.declined, []); +}); + +add_task(async function test_service_disableAfterUpdateState() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + const metaWBO = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + declined: [], + engines: { steam: {} }, + }); + const server = httpd_setup({ + "/1.1/johndoe/storage/meta/global": metaWBO.handler(), + }); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Now enabled, reflecting what's on the server. + Assert.ok(engine.enabled); + // Disable the engine, as though via CWTS. + engine.enabled = false; + + // Do the "reconcile local and remote states" dance. + let engineSync = new EngineSynchronizer(Service); + await engineSync._updateEnabledEngines(); + await Service._maybeUpdateDeclined(); + // engine should remain disabled. + Assert.ok(!engine.enabled); + // engine should now appear in declined on the server. + Assert.deepEqual(metaWBO.data.declined, ["steam"]); + // and should have been removed from engines. + Assert.deepEqual(metaWBO.data.engines, {}); +}); + +add_task(async function test_service_updateLocalEnginesState_no_meta_global() { + Service.syncID = "abcdefghij"; + const engine = Service.engineManager.get("steam"); + // The server doesn't contain /meta/global (sync was never enabled). + const server = httpd_setup({}); + await SyncTestingInfrastructure(server, "johndoe"); + + // Disconnect sync. + await Service.startOver(); + Service._ignorePrefObserver = true; + // Steam engine is enabled on our machine. + engine.enabled = true; + Service._ignorePrefObserver = false; + Service.identity._findCluster = () => server.baseURI + "/1.1/johndoe/"; + + // Update engine state from the server. + await Service.updateLocalEnginesState(); + // Still enabled. + Assert.ok(engine.enabled); +}); diff --git a/services/sync/tests/unit/test_service_verifyLogin.js b/services/sync/tests/unit/test_service_verifyLogin.js new file mode 100644 index 0000000000..9892df76dc --- /dev/null +++ b/services/sync/tests/unit/test_service_verifyLogin.js @@ -0,0 +1,116 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function login_handling(handler) { + return function (request, response) { + if (has_hawk_header(request)) { + handler(request, response); + } else { + let body = "Unauthorized"; + response.setStatusLine(request.httpVersion, 401, "Unauthorized"); + response.bodyOutputStream.write(body, body.length); + } + }; +} + +function service_unavailable(request, response) { + let body = "Service Unavailable"; + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + response.setHeader("Retry-After", "42"); + response.bodyOutputStream.write(body, body.length); +} + +function run_test() { + Log.repository.rootLogger.addAppender(new Log.DumpAppender()); + + run_next_test(); +} + +add_task(async function test_verifyLogin() { + // This test expects a clean slate -- no saved passphrase. + Services.logins.removeAllUserFacingLogins(); + let johnHelper = track_collections_helper(); + let johnU = johnHelper.with_updated_collection; + + do_test_pending(); + + let server = httpd_setup({ + "/1.1/johndoe/info/collections": login_handling(johnHelper.handler), + "/1.1/janedoe/info/collections": service_unavailable, + + "/1.1/johndoe/storage/crypto/keys": johnU( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": johnU( + "meta", + new ServerWBO("global").handler() + ), + }); + + try { + _("Force the initial state."); + Service.status.service = STATUS_OK; + Assert.equal(Service.status.service, STATUS_OK); + + _("Credentials won't check out because we're not configured yet."); + Service.status.resetSync(); + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Service.status.login, LOGIN_FAILED_NO_USERNAME); + + _("Success if syncBundleKey is set."); + Service.status.resetSync(); + await configureIdentity({ username: "johndoe" }, server); + Assert.ok(await Service.verifyLogin()); + Assert.equal(Service.status.service, STATUS_OK); + Assert.equal(Service.status.login, LOGIN_SUCCEEDED); + + _( + "If verifyLogin() encounters a server error, it flips on the backoff flag and notifies observers on a 503 with Retry-After." + ); + Service.status.resetSync(); + await configureIdentity({ username: "janedoe" }, server); + Service._updateCachedURLs(); + Assert.ok(!Service.status.enforceBackoff); + let backoffInterval; + Svc.Obs.add( + "weave:service:backoff:interval", + function observe(subject, data) { + Svc.Obs.remove("weave:service:backoff:interval", observe); + backoffInterval = subject; + } + ); + Assert.equal(false, await Service.verifyLogin()); + Assert.ok(Service.status.enforceBackoff); + Assert.equal(backoffInterval, 42); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, SERVER_MAINTENANCE); + + _( + "Ensure a network error when finding the cluster sets the right Status bits." + ); + Service.status.resetSync(); + Service.clusterURL = ""; + Service.identity._findCluster = () => "http://localhost:12345/"; + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + + _( + "Ensure a network error when getting the collection info sets the right Status bits." + ); + Service.status.resetSync(); + Service.clusterURL = "http://localhost:12345/"; + Assert.equal(false, await Service.verifyLogin()); + Assert.equal(Service.status.service, LOGIN_FAILED); + Assert.equal(Service.status.login, LOGIN_FAILED_NETWORK_ERROR); + } finally { + Svc.Prefs.resetBranch(""); + server.stop(do_test_finished); + } +}); diff --git a/services/sync/tests/unit/test_service_wipeClient.js b/services/sync/tests/unit/test_service_wipeClient.js new file mode 100644 index 0000000000..aa48868ca0 --- /dev/null +++ b/services/sync/tests/unit/test_service_wipeClient.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +function CanDecryptEngine() { + SyncEngine.call(this, "CanDecrypt", Service); +} +CanDecryptEngine.prototype = { + // Override these methods with mocks for the test + async canDecrypt() { + return true; + }, + + wasWiped: false, + async wipeClient() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(CanDecryptEngine.prototype, SyncEngine.prototype); + +function CannotDecryptEngine() { + SyncEngine.call(this, "CannotDecrypt", Service); +} +CannotDecryptEngine.prototype = { + // Override these methods with mocks for the test + async canDecrypt() { + return false; + }, + + wasWiped: false, + async wipeClient() { + this.wasWiped = true; + }, +}; +Object.setPrototypeOf(CannotDecryptEngine.prototype, SyncEngine.prototype); + +let canDecryptEngine; +let cannotDecryptEngine; + +add_task(async function setup() { + await Service.engineManager.clear(); + + await Service.engineManager.register(CanDecryptEngine); + await Service.engineManager.register(CannotDecryptEngine); + canDecryptEngine = Service.engineManager.get("candecrypt"); + cannotDecryptEngine = Service.engineManager.get("cannotdecrypt"); +}); + +add_task(async function test_withEngineList() { + try { + _("Ensure initial scenario."); + Assert.ok(!canDecryptEngine.wasWiped); + Assert.ok(!cannotDecryptEngine.wasWiped); + + _("Wipe local engine data."); + await Service.wipeClient(["candecrypt", "cannotdecrypt"]); + + _("Ensure only the engine that can decrypt was wiped."); + Assert.ok(canDecryptEngine.wasWiped); + Assert.ok(!cannotDecryptEngine.wasWiped); + } finally { + canDecryptEngine.wasWiped = false; + cannotDecryptEngine.wasWiped = false; + await Service.startOver(); + } +}); + +add_task(async function test_startOver_clears_keys() { + syncTestLogging(); + await generateNewKeys(Service.collectionKeys); + Assert.ok(!!Service.collectionKeys.keyForCollection()); + await Service.startOver(); + syncTestLogging(); + Assert.ok(!Service.collectionKeys.keyForCollection()); +}); diff --git a/services/sync/tests/unit/test_service_wipeServer.js b/services/sync/tests/unit/test_service_wipeServer.js new file mode 100644 index 0000000000..bf8b7e192e --- /dev/null +++ b/services/sync/tests/unit/test_service_wipeServer.js @@ -0,0 +1,228 @@ +Svc.Prefs.set("registerEngines", ""); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +// configure the identity we use for this test. +const identityConfig = makeIdentityConfig({ username: "johndoe" }); + +function FakeCollection() { + this.deleted = false; +} +FakeCollection.prototype = { + handler() { + let self = this; + return function (request, response) { + let body = ""; + self.timestamp = new_timestamp(); + let timestamp = "" + self.timestamp; + if (request.method == "DELETE") { + body = timestamp; + self.deleted = true; + } + response.setHeader("X-Weave-Timestamp", timestamp); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + }; + }, +}; + +async function setUpTestFixtures(server) { + Service.clusterURL = server.baseURI + "/"; + + await configureIdentity(identityConfig); +} + +add_task(async function test_wipeServer_list_success() { + _("Service.wipeServer() deletes collections given as argument."); + + let steam_coll = new FakeCollection(); + let diesel_coll = new FakeCollection(); + + let server = httpd_setup({ + "/1.1/johndoe/storage/steam": steam_coll.handler(), + "/1.1/johndoe/storage/diesel": diesel_coll.handler(), + "/1.1/johndoe/storage/petrol": httpd_handler(404, "Not Found"), + }); + + try { + await setUpTestFixtures(server); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + + _("Confirm initial environment."); + Assert.ok(!steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + + _( + "wipeServer() will happily ignore the non-existent collection and use the timestamp of the last DELETE that was successful." + ); + let timestamp = await Service.wipeServer(["steam", "diesel", "petrol"]); + Assert.equal(timestamp, diesel_coll.timestamp); + + _( + "wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted." + ); + Assert.ok(steam_coll.deleted); + Assert.ok(diesel_coll.deleted); + } finally { + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_wipeServer_list_503() { + _("Service.wipeServer() deletes collections given as argument."); + + let steam_coll = new FakeCollection(); + let diesel_coll = new FakeCollection(); + + let server = httpd_setup({ + "/1.1/johndoe/storage/steam": steam_coll.handler(), + "/1.1/johndoe/storage/petrol": httpd_handler(503, "Service Unavailable"), + "/1.1/johndoe/storage/diesel": diesel_coll.handler(), + }); + + try { + await setUpTestFixtures(server); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + + _("Confirm initial environment."); + Assert.ok(!steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + + _( + "wipeServer() will happily ignore the non-existent collection, delete the 'steam' collection and abort after an receiving an error on the 'petrol' collection." + ); + let error; + try { + await Service.wipeServer(["non-existent", "steam", "petrol", "diesel"]); + do_throw("Should have thrown!"); + } catch (ex) { + error = ex; + } + _("wipeServer() threw this exception: " + error); + Assert.equal(error.status, 503); + + _( + "wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted." + ); + Assert.ok(steam_coll.deleted); + Assert.ok(!diesel_coll.deleted); + } finally { + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_wipeServer_all_success() { + _("Service.wipeServer() deletes all the things."); + + /** + * Handle the bulk DELETE request sent by wipeServer. + */ + let deleted = false; + let serverTimestamp; + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + deleted = true; + serverTimestamp = return_timestamp(request, response); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + let returnedTimestamp = await Service.wipeServer(); + Assert.ok(deleted); + Assert.equal(returnedTimestamp, serverTimestamp); + + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); +}); + +add_task(async function test_wipeServer_all_404() { + _("Service.wipeServer() accepts a 404."); + + /** + * Handle the bulk DELETE request sent by wipeServer. Returns a 404. + */ + let deleted = false; + let serverTimestamp; + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + deleted = true; + serverTimestamp = new_timestamp(); + response.setHeader("X-Weave-Timestamp", "" + serverTimestamp); + response.setStatusLine(request.httpVersion, 404, "Not Found"); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + let returnedTimestamp = await Service.wipeServer(); + Assert.ok(deleted); + Assert.equal(returnedTimestamp, serverTimestamp); + + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); +}); + +add_task(async function test_wipeServer_all_503() { + _("Service.wipeServer() throws if it encounters a non-200/404 response."); + + /** + * Handle the bulk DELETE request sent by wipeServer. Returns a 503. + */ + function storageHandler(request, response) { + Assert.equal("DELETE", request.method); + Assert.ok(request.hasHeader("X-Confirm-Delete")); + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + } + + let server = httpd_setup({ + "/1.1/johndoe/storage": storageHandler, + }); + await setUpTestFixtures(server); + + _("Try deletion."); + let error; + try { + await SyncTestingInfrastructure(server, "johndoe", "irrelevant"); + await Service.wipeServer(); + do_throw("Should have thrown!"); + } catch (ex) { + error = ex; + } + Assert.equal(error.status, 503); + + await promiseStopServer(server); + Svc.Prefs.resetBranch(""); +}); + +add_task(async function test_wipeServer_all_connectionRefused() { + _("Service.wipeServer() throws if it encounters a network problem."); + let server = httpd_setup({}); + await setUpTestFixtures(server); + + Service.clusterURL = "http://localhost:4352/"; + + _("Try deletion."); + try { + await Service.wipeServer(); + do_throw("Should have thrown!"); + } catch (ex) { + Assert.equal(ex.result, Cr.NS_ERROR_CONNECTION_REFUSED); + } + + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_status.js b/services/sync/tests/unit/test_status.js new file mode 100644 index 0000000000..5bcfa182c3 --- /dev/null +++ b/services/sync/tests/unit/test_status.js @@ -0,0 +1,83 @@ +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +function run_test() { + // Check initial states + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + if (Status.engines.length) { + do_throw("Status.engines should be empty."); + } + Assert.equal(Status.partial, false); + + // Check login status + for (let code of [LOGIN_FAILED_NO_USERNAME, LOGIN_FAILED_NO_PASSPHRASE]) { + Status.login = code; + Assert.equal(Status.login, code); + Assert.equal(Status.service, CLIENT_NOT_CONFIGURED); + Status.resetSync(); + } + + Status.login = LOGIN_FAILED; + Assert.equal(Status.login, LOGIN_FAILED); + Assert.equal(Status.service, LOGIN_FAILED); + Status.resetSync(); + + Status.login = LOGIN_SUCCEEDED; + Assert.equal(Status.login, LOGIN_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + Status.resetSync(); + + // Check sync status + Status.sync = SYNC_FAILED; + Assert.equal(Status.sync, SYNC_FAILED); + Assert.equal(Status.service, SYNC_FAILED); + + Status.sync = SYNC_SUCCEEDED; + Assert.equal(Status.sync, SYNC_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + + Status.resetSync(); + + // Check engine status + Status.engines = ["testEng1", ENGINE_SUCCEEDED]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.service, STATUS_OK); + + Status.engines = ["testEng2", ENGINE_DOWNLOAD_FAIL]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.engines.testEng2, ENGINE_DOWNLOAD_FAIL); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + Status.engines = ["testEng3", ENGINE_SUCCEEDED]; + Assert.equal(Status.engines.testEng1, ENGINE_SUCCEEDED); + Assert.equal(Status.engines.testEng2, ENGINE_DOWNLOAD_FAIL); + Assert.equal(Status.engines.testEng3, ENGINE_SUCCEEDED); + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + // Check resetSync + Status.sync = SYNC_FAILED; + Status.resetSync(); + + Assert.equal(Status.service, STATUS_OK); + Assert.equal(Status.sync, SYNC_SUCCEEDED); + if (Status.engines.length) { + do_throw("Status.engines should be empty."); + } + + // Check resetBackoff + Status.enforceBackoff = true; + Status.backOffInterval = 4815162342; + Status.backOffInterval = 42; + Status.resetBackoff(); + + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); +} diff --git a/services/sync/tests/unit/test_status_checkSetup.js b/services/sync/tests/unit/test_status_checkSetup.js new file mode 100644 index 0000000000..f42736bf1e --- /dev/null +++ b/services/sync/tests/unit/test_status_checkSetup.js @@ -0,0 +1,24 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +add_task(async function test_status_checkSetup() { + try { + _("Fresh setup, we're not configured."); + Assert.equal(Status.checkSetup(), CLIENT_NOT_CONFIGURED); + Assert.equal(Status.login, LOGIN_FAILED_NO_USERNAME); + Status.resetSync(); + + _("Let's provide the syncKeyBundle"); + await configureIdentity(); + + _("checkSetup()"); + Assert.equal(Status.checkSetup(), STATUS_OK); + Status.resetSync(); + } finally { + Svc.Prefs.resetBranch(""); + } +}); diff --git a/services/sync/tests/unit/test_sync_auth_manager.js b/services/sync/tests/unit/test_sync_auth_manager.js new file mode 100644 index 0000000000..8b9bffe377 --- /dev/null +++ b/services/sync/tests/unit/test_sync_auth_manager.js @@ -0,0 +1,1053 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AuthenticationError, SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { initializeIdentityWithTokenServerResponse } = + ChromeUtils.importESModule( + "resource://testing-common/services/sync/fxa_utils.sys.mjs" + ); +const { HawkClient } = ChromeUtils.importESModule( + "resource://services-common/hawkclient.sys.mjs" +); +const { FxAccounts } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const { FxAccountsClient } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccountsClient.sys.mjs" +); +const { + ERRNO_INVALID_AUTH_TOKEN, + ONLOGIN_NOTIFICATION, + ONVERIFIED_NOTIFICATION, +} = ChromeUtils.import("resource://gre/modules/FxAccountsCommon.js"); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); +const { TokenServerClient, TokenServerClientServerError } = + ChromeUtils.importESModule( + "resource://services-common/tokenserverclient.sys.mjs" + ); +const { AccountState } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); + +const SECOND_MS = 1000; +const MINUTE_MS = SECOND_MS * 60; +const HOUR_MS = MINUTE_MS * 60; + +const MOCK_ACCESS_TOKEN = + "e3c5caf17f27a0d9e351926a928938b3737df43e91d4992a5a5fca9a7bdef8ba"; + +var globalIdentityConfig = makeIdentityConfig(); +var globalSyncAuthManager = new SyncAuthManager(); +configureFxAccountIdentity(globalSyncAuthManager, globalIdentityConfig); + +/** + * Mock client clock and skew vs server in FxAccounts signed-in user module and + * API client. sync_auth.js queries these values to construct HAWK + * headers. We will use this to test clock skew compensation in these headers + * below. + */ +var MockFxAccountsClient = function () { + FxAccountsClient.apply(this); +}; +MockFxAccountsClient.prototype = { + accountStatus() { + return Promise.resolve(true); + }, + getScopedKeyData() { + return Promise.resolve({ + "https://identity.mozilla.com/apps/oldsync": { + identifier: "https://identity.mozilla.com/apps/oldsync", + keyRotationSecret: + "0000000000000000000000000000000000000000000000000000000000000000", + keyRotationTimestamp: 1234567890123, + }, + }); + }, +}; +Object.setPrototypeOf( + MockFxAccountsClient.prototype, + FxAccountsClient.prototype +); + +add_test(function test_initial_state() { + _("Verify initial state"); + Assert.ok(!globalSyncAuthManager._token); + Assert.ok(!globalSyncAuthManager._hasValidToken()); + run_next_test(); +}); + +add_task(async function test_initialialize() { + _("Verify start after fetching token"); + await globalSyncAuthManager._ensureValidToken(); + Assert.ok(!!globalSyncAuthManager._token); + Assert.ok(globalSyncAuthManager._hasValidToken()); +}); + +add_task(async function test_refreshOAuthTokenOn401() { + _("Refreshes the FXA OAuth token after a 401."); + let getTokenCount = 0; + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + syncAuthManager._fxaService.getOAuthToken = () => { + ++getTokenCount; + return Promise.resolve(MOCK_ACCESS_TOKEN); + }; + + let didReturn401 = false; + let didReturn200 = false; + let mockTSC = mockTokenServer(() => { + if (getTokenCount <= 1) { + didReturn401 = true; + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + didReturn200 = true; + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "id", + key: "key", + api_endpoint: "http://example.com/", + uid: "uid", + duration: 300, + }), + }; + }); + + syncAuthManager._tokenServerClient = mockTSC; + + await syncAuthManager._ensureValidToken(); + + Assert.equal(getTokenCount, 2); + Assert.ok(didReturn401); + Assert.ok(didReturn200); + Assert.ok(syncAuthManager._token); + Assert.ok(syncAuthManager._hasValidToken()); +}); + +add_task(async function test_initialializeWithAuthErrorAndDeletedAccount() { + _("Verify sync state with auth error + account deleted"); + + var identityConfig = makeIdentityConfig(); + var syncAuthManager = new SyncAuthManager(); + + // Use the real `getOAuthToken` method that calls + // `mockFxAClient.accessTokenWithSessionToken`. + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + delete fxaInternal.getOAuthToken; + + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + + let accessTokenWithSessionTokenCalled = false; + let accountStatusCalled = false; + let sessionStatusCalled = false; + + let AuthErrorMockFxAClient = function () { + FxAccountsClient.apply(this); + }; + AuthErrorMockFxAClient.prototype = { + accessTokenWithSessionToken() { + accessTokenWithSessionTokenCalled = true; + return Promise.reject({ + code: 401, + errno: ERRNO_INVALID_AUTH_TOKEN, + }); + }, + accountStatus() { + accountStatusCalled = true; + return Promise.resolve(false); + }, + sessionStatus() { + sessionStatusCalled = true; + return Promise.resolve(false); + }, + }; + Object.setPrototypeOf( + AuthErrorMockFxAClient.prototype, + FxAccountsClient.prototype + ); + + let mockFxAClient = new AuthErrorMockFxAClient(); + syncAuthManager._fxaService._internal._fxAccountsClient = mockFxAClient; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + AuthenticationError, + "should reject due to an auth error" + ); + + Assert.ok(accessTokenWithSessionTokenCalled); + Assert.ok(sessionStatusCalled); + Assert.ok(accountStatusCalled); + Assert.ok(!syncAuthManager._token); + Assert.ok(!syncAuthManager._hasValidToken()); +}); + +add_task(async function test_getResourceAuthenticator() { + _( + "SyncAuthManager supplies a Resource Authenticator callback which returns a Hawk header." + ); + configureFxAccountIdentity(globalSyncAuthManager); + let authenticator = globalSyncAuthManager.getResourceAuthenticator(); + Assert.ok(!!authenticator); + let req = { + uri: CommonUtils.makeURI("https://example.net/somewhere/over/the/rainbow"), + method: "GET", + }; + let output = await authenticator(req, "GET"); + Assert.ok("headers" in output); + Assert.ok("authorization" in output.headers); + Assert.ok(output.headers.authorization.startsWith("Hawk")); + _("Expected internal state after successful call."); + Assert.equal( + globalSyncAuthManager._token.uid, + globalIdentityConfig.fxaccount.token.uid + ); +}); + +add_task(async function test_resourceAuthenticatorSkew() { + _( + "SyncAuthManager Resource Authenticator compensates for clock skew in Hawk header." + ); + + // Clock is skewed 12 hours into the future + // We pick a date in the past so we don't risk concealing bugs in code that + // uses new Date() instead of our given date. + let now = + new Date("Fri Apr 09 2004 00:00:00 GMT-0700").valueOf() + 12 * HOUR_MS; + let syncAuthManager = new SyncAuthManager(); + let hawkClient = new HawkClient("https://example.net/v1", "/foo"); + + // mock fxa hawk client skew + hawkClient.now = function () { + dump("mocked client now: " + now + "\n"); + return now; + }; + // Imagine there's already been one fxa request and the hawk client has + // already detected skew vs the fxa auth server. + let localtimeOffsetMsec = -1 * 12 * HOUR_MS; + hawkClient._localtimeOffsetMsec = localtimeOffsetMsec; + + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = hawkClient; + + // Sanity check + Assert.equal(hawkClient.now(), now); + Assert.equal(hawkClient.localtimeOffsetMsec, localtimeOffsetMsec); + + // Properly picked up by the client + Assert.equal(fxaClient.now(), now); + Assert.equal(fxaClient.localtimeOffsetMsec, localtimeOffsetMsec); + + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal._now_is = now; + fxaInternal.fxAccountsClient = fxaClient; + + // Mocks within mocks... + configureFxAccountIdentity( + syncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + Assert.equal( + syncAuthManager._fxaService._internal.localtimeOffsetMsec, + localtimeOffsetMsec + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + Assert.equal( + syncAuthManager._fxaService._internal.localtimeOffsetMsec, + localtimeOffsetMsec + ); + + let request = new Resource("https://example.net/i/like/pie/"); + let authenticator = syncAuthManager.getResourceAuthenticator(); + let output = await authenticator(request, "GET"); + dump("output" + JSON.stringify(output)); + let authHeader = output.headers.authorization; + Assert.ok(authHeader.startsWith("Hawk")); + + // Skew correction is applied in the header and we're within the two-minute + // window. + Assert.equal(getTimestamp(authHeader), now - 12 * HOUR_MS); + Assert.ok(getTimestampDelta(authHeader, now) - 12 * HOUR_MS < 2 * MINUTE_MS); +}); + +add_task(async function test_RESTResourceAuthenticatorSkew() { + _( + "SyncAuthManager REST Resource Authenticator compensates for clock skew in Hawk header." + ); + + // Clock is skewed 12 hours into the future from our arbitary date + let now = + new Date("Fri Apr 09 2004 00:00:00 GMT-0700").valueOf() + 12 * HOUR_MS; + let syncAuthManager = new SyncAuthManager(); + let hawkClient = new HawkClient("https://example.net/v1", "/foo"); + + // mock fxa hawk client skew + hawkClient.now = function () { + return now; + }; + // Imagine there's already been one fxa request and the hawk client has + // already detected skew vs the fxa auth server. + hawkClient._localtimeOffsetMsec = -1 * 12 * HOUR_MS; + + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = hawkClient; + + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal._now_is = now; + fxaInternal.fxAccountsClient = fxaClient; + + configureFxAccountIdentity( + syncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + Assert.equal(syncAuthManager._fxaService._internal.now(), now); + + let request = new Resource("https://example.net/i/like/pie/"); + let authenticator = syncAuthManager.getResourceAuthenticator(); + let output = await authenticator(request, "GET"); + dump("output" + JSON.stringify(output)); + let authHeader = output.headers.authorization; + Assert.ok(authHeader.startsWith("Hawk")); + + // Skew correction is applied in the header and we're within the two-minute + // window. + Assert.equal(getTimestamp(authHeader), now - 12 * HOUR_MS); + Assert.ok(getTimestampDelta(authHeader, now) - 12 * HOUR_MS < 2 * MINUTE_MS); +}); + +add_task(async function test_ensureLoggedIn() { + configureFxAccountIdentity(globalSyncAuthManager); + await globalSyncAuthManager._ensureValidToken(); + Assert.equal(Status.login, LOGIN_SUCCEEDED, "original initialize worked"); + Assert.ok(globalSyncAuthManager._token); + + // arrange for no logged in user. + let fxa = globalSyncAuthManager._fxaService; + let signedInUser = + fxa._internal.currentAccountState.storageManager.accountData; + fxa._internal.currentAccountState.storageManager.accountData = null; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + /no user is logged in/, + "expecting rejection due to no user" + ); + // Restore the logged in user to what it was. + fxa._internal.currentAccountState.storageManager.accountData = signedInUser; + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + await globalSyncAuthManager._ensureValidToken(true); + Assert.equal(Status.login, LOGIN_SUCCEEDED, "final ensureLoggedIn worked"); +}); + +add_task(async function test_syncState() { + // Avoid polling for an unverified user. + let identityConfig = makeIdentityConfig(); + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + fxaInternal.startVerifiedCheck = () => {}; + configureFxAccountIdentity( + globalSyncAuthManager, + globalIdentityConfig, + fxaInternal + ); + + // arrange for no logged in user. + let fxa = globalSyncAuthManager._fxaService; + let signedInUser = + fxa._internal.currentAccountState.storageManager.accountData; + fxa._internal.currentAccountState.storageManager.accountData = null; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + /no user is logged in/, + "expecting rejection due to no user" + ); + // Restore to an unverified user. + Services.prefs.setStringPref("services.sync.username", signedInUser.email); + signedInUser.verified = false; + fxa._internal.currentAccountState.storageManager.accountData = signedInUser; + Status.login = LOGIN_FAILED_LOGIN_REJECTED; + // The sync_auth observers are async, so call them directly. + await globalSyncAuthManager.observe(null, ONLOGIN_NOTIFICATION, ""); + Assert.equal( + Status.login, + LOGIN_FAILED_LOGIN_REJECTED, + "should not have changed the login state for an unverified user" + ); + + // now pretend the user because verified. + signedInUser.verified = true; + await globalSyncAuthManager.observe(null, ONVERIFIED_NOTIFICATION, ""); + Assert.equal( + Status.login, + LOGIN_SUCCEEDED, + "should have changed the login state to success" + ); +}); + +add_task(async function test_tokenExpiration() { + _("SyncAuthManager notices token expiration:"); + let bimExp = new SyncAuthManager(); + configureFxAccountIdentity(bimExp, globalIdentityConfig); + + let authenticator = bimExp.getResourceAuthenticator(); + Assert.ok(!!authenticator); + let req = { + uri: CommonUtils.makeURI("https://example.net/somewhere/over/the/rainbow"), + method: "GET", + }; + await authenticator(req, "GET"); + + // Mock the clock. + _("Forcing the token to expire ..."); + Object.defineProperty(bimExp, "_now", { + value: function customNow() { + return Date.now() + 3000001; + }, + writable: true, + }); + Assert.ok(bimExp._token.expiration < bimExp._now()); + _("... means SyncAuthManager knows to re-fetch it on the next call."); + Assert.ok(!bimExp._hasValidToken()); +}); + +add_task(async function test_getTokenErrors() { + _("SyncAuthManager correctly handles various failures to get a token."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + initializeIdentityWithTokenServerResponse({ + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }); + let syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + AuthenticationError, + "should reject due to 401" + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); + + // XXX - other interesting responses to return? + + // And for good measure, some totally "unexpected" errors - we generally + // assume these problems are going to magically go away at some point. + _( + "Arrange for an empty body with a 200 response - should reflect a network error." + ); + initializeIdentityWithTokenServerResponse({ + status: 200, + headers: [], + body: "", + }); + syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to non-JSON response" + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "login state is LOGIN_FAILED_NETWORK_ERROR" + ); +}); + +add_task(async function test_refreshAccessTokenOn401() { + _("SyncAuthManager refreshes the FXA OAuth access token after a 401."); + var identityConfig = makeIdentityConfig(); + var syncAuthManager = new SyncAuthManager(); + // Use the real `getOAuthToken` method that calls + // `mockFxAClient.accessTokenWithSessionToken`. + let fxaInternal = makeFxAccountsInternalMock(identityConfig); + delete fxaInternal.getOAuthToken; + configureFxAccountIdentity(syncAuthManager, identityConfig, fxaInternal); + syncAuthManager._fxaService._internal.initialize(); + + let getTokenCount = 0; + + let CheckSignMockFxAClient = function () { + FxAccountsClient.apply(this); + }; + CheckSignMockFxAClient.prototype = { + accessTokenWithSessionToken() { + ++getTokenCount; + return Promise.resolve({ access_token: "token" }); + }, + }; + Object.setPrototypeOf( + CheckSignMockFxAClient.prototype, + FxAccountsClient.prototype + ); + + let mockFxAClient = new CheckSignMockFxAClient(); + syncAuthManager._fxaService._internal._fxAccountsClient = mockFxAClient; + + let didReturn401 = false; + let didReturn200 = false; + let mockTSC = mockTokenServer(() => { + if (getTokenCount <= 1) { + didReturn401 = true; + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + didReturn200 = true; + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "id", + key: "key", + api_endpoint: "http://example.com/", + uid: "uid", + duration: 300, + }), + }; + }); + + syncAuthManager._tokenServerClient = mockTSC; + + await syncAuthManager._ensureValidToken(); + + Assert.equal(getTokenCount, 2); + Assert.ok(didReturn401); + Assert.ok(didReturn200); + Assert.ok(syncAuthManager._token); + Assert.ok(syncAuthManager._hasValidToken()); +}); + +add_task(async function test_getTokenErrorWithRetry() { + _("tokenserver sends an observer notification on various backoff headers."); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a Retry-After header."); + initializeIdentityWithTokenServerResponse({ + status: 503, + headers: { "content-type": "application/json", "retry-after": "100" }, + body: JSON.stringify({}), + }); + let syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); + + _("Arrange for a 200 with an X-Backoff header."); + Status.backoffInterval = 0; + initializeIdentityWithTokenServerResponse({ + status: 503, + headers: { "content-type": "application/json", "x-backoff": "200" }, + body: JSON.stringify({}), + }); + syncAuthManager = Service.identity; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to no token in response" + ); + + // The observer should have fired - check it got the value in the response. + Assert.ok(Status.backoffInterval >= 200000); +}); + +add_task(async function test_getKeysErrorWithBackoff() { + _( + "Auth server (via hawk) sends an observer notification on backoff headers." + ); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a X-Backoff header."); + + let config = makeIdentityConfig(); + // We want no kSync, kXCS, kExtSync or kExtKbHash so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + delete config.fxaccount.user.kSync; + delete config.fxaccount.user.kXCS; + delete config.fxaccount.user.kExtSync; + delete config.fxaccount.user.kExtKbHash; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json", "x-backoff": "100" }, + body: "{}", + }; + } + ); + + let syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); +}); + +add_task(async function test_getKeysErrorWithRetry() { + _("Auth server (via hawk) sends an observer notification on retry headers."); + + // Set Sync's backoffInterval to zero - after we simulated the backoff header + // it should reflect the value we sent. + Status.backoffInterval = 0; + _("Arrange for a 503 with a Retry-After header."); + + let config = makeIdentityConfig(); + // We want no kSync, kXCS, kExtSync or kExtKbHash so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + delete config.fxaccount.user.kSync; + delete config.fxaccount.user.kXCS; + delete config.fxaccount.user.kExtSync; + delete config.fxaccount.user.kExtKbHash; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json", "retry-after": "100" }, + body: "{}", + }; + } + ); + + let syncAuthManager = Service.identity; + await Assert.rejects( + syncAuthManager._ensureValidToken(), + TokenServerClientServerError, + "should reject due to 503" + ); + + // The observer should have fired - check it got the value in the response. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected"); + // Sync will have the value in ms with some slop - so check it is at least that. + Assert.ok(Status.backoffInterval >= 100000); +}); + +add_task(async function test_getHAWKErrors() { + _("SyncAuthManager correctly handles various HAWK failures."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + let config = makeIdentityConfig(); + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + if (uri == "http://mockedserver:9999/oauth/token") { + Assert.equal(method, "post"); + return { + status: 401, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + code: 401, + errno: 110, + error: "invalid token", + }), + }; + } + // For any follow-up requests that check account status. + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({}), + }; + } + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); + + // XXX - other interesting responses to return? + + // And for good measure, some totally "unexpected" errors - we generally + // assume these problems are going to magically go away at some point. + _( + "Arrange for an empty body with a 200 response - should reflect a network error." + ); + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "post"); + Assert.equal(uri, "http://mockedserver:9999/oauth/token"); + return { + status: 200, + headers: [], + body: "", + }; + } + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "login state is LOGIN_FAILED_NETWORK_ERROR" + ); +}); + +add_task(async function test_getGetKeysFailing401() { + _("SyncAuthManager correctly handles 401 responses fetching keys."); + + _("Arrange for a 401 - Sync should reflect an auth error."); + let config = makeIdentityConfig(); + // We want no kSync, kXCS, kExtSync or kExtKbHash so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + delete config.fxaccount.user.kSync; + delete config.fxaccount.user.kXCS; + delete config.fxaccount.user.kExtSync; + delete config.fxaccount.user.kExtKbHash; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 401, + headers: { "content-type": "application/json" }, + body: "{}", + }; + } + ); + Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected"); +}); + +add_task(async function test_getGetKeysFailing503() { + _("SyncAuthManager correctly handles 5XX responses fetching keys."); + + _("Arrange for a 503 - Sync should reflect a network error."); + let config = makeIdentityConfig(); + // We want no kSync, kXCS, kExtSync or kExtKbHash so we attempt to fetch them. + delete config.fxaccount.user.scopedKeys; + delete config.fxaccount.user.kSync; + delete config.fxaccount.user.kXCS; + delete config.fxaccount.user.kExtSync; + delete config.fxaccount.user.kExtKbHash; + config.fxaccount.user.keyFetchToken = "keyfetchtoken"; + await initializeIdentityWithHAWKResponseFactory( + config, + function (method, data, uri) { + Assert.equal(method, "get"); + Assert.equal(uri, "http://mockedserver:9999/account/keys"); + return { + status: 503, + headers: { "content-type": "application/json" }, + body: "{}", + }; + } + ); + Assert.equal( + Status.login, + LOGIN_FAILED_NETWORK_ERROR, + "state reflects network error" + ); +}); + +add_task(async function test_getKeysMissing() { + _( + "SyncAuthManager correctly handles getKeyForScope succeeding but not returning the key." + ); + + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + // our mock identity config already has kSync, kXCS, kExtSync and kExtKbHash - remove them or we never + // try and fetch them. + delete identityConfig.fxaccount.user.scopedKeys; + delete identityConfig.fxaccount.user.kSync; + delete identityConfig.fxaccount.user.kXCS; + delete identityConfig.fxaccount.user.kExtSync; + delete identityConfig.fxaccount.user.kExtKbHash; + identityConfig.fxaccount.user.keyFetchToken = "keyFetchToken"; + + configureFxAccountIdentity(syncAuthManager, identityConfig); + + // Mock a fxAccounts object + let fxa = new FxAccounts({ + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(identityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + // And the keys object with a mock that returns no keys. + keys: { + getKeyForScope() { + return Promise.resolve(null); + }, + }, + }); + + syncAuthManager._fxaService = fxa; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + /browser does not have the sync key, cannot sync/ + ); +}); + +add_task(async function test_getKeysUnexpecedError() { + _( + "SyncAuthManager correctly handles getKeyForScope throwing an unexpected error." + ); + + let syncAuthManager = new SyncAuthManager(); + let identityConfig = makeIdentityConfig(); + // our mock identity config already has kSync, kXCS, kExtSync and kExtKbHash - remove them or we never + // try and fetch them. + delete identityConfig.fxaccount.user.scopedKeys; + delete identityConfig.fxaccount.user.kSync; + delete identityConfig.fxaccount.user.kXCS; + delete identityConfig.fxaccount.user.kExtSync; + delete identityConfig.fxaccount.user.kExtKbHash; + identityConfig.fxaccount.user.keyFetchToken = "keyFetchToken"; + + configureFxAccountIdentity(syncAuthManager, identityConfig); + + // Mock a fxAccounts object + let fxa = new FxAccounts({ + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(identityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + // And the keys object with a mock that returns no keys. + keys: { + async getKeyForScope() { + throw new Error("well that was unexpected"); + }, + }, + }); + + syncAuthManager._fxaService = fxa; + + await Assert.rejects( + syncAuthManager._ensureValidToken(), + /well that was unexpected/ + ); +}); + +add_task(async function test_signedInUserMissing() { + _( + "SyncAuthManager detects getSignedInUser returning incomplete account data" + ); + + let syncAuthManager = new SyncAuthManager(); + // Delete stored keys and the key fetch token. + delete globalIdentityConfig.fxaccount.user.scopedKeys; + delete globalIdentityConfig.fxaccount.user.kSync; + delete globalIdentityConfig.fxaccount.user.kXCS; + delete globalIdentityConfig.fxaccount.user.kExtSync; + delete globalIdentityConfig.fxaccount.user.kExtKbHash; + delete globalIdentityConfig.fxaccount.user.keyFetchToken; + + configureFxAccountIdentity(syncAuthManager, globalIdentityConfig); + + let fxa = new FxAccounts({ + fetchAndUnwrapKeys() { + return Promise.resolve({}); + }, + fxAccountsClient: new MockFxAccountsClient(), + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(globalIdentityConfig.fxaccount.user); + return new AccountState(storageManager); + }, + }); + + syncAuthManager._fxaService = fxa; + + let status = await syncAuthManager.unlockAndVerifyAuthState(); + Assert.equal(status, LOGIN_FAILED_LOGIN_REJECTED); +}); + +// End of tests +// Utility functions follow + +// Create a new sync_auth object and initialize it with a +// hawk mock that simulates HTTP responses. +// The callback function will be called each time the mocked hawk server wants +// to make a request. The result of the callback should be the mock response +// object that will be returned to hawk. +// A token server mock will be used that doesn't hit a server, so we move +// directly to a hawk request. +async function initializeIdentityWithHAWKResponseFactory( + config, + cbGetResponse +) { + // A mock request object. + function MockRESTRequest(uri, credentials, extra) { + this._uri = uri; + this._credentials = credentials; + this._extra = extra; + } + MockRESTRequest.prototype = { + setHeader() {}, + async post(data) { + this.response = cbGetResponse( + "post", + data, + this._uri, + this._credentials, + this._extra + ); + return this.response; + }, + async get() { + // Skip /status requests (sync_auth checks if the account still + // exists after an auth error) + if (this._uri.startsWith("http://mockedserver:9999/account/status")) { + this.response = { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ exists: true }), + }; + } else { + this.response = cbGetResponse( + "get", + null, + this._uri, + this._credentials, + this._extra + ); + } + return this.response; + }, + }; + + // The hawk client. + function MockedHawkClient() {} + MockedHawkClient.prototype = new HawkClient("http://mockedserver:9999"); + MockedHawkClient.prototype.constructor = MockedHawkClient; + MockedHawkClient.prototype.newHAWKAuthenticatedRESTRequest = function ( + uri, + credentials, + extra + ) { + return new MockRESTRequest(uri, credentials, extra); + }; + // Arrange for the same observerPrefix as FxAccountsClient uses + MockedHawkClient.prototype.observerPrefix = "FxA:hawk"; + + // tie it all together - configureFxAccountIdentity isn't useful here :( + let fxaClient = new MockFxAccountsClient(); + fxaClient.hawk = new MockedHawkClient(); + let internal = { + fxAccountsClient: fxaClient, + newAccountState(credentials) { + // We only expect this to be called with null indicating the (mock) + // storage should be read. + if (credentials) { + throw new Error("Not expecting to have credentials passed"); + } + let storageManager = new MockFxaStorageManager(); + storageManager.initialize(config.fxaccount.user); + return new AccountState(storageManager); + }, + }; + let fxa = new FxAccounts(internal); + + globalSyncAuthManager._fxaService = fxa; + await Assert.rejects( + globalSyncAuthManager._ensureValidToken(true), + // TODO: Ideally this should have a specific check for an error. + () => true, + "expecting rejection due to hawk error" + ); +} + +function getTimestamp(hawkAuthHeader) { + return parseInt(/ts="(\d+)"/.exec(hawkAuthHeader)[1], 10) * SECOND_MS; +} + +function getTimestampDelta(hawkAuthHeader, now = Date.now()) { + return Math.abs(getTimestamp(hawkAuthHeader) - now); +} + +function mockTokenServer(func) { + let requestLog = Log.repository.getLogger("testing.mock-rest"); + if (!requestLog.appenders.length) { + // might as well see what it says :) + requestLog.addAppender(new Log.DumpAppender()); + requestLog.level = Log.Level.Trace; + } + function MockRESTRequest(url) {} + MockRESTRequest.prototype = { + _log: requestLog, + setHeader() {}, + async get() { + this.response = func(); + return this.response; + }, + }; + // The mocked TokenServer client which will get the response. + function MockTSC() {} + MockTSC.prototype = new TokenServerClient(); + MockTSC.prototype.constructor = MockTSC; + MockTSC.prototype.newRESTRequest = function (url) { + return new MockRESTRequest(url); + }; + // Arrange for the same observerPrefix as sync_auth uses. + MockTSC.prototype.observerPrefix = "weave:service"; + return new MockTSC(); +} diff --git a/services/sync/tests/unit/test_syncedtabs.js b/services/sync/tests/unit/test_syncedtabs.js new file mode 100644 index 0000000000..79ab3e0686 --- /dev/null +++ b/services/sync/tests/unit/test_syncedtabs.js @@ -0,0 +1,342 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- + * vim:set ts=2 sw=2 sts=2 et: + */ +"use strict"; + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { SyncedTabs } = ChromeUtils.importESModule( + "resource://services-sync/SyncedTabs.sys.mjs" +); + +Log.repository.getLogger("Sync.RemoteTabs").addAppender(new Log.DumpAppender()); + +// A mock "Tabs" engine which the SyncedTabs module will use instead of the real +// engine. We pass a constructor that Sync creates. +function MockTabsEngine() { + this.clients = {}; // We'll set this dynamically + // Mock fxAccounts + recentDeviceList as if we hit the FxA server + this.fxAccounts = { + device: { + recentDeviceList: [ + { + id: 1, + name: "updated desktop name", + availableCommands: { + "https://identity.mozilla.com/cmd/open-uri": "baz", + }, + }, + { + id: 2, + name: "updated mobile name", + availableCommands: { + "https://identity.mozilla.com/cmd/open-uri": "boo", + }, + }, + ], + }, + }; +} + +MockTabsEngine.prototype = { + name: "tabs", + enabled: true, + + getAllClients() { + return Object.values(this.clients); + }, + + getOpenURLs() { + return new Set(); + }, +}; + +let tabsEngine; + +// A clients engine that doesn't need to be a constructor. +let MockClientsEngine = { + clientSettings: null, // Set in `configureClients`. + + isMobile(guid) { + if (!guid.endsWith("desktop") && !guid.endsWith("mobile")) { + throw new Error( + "this module expected guids to end with 'desktop' or 'mobile'" + ); + } + return guid.endsWith("mobile"); + }, + remoteClientExists(id) { + return this.clientSettings[id] !== false; + }, + getClientName(id) { + if (this.clientSettings[id]) { + return this.clientSettings[id]; + } + let client = tabsEngine.clients[id]; + let fxaDevice = tabsEngine.fxAccounts.device.recentDeviceList.find( + device => device.id === client.fxaDeviceId + ); + return fxaDevice ? fxaDevice.name : client.clientName; + }, + + getClientFxaDeviceId(id) { + if (this.clientSettings[id]) { + return this.clientSettings[id]; + } + return tabsEngine.clients[id].fxaDeviceId; + }, + + getClientType(id) { + return "desktop"; + }, +}; + +function configureClients(clients, clientSettings = {}) { + // each client record is expected to have an id. + for (let [guid, client] of Object.entries(clients)) { + client.id = guid; + } + tabsEngine.clients = clients; + // Apply clients collection overrides. + MockClientsEngine.clientSettings = clientSettings; + // Send an observer that pretends the engine just finished a sync. + Services.obs.notifyObservers(null, "weave:engine:sync:finish", "tabs"); +} + +add_task(async function setup() { + await Weave.Service.promiseInitialized; + // Configure Sync with our mock tabs engine and force it to become initialized. + await Weave.Service.engineManager.unregister("tabs"); + await Weave.Service.engineManager.register(MockTabsEngine); + Weave.Service.clientsEngine = MockClientsEngine; + tabsEngine = Weave.Service.engineManager.get("tabs"); + + // Tell the Sync XPCOM service it is initialized. + let weaveXPCService = Cc["@mozilla.org/weave/service;1"].getService( + Ci.nsISupports + ).wrappedJSObject; + weaveXPCService.ready = true; +}); + +// The tests. +add_task(async function test_noClients() { + // no clients, can't be tabs. + await configureClients({}); + + let tabs = await SyncedTabs.getTabClients(); + equal(Object.keys(tabs).length, 0); +}); + +add_task(async function test_clientWithTabs() { + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + lastUsed: 1655745700, // Mon, 20 Jun 2022 17:21:40 GMT + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[0].tabs[0].icon, "http://foo.com/favicon"); + equal(clients[0].tabs[0].lastUsed, 1655745700); + // second client has no tabs. + equal(clients[1].tabs.length, 0); +}); + +add_task(async function test_staleClientWithTabs() { + await configureClients( + { + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + lastUsed: 1655745750, + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [], + }, + guid_stale_mobile: { + clientName: "My Deleted Phone", + tabs: [], + }, + guid_stale_desktop: { + clientName: "My Deleted Laptop", + tabs: [ + { + urlHistory: ["https://bar.com/"], + icon: "https://bar.com/favicon", + lastUsed: 1655745700, + }, + ], + }, + guid_stale_name_desktop: { + clientName: "My Generic Device", + tabs: [ + { + urlHistory: ["https://example.edu/"], + icon: "https://example.edu/favicon", + lastUsed: 1655745800, + }, + ], + }, + }, + { + guid_stale_mobile: false, + guid_stale_desktop: false, + // We should always use the device name from the clients collection, instead + // of the possibly stale tabs collection. + guid_stale_name_desktop: "My Laptop", + } + ); + let clients = await SyncedTabs.getTabClients(); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients.length, 3); + equal(clients[0].name, "My Desktop"); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[0].tabs[0].lastUsed, 1655745750); + equal(clients[1].name, "My Laptop"); + equal(clients[1].tabs.length, 1); + equal(clients[1].tabs[0].url, "https://example.edu/"); + equal(clients[1].tabs[0].lastUsed, 1655745800); + equal(clients[2].name, "My Phone"); + equal(clients[2].tabs.length, 0); +}); + +add_task(async function test_clientWithTabsIconsDisabled() { + Services.prefs.setBoolPref("services.sync.syncedTabs.showRemoteIcons", false); + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 1); + clients.sort((a, b) => { + return a.name.localeCompare(b.name); + }); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + // Expect the default favicon due to the pref being false. + equal(clients[0].tabs[0].icon, "page-icon:http://foo.com/"); + Services.prefs.clearUserPref("services.sync.syncedTabs.showRemoteIcons"); +}); + +add_task(async function test_filter() { + // Nothing matches. + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + { + urlHistory: ["http://bar.com/"], + title: "Another page.", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients("foo"); + equal(clients.length, 1); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + // check it matches the title. + clients = await SyncedTabs.getTabClients("test"); + equal(clients.length, 1); + equal(clients[0].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); +}); + +add_task(async function test_duplicatesTabsAcrossClients() { + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + ], + }, + guid_mobile: { + clientName: "My Phone", + tabs: [ + { + urlHistory: ["http://foo.com/"], + title: "A test page.", + }, + ], + }, + }); + + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + equal(clients[0].tabs.length, 1); + equal(clients[1].tabs.length, 1); + equal(clients[0].tabs[0].url, "http://foo.com/"); + equal(clients[1].tabs[0].url, "http://foo.com/"); +}); + +add_task(async function test_clientsTabUpdatedName() { + // See the "fxAccounts" object in the MockEngine above for the device list + await configureClients({ + guid_desktop: { + clientName: "My Desktop", + tabs: [ + { + urlHistory: ["http://foo.com/"], + icon: "http://foo.com/favicon", + }, + ], + fxaDeviceId: 1, + }, + guid_mobile: { + clientName: "My Phone", + tabs: [ + { + urlHistory: ["http://bar.com/"], + icon: "http://bar.com/favicon", + }, + ], + fxaDeviceId: 2, + }, + }); + let clients = await SyncedTabs.getTabClients(); + equal(clients.length, 2); + equal(clients[0].name, "updated desktop name"); + equal(clients[1].name, "updated mobile name"); +}); diff --git a/services/sync/tests/unit/test_syncengine.js b/services/sync/tests/unit/test_syncengine.js new file mode 100644 index 0000000000..07fcfdcff1 --- /dev/null +++ b/services/sync/tests/unit/test_syncengine.js @@ -0,0 +1,281 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +async function makeSteamEngine() { + let engine = new SyncEngine("Steam", Service); + await engine.initialize(); + return engine; +} + +function guidSetOfSize(length) { + return new SerializableSet(Array.from({ length }, () => Utils.makeGUID())); +} + +function assertSetsEqual(a, b) { + // Assert.deepEqual doesn't understand Set. + Assert.deepEqual(Array.from(a).sort(), Array.from(b).sort()); +} + +async function testSteamEngineStorage(test) { + try { + let setupEngine = await makeSteamEngine(); + + if (test.setup) { + await test.setup(setupEngine); + } + + // Finalize the engine to flush the backlog and previous failed to disk. + await setupEngine.finalize(); + + if (test.beforeCheck) { + await test.beforeCheck(); + } + + let checkEngine = await makeSteamEngine(); + await test.check(checkEngine); + + await checkEngine.resetClient(); + await checkEngine.finalize(); + } finally { + Svc.Prefs.resetBranch(""); + } +} + +let server; + +add_task(async function setup() { + server = httpd_setup({}); +}); + +add_task(async function test_url_attributes() { + _("SyncEngine url attributes"); + await SyncTestingInfrastructure(server); + Service.clusterURL = "https://cluster/1.1/foo/"; + let engine = await makeSteamEngine(); + try { + Assert.equal(engine.storageURL, "https://cluster/1.1/foo/storage/"); + Assert.equal(engine.engineURL, "https://cluster/1.1/foo/storage/steam"); + Assert.equal(engine.metaURL, "https://cluster/1.1/foo/storage/meta/global"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_syncID() { + _("SyncEngine.syncID corresponds to preference"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal(Svc.Prefs.get("steam.syncID"), undefined); + Assert.equal(await engine.getSyncID(), ""); + + // Performing the first get on the attribute will generate a new GUID. + Assert.equal(await engine.resetLocalSyncID(), "fake-guid-00"); + Assert.equal(Svc.Prefs.get("steam.syncID"), "fake-guid-00"); + + Svc.Prefs.set("steam.syncID", Utils.makeGUID()); + Assert.equal(Svc.Prefs.get("steam.syncID"), "fake-guid-01"); + Assert.equal(await engine.getSyncID(), "fake-guid-01"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_lastSync() { + _("SyncEngine.lastSync corresponds to preferences"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal(Svc.Prefs.get("steam.lastSync"), undefined); + Assert.equal(await engine.getLastSync(), 0); + + // Floats are properly stored as floats and synced with the preference + await engine.setLastSync(123.45); + Assert.equal(await engine.getLastSync(), 123.45); + Assert.equal(Svc.Prefs.get("steam.lastSync"), "123.45"); + + // Integer is properly stored + await engine.setLastSync(67890); + Assert.equal(await engine.getLastSync(), 67890); + Assert.equal(Svc.Prefs.get("steam.lastSync"), "67890"); + + // resetLastSync() resets the value (and preference) to 0 + await engine.resetLastSync(); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(Svc.Prefs.get("steam.lastSync"), "0"); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_toFetch() { + _("SyncEngine.toFetch corresponds to file on disk"); + await SyncTestingInfrastructure(server); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(3), + setup(engine) { + // Ensure pristine environment + Assert.equal(engine.toFetch.size, 0); + + // Write file to disk + engine.toFetch = this.toFetch; + Assert.equal(engine.toFetch, this.toFetch); + }, + check(engine) { + // toFetch is written asynchronously + assertSetsEqual(engine.toFetch, this.toFetch); + }, + }); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(4), + toFetch2: guidSetOfSize(5), + setup(engine) { + // Make sure it work for consecutive writes before the callback is executed. + engine.toFetch = this.toFetch; + Assert.equal(engine.toFetch, this.toFetch); + + engine.toFetch = this.toFetch2; + Assert.equal(engine.toFetch, this.toFetch2); + }, + check(engine) { + assertSetsEqual(engine.toFetch, this.toFetch2); + }, + }); + + await testSteamEngineStorage({ + toFetch: guidSetOfSize(2), + async beforeCheck() { + let toFetchPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "toFetch", + "steam.json" + ); + await IOUtils.writeJSON(toFetchPath, this.toFetch, { + tmpPath: toFetchPath + ".tmp", + }); + }, + check(engine) { + // Read file from disk + assertSetsEqual(engine.toFetch, this.toFetch); + }, + }); +}); + +add_task(async function test_previousFailed() { + _("SyncEngine.previousFailed corresponds to file on disk"); + await SyncTestingInfrastructure(server); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(3), + setup(engine) { + // Ensure pristine environment + Assert.equal(engine.previousFailed.size, 0); + + // Write file to disk + engine.previousFailed = this.previousFailed; + Assert.equal(engine.previousFailed, this.previousFailed); + }, + check(engine) { + // previousFailed is written asynchronously + assertSetsEqual(engine.previousFailed, this.previousFailed); + }, + }); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(4), + previousFailed2: guidSetOfSize(5), + setup(engine) { + // Make sure it work for consecutive writes before the callback is executed. + engine.previousFailed = this.previousFailed; + Assert.equal(engine.previousFailed, this.previousFailed); + + engine.previousFailed = this.previousFailed2; + Assert.equal(engine.previousFailed, this.previousFailed2); + }, + check(engine) { + assertSetsEqual(engine.previousFailed, this.previousFailed2); + }, + }); + + await testSteamEngineStorage({ + previousFailed: guidSetOfSize(2), + async beforeCheck() { + let previousFailedPath = PathUtils.join( + PathUtils.profileDir, + "weave", + "failed", + "steam.json" + ); + await IOUtils.writeJSON(previousFailedPath, this.previousFailed, { + tmpPath: previousFailedPath + ".tmp", + }); + }, + check(engine) { + // Read file from disk + assertSetsEqual(engine.previousFailed, this.previousFailed); + }, + }); +}); + +add_task(async function test_resetClient() { + _("SyncEngine.resetClient resets lastSync and toFetch"); + await SyncTestingInfrastructure(server); + let engine = await makeSteamEngine(); + try { + // Ensure pristine environment + Assert.equal(Svc.Prefs.get("steam.lastSync"), undefined); + Assert.equal(engine.toFetch.size, 0); + + await engine.setLastSync(123.45); + engine.toFetch = guidSetOfSize(4); + engine.previousFailed = guidSetOfSize(3); + + await engine.resetClient(); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + } finally { + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function test_wipeServer() { + _("SyncEngine.wipeServer deletes server data and resets the client."); + let engine = await makeSteamEngine(); + + const PAYLOAD = 42; + let steamCollection = new ServerWBO("steam", PAYLOAD); + let steamServer = httpd_setup({ + "/1.1/foo/storage/steam": steamCollection.handler(), + }); + await SyncTestingInfrastructure(steamServer); + do_test_pending(); + + try { + // Some data to reset. + await engine.setLastSync(123.45); + engine.toFetch = guidSetOfSize(3); + + _("Wipe server data and reset client."); + await engine.wipeServer(); + Assert.equal(steamCollection.payload, undefined); + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + } finally { + steamServer.stop(do_test_finished); + Svc.Prefs.resetBranch(""); + } +}); + +add_task(async function finish() { + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_syncengine_sync.js b/services/sync/tests/unit/test_syncengine_sync.js new file mode 100644 index 0000000000..031b494ad4 --- /dev/null +++ b/services/sync/tests/unit/test_syncengine_sync.js @@ -0,0 +1,1779 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Weave } = ChromeUtils.importESModule( + "resource://services-sync/main.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); + +function makeRotaryEngine() { + return new RotaryEngine(Service); +} + +async function clean(engine) { + Svc.Prefs.resetBranch(""); + Svc.Prefs.set("log.logger.engine.rotary", "Trace"); + Service.recordManager.clearCache(); + await engine._tracker.clearChangedIDs(); + await engine.finalize(); +} + +async function cleanAndGo(engine, server) { + await clean(engine); + await promiseStopServer(server); +} + +async function promiseClean(engine, server) { + await clean(engine); + await promiseStopServer(server); +} + +async function createServerAndConfigureClient() { + let engine = new RotaryEngine(Service); + let syncID = await engine.resetLocalSyncID(); + + let contents = { + meta: { + global: { engines: { rotary: { version: engine.version, syncID } } }, + }, + crypto: {}, + rotary: {}, + }; + + const USER = "foo"; + let server = new SyncServer(); + server.registerUser(USER, "password"); + server.createContents(USER, contents); + server.start(); + + await SyncTestingInfrastructure(server, USER); + Service._updateCachedURLs(); + + return [engine, server, USER]; +} + +/* + * Tests + * + * SyncEngine._sync() is divided into four rather independent steps: + * + * - _syncStartup() + * - _processIncoming() + * - _uploadOutgoing() + * - _syncFinish() + * + * In the spirit of unit testing, these are tested individually for + * different scenarios below. + */ + +add_task(async function setup() { + await generateNewKeys(Service.collectionKeys); + Svc.Prefs.set("log.logger.engine.rotary", "Trace"); +}); + +add_task(async function test_syncStartup_emptyOrOutdatedGlobalsResetsSync() { + _( + "SyncEngine._syncStartup resets sync and wipes server data if there's no or an outdated global record" + ); + + // Some server side data that's going to be wiped + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { rekolok: "Rekonstruktionslokomotive" }; + try { + // Confirm initial environment + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.rekolok, undefined); + let metaGlobal = await Service.recordManager.get(engine.metaURL); + Assert.equal(metaGlobal.payload.engines, undefined); + Assert.ok(!!collection.payload("flying")); + Assert.ok(!!collection.payload("scotsman")); + + await engine.setLastSync(Date.now() / 1000); + + // Trying to prompt a wipe -- we no longer track CryptoMeta per engine, + // so it has nothing to check. + await engine._syncStartup(); + + // The meta/global WBO has been filled with data about the engine + let engineData = metaGlobal.payload.engines.rotary; + Assert.equal(engineData.version, engine.version); + Assert.equal(engineData.syncID, await engine.getSyncID()); + + // Sync was reset and server data was wiped + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncStartup_serverHasNewerVersion() { + _("SyncEngine._syncStartup "); + + let global = new ServerWBO("global", { + engines: { rotary: { version: 23456 } }, + }); + let server = httpd_setup({ + "/1.1/foo/storage/meta/global": global.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // The server has a newer version of the data and our engine can + // handle. That should give us an exception. + let error; + try { + await engine._syncStartup(); + } catch (ex) { + error = ex; + } + Assert.equal(error.failureCode, VERSION_OUT_OF_DATE); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncStartup_syncIDMismatchResetsClient() { + _("SyncEngine._syncStartup resets sync if syncIDs don't match"); + + let server = sync_httpd_setup({}); + + await SyncTestingInfrastructure(server); + + // global record with a different syncID than our engine has + let engine = makeRotaryEngine(); + let global = new ServerWBO("global", { + engines: { rotary: { version: engine.version, syncID: "foobar" } }, + }); + server.registerPathHandler("/1.1/foo/storage/meta/global", global.handler()); + + try { + // Confirm initial environment + Assert.equal(await engine.getSyncID(), ""); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.rekolok, undefined); + + await engine.setLastSync(Date.now() / 1000); + await engine._syncStartup(); + + // The engine has assumed the server's syncID + Assert.equal(await engine.getSyncID(), "foobar"); + + // Sync was reset + Assert.equal(await engine.getLastSync(), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_emptyServer() { + _("SyncEngine._processIncoming working with an empty server backend"); + + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // Merely ensure that this code path is run without any errors + await engine._processIncoming(); + Assert.equal(await engine.getLastSync(), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_createFromServer() { + _("SyncEngine._processIncoming creates new records from server data"); + + // Some server records that will be downloaded + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + + // Two pathological cases involving relative URIs gone wrong. + let pathologicalPayload = encryptPayload({ + id: "../pathological", + denomination: "Pathological Case", + }); + collection.insert("../pathological", pathologicalPayload); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.lastModified, null); + Assert.equal(engine._store.items.flying, undefined); + Assert.equal(engine._store.items.scotsman, undefined); + Assert.equal(engine._store.items["../pathological"], undefined); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Timestamps of last sync and last server modification are set. + Assert.ok((await engine.getLastSync()) > 0); + Assert.ok(engine.lastModified > 0); + + // Local records have been created from the server data. + Assert.equal(engine._store.items.flying, "LNER Class A3 4472"); + Assert.equal(engine._store.items.scotsman, "Flying Scotsman"); + Assert.equal(engine._store.items["../pathological"], "Pathological Case"); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_reconcile() { + _("SyncEngine._processIncoming updates local records"); + + let collection = new ServerCollection(); + + // This server record is newer than the corresponding client one, + // so it'll update its data. + collection.insert( + "newrecord", + encryptPayload({ id: "newrecord", denomination: "New stuff..." }) + ); + + // This server record is newer than the corresponding client one, + // so it'll update its data. + collection.insert( + "newerserver", + encryptPayload({ id: "newerserver", denomination: "New data!" }) + ); + + // This server record is 2 mins older than the client counterpart + // but identical to it, so we're expecting the client record's + // changedID to be reset. + collection.insert( + "olderidentical", + encryptPayload({ + id: "olderidentical", + denomination: "Older but identical", + }) + ); + collection._wbos.olderidentical.modified -= 120; + + // This item simply has different data than the corresponding client + // record (which is unmodified), so it will update the client as well + collection.insert( + "updateclient", + encryptPayload({ id: "updateclient", denomination: "Get this!" }) + ); + + // This is a dupe of 'original'. + collection.insert( + "duplication", + encryptPayload({ id: "duplication", denomination: "Original Entry" }) + ); + + // This record is marked as deleted, so we're expecting the client + // record to be removed. + collection.insert( + "nukeme", + encryptPayload({ id: "nukeme", denomination: "Nuke me!", deleted: true }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { + newerserver: "New data, but not as new as server!", + olderidentical: "Older but identical", + updateclient: "Got data?", + original: "Original Entry", + long_original: "Long Original Entry", + nukeme: "Nuke me!", + }; + // Make this record 1 min old, thus older than the one on the server + await engine._tracker.addChangedID("newerserver", Date.now() / 1000 - 60); + // This record has been changed 2 mins later than the one on the server + await engine._tracker.addChangedID("olderidentical", Date.now() / 1000); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(engine._store.items.newrecord, undefined); + Assert.equal( + engine._store.items.newerserver, + "New data, but not as new as server!" + ); + Assert.equal(engine._store.items.olderidentical, "Older but identical"); + Assert.equal(engine._store.items.updateclient, "Got data?"); + Assert.equal(engine._store.items.nukeme, "Nuke me!"); + let changes = await engine._tracker.getChangedIDs(); + Assert.ok(changes.olderidentical > 0); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Timestamps of last sync and last server modification are set. + Assert.ok((await engine.getLastSync()) > 0); + Assert.ok(engine.lastModified > 0); + + // The new record is created. + Assert.equal(engine._store.items.newrecord, "New stuff..."); + + // The 'newerserver' record is updated since the server data is newer. + Assert.equal(engine._store.items.newerserver, "New data!"); + + // The data for 'olderidentical' is identical on the server, so + // it's no longer marked as changed anymore. + Assert.equal(engine._store.items.olderidentical, "Older but identical"); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.olderidentical, undefined); + + // Updated with server data. + Assert.equal(engine._store.items.updateclient, "Get this!"); + + // The incoming ID is preferred. + Assert.equal(engine._store.items.original, undefined); + Assert.equal(engine._store.items.duplication, "Original Entry"); + Assert.notEqual(engine._delete.ids.indexOf("original"), -1); + + // The 'nukeme' record marked as deleted is removed. + Assert.equal(engine._store.items.nukeme, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_reconcile_local_deleted() { + _("Ensure local, duplicate ID is deleted on server."); + + // When a duplicate is resolved, the local ID (which is never taken) should + // be deleted on the server. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + record = encryptPayload({ id: "DUPE_LOCAL", denomination: "local" }); + wbo = new ServerWBO("DUPE_LOCAL", record, now - 1); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + await engine._sync(); + + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + Assert.notEqual(undefined, collection.wbo("DUPE_INCOMING")); + + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_reconcile_equivalent() { + _("Ensure proper handling of incoming records that match local."); + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ id: "entry", denomination: "denomination" }); + let wbo = new ServerWBO("entry", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + engine._store.items = { entry: "denomination" }; + Assert.ok(await engine._store.itemExists("entry")); + + await engine._sync(); + + do_check_attribute_count(engine._store.items, 1); + + await cleanAndGo(engine, server); +}); + +add_task( + async function test_processIncoming_reconcile_locally_deleted_dupe_new() { + _( + "Ensure locally deleted duplicate record newer than incoming is handled." + ); + + // This is a somewhat complicated test. It ensures that if a client receives + // a modified record for an item that is deleted locally but with a different + // ID that the incoming record is ignored. This is a corner case for record + // handling, but it needs to be supported. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + // Simulate a locally-deleted item. + engine._store.items = {}; + await engine._tracker.addChangedID("DUPE_LOCAL", now + 3); + Assert.equal(false, await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal(false, await engine._store.itemExists("DUPE_INCOMING")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // After the sync, the server's payload for the original ID should be marked + // as deleted. + do_check_empty(engine._store.items); + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(null, wbo); + let payload = wbo.getCleartext(); + Assert.ok(payload.deleted); + + await cleanAndGo(engine, server); + } +); + +add_task( + async function test_processIncoming_reconcile_locally_deleted_dupe_old() { + _( + "Ensure locally deleted duplicate record older than incoming is restored." + ); + + // This is similar to the above test except it tests the condition where the + // incoming record is newer than the local deletion, therefore overriding it. + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + // Simulate a locally-deleted item. + engine._store.items = {}; + await engine._tracker.addChangedID("DUPE_LOCAL", now + 1); + Assert.equal(false, await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal(false, await engine._store.itemExists("DUPE_INCOMING")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + await engine._sync(); + + // Since the remote change is newer, the incoming item should exist locally. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + Assert.equal("incoming", engine._store.items.DUPE_INCOMING); + + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + let payload = wbo.getCleartext(); + Assert.equal("incoming", payload.denomination); + + await cleanAndGo(engine, server); + } +); + +add_task(async function test_processIncoming_reconcile_changed_dupe() { + _("Ensure that locally changed duplicate record is handled properly."); + + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + // The local record is newer than the incoming one, so it should be retained. + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + await engine._tracker.addChangedID("DUPE_LOCAL", now + 3); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // The ID should have been changed to incoming. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + // On the server, the local ID should be deleted and the incoming ID should + // have its payload set to what was in the local record. + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(undefined, wbo); + let payload = wbo.getCleartext(); + Assert.equal("local", payload.denomination); + + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_reconcile_changed_dupe_new() { + _("Ensure locally changed duplicate record older than incoming is ignored."); + + // This test is similar to the above except the incoming record is younger + // than the local record. The incoming record should be authoritative. + let [engine, server, user] = await createServerAndConfigureClient(); + + let now = Date.now() / 1000 - 10; + await engine.setLastSync(now); + engine.lastModified = now + 1; + + let record = encryptPayload({ + id: "DUPE_INCOMING", + denomination: "incoming", + }); + let wbo = new ServerWBO("DUPE_INCOMING", record, now + 2); + server.insertWBO(user, "rotary", wbo); + + await engine._store.create({ id: "DUPE_LOCAL", denomination: "local" }); + await engine._tracker.addChangedID("DUPE_LOCAL", now + 1); + Assert.ok(await engine._store.itemExists("DUPE_LOCAL")); + Assert.equal("DUPE_LOCAL", await engine._findDupe({ id: "DUPE_INCOMING" })); + + engine.lastModified = server.getCollection(user, engine.name).timestamp; + await engine._sync(); + + // The ID should have been changed to incoming. + do_check_attribute_count(engine._store.items, 1); + Assert.ok("DUPE_INCOMING" in engine._store.items); + + // On the server, the local ID should be deleted and the incoming ID should + // have its payload retained. + let collection = server.getCollection(user, "rotary"); + Assert.equal(1, collection.count()); + wbo = collection.wbo("DUPE_INCOMING"); + Assert.notEqual(undefined, wbo); + let payload = wbo.getCleartext(); + Assert.equal("incoming", payload.denomination); + await cleanAndGo(engine, server); +}); + +add_task(async function test_processIncoming_resume_toFetch() { + _( + "toFetch and previousFailed items left over from previous syncs are fetched on the next sync, along with new items." + ); + + const LASTSYNC = Date.now() / 1000; + + // Server records that will be downloaded + let collection = new ServerCollection(); + collection.insert( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection.insert( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection.insert( + "rekolok", + encryptPayload({ id: "rekolok", denomination: "Rekonstruktionslokomotive" }) + ); + for (let i = 0; i < 3; i++) { + let id = "failed" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + let wbo = new ServerWBO(id, payload); + wbo.modified = LASTSYNC - 10; + collection.insertWBO(wbo); + } + + collection.wbo("flying").modified = collection.wbo("scotsman").modified = + LASTSYNC - 10; + collection._wbos.rekolok.modified = LASTSYNC + 10; + + // Time travel 10 seconds into the future but still download the above WBOs. + let engine = makeRotaryEngine(); + await engine.setLastSync(LASTSYNC); + engine.toFetch = new SerializableSet(["flying", "scotsman"]); + engine.previousFailed = new SerializableSet([ + "failed0", + "failed1", + "failed2", + ]); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment + Assert.equal(engine._store.items.flying, undefined); + Assert.equal(engine._store.items.scotsman, undefined); + Assert.equal(engine._store.items.rekolok, undefined); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Local records have been created from the server data. + Assert.equal(engine._store.items.flying, "LNER Class A3 4472"); + Assert.equal(engine._store.items.scotsman, "Flying Scotsman"); + Assert.equal(engine._store.items.rekolok, "Rekonstruktionslokomotive"); + Assert.equal(engine._store.items.failed0, "Record No. 0"); + Assert.equal(engine._store.items.failed1, "Record No. 1"); + Assert.equal(engine._store.items.failed2, "Record No. 2"); + Assert.equal(engine.previousFailed.size, 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_notify_count() { + _("Ensure that failed records are reported only once."); + + const NUMBER_OF_RECORDS = 15; + + // Engine that fails every 5 records. + let engine = makeRotaryEngine(); + engine._store._applyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = async function (records, countTelemetry) { + let sortedRecords = records.sort((a, b) => (a.id > b.id ? 1 : -1)); + let recordsToApply = [], + recordsToFail = []; + for (let i = 0; i < sortedRecords.length; i++) { + (i % 5 === 0 ? recordsToFail : recordsToApply).push(sortedRecords[i]); + } + recordsToFail.forEach(() => { + countTelemetry.addIncomingFailedReason("failed message"); + }); + await engine._store._applyIncomingBatch(recordsToApply, countTelemetry); + + return recordsToFail.map(record => record.id); + }; + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i.toString(10).padStart(2, "0"); + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + } + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment. + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + let called = 0; + let counts; + function onApplied(count) { + _("Called with " + JSON.stringify(counts)); + counts = count; + called++; + } + Svc.Obs.add("weave:engine:sync:applied", onApplied); + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + // Confirm failures. + do_check_attribute_count(engine._store.items, 12); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + ["record-no-00", "record-no-05", "record-no-10"].sort() + ); + + // There are newly failed records and they are reported. + Assert.equal(called, 1); + Assert.equal(counts.failed, 3); + Assert.equal(counts.failedReasons[0].count, 3); + Assert.equal(counts.failedReasons[0].name, "failed message"); + Assert.equal(counts.applied, 15); + Assert.equal(counts.newFailed, 3); + Assert.equal(counts.succeeded, 12); + + // Sync again, 1 of the failed items are the same, the rest didn't fail. + await engine._processIncoming(); + + // Confirming removed failures. + do_check_attribute_count(engine._store.items, 14); + // After failing twice the record that failed again [record-no-00] + // should NOT be stored to try again + Assert.deepEqual(Array.from(engine.previousFailed), []); + + Assert.equal(called, 2); + Assert.equal(counts.failed, 1); + Assert.equal(counts.failedReasons[0].count, 1); + Assert.equal(counts.failedReasons[0].name, "failed message"); + Assert.equal(counts.applied, 3); + Assert.equal(counts.newFailed, 0); + Assert.equal(counts.succeeded, 2); + + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_previousFailed() { + _("Ensure that failed records are retried."); + + const NUMBER_OF_RECORDS = 14; + + // Engine that alternates between failing and applying every 2 records. + let engine = makeRotaryEngine(); + engine._store._applyIncomingBatch = engine._store.applyIncomingBatch; + engine._store.applyIncomingBatch = async function (records, countTelemetry) { + let sortedRecords = records.sort((a, b) => (a.id > b.id ? 1 : -1)); + let recordsToApply = [], + recordsToFail = []; + let chunks = Array.from(PlacesUtils.chunkArray(sortedRecords, 2)); + for (let i = 0; i < chunks.length; i++) { + (i % 2 === 0 ? recordsToFail : recordsToApply).push(...chunks[i]); + } + await engine._store._applyIncomingBatch(recordsToApply, countTelemetry); + return recordsToFail.map(record => record.id); + }; + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i.toString(10).padStart(2, "0"); + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + collection.insert(id, payload); + } + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial environment. + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + // Initial failed items in previousFailed to be reset. + let previousFailed = new SerializableSet([ + Utils.makeGUID(), + Utils.makeGUID(), + Utils.makeGUID(), + ]); + engine.previousFailed = previousFailed; + Assert.equal(engine.previousFailed, previousFailed); + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + // Expected result: 4 sync batches with 2 failures each => 8 failures + do_check_attribute_count(engine._store.items, 6); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + [ + "record-no-00", + "record-no-01", + "record-no-04", + "record-no-05", + "record-no-08", + "record-no-09", + "record-no-12", + "record-no-13", + ].sort() + ); + + // Sync again with the same failed items (records 0, 1, 8, 9). + await engine._processIncoming(); + + do_check_attribute_count(engine._store.items, 10); + // A second sync with the same failed items should NOT add the same items again. + // Items that did not fail a second time should no longer be in previousFailed. + Assert.deepEqual(Array.from(engine.previousFailed).sort(), []); + + // Refetched items that didn't fail the second time are in engine._store.items. + Assert.equal(engine._store.items["record-no-04"], "Record No. 4"); + Assert.equal(engine._store.items["record-no-05"], "Record No. 5"); + Assert.equal(engine._store.items["record-no-12"], "Record No. 12"); + Assert.equal(engine._store.items["record-no-13"], "Record No. 13"); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_failed_records() { + _( + "Ensure that failed records from _reconcile and applyIncomingBatch are refetched." + ); + + // Let's create three and a bit batches worth of server side records. + let APPLY_BATCH_SIZE = 50; + let collection = new ServerCollection(); + const NUMBER_OF_RECORDS = APPLY_BATCH_SIZE * 3 + 5; + for (let i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + let wbo = new ServerWBO(id, payload); + wbo.modified = Date.now() / 1000 + 60 * (i - APPLY_BATCH_SIZE * 3); + collection.insertWBO(wbo); + } + + // Engine that batches but likes to throw on a couple of records, + // two in each batch: the even ones fail in reconcile, the odd ones + // in applyIncoming. + const BOGUS_RECORDS = [ + "record-no-" + 42, + "record-no-" + 23, + "record-no-" + (42 + APPLY_BATCH_SIZE), + "record-no-" + (23 + APPLY_BATCH_SIZE), + "record-no-" + (42 + APPLY_BATCH_SIZE * 2), + "record-no-" + (23 + APPLY_BATCH_SIZE * 2), + "record-no-" + (2 + APPLY_BATCH_SIZE * 3), + "record-no-" + (1 + APPLY_BATCH_SIZE * 3), + ]; + let engine = makeRotaryEngine(); + + engine.__reconcile = engine._reconcile; + engine._reconcile = async function _reconcile(record) { + if (BOGUS_RECORDS.indexOf(record.id) % 2 == 0) { + throw new Error("I don't like this record! Baaaaaah!"); + } + return this.__reconcile.apply(this, arguments); + }; + engine._store._applyIncoming = engine._store.applyIncoming; + engine._store.applyIncoming = async function (record) { + if (BOGUS_RECORDS.indexOf(record.id) % 2 == 1) { + throw new Error("I don't like this record! Baaaaaah!"); + } + return this._applyIncoming.apply(this, arguments); + }; + + // Keep track of requests made of a collection. + let count = 0; + let uris = []; + function recording_handler(recordedCollection) { + let h = recordedCollection.handler(); + return function (req, res) { + ++count; + uris.push(req.path + "?" + req.queryString); + return h(req, res); + }; + } + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": recording_handler(collection), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + // Confirm initial environment + Assert.equal(await engine.getLastSync(), 0); + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + do_check_empty(engine._store.items); + + let observerSubject; + let observerData; + Svc.Obs.add("weave:engine:sync:applied", function onApplied(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + observerSubject = subject; + observerData = data; + }); + + await engine._syncStartup(); + await engine._processIncoming(); + + // Ensure that all records but the bogus 4 have been applied. + do_check_attribute_count( + engine._store.items, + NUMBER_OF_RECORDS - BOGUS_RECORDS.length + ); + + // Ensure that the bogus records will be fetched again on the next sync. + Assert.equal(engine.previousFailed.size, BOGUS_RECORDS.length); + Assert.deepEqual( + Array.from(engine.previousFailed).sort(), + BOGUS_RECORDS.sort() + ); + + // Ensure the observer was notified + Assert.equal(observerData, engine.name); + Assert.equal(observerSubject.failed, BOGUS_RECORDS.length); + Assert.equal(observerSubject.newFailed, BOGUS_RECORDS.length); + + // Testing batching of failed item fetches. + // Try to sync again. Ensure that we split the request into chunks to avoid + // URI length limitations. + async function batchDownload(batchSize) { + count = 0; + uris = []; + engine.guidFetchBatchSize = batchSize; + await engine._processIncoming(); + _("Tried again. Requests: " + count + "; URIs: " + JSON.stringify(uris)); + return count; + } + + // There are 8 bad records, so this needs 3 fetches. + _("Test batching with ID batch size 3, normal mobile batch size."); + Assert.equal(await batchDownload(3), 3); + + // Since there the previous batch failed again, there should be + // no more records to fetch + _("Test that the second time a record failed to sync, gets ignored"); + Assert.equal(await batchDownload(BOGUS_RECORDS.length), 0); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_processIncoming_decrypt_failed() { + _("Ensure that records failing to decrypt are either replaced or refetched."); + + // Some good and some bogus records. One doesn't contain valid JSON, + // the other will throw during decrypt. + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection._wbos.nojson = new ServerWBO("nojson", "This is invalid JSON"); + collection._wbos.nojson2 = new ServerWBO("nojson2", "This is invalid JSON"); + collection._wbos.scotsman = new ServerWBO( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection._wbos.nodecrypt = new ServerWBO("nodecrypt", "Decrypt this!"); + collection._wbos.nodecrypt2 = new ServerWBO("nodecrypt2", "Decrypt this!"); + + // Patch the fake crypto service to throw on the record above. + Weave.Crypto._decrypt = Weave.Crypto.decrypt; + Weave.Crypto.decrypt = function (ciphertext) { + if (ciphertext == "Decrypt this!") { + throw new Error( + "Derp! Cipher finalized failed. Im ur crypto destroyin ur recordz." + ); + } + return this._decrypt.apply(this, arguments); + }; + + // Some broken records also exist locally. + let engine = makeRotaryEngine(); + engine.enabled = true; + engine._store.items = { nojson: "Valid JSON", nodecrypt: "Valid ciphertext" }; + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + try { + // Confirm initial state + Assert.equal(engine.toFetch.size, 0); + Assert.equal(engine.previousFailed.size, 0); + + let observerSubject; + let observerData; + Svc.Obs.add("weave:engine:sync:applied", function onApplied(subject, data) { + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + observerSubject = subject; + observerData = data; + }); + + await engine.setLastSync(collection.wbo("nojson").modified - 1); + let ping = await sync_engine_and_validate_telem(engine, true); + Assert.equal(ping.engines[0].incoming.applied, 2); + Assert.equal(ping.engines[0].incoming.failed, 4); + console.log("incoming telem: ", ping.engines[0].incoming); + Assert.equal( + ping.engines[0].incoming.failedReasons[0].name, + "No ciphertext: nothing to decrypt?" + ); + // There should be 4 of the same error + Assert.equal(ping.engines[0].incoming.failedReasons[0].count, 4); + + Assert.equal(engine.previousFailed.size, 4); + Assert.ok(engine.previousFailed.has("nojson")); + Assert.ok(engine.previousFailed.has("nojson2")); + Assert.ok(engine.previousFailed.has("nodecrypt")); + Assert.ok(engine.previousFailed.has("nodecrypt2")); + + // Ensure the observer was notified + Assert.equal(observerData, engine.name); + Assert.equal(observerSubject.applied, 2); + Assert.equal(observerSubject.failed, 4); + Assert.equal(observerSubject.failedReasons[0].count, 4); + } finally { + await promiseClean(engine, server); + } +}); + +add_task(async function test_uploadOutgoing_toEmptyServer() { + _("SyncEngine._uploadOutgoing uploads new records to server"); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + // Mark one of these records as changed + await engine._tracker.addChangedID("scotsman", 0); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + + await engine._syncStartup(); + await engine._uploadOutgoing(); + + // Ensure the marked record ('scotsman') has been uploaded and is + // no longer marked. + Assert.equal(collection.payload("flying"), undefined); + Assert.ok(!!collection.payload("scotsman")); + Assert.equal(collection.cleartext("scotsman").id, "scotsman"); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.scotsman, undefined); + + // The 'flying' record wasn't marked so it wasn't uploaded + Assert.equal(collection.payload("flying"), undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +async function test_uploadOutgoing_max_record_payload_bytes( + allowSkippedRecord +) { + _( + "SyncEngine._uploadOutgoing throws when payload is bigger than max_record_payload_bytes" + ); + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + "/1.1/foo/storage/rotary/flying": collection.wbo("flying").handler(), + "/1.1/foo/storage/rotary/scotsman": collection.wbo("scotsman").handler(), + }); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = allowSkippedRecord; + engine._store.items = { flying: "a".repeat(1024 * 1024), scotsman: "abcd" }; + + await engine._tracker.addChangedID("flying", 1000); + await engine._tracker.addChangedID("scotsman", 1000); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(1); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + Assert.equal(collection.payload("scotsman"), undefined); + + await engine._syncStartup(); + await engine._uploadOutgoing(); + + if (!allowSkippedRecord) { + do_throw("should not get here"); + } + + await engine.trackRemainingChanges(); + + // Check we uploaded the other record to the server + Assert.ok(collection.payload("scotsman")); + // And that we won't try to upload the huge record next time. + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + } catch (e) { + if (allowSkippedRecord) { + do_throw("should not get here"); + } + + await engine.trackRemainingChanges(); + + // Check that we will try to upload the huge record next time + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, 1000); + } finally { + // Check we didn't upload the oversized record to the server + Assert.equal(collection.payload("flying"), undefined); + await cleanAndGo(engine, server); + } +} + +add_task( + async function test_uploadOutgoing_max_record_payload_bytes_disallowSkippedRecords() { + return test_uploadOutgoing_max_record_payload_bytes(false); + } +); + +add_task( + async function test_uploadOutgoing_max_record_payload_bytes_allowSkippedRecords() { + return test_uploadOutgoing_max_record_payload_bytes(true); + } +); + +add_task(async function test_uploadOutgoing_failed() { + _( + "SyncEngine._uploadOutgoing doesn't clear the tracker of objects that failed to upload." + ); + + let collection = new ServerCollection(); + // We only define the "flying" WBO on the server, not the "scotsman" + // and "peppercorn" ones. + collection._wbos.flying = new ServerWBO("flying"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + peppercorn: "Peppercorn Class", + }; + // Mark these records as changed + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + const PEPPERCORN_CHANGED = 34567; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + await engine._tracker.addChangedID("peppercorn", PEPPERCORN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + let changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, FLYING_CHANGED); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + Assert.equal(changes.peppercorn, PEPPERCORN_CHANGED); + + engine.enabled = true; + await sync_engine_and_validate_telem(engine, true); + + // Ensure the 'flying' record has been uploaded and is no longer marked. + Assert.ok(!!collection.payload("flying")); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + + // The 'scotsman' and 'peppercorn' records couldn't be uploaded so + // they weren't cleared from the tracker. + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + Assert.equal(changes.peppercorn, PEPPERCORN_CHANGED); + } finally { + await promiseClean(engine, server); + } +}); + +async function createRecordFailTelemetry(allowSkippedRecord) { + Services.prefs.setStringPref("services.sync.username", "foo"); + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + collection._wbos.scotsman = new ServerWBO("scotsman"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = allowSkippedRecord; + let oldCreateRecord = engine._store.createRecord; + engine._store.createRecord = async (id, col) => { + if (id != "flying") { + throw new Error("oops"); + } + return oldCreateRecord.call(engine._store, id, col); + }; + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + }; + // Mark these records as changed + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let ping; + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + // Confirm initial environment + Assert.equal(collection.payload("flying"), undefined); + let changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, FLYING_CHANGED); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + + engine.enabled = true; + ping = await sync_engine_and_validate_telem(engine, true, onErrorPing => { + ping = onErrorPing; + }); + + if (!allowSkippedRecord) { + do_throw("should not get here"); + } + + // Ensure the 'flying' record has been uploaded and is no longer marked. + Assert.ok(!!collection.payload("flying")); + changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.flying, undefined); + } catch (err) { + if (allowSkippedRecord) { + do_throw("should not get here"); + } + + // Ensure the 'flying' record has not been uploaded and is still marked + Assert.ok(!collection.payload("flying")); + const changes = await engine._tracker.getChangedIDs(); + Assert.ok(changes.flying); + } finally { + // We reported in telemetry that we failed a record + Assert.equal(ping.engines[0].outgoing[0].failed, 1); + Assert.equal(ping.engines[0].outgoing[0].failedReasons[0].name, "oops"); + + // In any case, the 'scotsman' record couldn't be created so it wasn't + // uploaded nor it was not cleared from the tracker. + Assert.ok(!collection.payload("scotsman")); + const changes = await engine._tracker.getChangedIDs(); + Assert.equal(changes.scotsman, SCOTSMAN_CHANGED); + + engine._store.createRecord = oldCreateRecord; + await promiseClean(engine, server); + } +} + +add_task( + async function test_uploadOutgoing_createRecord_throws_reported_telemetry() { + _( + "SyncEngine._uploadOutgoing reports a failed record to telemetry if createRecord throws" + ); + await createRecordFailTelemetry(true); + } +); + +add_task( + async function test_uploadOutgoing_createRecord_throws_dontAllowSkipRecord() { + _( + "SyncEngine._uploadOutgoing will throw if createRecord throws and allowSkipRecord is set to false" + ); + await createRecordFailTelemetry(false); + } +); + +add_task(async function test_uploadOutgoing_largeRecords() { + _( + "SyncEngine._uploadOutgoing throws on records larger than the max record payload size" + ); + + let collection = new ServerCollection(); + + let engine = makeRotaryEngine(); + engine.allowSkippedRecord = false; + engine._store.items["large-item"] = "Y".repeat( + Service.getMaxRecordPayloadSize() * 2 + ); + await engine._tracker.addChangedID("large-item", 0); + collection.insert("large-item"); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + try { + await engine._syncStartup(); + let error = null; + try { + await engine._uploadOutgoing(); + } catch (e) { + error = e; + } + ok(!!error); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncFinish_deleteByIds() { + _( + "SyncEngine._syncFinish deletes server records slated for deletion (list of record IDs)." + ); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + collection._wbos.scotsman = new ServerWBO( + "scotsman", + encryptPayload({ id: "scotsman", denomination: "Flying Scotsman" }) + ); + collection._wbos.rekolok = new ServerWBO( + "rekolok", + encryptPayload({ id: "rekolok", denomination: "Rekonstruktionslokomotive" }) + ); + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + engine._delete = { ids: ["flying", "rekolok"] }; + await engine._syncFinish(); + + // The 'flying' and 'rekolok' records were deleted while the + // 'scotsman' one wasn't. + Assert.equal(collection.payload("flying"), undefined); + Assert.ok(!!collection.payload("scotsman")); + Assert.equal(collection.payload("rekolok"), undefined); + + // The deletion todo list has been reset. + Assert.equal(engine._delete.ids, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncFinish_deleteLotsInBatches() { + _( + "SyncEngine._syncFinish deletes server records in batches of 100 (list of record IDs)." + ); + + let collection = new ServerCollection(); + + // Let's count how many times the client does a DELETE request to the server + var noOfUploads = 0; + collection.delete = (function (orig) { + return function () { + noOfUploads++; + return orig.apply(this, arguments); + }; + })(collection.delete); + + // Create a bunch of records on the server + let now = Date.now(); + for (var i = 0; i < 234; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + i }); + let wbo = new ServerWBO(id, payload); + wbo.modified = now / 1000 - 60 * (i + 110); + collection.insertWBO(wbo); + } + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let engine = makeRotaryEngine(); + try { + // Confirm initial environment + Assert.equal(noOfUploads, 0); + + // Declare what we want to have deleted: all records no. 100 and + // up and all records that are less than 200 mins old (which are + // records 0 thru 90). + engine._delete = { ids: [], newer: now / 1000 - 60 * 200.5 }; + for (i = 100; i < 234; i++) { + engine._delete.ids.push("record-no-" + i); + } + + await engine._syncFinish(); + + // Ensure that the appropriate server data has been wiped while + // preserving records 90 thru 200. + for (i = 0; i < 234; i++) { + let id = "record-no-" + i; + if (i <= 90 || i >= 100) { + Assert.equal(collection.payload(id), undefined); + } else { + Assert.ok(!!collection.payload(id)); + } + } + + // The deletion was done in batches + Assert.equal(noOfUploads, 2 + 1); + + // The deletion todo list has been reset. + Assert.equal(engine._delete.ids, undefined); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_sync_partialUpload() { + _("SyncEngine.sync() keeps changedIDs that couldn't be uploaded."); + + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + let oldServerConfiguration = Service.serverConfiguration; + Service.serverConfiguration = { + max_post_records: 100, + }; + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = makeRotaryEngine(); + + // Let the third upload fail completely + var noOfUploads = 0; + collection.post = (function (orig) { + return function () { + if (noOfUploads == 2) { + throw new Error("FAIL!"); + } + noOfUploads++; + return orig.apply(this, arguments); + }; + })(collection.post); + + // Create a bunch of records (and server side handlers) + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + engine._store.items[id] = "Record No. " + i; + await engine._tracker.addChangedID(id, i); + // Let two items in the first upload batch fail. + if (i != 23 && i != 42) { + collection.insert(id); + } + } + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + + engine.enabled = true; + let error; + try { + await sync_engine_and_validate_telem(engine, true); + } catch (ex) { + error = ex; + } + + ok(!!error); + + const changes = await engine._tracker.getChangedIDs(); + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + // Ensure failed records are back in the tracker: + // * records no. 23 and 42 were rejected by the server, + // * records after the third batch and higher couldn't be uploaded because + // we failed hard on the 3rd upload. + if (i == 23 || i == 42 || i >= 200) { + Assert.equal(changes[id], i); + } else { + Assert.equal(false, id in changes); + } + } + } finally { + Service.serverConfiguration = oldServerConfiguration; + await promiseClean(engine, server); + } +}); + +add_task(async function test_canDecrypt_noCryptoKeys() { + _( + "SyncEngine.canDecrypt returns false if the engine fails to decrypt items on the server, e.g. due to a missing crypto key collection." + ); + + // Wipe collection keys so we can test the desired scenario. + Service.collectionKeys.clear(); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + let engine = makeRotaryEngine(); + try { + Assert.equal(false, await engine.canDecrypt()); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_canDecrypt_true() { + _( + "SyncEngine.canDecrypt returns true if the engine can decrypt the items on the server." + ); + + await generateNewKeys(Service.collectionKeys); + + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO( + "flying", + encryptPayload({ id: "flying", denomination: "LNER Class A3 4472" }) + ); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + let engine = makeRotaryEngine(); + try { + Assert.ok(await engine.canDecrypt()); + } finally { + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_syncapplied_observer() { + const NUMBER_OF_RECORDS = 10; + + let engine = makeRotaryEngine(); + + // Create a batch of server side records. + let collection = new ServerCollection(); + for (var i = 0; i < NUMBER_OF_RECORDS; i++) { + let id = "record-no-" + i; + let payload = encryptPayload({ id, denomination: "Record No. " + id }); + collection.insert(id, payload); + } + + let server = httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + let numApplyCalls = 0; + let engine_name; + let count; + function onApplied(subject, data) { + numApplyCalls++; + engine_name = data; + count = subject; + } + + Svc.Obs.add("weave:engine:sync:applied", onApplied); + + try { + Service.scheduler.hasIncomingItems = false; + + // Do sync. + await engine._syncStartup(); + await engine._processIncoming(); + + do_check_attribute_count(engine._store.items, 10); + + Assert.equal(numApplyCalls, 1); + Assert.equal(engine_name, "rotary"); + Assert.equal(count.applied, 10); + + Assert.ok(Service.scheduler.hasIncomingItems); + } finally { + await cleanAndGo(engine, server); + Service.scheduler.hasIncomingItems = false; + Svc.Obs.remove("weave:engine:sync:applied", onApplied); + } +}); diff --git a/services/sync/tests/unit/test_syncscheduler.js b/services/sync/tests/unit/test_syncscheduler.js new file mode 100644 index 0000000000..7faccbc911 --- /dev/null +++ b/services/sync/tests/unit/test_syncscheduler.js @@ -0,0 +1,1151 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FxAccounts } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const { SyncAuthManager } = ChromeUtils.importESModule( + "resource://services-sync/sync_auth.sys.mjs" +); +const { SyncScheduler } = ChromeUtils.importESModule( + "resource://services-sync/policies.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { Status } = ChromeUtils.importESModule( + "resource://services-sync/status.sys.mjs" +); + +function CatapultEngine() { + SyncEngine.call(this, "Catapult", Service); +} +CatapultEngine.prototype = { + exception: null, // tests fill this in + async _sync() { + throw this.exception; + }, +}; +Object.setPrototypeOf(CatapultEngine.prototype, SyncEngine.prototype); + +var scheduler = new SyncScheduler(Service); +let clientsEngine; + +async function sync_httpd_setup() { + let clientsSyncID = await clientsEngine.resetLocalSyncID(); + let global = new ServerWBO("global", { + syncID: Service.syncID, + storageVersion: STORAGE_VERSION, + engines: { + clients: { version: clientsEngine.version, syncID: clientsSyncID }, + }, + }); + let clientsColl = new ServerCollection({}, true); + + // Tracking info/collections. + let collectionsHelper = track_collections_helper(); + let upd = collectionsHelper.with_updated_collection; + + return httpd_setup({ + "/1.1/johndoe@mozilla.com/storage/meta/global": upd( + "meta", + global.handler() + ), + "/1.1/johndoe@mozilla.com/info/collections": collectionsHelper.handler, + "/1.1/johndoe@mozilla.com/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe@mozilla.com/storage/clients": upd( + "clients", + clientsColl.handler() + ), + }); +} + +async function setUp(server) { + await configureIdentity({ username: "johndoe@mozilla.com" }, server); + + await generateNewKeys(Service.collectionKeys); + let serverKeys = Service.collectionKeys.asWBO("crypto", "keys"); + await serverKeys.encrypt(Service.identity.syncKeyBundle); + let result = ( + await serverKeys.upload(Service.resource(Service.cryptoKeysURL)) + ).success; + return result; +} + +async function cleanUpAndGo(server) { + await Async.promiseYield(); + await clientsEngine._store.wipe(); + await Service.startOver(); + // Re-enable logging, which we just disabled. + syncTestLogging(); + if (server) { + await promiseStopServer(server); + } +} + +add_task(async function setup() { + await Service.promiseInitialized; + clientsEngine = Service.clientsEngine; + // Don't remove stale clients when syncing. This is a test-only workaround + // that lets us add clients directly to the store, without losing them on + // the next sync. + clientsEngine._removeRemoteClient = async id => {}; + await Service.engineManager.clear(); + + validate_all_future_pings(); + + scheduler.setDefaults(); + + await Service.engineManager.register(CatapultEngine); +}); + +add_test(function test_prefAttributes() { + _("Test various attributes corresponding to preferences."); + + const INTERVAL = 42 * 60 * 1000; // 42 minutes + const THRESHOLD = 3142; + const SCORE = 2718; + const TIMESTAMP1 = 1275493471649; + + _( + "The 'nextSync' attribute stores a millisecond timestamp rounded down to the nearest second." + ); + Assert.equal(scheduler.nextSync, 0); + scheduler.nextSync = TIMESTAMP1; + Assert.equal(scheduler.nextSync, Math.floor(TIMESTAMP1 / 1000) * 1000); + + _("'syncInterval' defaults to singleDeviceInterval."); + Assert.equal(Svc.Prefs.get("syncInterval"), undefined); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + _("'syncInterval' corresponds to a preference setting."); + scheduler.syncInterval = INTERVAL; + Assert.equal(scheduler.syncInterval, INTERVAL); + Assert.equal(Svc.Prefs.get("syncInterval"), INTERVAL); + + _( + "'syncThreshold' corresponds to preference, defaults to SINGLE_USER_THRESHOLD" + ); + Assert.equal(Svc.Prefs.get("syncThreshold"), undefined); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + scheduler.syncThreshold = THRESHOLD; + Assert.equal(scheduler.syncThreshold, THRESHOLD); + + _("'globalScore' corresponds to preference, defaults to zero."); + Assert.equal(Svc.Prefs.get("globalScore"), 0); + Assert.equal(scheduler.globalScore, 0); + scheduler.globalScore = SCORE; + Assert.equal(scheduler.globalScore, SCORE); + Assert.equal(Svc.Prefs.get("globalScore"), SCORE); + + _("Intervals correspond to default preferences."); + Assert.equal( + scheduler.singleDeviceInterval, + Svc.Prefs.get("scheduler.fxa.singleDeviceInterval") * 1000 + ); + Assert.equal( + scheduler.idleInterval, + Svc.Prefs.get("scheduler.idleInterval") * 1000 + ); + Assert.equal( + scheduler.activeInterval, + Svc.Prefs.get("scheduler.activeInterval") * 1000 + ); + Assert.equal( + scheduler.immediateInterval, + Svc.Prefs.get("scheduler.immediateInterval") * 1000 + ); + + _("Custom values for prefs will take effect after a restart."); + Svc.Prefs.set("scheduler.fxa.singleDeviceInterval", 420); + Svc.Prefs.set("scheduler.idleInterval", 230); + Svc.Prefs.set("scheduler.activeInterval", 180); + Svc.Prefs.set("scheduler.immediateInterval", 31415); + scheduler.setDefaults(); + Assert.equal(scheduler.idleInterval, 230000); + Assert.equal(scheduler.singleDeviceInterval, 420000); + Assert.equal(scheduler.activeInterval, 180000); + Assert.equal(scheduler.immediateInterval, 31415000); + + _("Custom values for interval prefs can't be less than 60 seconds."); + Svc.Prefs.set("scheduler.fxa.singleDeviceInterval", 42); + Svc.Prefs.set("scheduler.idleInterval", 50); + Svc.Prefs.set("scheduler.activeInterval", 50); + Svc.Prefs.set("scheduler.immediateInterval", 10); + scheduler.setDefaults(); + Assert.equal(scheduler.idleInterval, 60000); + Assert.equal(scheduler.singleDeviceInterval, 60000); + Assert.equal(scheduler.activeInterval, 60000); + Assert.equal(scheduler.immediateInterval, 60000); + + Svc.Prefs.resetBranch(""); + scheduler.setDefaults(); + run_next_test(); +}); + +add_task(async function test_sync_skipped_low_score_no_resync() { + enableValidationPrefs(); + let server = await sync_httpd_setup(); + + function SkipEngine() { + SyncEngine.call(this, "Skip", Service); + this.syncs = 0; + } + + SkipEngine.prototype = { + _sync() { + do_throw("Should have been skipped"); + }, + shouldSkipSync() { + return true; + }, + }; + Object.setPrototypeOf(SkipEngine.prototype, SyncEngine.prototype); + await Service.engineManager.register(SkipEngine); + + let engine = Service.engineManager.get("skip"); + engine.enabled = true; + engine._tracker._score = 30; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + let resyncDoneObserver = promiseOneObserver("weave:service:resyncs-finished"); + + let synced = false; + function onSyncStarted() { + Assert.ok(!synced, "Only should sync once"); + synced = true; + } + + await Service.sync(); + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Svc.Obs.add("weave:service:sync:start", onSyncStarted); + await resyncDoneObserver; + + Svc.Obs.remove("weave:service:sync:start", onSyncStarted); + engine._tracker._store = 0; + await cleanUpAndGo(server); +}); + +add_task(async function test_updateClientMode() { + _( + "Test updateClientMode adjusts scheduling attributes based on # of clients appropriately" + ); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Trigger a change in interval & threshold by noting there are multiple clients. + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.updateClientMode(); + + Assert.equal(scheduler.syncThreshold, MULTI_DEVICE_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Resets the number of clients to 0. + await clientsEngine.resetClient(); + Svc.Prefs.reset("clients.devices.mobile"); + scheduler.updateClientMode(); + + // Goes back to single user if # clients is 1. + Assert.equal(scheduler.numClients, 1); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + await cleanUpAndGo(); +}); + +add_task(async function test_masterpassword_locked_retry_interval() { + enableValidationPrefs(); + + _( + "Test Status.login = MASTER_PASSWORD_LOCKED results in reschedule at MASTER_PASSWORD interval" + ); + let loginFailed = false; + Svc.Obs.add("weave:service:login:error", function onLoginError() { + Svc.Obs.remove("weave:service:login:error", onLoginError); + loginFailed = true; + }); + + let rescheduleInterval = false; + + let oldScheduleAtInterval = SyncScheduler.prototype.scheduleAtInterval; + SyncScheduler.prototype.scheduleAtInterval = function (interval) { + rescheduleInterval = true; + Assert.equal(interval, MASTER_PASSWORD_LOCKED_RETRY_INTERVAL); + }; + + let oldVerifyLogin = Service.verifyLogin; + Service.verifyLogin = async function () { + Status.login = MASTER_PASSWORD_LOCKED; + return false; + }; + + let server = await sync_httpd_setup(); + await setUp(server); + + await Service.sync(); + + Assert.ok(loginFailed); + Assert.equal(Status.login, MASTER_PASSWORD_LOCKED); + Assert.ok(rescheduleInterval); + + Service.verifyLogin = oldVerifyLogin; + SyncScheduler.prototype.scheduleAtInterval = oldScheduleAtInterval; + + await cleanUpAndGo(server); +}); + +add_task(async function test_calculateBackoff() { + Assert.equal(Status.backoffInterval, 0); + + // Test no interval larger than the maximum backoff is used if + // Status.backoffInterval is smaller. + Status.backoffInterval = 5; + let backoffInterval = Utils.calculateBackoff( + 50, + MAXIMUM_BACKOFF_INTERVAL, + Status.backoffInterval + ); + + Assert.equal(backoffInterval, MAXIMUM_BACKOFF_INTERVAL); + + // Test Status.backoffInterval is used if it is + // larger than MAXIMUM_BACKOFF_INTERVAL. + Status.backoffInterval = MAXIMUM_BACKOFF_INTERVAL + 10; + backoffInterval = Utils.calculateBackoff( + 50, + MAXIMUM_BACKOFF_INTERVAL, + Status.backoffInterval + ); + + Assert.equal(backoffInterval, MAXIMUM_BACKOFF_INTERVAL + 10); + + await cleanUpAndGo(); +}); + +add_task(async function test_scheduleNextSync_nowOrPast() { + enableValidationPrefs(); + + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + + let server = await sync_httpd_setup(); + await setUp(server); + + // We're late for a sync... + scheduler.scheduleNextSync(-1); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_scheduleNextSync_future_noBackoff() { + enableValidationPrefs(); + + _( + "scheduleNextSync() uses the current syncInterval if no interval is provided." + ); + // Test backoffInterval is 0 as expected. + Assert.equal(Status.backoffInterval, 0); + + _("Test setting sync interval when nextSync == 0"); + scheduler.nextSync = 0; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= scheduler.syncInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.syncInterval); + + _("Test setting sync interval when nextSync != 0"); + scheduler.nextSync = Date.now() + scheduler.singleDeviceInterval; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= scheduler.syncInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.syncInterval); + + _( + "Scheduling requests for intervals larger than the current one will be ignored." + ); + // Request a sync at a longer interval. The sync that's already scheduled + // for sooner takes precedence. + let nextSync = scheduler.nextSync; + let timerDelay = scheduler.syncTimer.delay; + let requestedInterval = scheduler.syncInterval * 10; + scheduler.scheduleNextSync(requestedInterval); + Assert.equal(scheduler.nextSync, nextSync); + Assert.equal(scheduler.syncTimer.delay, timerDelay); + + // We can schedule anything we want if there isn't a sync scheduled. + scheduler.nextSync = 0; + scheduler.scheduleNextSync(requestedInterval); + Assert.ok(scheduler.nextSync <= Date.now() + requestedInterval); + Assert.equal(scheduler.syncTimer.delay, requestedInterval); + + // Request a sync at the smallest possible interval (0 triggers now). + scheduler.scheduleNextSync(1); + Assert.ok(scheduler.nextSync <= Date.now() + 1); + Assert.equal(scheduler.syncTimer.delay, 1); + + await cleanUpAndGo(); +}); + +add_task(async function test_scheduleNextSync_future_backoff() { + enableValidationPrefs(); + + _("scheduleNextSync() will honour backoff in all scheduling requests."); + // Let's take a backoff interval that's bigger than the default sync interval. + const BACKOFF = 7337; + Status.backoffInterval = scheduler.syncInterval + BACKOFF; + + _("Test setting sync interval when nextSync == 0"); + scheduler.nextSync = 0; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + _("Test setting sync interval when nextSync != 0"); + scheduler.nextSync = Date.now() + scheduler.singleDeviceInterval; + scheduler.scheduleNextSync(); + + // nextSync - Date.now() might be smaller than expectedInterval + // since some time has passed since we called scheduleNextSync(). + Assert.ok(scheduler.nextSync - Date.now() <= Status.backoffInterval); + Assert.ok(scheduler.syncTimer.delay <= Status.backoffInterval); + + // Request a sync at a longer interval. The sync that's already scheduled + // for sooner takes precedence. + let nextSync = scheduler.nextSync; + let timerDelay = scheduler.syncTimer.delay; + let requestedInterval = scheduler.syncInterval * 10; + Assert.ok(requestedInterval > Status.backoffInterval); + scheduler.scheduleNextSync(requestedInterval); + Assert.equal(scheduler.nextSync, nextSync); + Assert.equal(scheduler.syncTimer.delay, timerDelay); + + // We can schedule anything we want if there isn't a sync scheduled. + scheduler.nextSync = 0; + scheduler.scheduleNextSync(requestedInterval); + Assert.ok(scheduler.nextSync <= Date.now() + requestedInterval); + Assert.equal(scheduler.syncTimer.delay, requestedInterval); + + // Request a sync at the smallest possible interval (0 triggers now). + scheduler.scheduleNextSync(1); + Assert.ok(scheduler.nextSync <= Date.now() + Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_handleSyncError() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Force sync to fail. + Svc.Prefs.set("firstSync", "notReady"); + + _("Ensure expected initial environment."); + Assert.equal(scheduler._syncErrors, 0); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(Status.backoffInterval, 0); + + // Trigger sync with an error several times & observe + // functionality of handleSyncError() + _("Test first error calls scheduleNextSync on default interval"); + await Service.sync(); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + Assert.equal(scheduler._syncErrors, 1); + Assert.ok(!Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Test second error still calls scheduleNextSync on default interval"); + await Service.sync(); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.singleDeviceInterval); + Assert.equal(scheduler.syncTimer.delay, scheduler.singleDeviceInterval); + Assert.equal(scheduler._syncErrors, 2); + Assert.ok(!Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Test third error sets Status.enforceBackoff and calls scheduleAtInterval"); + await Service.sync(); + let maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.equal(Status.backoffInterval, 0); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + Assert.equal(scheduler._syncErrors, 3); + Assert.ok(Status.enforceBackoff); + + // Status.enforceBackoff is false but there are still errors. + Status.resetBackoff(); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 3); + scheduler.syncTimer.clear(); + + _( + "Test fourth error still calls scheduleAtInterval even if enforceBackoff was reset" + ); + await Service.sync(); + maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + Assert.equal(scheduler._syncErrors, 4); + Assert.ok(Status.enforceBackoff); + scheduler.syncTimer.clear(); + + _("Arrange for a successful sync to reset the scheduler error count"); + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + Svc.Prefs.set("firstSync", "wipeRemote"); + scheduler.scheduleNextSync(-1); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_client_sync_finish_updateClientMode() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Confirm defaults. + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.ok(!scheduler.idle); + + // Trigger a change in interval & threshold by adding a client. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + Assert.equal(false, scheduler.numClients > 1); + scheduler.updateClientMode(); + await Service.sync(); + + Assert.equal(scheduler.syncThreshold, MULTI_DEVICE_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + // Resets the number of clients to 0. + await clientsEngine.resetClient(); + // Also re-init the server, or we suck our "foo" client back down. + await setUp(server); + + await Service.sync(); + + // Goes back to single user if # clients is 1. + Assert.equal(scheduler.numClients, 1); + Assert.equal(scheduler.syncThreshold, SINGLE_USER_THRESHOLD); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + Assert.equal(false, scheduler.numClients > 1); + Assert.ok(!scheduler.idle); + + await cleanUpAndGo(server); +}); + +add_task(async function test_autoconnect_nextSync_past() { + enableValidationPrefs(); + + let promiseObserved = promiseOneObserver("weave:service:sync:finish"); + // nextSync will be 0 by default, so it's way in the past. + + let server = await sync_httpd_setup(); + await setUp(server); + + scheduler.autoConnect(); + await promiseObserved; + await cleanUpAndGo(server); +}); + +add_task(async function test_autoconnect_nextSync_future() { + enableValidationPrefs(); + + let previousSync = Date.now() + scheduler.syncInterval / 2; + scheduler.nextSync = previousSync; + // nextSync rounds to the nearest second. + let expectedSync = scheduler.nextSync; + let expectedInterval = expectedSync - Date.now() - 1000; + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + await configureIdentity({ username: "johndoe@mozilla.com" }); + scheduler.autoConnect(); + await promiseZeroTimer(); + + Assert.equal(scheduler.nextSync, expectedSync); + Assert.ok(scheduler.syncTimer.delay >= expectedInterval); + + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(); +}); + +add_task(async function test_autoconnect_mp_locked() { + let server = await sync_httpd_setup(); + await setUp(server); + + // Pretend user did not unlock master password. + let origLocked = Utils.mpLocked; + Utils.mpLocked = () => true; + + let origEnsureMPUnlocked = Utils.ensureMPUnlocked; + Utils.ensureMPUnlocked = () => { + _("Faking Master Password entry cancelation."); + return false; + }; + let origFxA = Service.identity._fxaService; + Service.identity._fxaService = new FxAccounts({ + currentAccountState: { + getUserAccountData(...args) { + return origFxA._internal.currentAccountState.getUserAccountData( + ...args + ); + }, + }, + keys: { + canGetKeyForScope() { + return false; + }, + }, + }); + // A locked master password will still trigger a sync, but then we'll hit + // MASTER_PASSWORD_LOCKED and hence MASTER_PASSWORD_LOCKED_RETRY_INTERVAL. + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + scheduler.autoConnect(); + await promiseObserved; + + await Async.promiseYield(); + + Assert.equal(Status.login, MASTER_PASSWORD_LOCKED); + + Utils.mpLocked = origLocked; + Utils.ensureMPUnlocked = origEnsureMPUnlocked; + Service.identity._fxaService = origFxA; + + await cleanUpAndGo(server); +}); + +add_task(async function test_no_autoconnect_during_wizard() { + let server = await sync_httpd_setup(); + await setUp(server); + + // Simulate the Sync setup wizard. + Svc.Prefs.set("firstSync", "notReady"); + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + scheduler.autoConnect(0); + await promiseZeroTimer(); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(server); +}); + +add_task(async function test_no_autoconnect_status_not_ok() { + let server = await sync_httpd_setup(); + Status.__authManager = Service.identity = new SyncAuthManager(); + + // Ensure we don't actually try to sync (or log in for that matter). + function onLoginStart() { + do_throw("Should not get here!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + scheduler.autoConnect(); + await promiseZeroTimer(); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + + Assert.equal(Status.service, CLIENT_NOT_CONFIGURED); + Assert.equal(Status.login, LOGIN_FAILED_NO_USERNAME); + + await cleanUpAndGo(server); +}); + +add_task(async function test_idle_adjustSyncInterval() { + // Confirm defaults. + Assert.equal(scheduler.idle, false); + + // Single device: nothing changes. + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + Assert.equal(scheduler.idle, true); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); + + // Multiple devices: switch to idle interval. + scheduler.idle = false; + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.updateClientMode(); + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + Assert.equal(scheduler.idle, true); + Assert.equal(scheduler.syncInterval, scheduler.idleInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_back_triggersSync() { + // Confirm defaults. + Assert.ok(!scheduler.idle); + Assert.equal(Status.backoffInterval, 0); + + // Set up: Define 2 clients and put the system in idle. + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + Assert.ok(scheduler.idle); + + // We don't actually expect the sync (or the login, for that matter) to + // succeed. We just want to ensure that it was attempted. + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Send an 'active' event to trigger sync soonish. + scheduler.observe(null, "active", Svc.Prefs.get("scheduler.idleTime")); + await promiseObserved; + await cleanUpAndGo(); +}); + +add_task(async function test_active_triggersSync_observesBackoff() { + // Confirm defaults. + Assert.ok(!scheduler.idle); + + // Set up: Set backoff, define 2 clients and put the system in idle. + const BACKOFF = 7337; + Status.backoffInterval = scheduler.idleInterval + BACKOFF; + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + Assert.equal(scheduler.idle, true); + + function onLoginStart() { + do_throw("Shouldn't have kicked off a sync!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + let promiseTimer = promiseNamedTimer( + IDLE_OBSERVER_BACK_DELAY * 1.5, + {}, + "timer" + ); + + // Send an 'active' event to try to trigger sync soonish. + scheduler.observe(null, "active", Svc.Prefs.get("scheduler.idleTime")); + await promiseTimer; + Svc.Obs.remove("weave:service:login:start", onLoginStart); + + Assert.ok(scheduler.nextSync <= Date.now() + Status.backoffInterval); + Assert.equal(scheduler.syncTimer.delay, Status.backoffInterval); + + await cleanUpAndGo(); +}); + +add_task(async function test_back_debouncing() { + _( + "Ensure spurious back-then-idle events, as observed on OS X, don't trigger a sync." + ); + + // Confirm defaults. + Assert.equal(scheduler.idle, false); + + // Set up: Define 2 clients and put the system in idle. + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + Assert.equal(scheduler.idle, true); + + function onLoginStart() { + do_throw("Shouldn't have kicked off a sync!"); + } + Svc.Obs.add("weave:service:login:start", onLoginStart); + + // Create spurious back-then-idle events as observed on OS X: + scheduler.observe(null, "active", Svc.Prefs.get("scheduler.idleTime")); + scheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime")); + + await promiseNamedTimer(IDLE_OBSERVER_BACK_DELAY * 1.5, {}, "timer"); + Svc.Obs.remove("weave:service:login:start", onLoginStart); + await cleanUpAndGo(); +}); + +add_task(async function test_no_sync_node() { + enableValidationPrefs(); + + // Test when Status.sync == NO_SYNC_NODE_FOUND + // it is not overwritten on sync:finish + let server = await sync_httpd_setup(); + await setUp(server); + + let oldfc = Service.identity._findCluster; + Service.identity._findCluster = () => null; + Service.clusterURL = ""; + try { + await Service.sync(); + Assert.equal(Status.sync, NO_SYNC_NODE_FOUND); + Assert.equal(scheduler.syncTimer.delay, NO_SYNC_NODE_INTERVAL); + + await cleanUpAndGo(server); + } finally { + Service.identity._findCluster = oldfc; + } +}); + +add_task(async function test_sync_failed_partial_500s() { + enableValidationPrefs(); + + _("Test a 5xx status calls handleSyncError."); + scheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF; + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 500 }; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + let maxInterval = scheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL); + Assert.equal(Status.backoffInterval, 0); + Assert.ok(Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 4); + Assert.ok(scheduler.nextSync <= Date.now() + maxInterval); + Assert.ok(scheduler.syncTimer.delay <= maxInterval); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_failed_partial_noresync() { + enableValidationPrefs(); + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = "Bad news"; + engine._tracker._score = MULTI_DEVICE_THRESHOLD + 1; + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + let resyncDoneObserver = promiseOneObserver("weave:service:resyncs-finished"); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + + function onSyncStarted() { + do_throw("Should not start resync when previous sync failed"); + } + + Svc.Obs.add("weave:service:sync:start", onSyncStarted); + await resyncDoneObserver; + + Svc.Obs.remove("weave:service:sync:start", onSyncStarted); + engine._tracker._store = 0; + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_failed_partial_400s() { + enableValidationPrefs(); + + _("Test a non-5xx status doesn't call handleSyncError."); + scheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF; + let server = await sync_httpd_setup(); + + let engine = Service.engineManager.get("catapult"); + engine.enabled = true; + engine.exception = { status: 400 }; + + // Have multiple devices for an active interval. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + + Assert.equal(Status.sync, SYNC_SUCCEEDED); + + Assert.ok(await setUp(server)); + + await Service.sync(); + + Assert.equal(Status.service, SYNC_FAILED_PARTIAL); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + + Assert.equal(Status.backoffInterval, 0); + Assert.ok(!Status.enforceBackoff); + Assert.equal(scheduler._syncErrors, 0); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.activeInterval); + Assert.ok(scheduler.syncTimer.delay <= scheduler.activeInterval); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_X_Weave_Backoff() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Use an odd value on purpose so that it doesn't happen to coincide with one + // of the sync intervals. + const BACKOFF = 7337; + + // Extend info/collections so that we can put it into server maintenance mode. + const INFO_COLLECTIONS = "/1.1/johndoe@mozilla.com/info/collections"; + let infoColl = server._handler._overridePaths[INFO_COLLECTIONS]; + let serverBackoff = false; + function infoCollWithBackoff(request, response) { + if (serverBackoff) { + response.setHeader("X-Weave-Backoff", "" + BACKOFF); + } + infoColl(request, response); + } + server.registerPathHandler(INFO_COLLECTIONS, infoCollWithBackoff); + + // Pretend we have two clients so that the regular sync interval is + // sufficiently low. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + let rec = await clientsEngine._store.createRecord("foo", "clients"); + await rec.encrypt(Service.collectionKeys.keyForCollection("clients")); + await rec.upload(Service.resource(clientsEngine.engineURL + rec.id)); + + // Sync once to log in and get everything set up. Let's verify our initial + // values. + await Service.sync(); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.syncInterval); + // Sanity check that we picked the right value for BACKOFF: + Assert.ok(scheduler.syncInterval < BACKOFF * 1000); + + // Turn on server maintenance and sync again. + serverBackoff = true; + await Service.sync(); + + Assert.ok(Status.backoffInterval >= BACKOFF * 1000); + // Allowing 20 seconds worth of of leeway between when Status.minimumNextSync + // was set and when this line gets executed. + let minimumExpectedDelay = (BACKOFF - 20) * 1000; + Assert.ok(Status.minimumNextSync >= Date.now() + minimumExpectedDelay); + + // Verify that the next sync is actually going to wait that long. + Assert.ok(scheduler.nextSync >= Date.now() + minimumExpectedDelay); + Assert.ok(scheduler.syncTimer.delay >= minimumExpectedDelay); + + await cleanUpAndGo(server); +}); + +add_task(async function test_sync_503_Retry_After() { + enableValidationPrefs(); + + let server = await sync_httpd_setup(); + await setUp(server); + + // Use an odd value on purpose so that it doesn't happen to coincide with one + // of the sync intervals. + const BACKOFF = 7337; + + // Extend info/collections so that we can put it into server maintenance mode. + const INFO_COLLECTIONS = "/1.1/johndoe@mozilla.com/info/collections"; + let infoColl = server._handler._overridePaths[INFO_COLLECTIONS]; + let serverMaintenance = false; + function infoCollWithMaintenance(request, response) { + if (!serverMaintenance) { + infoColl(request, response); + return; + } + response.setHeader("Retry-After", "" + BACKOFF); + response.setStatusLine(request.httpVersion, 503, "Service Unavailable"); + } + server.registerPathHandler(INFO_COLLECTIONS, infoCollWithMaintenance); + + // Pretend we have two clients so that the regular sync interval is + // sufficiently low. + await clientsEngine._store.create({ + id: "foo", + cleartext: { os: "mobile", version: "0.01", type: "desktop" }, + }); + let rec = await clientsEngine._store.createRecord("foo", "clients"); + await rec.encrypt(Service.collectionKeys.keyForCollection("clients")); + await rec.upload(Service.resource(clientsEngine.engineURL + rec.id)); + + // Sync once to log in and get everything set up. Let's verify our initial + // values. + await Service.sync(); + Assert.ok(!Status.enforceBackoff); + Assert.equal(Status.backoffInterval, 0); + Assert.equal(Status.minimumNextSync, 0); + Assert.equal(scheduler.syncInterval, scheduler.activeInterval); + Assert.ok(scheduler.nextSync <= Date.now() + scheduler.syncInterval); + // Sanity check that we picked the right value for BACKOFF: + Assert.ok(scheduler.syncInterval < BACKOFF * 1000); + + // Turn on server maintenance and sync again. + serverMaintenance = true; + await Service.sync(); + + Assert.ok(Status.enforceBackoff); + Assert.ok(Status.backoffInterval >= BACKOFF * 1000); + // Allowing 3 seconds worth of of leeway between when Status.minimumNextSync + // was set and when this line gets executed. + let minimumExpectedDelay = (BACKOFF - 3) * 1000; + Assert.ok(Status.minimumNextSync >= Date.now() + minimumExpectedDelay); + + // Verify that the next sync is actually going to wait that long. + Assert.ok(scheduler.nextSync >= Date.now() + minimumExpectedDelay); + Assert.ok(scheduler.syncTimer.delay >= minimumExpectedDelay); + + await cleanUpAndGo(server); +}); + +add_task(async function test_loginError_recoverable_reschedules() { + _("Verify that a recoverable login error schedules a new sync."); + await configureIdentity({ username: "johndoe@mozilla.com" }); + Service.clusterURL = "http://localhost:1234/"; + Status.resetSync(); // reset Status.login + + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Let's set it up so that a sync is overdue, both in terms of previously + // scheduled syncs and the global score. We still do not expect an immediate + // sync because we just tried (duh). + scheduler.nextSync = Date.now() - 100000; + scheduler.globalScore = SINGLE_USER_THRESHOLD + 1; + function onSyncStart() { + do_throw("Shouldn't have started a sync!"); + } + Svc.Obs.add("weave:service:sync:start", onSyncStart); + + // Sanity check. + Assert.equal(scheduler.syncTimer, null); + Assert.equal(Status.checkSetup(), STATUS_OK); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + + scheduler.scheduleNextSync(0); + await promiseObserved; + await Async.promiseYield(); + + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + + let expectedNextSync = Date.now() + scheduler.syncInterval; + Assert.ok(scheduler.nextSync > Date.now()); + Assert.ok(scheduler.nextSync <= expectedNextSync); + Assert.ok(scheduler.syncTimer.delay > 0); + Assert.ok(scheduler.syncTimer.delay <= scheduler.syncInterval); + + Svc.Obs.remove("weave:service:sync:start", onSyncStart); + await cleanUpAndGo(); +}); + +add_task(async function test_loginError_fatal_clearsTriggers() { + _("Verify that a fatal login error clears sync triggers."); + await configureIdentity({ username: "johndoe@mozilla.com" }); + + let server = httpd_setup({ + "/1.1/johndoe@mozilla.com/info/collections": httpd_handler( + 401, + "Unauthorized" + ), + }); + + Service.clusterURL = server.baseURI + "/"; + Status.resetSync(); // reset Status.login + + let promiseObserved = promiseOneObserver("weave:service:login:error"); + + // Sanity check. + Assert.equal(scheduler.nextSync, 0); + Assert.equal(scheduler.syncTimer, null); + Assert.equal(Status.checkSetup(), STATUS_OK); + Assert.equal(Status.login, LOGIN_SUCCEEDED); + + scheduler.scheduleNextSync(0); + await promiseObserved; + await Async.promiseYield(); + + // For the FxA identity, a 401 on info/collections means a transient + // error, probably due to an inability to fetch a token. + Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR); + // syncs should still be scheduled. + Assert.ok(scheduler.nextSync > Date.now()); + Assert.ok(scheduler.syncTimer.delay > 0); + + await cleanUpAndGo(server); +}); + +add_task(async function test_proper_interval_on_only_failing() { + _("Ensure proper behavior when only failed records are applied."); + + // If an engine reports that no records succeeded, we shouldn't decrease the + // sync interval. + Assert.ok(!scheduler.hasIncomingItems); + const INTERVAL = 10000000; + scheduler.syncInterval = INTERVAL; + + Svc.Obs.notify("weave:service:sync:applied", { + applied: 2, + succeeded: 0, + failed: 2, + newFailed: 2, + reconciled: 0, + }); + + await Async.promiseYield(); + scheduler.adjustSyncInterval(); + Assert.ok(!scheduler.hasIncomingItems); + Assert.equal(scheduler.syncInterval, scheduler.singleDeviceInterval); +}); + +add_task(async function test_link_status_change() { + _("Check that we only attempt to sync when link status is up"); + try { + sinon.spy(scheduler, "scheduleNextSync"); + + Svc.Obs.notify("network:link-status-changed", null, "down"); + equal(scheduler.scheduleNextSync.callCount, 0); + + Svc.Obs.notify("network:link-status-changed", null, "change"); + equal(scheduler.scheduleNextSync.callCount, 0); + + Svc.Obs.notify("network:link-status-changed", null, "up"); + equal(scheduler.scheduleNextSync.callCount, 1); + + Svc.Obs.notify("network:link-status-changed", null, "change"); + equal(scheduler.scheduleNextSync.callCount, 1); + } finally { + scheduler.scheduleNextSync.restore(); + } +}); diff --git a/services/sync/tests/unit/test_tab_engine.js b/services/sync/tests/unit/test_tab_engine.js new file mode 100644 index 0000000000..298f8f799d --- /dev/null +++ b/services/sync/tests/unit/test_tab_engine.js @@ -0,0 +1,225 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +let engine; +// We'll need the clients engine for testing as tabs is closely related +let clientsEngine; + +async function syncClientsEngine(server) { + clientsEngine._lastFxADevicesFetch = 0; + clientsEngine.lastModified = server.getCollection("foo", "clients").timestamp; + await clientsEngine._sync(); +} + +async function makeRemoteClients() { + let server = await serverForFoo(clientsEngine); + await configureIdentity({ username: "foo" }, server); + await Service.login(); + + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let remoteId = Utils.makeGUID(); + let remoteId2 = Utils.makeGUID(); + let collection = server.getCollection("foo", "clients"); + + _("Create remote client records"); + collection.insertRecord({ + id: remoteId, + name: "Remote client", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId, + fxaDeviceName: "Fxa - Remote client", + protocols: ["1.5"], + }); + + collection.insertRecord({ + id: remoteId2, + name: "Remote client 2", + type: "desktop", + commands: [], + version: "48", + fxaDeviceId: remoteId2, + fxaDeviceName: "Fxa - Remote client 2", + protocols: ["1.5"], + }); + + let fxAccounts = clientsEngine.fxAccounts; + clientsEngine.fxAccounts = { + notifyDevices() { + return Promise.resolve(true); + }, + device: { + getLocalId() { + return fxAccounts.device.getLocalId(); + }, + getLocalName() { + return fxAccounts.device.getLocalName(); + }, + getLocalType() { + return fxAccounts.device.getLocalType(); + }, + recentDeviceList: [{ id: remoteId, name: "remote device" }], + refreshDeviceList() { + return Promise.resolve(true); + }, + }, + _internal: { + now() { + return Date.now(); + }, + }, + }; + + await syncClientsEngine(server); +} + +add_task(async function setup() { + clientsEngine = Service.clientsEngine; + // Make some clients to test with + await makeRemoteClients(); + + // Make the tabs engine for all the tests to use + engine = Service.engineManager.get("tabs"); + await engine.initialize(); + + // Since these are xpcshell tests, we'll need to mock this + TabProvider.shouldSkipWindow = mockShouldSkipWindow; +}); + +add_task(async function test_tab_engine_skips_incoming_local_record() { + _("Ensure incoming records that match local client ID are never applied."); + + let localID = clientsEngine.localID; + let collection = new ServerCollection(); + + _("Creating remote tab record with local client ID"); + let localRecord = encryptPayload({ + id: localID, + clientName: "local", + tabs: [ + { + title: "title", + urlHistory: ["http://foo.com/"], + icon: "", + lastUsed: 2000, + }, + ], + }); + collection.insert(localID, localRecord); + + _("Creating remote tab record with a different client ID"); + let remoteID = "fake-guid-00"; // remote should match one of the test clients + let remoteRecord = encryptPayload({ + id: remoteID, + clientName: "not local", + tabs: [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + lastUsed: 3000, + }, + ], + }); + collection.insert(remoteID, remoteRecord); + + _("Setting up Sync server"); + let server = sync_httpd_setup({ + "/1.1/foo/storage/tabs": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { + tabs: { version: engine.version, syncID }, + }; + + await generateNewKeys(Service.collectionKeys); + + let promiseFinished = new Promise(resolve => { + let syncFinish = engine._syncFinish; + engine._syncFinish = async function () { + let remoteTabs = await engine._rustStore.getAll(); + equal( + remoteTabs.length, + 1, + "Remote client record was applied and local wasn't" + ); + let record = remoteTabs[0]; + equal(record.clientId, remoteID, "Remote client ID matches"); + + _("Ensure getAllClients returns the correct shape"); + let clients = await engine.getAllClients(); + equal(clients.length, 1); + let client = clients[0]; + equal(client.id, "fake-guid-00"); + equal(client.name, "Remote client"); + equal(client.type, "desktop"); + Assert.ok(client.lastModified); // lastModified should be filled in once serverModified is populated from the server + deepEqual(client.tabs, [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + lastUsed: 3000, + }, + ]); + await syncFinish.call(engine); + resolve(); + }; + }); + + _("Start sync"); + Service.scheduler.hasIncomingItems = false; + await engine._sync(); + await promiseFinished; + // Bug 1800185 - we don't want the sync scheduler to see these records as incoming. + Assert.ok(!Service.scheduler.hasIncomingItems); +}); + +// Ensure we trim tabs in the case of going past the max payload size allowed +add_task(async function test_too_many_tabs() { + let a_lot_of_tabs = []; + + for (let i = 0; i < 4000; ++i) { + a_lot_of_tabs.push( + `http://example${i}.com/some-super-long-url-chain-to-help-with-bytes` + ); + } + + TabProvider.getWindowEnumerator = mockGetWindowEnumerator.bind( + this, + a_lot_of_tabs + ); + + let encoder = Utils.utf8Encoder; + // see tryfitItems(..) in util.js + const computeSerializedSize = records => + encoder.encode(JSON.stringify(records)).byteLength; + + const maxPayloadSize = Service.getMaxRecordPayloadSize(); + const maxSerializedSize = (maxPayloadSize / 4) * 3 - 1500; + // We are over max payload size + Assert.ok(computeSerializedSize(a_lot_of_tabs) > maxSerializedSize); + let tabs = await engine.getTabsWithinPayloadSize(); + // We are now under max payload size + Assert.ok(computeSerializedSize(tabs) < maxSerializedSize); +}); diff --git a/services/sync/tests/unit/test_tab_provider.js b/services/sync/tests/unit/test_tab_provider.js new file mode 100644 index 0000000000..bbf68dea33 --- /dev/null +++ b/services/sync/tests/unit/test_tab_provider.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); + +add_task(async function test_getAllTabs() { + let provider = TabProvider; + provider.shouldSkipWindow = mockShouldSkipWindow; + + let tabs; + + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://bar.com", + ]); + + _("Get all tabs."); + tabs = await provider.getAllTabsWithEstimatedMax( + false, + Number.MAX_SAFE_INTEGER + ); + _("Tabs: " + JSON.stringify(tabs)); + equal(tabs.length, 1); + equal(tabs[0].title, "title"); + equal(tabs[0].urlHistory.length, 1); + equal(tabs[0].urlHistory[0], "http://bar.com/"); + equal(tabs[0].icon, ""); + equal(tabs[0].lastUsed, 2); // windowenumerator returns in ms but the getAllTabs..() returns in seconds + + _("Get all tabs, and check that filtering works."); + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + "about:foo", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + _("Filtered: " + JSON.stringify(tabs)); + equal(tabs.length, 1); + + _("Get all tabs, and check that they are properly sorted"); + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + "http://bar.com", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + _("Ordered: " + JSON.stringify(tabs)); + equal(tabs[0].lastUsed > tabs[1].lastUsed, true); + + // reader mode URLs are provided. + provider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "about:reader?url=http%3A%2F%2Ffoo.com%2F", + ]); + tabs = await provider.getAllTabsWithEstimatedMax( + true, + Number.MAX_SAFE_INTEGER + ); + equal(tabs[0].urlHistory[0], "http://foo.com/"); +}); diff --git a/services/sync/tests/unit/test_tab_quickwrite.js b/services/sync/tests/unit/test_tab_quickwrite.js new file mode 100644 index 0000000000..2a1c75c8c6 --- /dev/null +++ b/services/sync/tests/unit/test_tab_quickwrite.js @@ -0,0 +1,204 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.importESModule("resource://services-sync/engines/tabs.sys.mjs"); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { TabProvider } = ChromeUtils.importESModule( + "resource://services-sync/engines/tabs.sys.mjs" +); + +const FAR_FUTURE = 4102405200000; // 2100/01/01 + +add_task(async function setup() { + // Since these are xpcshell tests, we'll need to mock ui features + TabProvider.shouldSkipWindow = mockShouldSkipWindow; + TabProvider.getWindowEnumerator = mockGetWindowEnumerator.bind(this, [ + "http://foo.com", + ]); +}); + +async function prepareServer() { + _("Setting up Sync server"); + Service.serverConfiguration = { + max_post_records: 100, + }; + + let server = new SyncServer(); + server.start(); + await SyncTestingInfrastructure(server, "username"); + server.registerUser("username"); + + let collection = server.createCollection("username", "tabs"); + await generateNewKeys(Service.collectionKeys); + + let engine = Service.engineManager.get("tabs"); + await engine.initialize(); + + return { server, collection, engine }; +} + +async function withPatchedValue(object, name, patchedVal, fn) { + _(`patching ${name}=${patchedVal}`); + let old = object[name]; + object[name] = patchedVal; + try { + await fn(); + } finally { + object[name] = old; + } +} + +add_task(async function test_tab_quickwrite_works() { + _("Ensure a simple quickWrite works."); + let { server, collection, engine } = await prepareServer(); + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + Assert.ok(await engine.quickWrite()); + // Validate we didn't bork lastSync + let lastSync = await engine.getLastSync(); + Assert.ok(lastSync < FAR_FUTURE); + Assert.equal(collection.count(), 1, "tab record was written"); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_bad_status() { + _("Ensure quickWrite silently aborts when we aren't setup correctly."); + let { server, engine } = await prepareServer(); + // Store the original lock to reset it back after this test + let lock = engine.lock; + // Arrange for this test to fail if it tries to take the lock. + engine.lock = function () { + throw new Error("this test should abort syncing before locking"); + }; + let quickWrite = engine.quickWrite.bind(engine); // lol javascript. + + await withPatchedValue(engine, "enabled", false, quickWrite); + await withPatchedValue(Service, "serverConfiguration", null, quickWrite); + + Services.prefs.clearUserPref("services.sync.username"); + await quickWrite(); + // Validate we didn't bork lastSync + let lastSync = await engine.getLastSync(); + Assert.ok(lastSync < FAR_FUTURE); + Service.status.resetSync(); + engine.lock = lock; + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickwrite_lock() { + _("Ensure we fail to quickWrite if the engine is locked."); + let { server, collection, engine } = await prepareServer(); + + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + engine.lock(); + Assert.ok(!(await engine.quickWrite())); + Assert.equal(collection.count(), 0, "didn't sync due to being locked"); + engine.unlock(); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickwrite_keeps_old_tabs() { + _("Ensure we don't delete other tabs on quickWrite (bug 1801295)."); + let { server, engine } = await prepareServer(); + + // need a first sync to ensure everything is setup correctly. + await Service.sync({ engines: ["tabs"] }); + + const id = "fake-guid-99"; + let remoteRecord = encryptPayload({ + id, + clientName: "not local", + tabs: [ + { + title: "title2", + urlHistory: ["http://bar.com/"], + icon: "", + lastUsed: 3000, + }, + ], + }); + + let collection = server.getCollection("username", "tabs"); + collection.insert(id, remoteRecord); + + await Service.sync({ engines: ["tabs"] }); + + // collection should now have 2 records - ours and the pretend remote one we inserted. + Assert.equal(collection.count(), 2, "starting with 2 tab records"); + + // So fxAccounts.device.recentDeviceList is not null. + engine.service.clientsEngine.fxAccounts.device._deviceListCache = { + devices: [], + }; + // trick the clients engine into thinking it has a remote client with the same guid. + engine.service.clientsEngine._store._remoteClients = {}; + engine.service.clientsEngine._store._remoteClients[id] = { + id, + fxaDeviceId: id, + }; + + let clients = await engine.getAllClients(); + Assert.equal(clients.length, 1); + + _("Doing a quick-write"); + Assert.ok(await engine.quickWrite()); + + // Should still have our client after a quickWrite. + _("Grabbing clients after the quick-write"); + clients = await engine.getAllClients(); + Assert.equal(clients.length, 1); + + engine.service.clientsEngine._store._remoteClients = {}; + + await promiseStopServer(server); +}); + +add_task(async function test_tab_lastSync() { + _("Ensure we restore the lastSync timestamp after a quick-write."); + let { server, collection, engine } = await prepareServer(); + + await engine.initialize(); + await engine.service.clientsEngine.initialize(); + + let origLastSync = engine.lastSync; + Assert.ok(await engine.quickWrite()); + Assert.equal(engine.lastSync, origLastSync); + Assert.equal(collection.count(), 1, "successful sync"); + engine.unlock(); + + await promiseStopServer(server); +}); + +add_task(async function test_tab_quickWrite_telemetry() { + _("Ensure we record the telemetry we expect."); + // hook into telemetry + let telem = get_sync_test_telemetry(); + telem.payloads = []; + let oldSubmit = telem.submit; + let submitPromise = new Promise((resolve, reject) => { + telem.submit = function (ping) { + telem.submit = oldSubmit; + resolve(ping); + }; + }); + + let { server, collection, engine } = await prepareServer(); + + Assert.equal(collection.count(), 0, "starting with 0 tab records"); + Assert.ok(await engine.quickWrite()); + Assert.equal(collection.count(), 1, "tab record was written"); + + let ping = await submitPromise; + let syncs = ping.syncs; + Assert.equal(syncs.length, 1); + let sync = syncs[0]; + Assert.equal(sync.why, "quick-write"); + Assert.equal(sync.engines.length, 1); + Assert.equal(sync.engines[0].name, "tabs"); + + await promiseStopServer(server); +}); diff --git a/services/sync/tests/unit/test_tab_tracker.js b/services/sync/tests/unit/test_tab_tracker.js new file mode 100644 index 0000000000..fe51156849 --- /dev/null +++ b/services/sync/tests/unit/test_tab_tracker.js @@ -0,0 +1,364 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +ChromeUtils.importESModule("resource://services-sync/engines/tabs.sys.mjs"); +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +const { SyncScheduler } = ChromeUtils.importESModule( + "resource://services-sync/policies.sys.mjs" +); + +var scheduler = new SyncScheduler(Service); +let clientsEngine; + +add_task(async function setup() { + await Service.promiseInitialized; + clientsEngine = Service.clientsEngine; + + scheduler.setDefaults(); +}); + +function fakeSvcWinMediator() { + // actions on windows are captured in logs + let logs = []; + delete Services.wm; + + function getNext() { + let elt = { addTopics: [], remTopics: [], numAPL: 0, numRPL: 0 }; + logs.push(elt); + return { + addEventListener(topic) { + elt.addTopics.push(topic); + }, + removeEventListener(topic) { + elt.remTopics.push(topic); + }, + gBrowser: { + addProgressListener() { + elt.numAPL++; + }, + removeProgressListener() { + elt.numRPL++; + }, + }, + }; + } + + Services.wm = { + getEnumerator() { + return [getNext(), getNext()]; + }, + }; + return logs; +} + +function fakeGetTabState(tab) { + return tab; +} + +function clearQuickWriteTimer(tracker) { + if (tracker.tabsQuickWriteTimer) { + tracker.tabsQuickWriteTimer.clear(); + } +} + +add_task(async function run_test() { + let engine = Service.engineManager.get("tabs"); + await engine.initialize(); + _("We assume that tabs have changed at startup."); + let tracker = engine._tracker; + tracker.getTabState = fakeGetTabState; + + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + let logs; + + _("Test listeners are registered on windows"); + logs = fakeSvcWinMediator(); + tracker.start(); + Assert.equal(logs.length, 2); + for (let log of logs) { + Assert.equal(log.addTopics.length, 3); + Assert.ok(log.addTopics.includes("TabOpen")); + Assert.ok(log.addTopics.includes("TabClose")); + Assert.ok(log.addTopics.includes("unload")); + Assert.equal(log.remTopics.length, 0); + Assert.equal(log.numAPL, 1, "Added 1 progress listener"); + Assert.equal(log.numRPL, 0, "Didn't remove a progress listener"); + } + + _("Test listeners are unregistered on windows"); + logs = fakeSvcWinMediator(); + await tracker.stop(); + Assert.equal(logs.length, 2); + for (let log of logs) { + Assert.equal(log.addTopics.length, 0); + Assert.equal(log.remTopics.length, 3); + Assert.ok(log.remTopics.includes("TabOpen")); + Assert.ok(log.remTopics.includes("TabClose")); + Assert.ok(log.remTopics.includes("unload")); + Assert.equal(log.numAPL, 0, "Didn't add a progress listener"); + Assert.equal(log.numRPL, 1, "Removed 1 progress listener"); + } + + _("Test tab listener"); + for (let evttype of ["TabOpen", "TabClose"]) { + // Pretend we just synced. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + } + + // Pretend we just synced. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + + tracker.onTab({ + type: "TabOpen", + originalTarget: "TabOpen", + target: { entries: [], currentURI: "about:config" }, + }); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + // Pretend we just synced and saw some progress listeners. + await tracker.clearChangedIDs(); + Assert.ok(!tracker.modified); + tracker.onLocationChange({ isTopLevel: false }, undefined, undefined, 0); + Assert.ok(!tracker.modified, "non-toplevel request didn't flag as modified"); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org"), + Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT + ); + Assert.ok( + tracker.modified, + "location change within the same document request did flag as modified" + ); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org") + ); + Assert.ok( + tracker.modified, + "location change for a new top-level document flagged as modified" + ); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); +}); + +add_task(async function run_sync_on_tab_change_test() { + let testPrefDelay = 20000; + + // This is the pref that determines sync delay after tab change + Svc.Prefs.set("syncedTabs.syncDelayAfterTabChange", testPrefDelay); + // We should only be syncing on tab change if + // the user has > 1 client + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 2); + + let engine = Service.engineManager.get("tabs"); + + _("We assume that tabs have changed at startup."); + let tracker = engine._tracker; + tracker.getTabState = fakeGetTabState; + + Assert.ok(tracker.modified); + Assert.ok( + Utils.deepEquals(Object.keys(await engine.getChangedIDs()), [ + clientsEngine.localID, + ]) + ); + + _("Test sync is scheduled after a tab change"); + for (let evttype of ["TabOpen", "TabClose"]) { + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be more delayed at or more than what the pref is set at + let nextSchedule = tracker.tabsQuickWriteTimer.delay; + Assert.ok(nextSchedule >= testPrefDelay); + } + + _("Test sync is NOT scheduled after an unsupported tab open"); + for (let evttype of ["TabOpen"]) { + // Send a fake tab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: ["about:newtab"], currentURI: null }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be scheduling <= pref value + Assert.ok(scheduler.nextSync - Date.now() <= testPrefDelay); + } + + _("Test navigating within the same tab does NOT trigger a sync"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org"), + Ci.nsIWebProgressListener.LOCATION_CHANGE_RELOAD + ); + Assert.ok( + !tracker.modified, + "location change for reloading doesn't trigger a sync" + ); + Assert.ok(!tracker.tabsQuickWriteTimer, "reload does not trigger a sync"); + + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + _("Test navigating to an about page does trigger sync"); + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("about:config") + ); + Assert.ok(tracker.modified, "about page does not trigger a tab modified"); + Assert.ok( + tracker.tabsQuickWriteTimer, + "about schema should trigger a sync happening soon" + ); + + _("Test adjusting the filterScheme pref works"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + Svc.Prefs.set( + "engine.tabs.filteredSchemes", + // Removing the about scheme for this test + "resource|chrome|file|blob|moz-extension" + ); + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("about:config") + ); + Assert.ok( + tracker.modified, + "about page triggers a modified after we changed the pref" + ); + Assert.ok( + tracker.tabsQuickWriteTimer, + "about page should schedule a quickWrite sync soon after we changed the pref" + ); + + _("Test no sync after tab change for accounts with <= 1 clients"); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + // Setting clients to only 1 so we don't sync after a tab change + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 0); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 1); + + tracker.onLocationChange( + { isTopLevel: true }, + undefined, + Services.io.newURI("https://www.mozilla.org") + ); + Assert.ok( + tracker.modified, + "location change for a new top-level document flagged as modified" + ); + Assert.ok( + !tracker.tabsQuickWriteTimer, + "We should NOT be syncing shortly because there is only one client" + ); + + _("Changing the pref adjusts the sync schedule"); + Svc.Prefs.set("syncedTabs.syncDelayAfterTabChange", 10000); // 10seconds + let delayPref = Svc.Prefs.get("syncedTabs.syncDelayAfterTabChange"); + let evttype = "TabOpen"; + Assert.equal(delayPref, 10000); // ensure our pref is at 10s + // Only have task continuity if we have more than 1 device + Svc.Prefs.set("clients.devices.desktop", 1); + Svc.Prefs.set("clients.devices.mobile", 1); + scheduler.updateClientMode(); + Assert.equal(scheduler.numClients, 2); + clearQuickWriteTimer(tracker); + + // Fire ontab event + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + + // Ensure the tracker fired + Assert.ok(tracker.modified); + // We should be scheduling <= preference value + Assert.equal(tracker.tabsQuickWriteTimer.delay, delayPref); + + _("We should not have a sync scheduled if pref is at 0"); + + Svc.Prefs.set("syncedTabs.syncDelayAfterTabChange", 0); + // Pretend we just synced + await tracker.clearChangedIDs(); + clearQuickWriteTimer(tracker); + + // Fire ontab event + evttype = "TabOpen"; + tracker.onTab({ + type: evttype, + originalTarget: evttype, + target: { entries: [], currentURI: "about:config" }, + }); + // Ensure the tracker fired + Assert.ok(tracker.modified); + + // We should NOT be scheduled for a sync soon + Assert.ok(!tracker.tabsQuickWriteTimer); + + scheduler.setDefaults(); + Svc.Prefs.resetBranch(""); +}); diff --git a/services/sync/tests/unit/test_telemetry.js b/services/sync/tests/unit/test_telemetry.js new file mode 100644 index 0000000000..02aceeaf89 --- /dev/null +++ b/services/sync/tests/unit/test_telemetry.js @@ -0,0 +1,1447 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); +const { WBORecord } = ChromeUtils.importESModule( + "resource://services-sync/record.sys.mjs" +); +const { Resource } = ChromeUtils.importESModule( + "resource://services-sync/resource.sys.mjs" +); +const { RotaryEngine } = ChromeUtils.importESModule( + "resource://testing-common/services/sync/rotaryengine.sys.mjs" +); +const { getFxAccountsSingleton } = ChromeUtils.importESModule( + "resource://gre/modules/FxAccounts.sys.mjs" +); +const fxAccounts = getFxAccountsSingleton(); + +function SteamStore(engine) { + Store.call(this, "Steam", engine); +} +Object.setPrototypeOf(SteamStore.prototype, Store.prototype); + +function SteamTracker(name, engine) { + LegacyTracker.call(this, name || "Steam", engine); +} +Object.setPrototypeOf(SteamTracker.prototype, LegacyTracker.prototype); + +function SteamEngine(service) { + SyncEngine.call(this, "steam", service); +} + +SteamEngine.prototype = { + _storeObj: SteamStore, + _trackerObj: SteamTracker, + _errToThrow: null, + problemsToReport: null, + async _sync() { + if (this._errToThrow) { + throw this._errToThrow; + } + }, + getValidator() { + return new SteamValidator(); + }, +}; +Object.setPrototypeOf(SteamEngine.prototype, SyncEngine.prototype); + +function BogusEngine(service) { + SyncEngine.call(this, "bogus", service); +} + +BogusEngine.prototype = Object.create(SteamEngine.prototype); + +class SteamValidator { + async canValidate() { + return true; + } + + async validate(engine) { + return { + problems: new SteamValidationProblemData(engine.problemsToReport), + version: 1, + duration: 0, + recordCount: 0, + }; + } +} + +class SteamValidationProblemData { + constructor(problemsToReport = []) { + this.problemsToReport = problemsToReport; + } + + getSummary() { + return this.problemsToReport; + } +} + +async function cleanAndGo(engine, server) { + await engine._tracker.clearChangedIDs(); + Svc.Prefs.resetBranch(""); + syncTestLogging(); + Service.recordManager.clearCache(); + await promiseStopServer(server); +} + +add_task(async function setup() { + // Avoid addon manager complaining about not being initialized + await Service.engineManager.unregister("addons"); + await Service.engineManager.unregister("extension-storage"); +}); + +add_task(async function test_basic() { + enableValidationPrefs(); + + let helper = track_collections_helper(); + let upd = helper.with_updated_collection; + + let handlers = { + "/1.1/johndoe/info/collections": helper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + }; + + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = upd( + coll, + new ServerCollection({}, true).handler() + ); + } + + let server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + + let ping = await wait_for_ping(() => Service.sync(), true, true); + + // Check the "os" block - we can't really check specific values, but can + // check it smells sane. + ok(ping.os, "there is an OS block"); + ok("name" in ping.os, "there is an OS name"); + ok("version" in ping.os, "there is an OS version"); + ok("locale" in ping.os, "there is an OS locale"); + + Svc.Prefs.resetBranch(""); + await promiseStopServer(server); +}); + +add_task(async function test_processIncoming_error() { + let engine = Service.engineManager.get("bookmarks"); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let collection = server.user("foo").collection("bookmarks"); + try { + // Create a bogus record that when synced down will provoke a + // network error which in turn provokes an exception in _processIncoming. + const BOGUS_GUID = "zzzzzzzzzzzz"; + let bogus_record = collection.insert(BOGUS_GUID, "I'm a bogus record!"); + bogus_record.get = function get() { + throw new Error("Sync this!"); + }; + // Make the 10 minutes old so it will only be synced in the toFetch phase. + bogus_record.modified = Date.now() / 1000 - 60 * 10; + await engine.setLastSync(Date.now() / 1000 - 60); + engine.toFetch = new SerializableSet([BOGUS_GUID]); + + let error, pingPayload, fullPing; + try { + await sync_engine_and_validate_telem( + engine, + true, + (errPing, fullErrPing) => { + pingPayload = errPing; + fullPing = fullErrPing; + } + ); + } catch (ex) { + error = ex; + } + ok(!!error); + ok(!!pingPayload); + + equal(fullPing.uid, "f".repeat(32)); // as setup by SyncTestingInfrastructure + deepEqual(pingPayload.failureReason, { + name: "httperror", + code: 500, + }); + + equal(pingPayload.engines.length, 1); + + equal(pingPayload.engines[0].name, "bookmarks-buffered"); + deepEqual(pingPayload.engines[0].failureReason, { + name: "httperror", + code: 500, + }); + } finally { + await store.wipe(); + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_uploading() { + let engine = Service.engineManager.get("bookmarks"); + await engine.initialize(); + let store = engine._store; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + let bmk = await PlacesUtils.bookmarks.insert({ + parentGuid: PlacesUtils.bookmarks.toolbarGuid, + url: "http://getfirefox.com/", + title: "Get Firefox!", + }); + + try { + let ping = await sync_engine_and_validate_telem(engine, false); + ok(!!ping); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "bookmarks-buffered"); + ok(!!ping.engines[0].outgoing); + greater(ping.engines[0].outgoing[0].sent, 0); + ok(!ping.engines[0].incoming); + + await PlacesUtils.bookmarks.update({ + guid: bmk.guid, + title: "New Title", + }); + + await store.wipe(); + await engine.resetClient(); + // We don't sync via the service, so don't re-hit info/collections, so + // lastModified remaning at zero breaks things subtly... + engine.lastModified = null; + + ping = await sync_engine_and_validate_telem(engine, false); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "bookmarks-buffered"); + equal(ping.engines[0].outgoing.length, 1); + ok(!!ping.engines[0].incoming); + } finally { + // Clean up. + await store.wipe(); + await cleanAndGo(engine, server); + } +}); + +add_task(async function test_upload_failed() { + let collection = new ServerCollection(); + collection._wbos.flying = new ServerWBO("flying"); + + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + + await SyncTestingInfrastructure(server); + await configureIdentity({ username: "foo" }, server); + + let engine = new RotaryEngine(Service); + engine._store.items = { + flying: "LNER Class A3 4472", + scotsman: "Flying Scotsman", + peppercorn: "Peppercorn Class", + }; + const FLYING_CHANGED = 12345; + const SCOTSMAN_CHANGED = 23456; + const PEPPERCORN_CHANGED = 34567; + await engine._tracker.addChangedID("flying", FLYING_CHANGED); + await engine._tracker.addChangedID("scotsman", SCOTSMAN_CHANGED); + await engine._tracker.addChangedID("peppercorn", PEPPERCORN_CHANGED); + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + await engine.setLastSync(123); // needs to be non-zero so that tracker is queried + let changes = await engine._tracker.getChangedIDs(); + _( + `test_upload_failed: Rotary tracker contents at first sync: ${JSON.stringify( + changes + )}` + ); + engine.enabled = true; + let ping = await sync_engine_and_validate_telem(engine, true); + ok(!!ping); + equal(ping.engines.length, 1); + equal(ping.engines[0].incoming, null); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 3, + failed: 2, + failedReasons: [ + { name: "scotsman", count: 1 }, + { name: "peppercorn", count: 1 }, + ], + }, + ]); + await engine.setLastSync(123); + + changes = await engine._tracker.getChangedIDs(); + _( + `test_upload_failed: Rotary tracker contents at second sync: ${JSON.stringify( + changes + )}` + ); + ping = await sync_engine_and_validate_telem(engine, true); + ok(!!ping); + equal(ping.engines.length, 1); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 2, + failed: 2, + failedReasons: [ + { name: "scotsman", count: 1 }, + { name: "peppercorn", count: 1 }, + ], + }, + ]); + } finally { + await cleanAndGo(engine, server); + await engine.finalize(); + } +}); + +add_task(async function test_sync_partialUpload() { + let collection = new ServerCollection(); + let server = sync_httpd_setup({ + "/1.1/foo/storage/rotary": collection.handler(), + }); + await SyncTestingInfrastructure(server); + await generateNewKeys(Service.collectionKeys); + + let engine = new RotaryEngine(Service); + await engine.setLastSync(123); + + // Create a bunch of records (and server side handlers) + for (let i = 0; i < 234; i++) { + let id = "record-no-" + i; + engine._store.items[id] = "Record No. " + i; + await engine._tracker.addChangedID(id, i); + // Let two items in the first upload batch fail. + if (i != 23 && i != 42) { + collection.insert(id); + } + } + + let syncID = await engine.resetLocalSyncID(); + let meta_global = Service.recordManager.set( + engine.metaURL, + new WBORecord(engine.metaURL) + ); + meta_global.payload.engines = { rotary: { version: engine.version, syncID } }; + + try { + let changes = await engine._tracker.getChangedIDs(); + _( + `test_sync_partialUpload: Rotary tracker contents at first sync: ${JSON.stringify( + changes + )}` + ); + engine.enabled = true; + let ping = await sync_engine_and_validate_telem(engine, true); + + ok(!!ping); + ok(!ping.failureReason); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "rotary"); + ok(!ping.engines[0].incoming); + ok(!ping.engines[0].failureReason); + deepEqual(ping.engines[0].outgoing, [ + { + sent: 234, + failed: 2, + failedReasons: [ + { name: "record-no-23", count: 1 }, + { name: "record-no-42", count: 1 }, + ], + }, + ]); + collection.post = function () { + throw new Error("Failure"); + }; + + engine._store.items["record-no-1000"] = "Record No. 1000"; + await engine._tracker.addChangedID("record-no-1000", 1000); + collection.insert("record-no-1000", 1000); + + await engine.setLastSync(123); + ping = null; + + changes = await engine._tracker.getChangedIDs(); + _( + `test_sync_partialUpload: Rotary tracker contents at second sync: ${JSON.stringify( + changes + )}` + ); + try { + // should throw + await sync_engine_and_validate_telem( + engine, + true, + errPing => (ping = errPing) + ); + } catch (e) {} + // It would be nice if we had a more descriptive error for this... + let uploadFailureError = { + name: "httperror", + code: 500, + }; + + ok(!!ping); + deepEqual(ping.failureReason, uploadFailureError); + equal(ping.engines.length, 1); + equal(ping.engines[0].name, "rotary"); + deepEqual(ping.engines[0].incoming, { + failed: 1, + failedReasons: [{ name: "No ciphertext: nothing to decrypt?", count: 1 }], + }); + ok(!ping.engines[0].outgoing); + deepEqual(ping.engines[0].failureReason, uploadFailureError); + } finally { + await cleanAndGo(engine, server); + await engine.finalize(); + } +}); + +add_task(async function test_generic_engine_fail() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let e = new Error("generic failure message"); + engine._errToThrow = e; + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_generic_engine_fail: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: String(e), + }); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_engine_fail_weird_errors() { + enableValidationPrefs(); + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + try { + let msg = "Bad things happened!"; + engine._errToThrow = { message: msg }; + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: "Bad things happened!", + }); + }); + let e = { msg }; + engine._errToThrow = e; + await sync_and_validate_telem(ping => { + deepEqual(ping.engines.find(err => err.name === "steam").failureReason, { + name: "unexpectederror", + error: JSON.stringify(e), + }); + }); + } finally { + await cleanAndGo(engine, server); + Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_overrideTelemetryName() { + enableValidationPrefs(["steam"]); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.overrideTelemetryName = "steam-but-better"; + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + + const problemsToReport = [ + { name: "someProblem", count: 123 }, + { name: "anotherProblem", count: 456 }, + ]; + + try { + info("Sync with validation problems"); + engine.problemsToReport = problemsToReport; + await sync_and_validate_telem(ping => { + let enginePing = ping.engines.find(e => e.name === "steam-but-better"); + ok(enginePing); + ok(!ping.engines.find(e => e.name === "steam")); + deepEqual( + enginePing.validation, + { + version: 1, + checked: 0, + problems: problemsToReport, + }, + "Should include validation report with overridden name" + ); + }); + + info("Sync without validation problems"); + engine.problemsToReport = null; + await sync_and_validate_telem(ping => { + let enginePing = ping.engines.find(e => e.name === "steam-but-better"); + ok(enginePing); + ok(!ping.engines.find(e => e.name === "steam")); + ok( + !enginePing.validation, + "Should not include validation report when there are no problems" + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_engine_fail_ioerror() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + // create an IOError to re-throw as part of Sync. + try { + // (Note that fakeservices.js has replaced Utils.jsonMove etc, but for + // this test we need the real one so we get real exceptions from the + // filesystem.) + await Utils._real_jsonMove("file-does-not-exist", "anything", {}); + } catch (ex) { + engine._errToThrow = ex; + } + ok(engine._errToThrow, "expecting exception"); + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_engine_fail_ioerror: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + // ensure the profile dir in the exception message has been stripped. + ok( + !failureReason.error.includes(PathUtils.profileDir), + failureReason.error + ); + ok(failureReason.error.includes("[profileDir]"), failureReason.error); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_error_detections() { + let telem = get_sync_test_telemetry(); + + // Non-network NS_ERROR_ codes get their own category. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_FAILURE)), + { name: "nserror", code: Cr.NS_ERROR_FAILURE } + ); + + // Some NS_ERROR_ code in the "network" module are treated as http errors. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_UNKNOWN_HOST)), + { name: "httperror", code: Cr.NS_ERROR_UNKNOWN_HOST } + ); + // Some NS_ERROR_ABORT is treated as network by our telemetry. + Assert.deepEqual( + telem.transformError(Components.Exception("", Cr.NS_ERROR_ABORT)), + { name: "httperror", code: Cr.NS_ERROR_ABORT } + ); +}); + +add_task(async function test_clean_urls() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + engine._errToThrow = new TypeError( + "http://www.google .com is not a valid URL." + ); + + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_clean_urls: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal(failureReason.error, "<URL> is not a valid URL."); + }); + // Handle other errors that include urls. + engine._errToThrow = + "Other error message that includes some:url/foo/bar/ in it."; + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal( + failureReason.error, + "Other error message that includes <URL> in it." + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +// Test sanitizing guid-related errors with the pattern of <guid: {guid}> +add_task(async function test_sanitize_bookmarks_guid() { + let { ErrorSanitizer } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + + for (let [original, expected] of [ + [ + "Can't insert Bookmark <guid: sknD84IdnSY2> into Folder <guid: odfninDdi93_3>", + "Can't insert Bookmark <GUID> into Folder <GUID>", + ], + [ + "Merge Error: Item <guid: H6fmPA16gZs9> can't contain itself", + "Merge Error: Item <GUID> can't contain itself", + ], + ]) { + const sanitized = ErrorSanitizer.cleanErrorMessage(original); + Assert.equal(sanitized, expected); + } +}); + +// Test sanitization of some hard-coded error strings. +add_task(async function test_clean_errors() { + let { ErrorSanitizer } = ChromeUtils.importESModule( + "resource://services-sync/telemetry.sys.mjs" + ); + + for (let [original, expected] of [ + [ + "Win error 112 during operation write on file [profileDir]\\weave\\addonsreconciler.json (Espacio en disco insuficiente. )", + "OS error [No space left on device] during operation write on file [profileDir]/weave/addonsreconciler.json", + ], + [ + "Unix error 28 during operation write on file [profileDir]/weave/addonsreconciler.json (No space left on device)", + "OS error [No space left on device] during operation write on file [profileDir]/weave/addonsreconciler.json", + ], + ]) { + const sanitized = ErrorSanitizer.cleanErrorMessage(original); + Assert.equal(sanitized, expected); + } +}); + +// Arrange for a sync to hit a "real" OS error during a sync and make sure it's sanitized. +add_task(async function test_clean_real_os_error() { + enableValidationPrefs(); + + // Simulate a real error. + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let path = PathUtils.join(PathUtils.profileDir, "no", "such", "path.json"); + try { + await IOUtils.readJSON(path); + throw new Error("should fail to read the file"); + } catch (ex) { + engine._errToThrow = ex; + } + + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_clean_urls: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + let failureReason = ping.engines.find( + e => e.name === "steam" + ).failureReason; + equal(failureReason.name, "unexpectederror"); + equal( + failureReason.error, + "OS error [File/Path not found] Could not open the file at [profileDir]/no/such/path.json" + ); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_initial_sync_engines() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + // These are the only ones who actually have things to sync at startup. + let telemetryEngineNames = ["clients", "prefs", "tabs", "bookmarks-buffered"]; + let server = await serverForEnginesWithKeys( + { foo: "password" }, + ["bookmarks", "prefs", "tabs"].map(name => Service.engineManager.get(name)) + ); + await SyncTestingInfrastructure(server); + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_initial_sync_engines: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + let ping = await wait_for_ping(() => Service.sync(), true); + + equal(ping.engines.find(e => e.name === "clients").outgoing[0].sent, 1); + equal(ping.engines.find(e => e.name === "tabs").outgoing[0].sent, 1); + + // for the rest we don't care about specifics + for (let e of ping.engines) { + if (!telemetryEngineNames.includes(engine.name)) { + continue; + } + greaterOrEqual(e.took, 1); + ok(!!e.outgoing); + equal(e.outgoing.length, 1); + notEqual(e.outgoing[0].sent, undefined); + equal(e.outgoing[0].failed, undefined); + equal(e.outgoing[0].failedReasons, undefined); + } + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_nserror() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + engine._errToThrow = Components.Exception( + "NS_ERROR_UNKNOWN_HOST", + Cr.NS_ERROR_UNKNOWN_HOST + ); + try { + const changes = await engine._tracker.getChangedIDs(); + _(`test_nserror: Steam tracker contents: ${JSON.stringify(changes)}`); + await sync_and_validate_telem(ping => { + deepEqual(ping.status, { + service: SYNC_FAILED_PARTIAL, + sync: LOGIN_FAILED_NETWORK_ERROR, + }); + let enginePing = ping.engines.find(e => e.name === "steam"); + deepEqual(enginePing.failureReason, { + name: "httperror", + code: Cr.NS_ERROR_UNKNOWN_HOST, + }); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_sync_why() { + enableValidationPrefs(); + + await Service.engineManager.register(SteamEngine); + let engine = Service.engineManager.get("steam"); + engine.enabled = true; + let server = await serverForFoo(engine); + await SyncTestingInfrastructure(server); + let e = new Error("generic failure message"); + engine._errToThrow = e; + + try { + const changes = await engine._tracker.getChangedIDs(); + _( + `test_generic_engine_fail: Steam tracker contents: ${JSON.stringify( + changes + )}` + ); + let ping = await wait_for_ping( + () => Service.sync({ why: "user" }), + true, + false + ); + _(JSON.stringify(ping)); + equal(ping.why, "user"); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_discarding() { + enableValidationPrefs(); + + let helper = track_collections_helper(); + let upd = helper.with_updated_collection; + let telem = get_sync_test_telemetry(); + telem.maxPayloadCount = 2; + telem.submissionInterval = Infinity; + let oldSubmit = telem.submit; + + let server; + try { + let handlers = { + "/1.1/johndoe/info/collections": helper.handler, + "/1.1/johndoe/storage/crypto/keys": upd( + "crypto", + new ServerWBO("keys").handler() + ), + "/1.1/johndoe/storage/meta/global": upd( + "meta", + new ServerWBO("global").handler() + ), + }; + + let collections = [ + "clients", + "bookmarks", + "forms", + "history", + "passwords", + "prefs", + "tabs", + ]; + + for (let coll of collections) { + handlers["/1.1/johndoe/storage/" + coll] = upd( + coll, + new ServerCollection({}, true).handler() + ); + } + + server = httpd_setup(handlers); + await configureIdentity({ username: "johndoe" }, server); + telem.submit = p => + ok( + false, + "Submitted telemetry ping when we should not have" + JSON.stringify(p) + ); + + for (let i = 0; i < 5; ++i) { + await Service.sync(); + } + telem.submit = oldSubmit; + telem.submissionInterval = -1; + let ping = await wait_for_ping(() => Service.sync(), true, true); // with this we've synced 6 times + equal(ping.syncs.length, 2); + equal(ping.discarded, 4); + } finally { + telem.maxPayloadCount = 500; + telem.submissionInterval = -1; + telem.submit = oldSubmit; + if (server) { + await promiseStopServer(server); + } + } +}); + +add_task(async function test_submit_interval() { + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + let numSubmissions = 0; + telem.submit = function () { + numSubmissions += 1; + }; + + function notify(what, data = null) { + Svc.Obs.notify(what, JSON.stringify(data)); + } + + try { + // submissionInterval is set such that each sync should submit + notify("weave:service:sync:start", { why: "testing" }); + notify("weave:service:sync:finish"); + Assert.equal(numSubmissions, 1, "should submit this ping due to interval"); + + // As should each event outside of a sync. + Service.recordTelemetryEvent("object", "method"); + Assert.equal(numSubmissions, 2); + + // But events while we are syncing should not. + notify("weave:service:sync:start", { why: "testing" }); + Service.recordTelemetryEvent("object", "method"); + Assert.equal(numSubmissions, 2, "no submission for this event"); + notify("weave:service:sync:finish"); + Assert.equal(numSubmissions, 3, "was submitted after sync finish"); + } finally { + telem.submit = oldSubmit; + } +}); + +add_task(async function test_no_foreign_engines_in_error_ping() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + engine._errToThrow = new Error("Oh no!"); + await SyncTestingInfrastructure(server); + try { + await sync_and_validate_telem(ping => { + equal(ping.status.service, SYNC_FAILED_PARTIAL); + ok(ping.engines.every(e => e.name !== "bogus")); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_no_foreign_engines_in_success_ping() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + await sync_and_validate_telem(ping => { + ok(ping.engines.every(e => e.name !== "bogus")); + }); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_events() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + + let telem = get_sync_test_telemetry(); + telem.submissionInterval = Infinity; + + try { + let serverTime = Resource.serverTime; + Service.recordTelemetryEvent("object", "method", "value", { foo: "bar" }); + let ping = await wait_for_ping(() => Service.sync(), true, true); + equal(ping.events.length, 1); + let [timestamp, category, method, object, value, extra] = ping.events[0]; + ok(typeof timestamp == "number" && timestamp > 0); // timestamp. + equal(category, "sync"); + equal(method, "method"); + equal(object, "object"); + equal(value, "value"); + deepEqual(extra, { foo: "bar", serverTime: String(serverTime) }); + ping = await wait_for_ping( + () => { + // Test with optional values. + Service.recordTelemetryEvent("object", "method"); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 4); + + ping = await wait_for_ping( + () => { + Service.recordTelemetryEvent("object", "method", "extra"); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 5); + + ping = await wait_for_ping( + () => { + Service.recordTelemetryEvent("object", "method", undefined, { + foo: "bar", + }); + }, + false, + true + ); + equal(ping.events.length, 1); + equal(ping.events[0].length, 6); + [timestamp, category, method, object, value, extra] = ping.events[0]; + equal(value, null); + + // Fake a submission due to shutdown. + ping = await wait_for_ping( + () => { + telem.submissionInterval = Infinity; + Service.recordTelemetryEvent("object", "method", undefined, { + foo: "bar", + }); + telem.finish("shutdown"); + }, + false, + true + ); + equal(ping.syncs.length, 0); + equal(ping.events.length, 1); + equal(ping.events[0].length, 6); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_histograms() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + let histId = "TELEMETRY_TEST_LINEAR"; + Services.obs.notifyObservers(null, "weave:telemetry:histogram", histId); + let ping = await wait_for_ping(() => Service.sync(), true, true); + equal(Object.keys(ping.histograms).length, 1); + equal(ping.histograms[histId].sum, 0); + equal(ping.histograms[histId].histogram_type, 1); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_invalid_events() { + enableValidationPrefs(); + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + async function checkNotRecorded(...args) { + Service.recordTelemetryEvent.call(args); + let ping = await wait_for_ping(() => Service.sync(), false, true); + equal(ping.events, undefined); + } + + await SyncTestingInfrastructure(server); + try { + let long21 = "l".repeat(21); + let long81 = "l".repeat(81); + let long86 = "l".repeat(86); + await checkNotRecorded("object"); + await checkNotRecorded("object", 2); + await checkNotRecorded(2, "method"); + await checkNotRecorded("object", "method", 2); + await checkNotRecorded("object", "method", "value", 2); + await checkNotRecorded("object", "method", "value", { foo: 2 }); + await checkNotRecorded(long21, "method", "value"); + await checkNotRecorded("object", long21, "value"); + await checkNotRecorded("object", "method", long81); + let badextra = {}; + badextra[long21] = "x"; + await checkNotRecorded("object", "method", "value", badextra); + badextra = { x: long86 }; + await checkNotRecorded("object", "method", "value", badextra); + for (let i = 0; i < 10; i++) { + badextra["name" + i] = "x"; + } + await checkNotRecorded("object", "method", "value", badextra); + } finally { + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_no_ping_for_self_hosters() { + enableValidationPrefs(); + + let telem = get_sync_test_telemetry(); + let oldSubmit = telem.submit; + + await Service.engineManager.register(BogusEngine); + let engine = Service.engineManager.get("bogus"); + engine.enabled = true; + let server = await serverForFoo(engine); + + await SyncTestingInfrastructure(server); + try { + let submitPromise = new Promise(resolve => { + telem.submit = function () { + let result = oldSubmit.apply(this, arguments); + resolve(result); + }; + }); + await Service.sync(); + let pingSubmitted = await submitPromise; + // The Sync testing infrastructure already sets up a custom token server, + // so we don't need to do anything to simulate a self-hosted user. + ok(!pingSubmitted, "Should not submit ping with custom token server URL"); + } finally { + telem.submit = oldSubmit; + await cleanAndGo(engine, server); + await Service.engineManager.unregister(engine); + } +}); + +add_task(async function test_fxa_device_telem() { + let t = get_sync_test_telemetry(); + let syncEnabled = true; + let oldGetClientsEngineRecords = t.getClientsEngineRecords; + let oldGetFxaDevices = t.getFxaDevices; + let oldSyncIsEnabled = t.syncIsEnabled; + let oldSanitizeFxaDeviceId = t.sanitizeFxaDeviceId; + t.syncIsEnabled = () => syncEnabled; + t.sanitizeFxaDeviceId = id => `So clean: ${id}`; + try { + let keep0 = Utils.makeGUID(); + let keep1 = Utils.makeGUID(); + let keep2 = Utils.makeGUID(); + let curdev = Utils.makeGUID(); + + let keep1Sync = Utils.makeGUID(); + let keep2Sync = Utils.makeGUID(); + let curdevSync = Utils.makeGUID(); + let fxaDevices = [ + { + id: curdev, + isCurrentDevice: true, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "current device", + }, + { + id: keep0, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 10, + pushEndpointExpired: false, + type: "mobile", + name: "dupe", + }, + // Valid 2 + { + id: keep1, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "valid2", + }, + // Valid 3 + { + id: keep2, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 5, + pushEndpointExpired: false, + type: "desktop", + name: "valid3", + }, + ]; + let clientInfo = [ + { + id: keep1Sync, + fxaDeviceId: keep1, + os: "Windows 30", + version: "Firefox 1 million", + }, + { + id: keep2Sync, + fxaDeviceId: keep2, + os: "firefox, but an os", + verison: "twelve", + }, + { + id: Utils.makeGUID(), + fxaDeviceId: null, + os: "apparently ios used to keep write these IDs as null.", + version: "Doesn't seem to anymore", + }, + { + id: curdevSync, + fxaDeviceId: curdev, + os: "emacs", + version: "22", + }, + { + id: Utils.makeGUID(), + fxaDeviceId: Utils.makeGUID(), + os: "not part of the fxa device set at all", + version: "foo bar baz", + }, + // keep0 intententionally omitted. + ]; + t.getClientsEngineRecords = () => clientInfo; + let devInfo = t.updateFxaDevices(fxaDevices); + equal(devInfo.deviceID, t.sanitizeFxaDeviceId(curdev)); + for (let d of devInfo.devices) { + ok(d.id.startsWith("So clean:")); + if (d.syncID) { + ok(d.syncID.startsWith("So clean:")); + } + } + equal(devInfo.devices.length, 4); + let k0 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep0)); + let k1 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep1)); + let k2 = devInfo.devices.find(d => d.id == t.sanitizeFxaDeviceId(keep2)); + + deepEqual(k0, { + id: t.sanitizeFxaDeviceId(keep0), + type: "mobile", + os: undefined, + version: undefined, + syncID: undefined, + }); + deepEqual(k1, { + id: t.sanitizeFxaDeviceId(keep1), + type: "desktop", + os: clientInfo[0].os, + version: clientInfo[0].version, + syncID: t.sanitizeFxaDeviceId(keep1Sync), + }); + deepEqual(k2, { + id: t.sanitizeFxaDeviceId(keep2), + type: "desktop", + os: clientInfo[1].os, + version: clientInfo[1].version, + syncID: t.sanitizeFxaDeviceId(keep2Sync), + }); + let newCurId = Utils.makeGUID(); + // Update the ID + fxaDevices[0].id = newCurId; + + let keep3 = Utils.makeGUID(); + fxaDevices.push({ + id: keep3, + isCurrentDevice: false, + lastAccessTime: Date.now() - 1000 * 60 * 60 * 24 * 1, + pushEndpointExpired: false, + type: "desktop", + name: "valid 4", + }); + devInfo = t.updateFxaDevices(fxaDevices); + + let afterSubmit = [keep0, keep1, keep2, keep3, newCurId] + .map(id => t.sanitizeFxaDeviceId(id)) + .sort(); + deepEqual(devInfo.devices.map(d => d.id).sort(), afterSubmit); + + // Reset this, as our override doesn't check for sync being enabled. + t.sanitizeFxaDeviceId = oldSanitizeFxaDeviceId; + syncEnabled = false; + fxAccounts.telemetry._setHashedUID(false); + devInfo = t.updateFxaDevices(fxaDevices); + equal(devInfo.deviceID, undefined); + equal(devInfo.devices.length, 5); + for (let d of devInfo.devices) { + equal(d.os, undefined); + equal(d.version, undefined); + equal(d.syncID, undefined); + // Type should still be present. + notEqual(d.type, undefined); + } + } finally { + t.getClientsEngineRecords = oldGetClientsEngineRecords; + t.getFxaDevices = oldGetFxaDevices; + t.syncIsEnabled = oldSyncIsEnabled; + t.sanitizeFxaDeviceId = oldSanitizeFxaDeviceId; + } +}); + +add_task(async function test_sanitize_fxa_device_id() { + let t = get_sync_test_telemetry(); + fxAccounts.telemetry._setHashedUID(false); + sinon.stub(t, "syncIsEnabled").callsFake(() => true); + const rawDeviceId = "raw one two three"; + try { + equal(t.sanitizeFxaDeviceId(rawDeviceId), null); + fxAccounts.telemetry._setHashedUID("mock uid"); + const sanitizedDeviceId = t.sanitizeFxaDeviceId(rawDeviceId); + ok(sanitizedDeviceId); + notEqual(sanitizedDeviceId, rawDeviceId); + } finally { + t.syncIsEnabled.restore(); + fxAccounts.telemetry._setHashedUID(false); + } +}); + +add_task(async function test_no_node_type() { + let server = sync_httpd_setup({}); + await configureIdentity(null, server); + + await sync_and_validate_telem(ping => { + ok(ping.syncNodeType === undefined); + }, true); + await promiseStopServer(server); +}); + +add_task(async function test_node_type() { + Service.identity.logout(); + let server = sync_httpd_setup({}); + await configureIdentity({ node_type: "the-node-type" }, server); + + await sync_and_validate_telem(ping => { + equal(ping.syncNodeType, "the-node-type"); + }, true); + await promiseStopServer(server); +}); + +add_task(async function test_node_type_change() { + let pingPromise = wait_for_pings(2); + + Service.identity.logout(); + let server = sync_httpd_setup({}); + await configureIdentity({ node_type: "first-node-type" }, server); + // Default to submitting each hour - we should still submit on node change. + let telem = get_sync_test_telemetry(); + telem.submissionInterval = 60 * 60 * 1000; + // reset the node type from previous test or our first sync will submit. + telem.lastSyncNodeType = null; + // do 2 syncs with the same node type. + await Service.sync(); + await Service.sync(); + // then another with a different node type. + Service.identity.logout(); + await configureIdentity({ node_type: "second-node-type" }, server); + await Service.sync(); + telem.finish(); + + let pings = await pingPromise; + equal(pings.length, 2); + equal(pings[0].syncs.length, 2, "2 syncs in first ping"); + equal(pings[0].syncNodeType, "first-node-type"); + equal(pings[1].syncs.length, 1, "1 sync in second ping"); + equal(pings[1].syncNodeType, "second-node-type"); + await promiseStopServer(server); +}); + +add_task(async function test_ids() { + let telem = get_sync_test_telemetry(); + Assert.ok(!telem._shouldSubmitForDataChange()); + fxAccounts.telemetry._setHashedUID("new_uid"); + Assert.ok(telem._shouldSubmitForDataChange()); + telem.maybeSubmitForDataChange(); + // now it's been submitted the new uid is current. + Assert.ok(!telem._shouldSubmitForDataChange()); +}); + +add_task(async function test_deletion_request_ping() { + async function assertRecordedSyncDeviceID(expected) { + // The scalar gets updated asynchronously, so wait a tick before checking. + await Promise.resolve(); + const scalars = + Services.telemetry.getSnapshotForScalars("deletion-request").parent || {}; + equal(scalars["deletion.request.sync_device_id"], expected); + } + + const MOCK_HASHED_UID = "00112233445566778899aabbccddeeff"; + const MOCK_DEVICE_ID1 = "ffeeddccbbaa99887766554433221100"; + const MOCK_DEVICE_ID2 = "aabbccddeeff99887766554433221100"; + + // Calculated by hand using SHA256(DEVICE_ID + HASHED_UID)[:32] + const SANITIZED_DEVICE_ID1 = "dd7c845006df9baa1c6d756926519c8c"; + const SANITIZED_DEVICE_ID2 = "0d06919a736fc029007e1786a091882c"; + + let currentDeviceID = null; + sinon.stub(fxAccounts.device, "getLocalId").callsFake(() => { + return Promise.resolve(currentDeviceID); + }); + let telem = get_sync_test_telemetry(); + sinon.stub(telem, "isProductionSyncUser").callsFake(() => true); + fxAccounts.telemetry._setHashedUID(false); + try { + // The scalar should start out undefined, since no user is actually logged in. + await assertRecordedSyncDeviceID(undefined); + + // If we start up without knowing the hashed UID, it should stay undefined. + telem.observe(null, "weave:service:ready"); + await assertRecordedSyncDeviceID(undefined); + + // But now let's say we've discovered the hashed UID from the server. + fxAccounts.telemetry._setHashedUID(MOCK_HASHED_UID); + currentDeviceID = MOCK_DEVICE_ID1; + + // Now when we load up, we'll record the sync device id. + telem.observe(null, "weave:service:ready"); + await assertRecordedSyncDeviceID(SANITIZED_DEVICE_ID1); + + // When the device-id changes we'll update it. + currentDeviceID = MOCK_DEVICE_ID2; + telem.observe(null, "fxaccounts:new_device_id"); + await assertRecordedSyncDeviceID(SANITIZED_DEVICE_ID2); + + // When the user signs out we'll clear it. + telem.observe(null, "fxaccounts:onlogout"); + await assertRecordedSyncDeviceID(""); + } finally { + fxAccounts.telemetry._setHashedUID(false); + telem.isProductionSyncUser.restore(); + fxAccounts.device.getLocalId.restore(); + } +}); diff --git a/services/sync/tests/unit/test_tracker_addChanged.js b/services/sync/tests/unit/test_tracker_addChanged.js new file mode 100644 index 0000000000..7f510794fc --- /dev/null +++ b/services/sync/tests/unit/test_tracker_addChanged.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function test_tracker_basics() { + let tracker = new LegacyTracker("Tracker", Service); + + let id = "the_id!"; + + _("Make sure nothing exists yet.."); + let changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], null); + + _("Make sure adding of time 0 works"); + await tracker.addChangedID(id, 0); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 0); + + _("A newer time will replace the old 0"); + await tracker.addChangedID(id, 10); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 10); + + _("An older time will not replace the newer 10"); + await tracker.addChangedID(id, 5); + changes = await tracker.getChangedIDs(); + Assert.equal(changes[id], 10); + + _("Adding without time defaults to current time"); + await tracker.addChangedID(id); + changes = await tracker.getChangedIDs(); + Assert.ok(changes[id] > 10); +}); + +add_task(async function test_tracker_persistence() { + let tracker = new LegacyTracker("Tracker", Service); + let id = "abcdef"; + + let promiseSave = new Promise((resolve, reject) => { + let save = tracker._storage._save; + tracker._storage._save = function () { + save.call(tracker._storage).then(resolve, reject); + }; + }); + + await tracker.addChangedID(id, 5); + + await promiseSave; + + _("IDs saved."); + const changes = await tracker.getChangedIDs(); + Assert.equal(5, changes[id]); + + let json = await Utils.jsonLoad(["changes", "tracker"], tracker); + Assert.equal(5, json[id]); +}); diff --git a/services/sync/tests/unit/test_uistate.js b/services/sync/tests/unit/test_uistate.js new file mode 100644 index 0000000000..0139536ac0 --- /dev/null +++ b/services/sync/tests/unit/test_uistate.js @@ -0,0 +1,324 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +const { UIState } = ChromeUtils.importESModule( + "resource://services-sync/UIState.sys.mjs" +); + +const UIStateInternal = UIState._internal; + +add_task(async function test_isReady_unconfigured() { + UIState.reset(); + + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + // On the first call, returns false + // Does trigger a refresh of the state - even though services.sync.username + // is undefined we still need to check the account state. + ok(!UIState.isReady()); + // resfreshState is called when idle - so only check after idle. + await new Promise(resolve => { + Services.tm.idleDispatchToMainThread(resolve); + }); + ok(refreshState.called); + refreshState.resetHistory(); + + // On subsequent calls, only return true + ok(UIState.isReady()); + ok(!refreshState.called); + + refreshState.restore(); +}); + +add_task(async function test_isReady_signedin() { + UIState.reset(); + Services.prefs.setCharPref("services.sync.username", "foo"); + + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + // On the first call, returns false and triggers a refresh of the state + ok(!UIState.isReady()); + await new Promise(resolve => { + Services.tm.idleDispatchToMainThread(resolve); + }); + ok(refreshState.calledOnce); + refreshState.resetHistory(); + + // On subsequent calls, only return true + ok(UIState.isReady()); + ok(!refreshState.called); + + refreshState.restore(); +}); + +add_task(async function test_refreshState_signedin() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setCharPref("services.sync.lastSync", now); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ + verified: true, + uid: "123", + email: "foo@bar.com", + displayName: "Foo Bar", + avatar: "https://foo/bar", + }), + hasLocalSession: () => Promise.resolve(true), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_SIGNED_IN); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, "Foo Bar"); + equal(state.avatarURL, "https://foo/bar"); + equal(state.lastSync, now); + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_syncButNoFxA() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setStringPref("services.sync.lastSync", now); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => Promise.resolve(null), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, undefined); + equal(state.email, "test@test.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); // only set when STATUS_SIGNED_IN. + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; + Services.prefs.clearUserPref("services.sync.lastSync"); + Services.prefs.clearUserPref("services.sync.username"); +}); + +add_task(async function test_refreshState_signedin_profile_unavailable() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + const now = new Date().toString(); + Services.prefs.setCharPref("services.sync.lastSync", now); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + UIStateInternal.syncing = false; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + _internal: { + profile: { + getProfile: () => { + return Promise.reject(new Error("Profile unavailable")); + }, + }, + }, + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_SIGNED_IN); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, now); + equal(state.syncing, false); + + UIStateInternal.fxAccounts = fxAccountsOrig; + Services.prefs.clearUserPref("services.sync.lastSync"); + Services.prefs.clearUserPref("services.sync.username"); +}); + +add_task(async function test_refreshState_unverified() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: false, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_NOT_VERIFIED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_unverified_nosession() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: false, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(false), + }; + + let state = await UIState.refresh(); + + // No session should "win" over the unverified state. + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_refreshState_loginFailed() { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + let loginFailed = sinon.stub(UIStateInternal, "_loginFailed"); + loginFailed.returns(true); + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + }; + + let state = await UIState.refresh(); + + equal(state.status, UIState.STATUS_LOGIN_FAILED); + equal(state.uid, "123"); + equal(state.email, "foo@bar.com"); + equal(state.displayName, undefined); + equal(state.avatarURL, undefined); + equal(state.lastSync, undefined); + + loginFailed.restore(); + UIStateInternal.fxAccounts = fxAccountsOrig; +}); + +add_task(async function test_observer_refreshState() { + let refreshState = sinon.spy(UIStateInternal, "refreshState"); + + let shouldRefresh = [ + "weave:service:login:got-hashed-id", + "weave:service:login:error", + "weave:service:ready", + "fxaccounts:onverified", + "fxaccounts:onlogin", + "fxaccounts:onlogout", + "fxaccounts:profilechange", + ]; + + for (let topic of shouldRefresh) { + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, topic); + await uiUpdateObserved; + ok(refreshState.calledOnce); + refreshState.resetHistory(); + } + + refreshState.restore(); +}); + +// Drive the UIState in a configured state. +async function configureUIState(syncing, lastSync = new Date()) { + UIState.reset(); + const fxAccountsOrig = UIStateInternal.fxAccounts; + + UIStateInternal._syncing = syncing; + Services.prefs.setCharPref("services.sync.lastSync", lastSync.toString()); + Services.prefs.setStringPref("services.sync.username", "test@test.com"); + + UIStateInternal.fxAccounts = { + getSignedInUser: () => + Promise.resolve({ verified: true, uid: "123", email: "foo@bar.com" }), + hasLocalSession: () => Promise.resolve(true), + }; + await UIState.refresh(); + UIStateInternal.fxAccounts = fxAccountsOrig; +} + +add_task(async function test_syncStarted() { + await configureUIState(false); + + const oldState = Object.assign({}, UIState.get()); + ok(!oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, "weave:service:sync:start"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(newState.syncing); +}); + +add_task(async function test_syncFinished() { + let yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + await configureUIState(true, yesterday); + + const oldState = Object.assign({}, UIState.get()); + ok(oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.prefs.setCharPref("services.sync.lastSync", new Date().toString()); + Services.obs.notifyObservers(null, "weave:service:sync:finish"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(!newState.syncing); + ok(new Date(newState.lastSync) > new Date(oldState.lastSync)); +}); + +add_task(async function test_syncError() { + let yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + await configureUIState(true, yesterday); + + const oldState = Object.assign({}, UIState.get()); + ok(oldState.syncing); + + let uiUpdateObserved = observeUIUpdate(); + Services.obs.notifyObservers(null, "weave:service:sync:error"); + await uiUpdateObserved; + + const newState = Object.assign({}, UIState.get()); + ok(!newState.syncing); + deepEqual(newState.lastSync, oldState.lastSync); +}); + +function observeUIUpdate() { + return new Promise(resolve => { + let obs = (aSubject, aTopic, aData) => { + Services.obs.removeObserver(obs, aTopic); + const state = UIState.get(); + resolve(state); + }; + Services.obs.addObserver(obs, UIState.ON_UPDATE); + }); +} diff --git a/services/sync/tests/unit/test_utils_catch.js b/services/sync/tests/unit/test_utils_catch.js new file mode 100644 index 0000000000..590d04527f --- /dev/null +++ b/services/sync/tests/unit/test_utils_catch.js @@ -0,0 +1,119 @@ +const { Service } = ChromeUtils.importESModule( + "resource://services-sync/service.sys.mjs" +); + +add_task(async function run_test() { + _("Make sure catch when copied to an object will correctly catch stuff"); + let ret, rightThis, didCall, didThrow, wasCovfefe, wasLocked; + let obj = { + _catch: Utils.catch, + _log: { + debug(str) { + didThrow = str.search(/^Exception/) == 0; + }, + info(str) { + wasLocked = str.indexOf("Cannot start sync: already syncing?") == 0; + }, + }, + + func() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + })(); + }, + + callbacky() { + return this._catch( + async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + }, + async function (ex) { + wasCovfefe = ex && ex.message == "covfefe"; + } + )(); + }, + + lockedy() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + Utils.throwLockException(null); + })(); + }, + + lockedy_chained() { + return this._catch(async function () { + rightThis = this == obj; + didCall = true; + Utils.throwLockException(null); + })(); + }, + }; + + _("Make sure a normal call will call and return"); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(!didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(!wasLocked); + + _( + "Make sure catch/throw results in debug call and caller doesn't need to handle exception" + ); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.throwy(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(!wasLocked); + + _("Test callback for exception testing."); + rightThis = didCall = didThrow = wasLocked = false; + ret = await obj.callbacky(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.ok(wasCovfefe); + Assert.ok(!wasLocked); + + _("Test the lock-aware catch that Service uses."); + obj._catch = Service._catch; + rightThis = didCall = didThrow = wasLocked = false; + wasCovfefe = undefined; + ret = await obj.lockedy(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(wasLocked); + + _("Test the lock-aware catch that Service uses with a chained promise."); + rightThis = didCall = didThrow = wasLocked = false; + wasCovfefe = undefined; + ret = await obj.lockedy_chained(); + Assert.equal(ret, undefined); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.ok(didThrow); + Assert.equal(wasCovfefe, undefined); + Assert.ok(wasLocked); +}); diff --git a/services/sync/tests/unit/test_utils_deepEquals.js b/services/sync/tests/unit/test_utils_deepEquals.js new file mode 100644 index 0000000000..218cc21b72 --- /dev/null +++ b/services/sync/tests/unit/test_utils_deepEquals.js @@ -0,0 +1,51 @@ +_("Make sure Utils.deepEquals correctly finds items that are deeply equal"); + +Services.prefs.setBoolPref("security.allow_eval_with_system_principal", true); +registerCleanupFunction(() => { + Services.prefs.clearUserPref("security.allow_eval_with_system_principal"); +}); + +function run_test() { + let data = + '[NaN, undefined, null, true, false, Infinity, 0, 1, "a", "b", {a: 1}, {a: "a"}, [{a: 1}], [{a: true}], {a: 1, b: 2}, [1, 2], [1, 2, 3]]'; + _("Generating two copies of data:", data); + /* eslint-disable no-eval */ + let d1 = eval(data); + let d2 = eval(data); + /* eslint-enable no-eval */ + + d1.forEach(function (a) { + _("Testing", a, typeof a, JSON.stringify([a])); + let numMatch = 0; + + d2.forEach(function (b) { + if (Utils.deepEquals(a, b)) { + numMatch++; + _("Found a match", b, typeof b, JSON.stringify([b])); + } + }); + + let expect = 1; + if (isNaN(a) && typeof a == "number") { + expect = 0; + _("Checking NaN should result in no matches"); + } + + _("Making sure we found the correct # match:", expect); + _("Actual matches:", numMatch); + Assert.equal(numMatch, expect); + }); + + _("Make sure adding undefined properties doesn't affect equalness"); + let a = {}; + let b = { a: undefined }; + Assert.ok(Utils.deepEquals(a, b)); + a.b = 5; + Assert.ok(!Utils.deepEquals(a, b)); + b.b = 5; + Assert.ok(Utils.deepEquals(a, b)); + a.c = undefined; + Assert.ok(Utils.deepEquals(a, b)); + b.d = undefined; + Assert.ok(Utils.deepEquals(a, b)); +} diff --git a/services/sync/tests/unit/test_utils_deferGetSet.js b/services/sync/tests/unit/test_utils_deferGetSet.js new file mode 100644 index 0000000000..6db812b844 --- /dev/null +++ b/services/sync/tests/unit/test_utils_deferGetSet.js @@ -0,0 +1,50 @@ +_( + "Make sure various combinations of deferGetSet arguments correctly defer getting/setting properties to another object" +); + +function run_test() { + let base = function () {}; + base.prototype = { + dst: {}, + + get a() { + return "a"; + }, + set b(val) { + this.dst.b = val + "!!!"; + }, + }; + let src = new base(); + + _("get/set a single property"); + Utils.deferGetSet(base, "dst", "foo"); + src.foo = "bar"; + Assert.equal(src.dst.foo, "bar"); + Assert.equal(src.foo, "bar"); + + _("editing the target also updates the source"); + src.dst.foo = "baz"; + Assert.equal(src.dst.foo, "baz"); + Assert.equal(src.foo, "baz"); + + _("handle multiple properties"); + Utils.deferGetSet(base, "dst", ["p1", "p2"]); + src.p1 = "v1"; + src.p2 = "v2"; + Assert.equal(src.p1, "v1"); + Assert.equal(src.dst.p1, "v1"); + Assert.equal(src.p2, "v2"); + Assert.equal(src.dst.p2, "v2"); + + _("make sure existing getter keeps its functionality"); + Utils.deferGetSet(base, "dst", "a"); + src.a = "not a"; + Assert.equal(src.dst.a, "not a"); + Assert.equal(src.a, "a"); + + _("make sure existing setter keeps its functionality"); + Utils.deferGetSet(base, "dst", "b"); + src.b = "b"; + Assert.equal(src.dst.b, "b!!!"); + Assert.equal(src.b, "b!!!"); +} diff --git a/services/sync/tests/unit/test_utils_json.js b/services/sync/tests/unit/test_utils_json.js new file mode 100644 index 0000000000..5bf26b2361 --- /dev/null +++ b/services/sync/tests/unit/test_utils_json.js @@ -0,0 +1,95 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +add_task(async function test_roundtrip() { + _("Do a simple write of an array to json and read"); + await Utils.jsonSave("foo", {}, ["v1", "v2"]); + + let foo = await Utils.jsonLoad("foo", {}); + Assert.equal(typeof foo, "object"); + Assert.equal(foo.length, 2); + Assert.equal(foo[0], "v1"); + Assert.equal(foo[1], "v2"); +}); + +add_task(async function test_string() { + _("Try saving simple strings"); + await Utils.jsonSave("str", {}, "hi"); + + let str = await Utils.jsonLoad("str", {}); + Assert.equal(typeof str, "string"); + Assert.equal(str.length, 2); + Assert.equal(str[0], "h"); + Assert.equal(str[1], "i"); +}); + +add_task(async function test_number() { + _("Try saving a number"); + await Utils.jsonSave("num", {}, 42); + + let num = await Utils.jsonLoad("num", {}); + Assert.equal(typeof num, "number"); + Assert.equal(num, 42); +}); + +add_task(async function test_nonexistent_file() { + _("Try loading a non-existent file."); + let val = await Utils.jsonLoad("non-existent", {}); + Assert.equal(val, undefined); +}); + +add_task(async function test_save_logging() { + _("Verify that writes are logged."); + let trace; + await Utils.jsonSave( + "log", + { + _log: { + trace(msg) { + trace = msg; + }, + }, + }, + "hi" + ); + Assert.ok(!!trace); +}); + +add_task(async function test_load_logging() { + _("Verify that reads and read errors are logged."); + + // Write a file with some invalid JSON + let file = await IOUtils.getFile(PathUtils.profileDir, "weave", "log.json"); + let fos = Cc["@mozilla.org/network/file-output-stream;1"].createInstance( + Ci.nsIFileOutputStream + ); + let flags = + FileUtils.MODE_WRONLY | FileUtils.MODE_CREATE | FileUtils.MODE_TRUNCATE; + fos.init(file, flags, FileUtils.PERMS_FILE, fos.DEFER_OPEN); + let stream = Cc["@mozilla.org/intl/converter-output-stream;1"].createInstance( + Ci.nsIConverterOutputStream + ); + stream.init(fos, "UTF-8"); + stream.writeString("invalid json!"); + stream.close(); + + let trace, debug; + let obj = { + _log: { + trace(msg) { + trace = msg; + }, + debug(msg) { + debug = msg; + }, + }, + }; + let val = await Utils.jsonLoad("log", obj); + Assert.ok(!val); + Assert.ok(!!trace); + Assert.ok(!!debug); +}); diff --git a/services/sync/tests/unit/test_utils_keyEncoding.js b/services/sync/tests/unit/test_utils_keyEncoding.js new file mode 100644 index 0000000000..30a8a4f2aa --- /dev/null +++ b/services/sync/tests/unit/test_utils_keyEncoding.js @@ -0,0 +1,23 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +function run_test() { + Assert.equal( + Utils.encodeKeyBase32("foobarbafoobarba"), + "mzxw6ytb9jrgcztpn5rgc4tcme" + ); + Assert.equal( + Utils.decodeKeyBase32("mzxw6ytb9jrgcztpn5rgc4tcme"), + "foobarbafoobarba" + ); + Assert.equal( + Utils.encodeKeyBase32( + "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + ), + "aeaqcaibaeaqcaibaeaqcaibae" + ); + Assert.equal( + Utils.decodeKeyBase32("aeaqcaibaeaqcaibaeaqcaibae"), + "\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01" + ); +} diff --git a/services/sync/tests/unit/test_utils_lock.js b/services/sync/tests/unit/test_utils_lock.js new file mode 100644 index 0000000000..71d6486ff9 --- /dev/null +++ b/services/sync/tests/unit/test_utils_lock.js @@ -0,0 +1,76 @@ +_("Make sure lock prevents calling with a shared lock"); + +// Utility that we only use here. + +function do_check_begins(thing, startsWith) { + if (!(thing && thing.indexOf && thing.indexOf(startsWith) == 0)) { + do_throw(thing + " doesn't begin with " + startsWith); + } +} + +add_task(async function run_test() { + let ret, rightThis, didCall; + let state, lockState, lockedState, unlockState; + let obj = { + _lock: Utils.lock, + lock() { + lockState = ++state; + if (this._locked) { + lockedState = ++state; + return false; + } + this._locked = true; + return true; + }, + unlock() { + unlockState = ++state; + this._locked = false; + }, + + func() { + return this._lock("Test utils lock", async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this._lock("Test utils lock throwy", async function () { + rightThis = this == obj; + didCall = true; + return this.throwy(); + })(); + }, + }; + + _("Make sure a normal call will call and return"); + rightThis = didCall = false; + state = 0; + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + Assert.equal(lockState, 1); + Assert.equal(unlockState, 2); + Assert.equal(state, 2); + + _("Make sure code that calls locked code throws"); + ret = null; + rightThis = didCall = false; + try { + ret = await obj.throwy(); + do_throw("throwy internal call should have thrown!"); + } catch (ex) { + // Should throw an Error, not a string. + do_check_begins(ex.message, "Could not acquire lock"); + } + Assert.equal(ret, null); + Assert.ok(rightThis); + Assert.ok(didCall); + _("Lock should be called twice so state 3 is skipped"); + Assert.equal(lockState, 4); + Assert.equal(lockedState, 5); + Assert.equal(unlockState, 6); + Assert.equal(state, 6); +}); diff --git a/services/sync/tests/unit/test_utils_makeGUID.js b/services/sync/tests/unit/test_utils_makeGUID.js new file mode 100644 index 0000000000..b1104c1114 --- /dev/null +++ b/services/sync/tests/unit/test_utils_makeGUID.js @@ -0,0 +1,44 @@ +const base64url = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; + +function run_test() { + _("Make sure makeGUID makes guids of the right length/characters"); + _("Create a bunch of guids to make sure they don't conflict"); + let guids = []; + for (let i = 0; i < 1000; i++) { + let newGuid = Utils.makeGUID(); + _("Generated " + newGuid); + + // Verify that the GUID's length is correct, even when it's URL encoded. + Assert.equal(newGuid.length, 12); + Assert.equal(encodeURIComponent(newGuid).length, 12); + + // Verify that the GUID only contains base64url characters + Assert.ok( + Array.prototype.every.call(newGuid, function (chr) { + return base64url.includes(chr); + }) + ); + + // Verify that Utils.checkGUID() correctly identifies them as valid. + Assert.ok(Utils.checkGUID(newGuid)); + + // Verify uniqueness within our sample of 1000. This could cause random + // failures, but they should be extremely rare. Otherwise we'd have a + // problem with GUID collisions. + Assert.ok( + guids.every(function (g) { + return g != newGuid; + }) + ); + guids.push(newGuid); + } + + _("Make sure checkGUID fails for invalid GUIDs"); + Assert.ok(!Utils.checkGUID(undefined)); + Assert.ok(!Utils.checkGUID(null)); + Assert.ok(!Utils.checkGUID("")); + Assert.ok(!Utils.checkGUID("blergh")); + Assert.ok(!Utils.checkGUID("ThisGUIDisWayTooLong")); + Assert.ok(!Utils.checkGUID("Invalid!!!!!")); +} diff --git a/services/sync/tests/unit/test_utils_notify.js b/services/sync/tests/unit/test_utils_notify.js new file mode 100644 index 0000000000..5c0c3702a6 --- /dev/null +++ b/services/sync/tests/unit/test_utils_notify.js @@ -0,0 +1,97 @@ +_("Make sure notify sends out the right notifications"); +add_task(async function run_test() { + let ret, rightThis, didCall; + let obj = { + notify: Utils.notify("foo:"), + _log: { + trace() {}, + }, + + func() { + return this.notify("bar", "baz", async function () { + rightThis = this == obj; + didCall = true; + return 5; + })(); + }, + + throwy() { + return this.notify("bad", "one", async function () { + rightThis = this == obj; + didCall = true; + throw new Error("covfefe"); + })(); + }, + }; + + let state = 0; + let makeObs = function (topic) { + let obj2 = { + observe(subject, obsTopic, data) { + this.state = ++state; + this.subject = subject; + this.topic = obsTopic; + this.data = data; + }, + }; + + Svc.Obs.add(topic, obj2); + return obj2; + }; + + _("Make sure a normal call will call and return with notifications"); + rightThis = didCall = false; + let fs = makeObs("foo:bar:start"); + let ff = makeObs("foo:bar:finish"); + let fe = makeObs("foo:bar:error"); + ret = await obj.func(); + Assert.equal(ret, 5); + Assert.ok(rightThis); + Assert.ok(didCall); + + Assert.equal(fs.state, 1); + Assert.equal(fs.subject, undefined); + Assert.equal(fs.topic, "foo:bar:start"); + Assert.equal(fs.data, "baz"); + + Assert.equal(ff.state, 2); + Assert.equal(ff.subject, 5); + Assert.equal(ff.topic, "foo:bar:finish"); + Assert.equal(ff.data, "baz"); + + Assert.equal(fe.state, undefined); + Assert.equal(fe.subject, undefined); + Assert.equal(fe.topic, undefined); + Assert.equal(fe.data, undefined); + + _("Make sure a throwy call will call and throw with notifications"); + ret = null; + rightThis = didCall = false; + let ts = makeObs("foo:bad:start"); + let tf = makeObs("foo:bad:finish"); + let te = makeObs("foo:bad:error"); + try { + ret = await obj.throwy(); + do_throw("throwy should have thrown!"); + } catch (ex) { + Assert.equal(ex.message, "covfefe"); + } + Assert.equal(ret, null); + Assert.ok(rightThis); + Assert.ok(didCall); + + Assert.equal(ts.state, 3); + Assert.equal(ts.subject, undefined); + Assert.equal(ts.topic, "foo:bad:start"); + Assert.equal(ts.data, "one"); + + Assert.equal(tf.state, undefined); + Assert.equal(tf.subject, undefined); + Assert.equal(tf.topic, undefined); + Assert.equal(tf.data, undefined); + + Assert.equal(te.state, 4); + Assert.equal(te.subject.message, "covfefe"); + Assert.equal(te.topic, "foo:bad:error"); + Assert.equal(te.data, "one"); +}); diff --git a/services/sync/tests/unit/test_utils_passphrase.js b/services/sync/tests/unit/test_utils_passphrase.js new file mode 100644 index 0000000000..fa58086113 --- /dev/null +++ b/services/sync/tests/unit/test_utils_passphrase.js @@ -0,0 +1,45 @@ +/* eslint no-tabs:"off" */ + +function run_test() { + _("Normalize passphrase recognizes hyphens."); + const pp = "26ect2thczm599m2ffqarbicjq"; + const hyphenated = "2-6ect2-thczm-599m2-ffqar-bicjq"; + Assert.equal(Utils.normalizePassphrase(hyphenated), pp); + + _("Skip whitespace."); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase("aaaaaaaaaaaaaaaaaaaaaaaaaa ") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa ") + ); + Assert.equal( + "aaaaaaaaaaaaaaaaaaaaaaaaaa", + Utils.normalizePassphrase(" a-aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ") + ); + Assert.ok(Utils.isPassphrase("aaaaaaaaaaaaaaaaaaaaaaaaaa ")); + Assert.ok(Utils.isPassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa")); + Assert.ok(Utils.isPassphrase(" aaaaaaaaaaaaaaaaaaaaaaaaaa ")); + Assert.ok(Utils.isPassphrase(" a-aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ")); + Assert.ok(!Utils.isPassphrase(" -aaaaa-aaaaa-aaaaa-aaaaa-aaaaa ")); + + _("Normalizing 20-char passphrases."); + Assert.equal( + Utils.normalizePassphrase("abcde-abcde-abcde-abcde"), + "abcdeabcdeabcdeabcde" + ); + Assert.equal( + Utils.normalizePassphrase("a-bcde-abcde-abcde-abcde"), + "a-bcde-abcde-abcde-abcde" + ); + Assert.equal( + Utils.normalizePassphrase(" abcde-abcde-abcde-abcde "), + "abcdeabcdeabcdeabcde" + ); +} diff --git a/services/sync/tests/unit/xpcshell.ini b/services/sync/tests/unit/xpcshell.ini new file mode 100644 index 0000000000..b771c1c9c8 --- /dev/null +++ b/services/sync/tests/unit/xpcshell.ini @@ -0,0 +1,212 @@ +[DEFAULT] +head = head_appinfo.js ../../../common/tests/unit/head_helpers.js head_helpers.js head_http_server.js head_errorhandler_common.js +firefox-appdir = browser +prefs = + identity.fxaccounts.enabled=true +support-files = + addon1-search.json + bootstrap1-search.json + missing-sourceuri.json + missing-xpi-search.json + rewrite-search.json + sync_ping_schema.json + systemaddon-search.json + !/services/common/tests/unit/head_helpers.js + !/toolkit/components/extensions/test/xpcshell/head_sync.js + +# The manifest is roughly ordered from low-level to high-level. When making +# systemic sweeping changes, this makes it easier to identify errors closer to +# the source. + +# Ensure we can import everything. +[test_load_modules.js] + +# util contains a bunch of functionality used throughout. +[test_utils_catch.js] +[test_utils_deepEquals.js] +[test_utils_deferGetSet.js] +[test_utils_keyEncoding.js] +[test_utils_json.js] +[test_utils_lock.js] +[test_utils_makeGUID.js] +run-sequentially = Disproportionately slows down full test run, bug 1450316 +[test_utils_notify.js] +[test_utils_passphrase.js] + +# We have a number of other libraries that are pretty much standalone. +[test_addon_utils.js] +run-sequentially = Restarts server, can't change pref. +tags = addons +[test_httpd_sync_server.js] + +# HTTP layers. +[test_resource.js] +[test_resource_header.js] +[test_resource_ua.js] + +# Generic Sync types. +[test_collection_getBatched.js] +[test_collections_recovery.js] +[test_keys.js] +[test_records_crypto.js] +[test_records_wbo.js] +[test_sync_auth_manager.js] + +# Engine APIs. +[test_engine.js] +[test_engine_abort.js] +[test_engine_changes_during_sync.js] +skip-if = appname == 'thunderbird' +[test_enginemanager.js] +[test_syncengine.js] +[test_syncengine_sync.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_tracker_addChanged.js] + +# Service semantics. +[test_service_attributes.js] +# Bug 752243: Profile cleanup frequently fails +skip-if = + os == "mac" + os == "linux" +[test_service_cluster.js] +[test_service_detect_upgrade.js] +skip-if = appname == 'thunderbird' +[test_service_login.js] +[test_service_startOver.js] +[test_service_startup.js] +[test_service_sync_401.js] +[test_service_sync_locked.js] +[test_service_sync_remoteSetup.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_service_sync_specified.js] +[test_service_sync_updateEnabledEngines.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_service_verifyLogin.js] +[test_service_wipeClient.js] +[test_service_wipeServer.js] +# Bug 752243: Profile cleanup frequently fails +skip-if = + os == "mac" + os == "linux" + +[test_corrupt_keys.js] +skip-if = appname == 'thunderbird' +[test_declined.js] +[test_errorhandler_1.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_errorhandler_2.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_errorhandler_filelog.js] +[test_errorhandler_sync_checkServerError.js] +[test_hmac_error.js] +[test_interval_triggers.js] +[test_node_reassignment.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_score_triggers.js] +[test_status.js] +[test_status_checkSetup.js] +[test_syncscheduler.js] +run-sequentially = Frequent timeouts, bug 1395148 + +# Firefox Accounts specific tests +[test_fxa_service_cluster.js] +[test_fxa_node_reassignment.js] +run-sequentially = Frequent timeouts, bug 1395148 + +# Finally, we test each engine. +[test_addons_engine.js] +run-sequentially = Hardcoded port in static files. +tags = addons +[test_addons_reconciler.js] +skip-if = appname == 'thunderbird' +tags = addons +[test_addons_store.js] +run-sequentially = Hardcoded port in static files. +tags = addons +[test_addons_tracker.js] +tags = addons +[test_addons_validator.js] +tags = addons +[test_bookmark_batch_fail.js] +skip-if = appname == 'thunderbird' +[test_bookmark_engine.js] +skip-if = + appname == 'thunderbird' + tsan # Runs unreasonably slow on TSan, bug 1612707 +[test_bookmark_order.js] +skip-if = appname == 'thunderbird' +[test_bookmark_places_query_rewriting.js] +skip-if = appname == 'thunderbird' +[test_bookmark_record.js] +skip-if = appname == 'thunderbird' +[test_bookmark_store.js] +skip-if = appname == 'thunderbird' +[test_bookmark_decline_undecline.js] +skip-if = appname == 'thunderbird' +[test_bookmark_tracker.js] +skip-if = + appname == 'thunderbird' + tsan # Runs unreasonably slow on TSan, bug 1612707 +requesttimeoutfactor = 4 +[test_bridged_engine.js] +[test_clients_engine.js] +run-sequentially = Frequent timeouts, bug 1395148 +[test_clients_escape.js] +[test_extension_storage_migration_telem.js] +skip-if = appname == 'thunderbird' +run-sequentially = extension-storage migration happens only once, and must be tested first. +[test_extension_storage_engine.js] +skip-if = appname == 'thunderbird' +run-sequentially = extension-storage migration happens only once, and must be tested first. +[test_extension_storage_engine_kinto.js] +skip-if = appname == 'thunderbird' +run-sequentially = extension-storage migration happens only once, and must be tested first. +[test_extension_storage_tracker_kinto.js] +skip-if = appname == 'thunderbird' +[test_forms_store.js] +skip-if = appname == 'thunderbird' +[test_forms_tracker.js] +skip-if = appname == 'thunderbird' +[test_form_validator.js] +skip-if = appname == 'thunderbird' +[test_history_engine.js] +skip-if = appname == 'thunderbird' +[test_history_store.js] +skip-if = appname == 'thunderbird' +[test_history_tracker.js] +skip-if = appname == 'thunderbird' +[test_password_engine.js] +[test_password_store.js] +[test_password_validator.js] +[test_password_tracker.js] +[test_prefs_engine.js] +skip-if = appname == 'thunderbird' +[test_prefs_store.js] +skip-if = appname == 'thunderbird' +support-files = prefs_test_prefs_store.js +[test_prefs_tracker.js] +skip-if = appname == 'thunderbird' +[test_tab_engine.js] +skip-if = appname == 'thunderbird' +[test_tab_quickwrite.js] +skip-if = appname == 'thunderbird' +[test_tab_provider.js] +skip-if = appname == 'thunderbird' +[test_tab_tracker.js] +skip-if = appname == 'thunderbird' + +[test_postqueue.js] + +# Synced tabs. +[test_syncedtabs.js] + +[test_telemetry.js] +skip-if = + appname == 'thunderbird' + tsan # Unreasonably slow, bug 1612707 +requesttimeoutfactor = 4 + +[test_uistate.js] +[test_412.js] +[test_disconnect_shutdown.js] |